mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added functionality to limit the maximum number of notices per day

Browse Source
This commit is contained in:
Mazyar
2026-02-22 10:55:03 +03:30
parent e6005ee0ee
commit 9c9245601b
3 changed files with 13 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/scraper/bootstrap/bootstrap.go
+2
View File
@@ -91,6 +91,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
CleanupAfter: config.TED.CleanupAfter, CleanupAfter: config.TED.CleanupAfter,
ScrapingInterval: config.TED.ScrapingInterval, ScrapingInterval: config.TED.ScrapingInterval,
AlertMail: config.AlertMail, AlertMail: config.AlertMail,
MaxNoticesPerDay: config.TED.MaxNoticesPerDay,
}, },
appLogger, appLogger,
mongoManager, mongoManager,
@@ -189,6 +190,7 @@ func RunOneTimeScraping(
CleanupAfter: config.TED.CleanupAfter, CleanupAfter: config.TED.CleanupAfter,
ScrapingInterval: config.TED.ScrapingInterval, ScrapingInterval: config.TED.ScrapingInterval,
AlertMail: config.AlertMail, AlertMail: config.AlertMail,
MaxNoticesPerDay: config.TED.MaxNoticesPerDay,
} }
// Initialize TED scraper // Initialize TED scraper
cmd/scraper/bootstrap/config.go
+1
View File
@@ -27,4 +27,5 @@ type ScraperConfig struct {
OneTime bool `env:"TED_ONE_TIME" envDefault:"false"` OneTime bool `env:"TED_ONE_TIME" envDefault:"false"`
FromDate string `env:"TED_FROM_DATE" envDefault:"02/01/2025"` FromDate string `env:"TED_FROM_DATE" envDefault:"02/01/2025"`
ToDate string `env:"TED_TO_DATE" envDefault:"13/09/2025"` ToDate string `env:"TED_TO_DATE" envDefault:"13/09/2025"`
MaxNoticesPerDay int `env:"TED_MAX_NOTICES_PER_DAY" envDefault:"0"`
} }
ted/scraper.go
+10
View File
@@ -28,6 +28,7 @@ type Config struct {
CleanupAfter time.Duration `mapstructure:"cleanup_after"` CleanupAfter time.Duration `mapstructure:"cleanup_after"`
ScrapingInterval string `mapstructure:"scraping_interval"` ScrapingInterval string `mapstructure:"scraping_interval"`
AlertMail string `mapstructure:"ALERT_MAIL"` AlertMail string `mapstructure:"ALERT_MAIL"`
MaxNoticesPerDay int `mapstructure:"max_notices_per_day"`
} }
// TEDScraper handles downloading and parsing TED XML files // TEDScraper handles downloading and parsing TED XML files
@@ -235,6 +236,15 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
continue continue
} }
// Skip processing if per-day limit reached (for UAT/low storage)
if s.config.MaxNoticesPerDay > 0 && result.SuccessCount >= s.config.MaxNoticesPerDay {
s.logger.Info("Reached max notices per day limit, stopping processing", map[string]interface{}{
"limit": s.config.MaxNoticesPerDay,
"ojs": ojs,
})
break
}
// Parse and process XML content // Parse and process XML content
if err := s.processXMLContent(ctx, content, fileName, result); err != nil { if err := s.processXMLContent(ctx, content, fileName, result); err != nil {
errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err) errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err)