From 9c9245601b934239aaeeb3062e0d7e2dbf13a92b Mon Sep 17 00:00:00 2001
From: Mazyar <m.nazemi@ravanertebat.com>
Date: Sun, 22 Feb 2026 10:55:03 +0330
Subject: [PATCH] added functionality to limit the maximum number of notices
 per day

---
 cmd/scraper/bootstrap/bootstrap.go |  2 ++
 cmd/scraper/bootstrap/config.go    |  1 +
 ted/scraper.go                     | 10 ++++++++++
 3 files changed, 13 insertions(+)

diff --git a/cmd/scraper/bootstrap/bootstrap.go b/cmd/scraper/bootstrap/bootstrap.go
index 7978cac..e21eb5a 100644
--- a/cmd/scraper/bootstrap/bootstrap.go
+++ b/cmd/scraper/bootstrap/bootstrap.go
@@ -91,6 +91,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
 			CleanupAfter:     config.TED.CleanupAfter,
 			ScrapingInterval: config.TED.ScrapingInterval,
 			AlertMail:        config.AlertMail,
+			MaxNoticesPerDay: config.TED.MaxNoticesPerDay,
 		},
 		appLogger,
 		mongoManager,
@@ -189,6 +190,7 @@ func RunOneTimeScraping(
 		CleanupAfter:     config.TED.CleanupAfter,
 		ScrapingInterval: config.TED.ScrapingInterval,
 		AlertMail:        config.AlertMail,
+		MaxNoticesPerDay: config.TED.MaxNoticesPerDay,
 	}
 
 	// Initialize TED scraper
diff --git a/cmd/scraper/bootstrap/config.go b/cmd/scraper/bootstrap/config.go
index 472f3dd..5dba704 100644
--- a/cmd/scraper/bootstrap/config.go
+++ b/cmd/scraper/bootstrap/config.go
@@ -27,4 +27,5 @@ type ScraperConfig struct {
 	OneTime          bool          `env:"TED_ONE_TIME" envDefault:"false"`
 	FromDate         string        `env:"TED_FROM_DATE" envDefault:"02/01/2025"`
 	ToDate           string        `env:"TED_TO_DATE" envDefault:"13/09/2025"`
+	MaxNoticesPerDay int           `env:"TED_MAX_NOTICES_PER_DAY" envDefault:"0"`
 }
diff --git a/ted/scraper.go b/ted/scraper.go
index 403fd72..143d3e2 100644
--- a/ted/scraper.go
+++ b/ted/scraper.go
@@ -28,6 +28,7 @@ type Config struct {
 	CleanupAfter     time.Duration `mapstructure:"cleanup_after"`
 	ScrapingInterval string        `mapstructure:"scraping_interval"`
 	AlertMail        string        `mapstructure:"ALERT_MAIL"`
+	MaxNoticesPerDay int           `mapstructure:"max_notices_per_day"`
 }
 
 // TEDScraper handles downloading and parsing TED XML files
@@ -235,6 +236,15 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
 			continue
 		}
 
+		// Skip processing if per-day limit reached (for UAT/low storage)
+		if s.config.MaxNoticesPerDay > 0 && result.SuccessCount >= s.config.MaxNoticesPerDay {
+			s.logger.Info("Reached max notices per day limit, stopping processing", map[string]interface{}{
+				"limit": s.config.MaxNoticesPerDay,
+				"ojs":   ojs,
+			})
+			break
+		}
+
 		// Parse and process XML content
 		if err := s.processXMLContent(ctx, content, fileName, result); err != nil {
 			errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err)