Merge pull request 'added functionality to limit the maximum number of notices per day' (#18) from TM-367 into develop

Reviewed-on: https://repo.ravanertebat.com/TM/tm_back/pulls/18

cmd/scraper/bootstrap/bootstrap.go

@@ -91,6 +91,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
 			CleanupAfter:     config.TED.CleanupAfter,
 			ScrapingInterval: config.TED.ScrapingInterval,
 			AlertMail:        config.AlertMail,
+			MaxNoticesPerDay: config.TED.MaxNoticesPerDay,
 		},
 		appLogger,
 		mongoManager,
@@ -189,6 +190,7 @@ func RunOneTimeScraping(
 		CleanupAfter:     config.TED.CleanupAfter,
 		ScrapingInterval: config.TED.ScrapingInterval,
 		AlertMail:        config.AlertMail,
+		MaxNoticesPerDay: config.TED.MaxNoticesPerDay,
 	}
 
 	// Initialize TED scraper

cmd/scraper/bootstrap/config.go

@@ -27,4 +27,5 @@ type ScraperConfig struct {
 	OneTime          bool          `env:"TED_ONE_TIME" envDefault:"false"`
 	FromDate         string        `env:"TED_FROM_DATE" envDefault:"02/01/2025"`
 	ToDate           string        `env:"TED_TO_DATE" envDefault:"13/09/2025"`
+	MaxNoticesPerDay int           `env:"TED_MAX_NOTICES_PER_DAY" envDefault:"0"`
 }

ted/scraper.go

@@ -28,6 +28,7 @@ type Config struct {
 	CleanupAfter     time.Duration `mapstructure:"cleanup_after"`
 	ScrapingInterval string        `mapstructure:"scraping_interval"`
 	AlertMail        string        `mapstructure:"ALERT_MAIL"`
+	MaxNoticesPerDay int           `mapstructure:"max_notices_per_day"`
 }
 
 // TEDScraper handles downloading and parsing TED XML files
@@ -235,6 +236,15 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
 			continue
 		}
 
+		// Skip processing if per-day limit reached (for UAT/low storage)
+		if s.config.MaxNoticesPerDay > 0 && result.SuccessCount >= s.config.MaxNoticesPerDay {
+			s.logger.Info("Reached max notices per day limit, stopping processing", map[string]interface{}{
+				"limit": s.config.MaxNoticesPerDay,
+				"ojs":   ojs,
+			})
+			break
+		}
+
 		// Parse and process XML content
 		if err := s.processXMLContent(ctx, content, fileName, result); err != nil {
 			errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err)