added functionality to limit the maximum number of notices per day

ted/scraper.go

@@ -28,6 +28,7 @@ type Config struct {
 	CleanupAfter     time.Duration `mapstructure:"cleanup_after"`
 	ScrapingInterval string        `mapstructure:"scraping_interval"`
 	AlertMail        string        `mapstructure:"ALERT_MAIL"`
+	MaxNoticesPerDay int           `mapstructure:"max_notices_per_day"`
 }
 
 // TEDScraper handles downloading and parsing TED XML files
@@ -235,6 +236,15 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
 			continue
 		}
 
+		// Skip processing if per-day limit reached (for UAT/low storage)
+		if s.config.MaxNoticesPerDay > 0 && result.SuccessCount >= s.config.MaxNoticesPerDay {
+			s.logger.Info("Reached max notices per day limit, stopping processing", map[string]interface{}{
+				"limit": s.config.MaxNoticesPerDay,
+				"ojs":   ojs,
+			})
+			break
+		}
+
 		// Parse and process XML content
 		if err := s.processXMLContent(ctx, content, fileName, result); err != nil {
 			errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err)