Enhance dashboard statistics with TED notice tracking

- Added a new field, ScrapedTEDNotices, to the StatisticsLifetimeTotals struct to track the total number of TED notices scraped.
- Updated the Statistics method in the statistics repository to include a background process for retrieving total scraped TED notices, improving the accuracy of dashboard statistics.
- Introduced new methods in the Counter to increment and retrieve daily counts for scraped TED notices, ensuring reliable metrics for reporting.
- Modified the TEDScraper to increment the TED notice scraped counter upon successful import, enhancing the tracking of scraping activity.

This update improves the dashboard's statistics by providing detailed insights into TED notice scraping activities, contributing to better data visibility and reporting.

ted/scraper.go

@@ -34,13 +34,14 @@ type Config struct {
 
 // TEDScraper handles downloading and parsing TED XML files
 type TEDScraper struct {
-	notify       notification.SDK
-	config       *Config
-	logger       logger.Logger
-	httpClient   *http.Client
-	xmlParser    *TEDParser
-	mongoManager *orm.ConnectionManager
-	noticeRepo   notice.Repository
+	notify         notification.SDK
+	config         *Config
+	logger         logger.Logger
+	httpClient     *http.Client
+	xmlParser      *TEDParser
+	mongoManager   *orm.ConnectionManager
+	noticeRepo     notice.Repository
+	metricsCounter *orm.Counter
 }
 
 // NewTEDScraper creates a new TED scraper instance
@@ -56,13 +57,14 @@ func NewTEDScraper(
 	}
 
 	return &TEDScraper{
-		notify:       notify,
-		config:       config,
-		logger:       logger,
-		httpClient:   httpClient,
-		xmlParser:    NewTEDParser(),
-		mongoManager: mongoManager,
-		noticeRepo:   noticeRepo,
+		notify:         notify,
+		config:         config,
+		logger:         logger,
+		httpClient:     httpClient,
+		xmlParser:      NewTEDParser(),
+		mongoManager:   mongoManager,
+		noticeRepo:     noticeRepo,
+		metricsCounter: orm.NewCounter(mongoManager),
 	}
 }
 
@@ -420,6 +422,18 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 			return fmt.Errorf("failed to import notice: %w", err)
 		}
 
+		// Track scraping activity in a persistent counter (independent of the "notices"
+		// collection) since the worker deletes processed notices, which would otherwise make
+		// scrape history unrecoverable for dashboard reporting.
+		if s.metricsCounter != nil {
+			if incErr := s.metricsCounter.IncrementTEDNoticeScraped(ctx); incErr != nil {
+				s.logger.Warn("Failed to increment TED scraped notice counter", map[string]interface{}{
+					"contract_notice_id": t.ContractNoticeID,
+					"error":              incErr.Error(),
+				})
+			}
+		}
+
 		s.logger.Info("Successfully imported new notice", map[string]interface{}{
 			"contract_notice_id": t.ContractNoticeID,
 		})