Enhance dashboard statistics with TED notice tracking

- Added a new field, ScrapedTEDNotices, to the StatisticsLifetimeTotals struct to track the total number of TED notices scraped.
- Updated the Statistics method in the statistics repository to include a background process for retrieving total scraped TED notices, improving the accuracy of dashboard statistics.
- Introduced new methods in the Counter to increment and retrieve daily counts for scraped TED notices, ensuring reliable metrics for reporting.
- Modified the TEDScraper to increment the TED notice scraped counter upon successful import, enhancing the tracking of scraping activity.

This update improves the dashboard's statistics by providing detailed insights into TED notice scraping activities, contributing to better data visibility and reporting.

pkg/mongo/counter.go

@@ -16,6 +16,11 @@ const (
 	AITranslationSuccessCounterKey              = "ai_translation_successful_requests"
 	AITranslationSuccessCounterKeyDailyJob      = "ai_translation_successful_requests_daily_job"
 	AITranslationSuccessCounterKeyManualTrigger = "ai_translation_successful_requests_manual_trigger"
+	// TEDNoticeScrapedCounterKey tracks TED notices successfully scraped/ingested. This is
+	// recorded independently of the "notices" collection because that collection is a transient
+	// processing queue: the worker deletes notices once they've been promoted into tenders, so
+	// counting from "notices" directly undercounts (or zeroes out) historical scrape activity.
+	TEDNoticeScrapedCounterKey = "ted_notice_scraped"
 )
 
 // Counter provides atomic increment/read operations for named metrics stored in MongoDB.
@@ -96,8 +101,36 @@ func AITranslationSuccessDailyCounterKey(day time.Time) string {
 	return AITranslationSuccessCounterKey + "_day:" + day.UTC().Format("2006-01-02")
 }
 
-// GetDailyCounts returns counter values for each UTC day from start through end inclusive.
+// TEDNoticeScrapedDailyCounterKey returns the metrics key for TED notices scraped on a given UTC day.
+func TEDNoticeScrapedDailyCounterKey(day time.Time) string {
+	return TEDNoticeScrapedCounterKey + "_day:" + day.UTC().Format("2006-01-02")
+}
+
+// IncrementTEDNoticeScraped atomically increments the lifetime and today's daily counters for
+// successfully scraped/ingested TED notices. Call this once per newly created notice document
+// (not on updates/refreshes) so totals reflect distinct scrape events.
+func (c *Counter) IncrementTEDNoticeScraped(ctx context.Context) error {
+	if _, err := c.Increment(ctx, TEDNoticeScrapedCounterKey); err != nil {
+		return err
+	}
+	_, err := c.Increment(ctx, TEDNoticeScrapedDailyCounterKey(time.Now().UTC()))
+	return err
+}
+
+// GetDailyCounts returns counter values for each UTC day from start through end inclusive, for
+// the AI translation daily counter.
 func (c *Counter) GetDailyCounts(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
+	return c.getDailyCounts(ctx, startDay, endDay, AITranslationSuccessDailyCounterKey)
+}
+
+// GetTEDNoticeScrapedDailyCounts returns counter values for each UTC day from start through end
+// inclusive, for the TED-notice-scraped daily counter.
+func (c *Counter) GetTEDNoticeScrapedDailyCounts(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
+	return c.getDailyCounts(ctx, startDay, endDay, TEDNoticeScrapedDailyCounterKey)
+}
+
+// getDailyCounts is the shared implementation behind the per-metric GetXDailyCounts helpers above.
+func (c *Counter) getDailyCounts(ctx context.Context, startDay, endDay time.Time, dailyKey func(time.Time) string) (map[string]int64, error) {
 	startDay = time.Date(startDay.Year(), startDay.Month(), startDay.Day(), 0, 0, 0, 0, time.UTC)
 	endDay = time.Date(endDay.Year(), endDay.Month(), endDay.Day(), 0, 0, 0, 0, time.UTC)
 	if endDay.Before(startDay) {
@@ -108,7 +141,7 @@ func (c *Counter) GetDailyCounts(ctx context.Context, startDay, endDay time.Time
 	dates := make([]string, 0)
 	for day := startDay; !day.After(endDay); day = day.AddDate(0, 0, 1) {
 		dates = append(dates, day.Format("2006-01-02"))
-		keys = append(keys, AITranslationSuccessDailyCounterKey(day))
+		keys = append(keys, dailyKey(day))
 	}
 
 	counts := make(map[string]int64, len(dates))