Enhance dashboard statistics with TED notice tracking

- Added a new field, ScrapedTEDNotices, to the StatisticsLifetimeTotals struct to track the total number of TED notices scraped.
- Updated the Statistics method in the statistics repository to include a background process for retrieving total scraped TED notices, improving the accuracy of dashboard statistics.
- Introduced new methods in the Counter to increment and retrieve daily counts for scraped TED notices, ensuring reliable metrics for reporting.
- Modified the TEDScraper to increment the TED notice scraped counter upon successful import, enhancing the tracking of scraping activity.

This update improves the dashboard's statistics by providing detailed insights into TED notice scraping activities, contributing to better data visibility and reporting.

internal/dashboard/entity.go

@@ -105,4 +105,5 @@ type StatisticsDailySeries struct {
 type StatisticsLifetimeTotals struct {
 	ScrapedDocuments  int64 `json:"scraped_documents"`
 	TranslatedTenders int64 `json:"translated_tenders"`
+	ScrapedTEDNotices int64 `json:"scraped_ted_notices"`
 }

internal/dashboard/statistics_repository.go

@@ -6,7 +6,6 @@ import (
 	"sync"
 	"time"
 
-	"tm/internal/notice"
 	"tm/pkg/ai_summarizer"
 	orm "tm/pkg/mongo"
 
@@ -14,8 +13,6 @@ import (
 	mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
 )
 
-const noticesCollectionName = "notices"
-
 // scrapedScopeCacheTTL avoids listing every MinIO object on each statistics request while
 // keeping counts aligned with the tender panel (MinIO is the source of truth for scraped docs).
 const scrapedScopeCacheTTL = 15 * time.Minute
@@ -49,6 +46,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		translatedNotices map[string]int64
 		totalDocuments    int64
 		totalTranslated   int64
+		totalScrapedTED   int64
 		scrapedScope      scrapedTendersScope
 	)
 
@@ -70,10 +68,10 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout)
 		defer cancel()
 
-		counts, err := r.dailyCountByTimestamp(qctx, noticesCollectionName, bson.M{
-			"source":     notice.TenderSourceTEDScraper,
-			"created_at": bson.M{"$gte": startUnix, "$gt": 0},
-		}, "created_at")
+		// Read from the persistent metrics counter rather than the "notices" collection: the
+		// notice worker deletes processed notices shortly after promoting them into tenders, so
+		// counting from "notices" directly would undercount (often to zero) recent scrape activity.
+		counts, err := r.metricsCounter.GetTEDNoticeScrapedDailyCounts(qctx, startDay, endDay)
 		if err != nil {
 			recordFailure("scraped_ted", err)
 			scrapedTED = map[string]int64{}
@@ -82,6 +80,20 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		scrapedTED = counts
 	}()
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout)
+		defer cancel()
+
+		total, err := r.metricsCounter.Get(qctx, orm.TEDNoticeScrapedCounterKey)
+		if err != nil {
+			recordFailure("scraped_ted_total", err)
+			return
+		}
+		totalScrapedTED = total
+	}()
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
@@ -167,37 +179,11 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		Totals: StatisticsLifetimeTotals{
 			ScrapedDocuments:  totalDocuments,
 			TranslatedTenders: totalTranslated,
+			ScrapedTEDNotices: totalScrapedTED,
 		},
 	}, nil
 }
 
-func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) {
-	pipeline := mongodriver.Pipeline{
-		{{Key: "$match", Value: match}},
-		{{Key: "$addFields", Value: bson.M{
-			"metric_ts": normalizeTimestampExpr(timestampField),
-		}}},
-		{{Key: "$group", Value: bson.M{
-			"_id": bson.M{
-				"$dateToString": bson.M{
-					"format":   "%Y-%m-%d",
-					"date":     bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
-					"timezone": "UTC",
-				},
-			},
-			"count": bson.M{"$sum": 1},
-		}}},
-	}
-
-	cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline)
-	if err != nil {
-		return nil, err
-	}
-	defer cursor.Close(ctx)
-
-	return decodeDailyCounts(ctx, cursor)
-}
-
 func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64, scope scrapedTendersScope) (map[string]int64, error) {
 	if scope.zero {
 		return map[string]int64{}, nil

pkg/mongo/counter.go

@@ -16,6 +16,11 @@ const (
 	AITranslationSuccessCounterKey              = "ai_translation_successful_requests"
 	AITranslationSuccessCounterKeyDailyJob      = "ai_translation_successful_requests_daily_job"
 	AITranslationSuccessCounterKeyManualTrigger = "ai_translation_successful_requests_manual_trigger"
+	// TEDNoticeScrapedCounterKey tracks TED notices successfully scraped/ingested. This is
+	// recorded independently of the "notices" collection because that collection is a transient
+	// processing queue: the worker deletes notices once they've been promoted into tenders, so
+	// counting from "notices" directly undercounts (or zeroes out) historical scrape activity.
+	TEDNoticeScrapedCounterKey = "ted_notice_scraped"
 )
 
 // Counter provides atomic increment/read operations for named metrics stored in MongoDB.
@@ -96,8 +101,36 @@ func AITranslationSuccessDailyCounterKey(day time.Time) string {
 	return AITranslationSuccessCounterKey + "_day:" + day.UTC().Format("2006-01-02")
 }
 
-// GetDailyCounts returns counter values for each UTC day from start through end inclusive.
+// TEDNoticeScrapedDailyCounterKey returns the metrics key for TED notices scraped on a given UTC day.
+func TEDNoticeScrapedDailyCounterKey(day time.Time) string {
+	return TEDNoticeScrapedCounterKey + "_day:" + day.UTC().Format("2006-01-02")
+}
+
+// IncrementTEDNoticeScraped atomically increments the lifetime and today's daily counters for
+// successfully scraped/ingested TED notices. Call this once per newly created notice document
+// (not on updates/refreshes) so totals reflect distinct scrape events.
+func (c *Counter) IncrementTEDNoticeScraped(ctx context.Context) error {
+	if _, err := c.Increment(ctx, TEDNoticeScrapedCounterKey); err != nil {
+		return err
+	}
+	_, err := c.Increment(ctx, TEDNoticeScrapedDailyCounterKey(time.Now().UTC()))
+	return err
+}
+
+// GetDailyCounts returns counter values for each UTC day from start through end inclusive, for
+// the AI translation daily counter.
 func (c *Counter) GetDailyCounts(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
+	return c.getDailyCounts(ctx, startDay, endDay, AITranslationSuccessDailyCounterKey)
+}
+
+// GetTEDNoticeScrapedDailyCounts returns counter values for each UTC day from start through end
+// inclusive, for the TED-notice-scraped daily counter.
+func (c *Counter) GetTEDNoticeScrapedDailyCounts(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
+	return c.getDailyCounts(ctx, startDay, endDay, TEDNoticeScrapedDailyCounterKey)
+}
+
+// getDailyCounts is the shared implementation behind the per-metric GetXDailyCounts helpers above.
+func (c *Counter) getDailyCounts(ctx context.Context, startDay, endDay time.Time, dailyKey func(time.Time) string) (map[string]int64, error) {
 	startDay = time.Date(startDay.Year(), startDay.Month(), startDay.Day(), 0, 0, 0, 0, time.UTC)
 	endDay = time.Date(endDay.Year(), endDay.Month(), endDay.Day(), 0, 0, 0, 0, time.UTC)
 	if endDay.Before(startDay) {
@@ -108,7 +141,7 @@ func (c *Counter) GetDailyCounts(ctx context.Context, startDay, endDay time.Time
 	dates := make([]string, 0)
 	for day := startDay; !day.After(endDay); day = day.AddDate(0, 0, 1) {
 		dates = append(dates, day.Format("2006-01-02"))
-		keys = append(keys, AITranslationSuccessDailyCounterKey(day))
+		keys = append(keys, dailyKey(day))
 	}
 
 	counts := make(map[string]int64, len(dates))

ted/scraper.go

@@ -34,13 +34,14 @@ type Config struct {
 
 // TEDScraper handles downloading and parsing TED XML files
 type TEDScraper struct {
-	notify       notification.SDK
-	config       *Config
-	logger       logger.Logger
-	httpClient   *http.Client
-	xmlParser    *TEDParser
-	mongoManager *orm.ConnectionManager
-	noticeRepo   notice.Repository
+	notify         notification.SDK
+	config         *Config
+	logger         logger.Logger
+	httpClient     *http.Client
+	xmlParser      *TEDParser
+	mongoManager   *orm.ConnectionManager
+	noticeRepo     notice.Repository
+	metricsCounter *orm.Counter
 }
 
 // NewTEDScraper creates a new TED scraper instance
@@ -56,13 +57,14 @@ func NewTEDScraper(
 	}
 
 	return &TEDScraper{
-		notify:       notify,
-		config:       config,
-		logger:       logger,
-		httpClient:   httpClient,
-		xmlParser:    NewTEDParser(),
-		mongoManager: mongoManager,
-		noticeRepo:   noticeRepo,
+		notify:         notify,
+		config:         config,
+		logger:         logger,
+		httpClient:     httpClient,
+		xmlParser:      NewTEDParser(),
+		mongoManager:   mongoManager,
+		noticeRepo:     noticeRepo,
+		metricsCounter: orm.NewCounter(mongoManager),
 	}
 }
 
@@ -420,6 +422,18 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 			return fmt.Errorf("failed to import notice: %w", err)
 		}
 
+		// Track scraping activity in a persistent counter (independent of the "notices"
+		// collection) since the worker deletes processed notices, which would otherwise make
+		// scrape history unrecoverable for dashboard reporting.
+		if s.metricsCounter != nil {
+			if incErr := s.metricsCounter.IncrementTEDNoticeScraped(ctx); incErr != nil {
+				s.logger.Warn("Failed to increment TED scraped notice counter", map[string]interface{}{
+					"contract_notice_id": t.ContractNoticeID,
+					"error":              incErr.Error(),
+				})
+			}
+		}
+
 		s.logger.Info("Successfully imported new notice", map[string]interface{}{
 			"contract_notice_id": t.ContractNoticeID,
 		})