Enhance dashboard statistics with TED notice tracking

- Added a new field, ScrapedTEDNotices, to the StatisticsLifetimeTotals struct to track the total number of TED notices scraped.
- Updated the Statistics method in the statistics repository to include a background process for retrieving total scraped TED notices, improving the accuracy of dashboard statistics.
- Introduced new methods in the Counter to increment and retrieve daily counts for scraped TED notices, ensuring reliable metrics for reporting.
- Modified the TEDScraper to increment the TED notice scraped counter upon successful import, enhancing the tracking of scraping activity.

This update improves the dashboard's statistics by providing detailed insights into TED notice scraping activities, contributing to better data visibility and reporting.

internal/dashboard/statistics_repository.go

@@ -6,7 +6,6 @@ import (
 	"sync"
 	"time"
 
-	"tm/internal/notice"
 	"tm/pkg/ai_summarizer"
 	orm "tm/pkg/mongo"
 
@@ -14,8 +13,6 @@ import (
 	mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
 )
 
-const noticesCollectionName = "notices"
-
 // scrapedScopeCacheTTL avoids listing every MinIO object on each statistics request while
 // keeping counts aligned with the tender panel (MinIO is the source of truth for scraped docs).
 const scrapedScopeCacheTTL = 15 * time.Minute
@@ -49,6 +46,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		translatedNotices map[string]int64
 		totalDocuments    int64
 		totalTranslated   int64
+		totalScrapedTED   int64
 		scrapedScope      scrapedTendersScope
 	)
 
@@ -70,10 +68,10 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout)
 		defer cancel()
 
-		counts, err := r.dailyCountByTimestamp(qctx, noticesCollectionName, bson.M{
-			"source":     notice.TenderSourceTEDScraper,
-			"created_at": bson.M{"$gte": startUnix, "$gt": 0},
-		}, "created_at")
+		// Read from the persistent metrics counter rather than the "notices" collection: the
+		// notice worker deletes processed notices shortly after promoting them into tenders, so
+		// counting from "notices" directly would undercount (often to zero) recent scrape activity.
+		counts, err := r.metricsCounter.GetTEDNoticeScrapedDailyCounts(qctx, startDay, endDay)
 		if err != nil {
 			recordFailure("scraped_ted", err)
 			scrapedTED = map[string]int64{}
@@ -82,6 +80,20 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		scrapedTED = counts
 	}()
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout)
+		defer cancel()
+
+		total, err := r.metricsCounter.Get(qctx, orm.TEDNoticeScrapedCounterKey)
+		if err != nil {
+			recordFailure("scraped_ted_total", err)
+			return
+		}
+		totalScrapedTED = total
+	}()
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
@@ -167,37 +179,11 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		Totals: StatisticsLifetimeTotals{
 			ScrapedDocuments:  totalDocuments,
 			TranslatedTenders: totalTranslated,
+			ScrapedTEDNotices: totalScrapedTED,
 		},
 	}, nil
 }
 
-func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) {
-	pipeline := mongodriver.Pipeline{
-		{{Key: "$match", Value: match}},
-		{{Key: "$addFields", Value: bson.M{
-			"metric_ts": normalizeTimestampExpr(timestampField),
-		}}},
-		{{Key: "$group", Value: bson.M{
-			"_id": bson.M{
-				"$dateToString": bson.M{
-					"format":   "%Y-%m-%d",
-					"date":     bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
-					"timezone": "UTC",
-				},
-			},
-			"count": bson.M{"$sum": 1},
-		}}},
-	}
-
-	cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline)
-	if err != nil {
-		return nil, err
-	}
-	defer cursor.Close(ctx)
-
-	return decodeDailyCounts(ctx, cursor)
-}
-
 func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64, scope scrapedTendersScope) (map[string]int64, error) {
 	if scope.zero {
 		return map[string]int64{}, nil