From 7f84746400a13cc1fedbecc08602ba6c70f0daf2 Mon Sep 17 00:00:00 2001
From: Mazyar <m.nazemi@ravanertebat.com>
Date: Mon, 15 Jun 2026 23:39:32 +0330
Subject: [PATCH] Refactor statistics repository to use created_at for
 historical data retrieval

- Updated the `Statistics` method to utilize `created_at` instead of `processing_metadata.scraped_at` for fetching daily counts, ensuring accurate historical data representation.
- Removed redundant conditions in the `scrapedDocumentsPerDay` method, streamlining the query logic for better performance and clarity.
- Added a new index on `source` and `created_at` to optimize database queries related to scraped documents.

This update enhances the accuracy of data retrieval in the dashboard statistics, improving the overall efficiency of the tender management system.
---
 internal/dashboard/statistics_repository.go | 18 ++++++++----------
 internal/notice/repository.go               |  4 ++++
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/internal/dashboard/statistics_repository.go b/internal/dashboard/statistics_repository.go
index e149d29..ae7fe3d 100644
--- a/internal/dashboard/statistics_repository.go
+++ b/internal/dashboard/statistics_repository.go
@@ -31,10 +31,12 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 	g, gctx := errgroup.WithContext(ctx)
 
 	g.Go(func() error {
+		// Use created_at (first ingest time). processing_metadata.scraped_at is reset on
+		// notice refresh merges, which would collapse historical scrapes onto the latest day.
 		counts, err := r.dailyCountByTimestamp(gctx, noticesCollectionName, bson.M{
-			"source": notice.TenderSourceTEDScraper,
-			"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
-		}, "processing_metadata.scraped_at")
+			"source":     notice.TenderSourceTEDScraper,
+			"created_at": bson.M{"$gte": startUnix, "$gt": 0},
+		}, "created_at")
 		if err != nil {
 			return fmt.Errorf("scraped ted per day: %w", err)
 		}
@@ -129,10 +131,6 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
 		{{Key: "$match", Value: bson.M{
 			"processing_metadata.documents_scraped": true,
 			"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
-			"$or": []bson.M{
-				{"processing_metadata.documents_scraped_at": bson.M{"$gte": startUnix}},
-				{"scraped_documents.scraped_at": bson.M{"$gte": startUnix}},
-			},
 		}}},
 		{{Key: "$unwind", Value: "$scraped_documents"}},
 		{{Key: "$addFields", Value: bson.M{
@@ -144,12 +142,12 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
 				},
 			},
 		}}},
-		{{Key: "$match", Value: bson.M{
-			"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
-		}}},
 		{{Key: "$addFields", Value: bson.M{
 			"metric_ts": normalizeTimestampExpr("metric_ts"),
 		}}},
+		{{Key: "$match", Value: bson.M{
+			"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
+		}}},
 		{{Key: "$group", Value: bson.M{
 			"_id": bson.M{
 				"$dateToString": bson.M{
diff --git a/internal/notice/repository.go b/internal/notice/repository.go
index 2b6895d..15adce6 100644
--- a/internal/notice/repository.go
+++ b/internal/notice/repository.go
@@ -43,6 +43,10 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
 			{Key: "source", Value: 1},
 			{Key: "processing_metadata.scraped_at", Value: 1},
 		}),
+		*orm.NewIndex("source_created_at_idx", bson.D{
+			{Key: "source", Value: 1},
+			{Key: "created_at", Value: 1},
+		}),
 		*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
 		*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
 		// One row per TED ContractNoticeID when present (parallel scrape races).