mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'Refactor statistics repository to use created_at for historical data retrieval' (#43) from dashboard-data into develop

Browse Source 
Reviewed-on: https://repo.ravanertebat.com/TM/tm_back/pulls/43
Reviewed-by: Hadi Barzegar <barzagarhadi@gmail.com>
This commit is contained in:
m.nazemi
2026-06-17 00:54:01 +03:30
parent 31d8efef0b 7f84746400
commit 58345ac4f3
2 changed files with 12 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/dashboard/statistics_repository.go
+8 -10
View File
@@ -31,10 +31,12 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
g, gctx := errgroup.WithContext(ctx)
g.Go(func() error {
// Use created_at (first ingest time). processing_metadata.scraped_at is reset on
// notice refresh merges, which would collapse historical scrapes onto the latest day.
counts, err := r.dailyCountByTimestamp(gctx, noticesCollectionName, bson.M{
"source": notice.TenderSourceTEDScraper,
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
}, "processing_metadata.scraped_at")
"source": notice.TenderSourceTEDScraper,
"created_at": bson.M{"$gte": startUnix, "$gt": 0},
}, "created_at")
if err != nil {
return fmt.Errorf("scraped ted per day: %w", err)
}
@@ -129,10 +131,6 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
{{Key: "$match", Value: bson.M{
"processing_metadata.documents_scraped": true,
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
"$or": []bson.M{
{"processing_metadata.documents_scraped_at": bson.M{"$gte": startUnix}},
{"scraped_documents.scraped_at": bson.M{"$gte": startUnix}},
},
}}},
{{Key: "$unwind", Value: "$scraped_documents"}},
{{Key: "$addFields", Value: bson.M{
@@ -144,12 +142,12 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
},
},
}}},
{{Key: "$match", Value: bson.M{
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
}}},
{{Key: "$addFields", Value: bson.M{
"metric_ts": normalizeTimestampExpr("metric_ts"),
}}},
{{Key: "$match", Value: bson.M{
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
}}},
{{Key: "$group", Value: bson.M{
"_id": bson.M{
"$dateToString": bson.M{
internal/notice/repository.go
+4
View File
@@ -43,6 +43,10 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
{Key: "source", Value: 1},
{Key: "processing_metadata.scraped_at", Value: 1},
}),
*orm.NewIndex("source_created_at_idx", bson.D{
{Key: "source", Value: 1},
{Key: "created_at", Value: 1},
}),
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
// One row per TED ContractNoticeID when present (parallel scrape races).