mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into TM-310

Browse Source
This commit is contained in:
m.nazemi
2026-06-17 00:54:35 +03:30
parent 902cc686a0 58345ac4f3
commit 243f667005
2 changed files with 12 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/dashboard/statistics_repository.go
+8 -10
View File
@@ -31,10 +31,12 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
g, gctx := errgroup.WithContext(ctx) g, gctx := errgroup.WithContext(ctx)
g.Go(func() error { g.Go(func() error {
// Use created_at (first ingest time). processing_metadata.scraped_at is reset on
// notice refresh merges, which would collapse historical scrapes onto the latest day.
counts, err := r.dailyCountByTimestamp(gctx, noticesCollectionName, bson.M{ counts, err := r.dailyCountByTimestamp(gctx, noticesCollectionName, bson.M{
"source": notice.TenderSourceTEDScraper, "source": notice.TenderSourceTEDScraper,
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0}, "created_at": bson.M{"$gte": startUnix, "$gt": 0},
}, "processing_metadata.scraped_at") }, "created_at")
if err != nil { if err != nil {
return fmt.Errorf("scraped ted per day: %w", err) return fmt.Errorf("scraped ted per day: %w", err)
} }
@@ -129,10 +131,6 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
{{Key: "$match", Value: bson.M{ {{Key: "$match", Value: bson.M{
"processing_metadata.documents_scraped": true, "processing_metadata.documents_scraped": true,
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}}, "scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
"$or": []bson.M{
{"processing_metadata.documents_scraped_at": bson.M{"$gte": startUnix}},
{"scraped_documents.scraped_at": bson.M{"$gte": startUnix}},
},
}}}, }}},
{{Key: "$unwind", Value: "$scraped_documents"}}, {{Key: "$unwind", Value: "$scraped_documents"}},
{{Key: "$addFields", Value: bson.M{ {{Key: "$addFields", Value: bson.M{
@@ -144,12 +142,12 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
}, },
}, },
}}}, }}},
{{Key: "$match", Value: bson.M{
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
}}},
{{Key: "$addFields", Value: bson.M{ {{Key: "$addFields", Value: bson.M{
"metric_ts": normalizeTimestampExpr("metric_ts"), "metric_ts": normalizeTimestampExpr("metric_ts"),
}}}, }}},
{{Key: "$match", Value: bson.M{
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
}}},
{{Key: "$group", Value: bson.M{ {{Key: "$group", Value: bson.M{
"_id": bson.M{ "_id": bson.M{
"$dateToString": bson.M{ "$dateToString": bson.M{
internal/notice/repository.go
+4
View File
@@ -43,6 +43,10 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
{Key: "source", Value: 1}, {Key: "source", Value: 1},
{Key: "processing_metadata.scraped_at", Value: 1}, {Key: "processing_metadata.scraped_at", Value: 1},
}), }),
*orm.NewIndex("source_created_at_idx", bson.D{
{Key: "source", Value: 1},
{Key: "created_at", Value: 1},
}),
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}), *orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}), *orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
// One row per TED ContractNoticeID when present (parallel scrape races). // One row per TED ContractNoticeID when present (parallel scrape races).