package dashboard import ( "context" "fmt" "time" "tm/internal/notice" orm "tm/pkg/mongo" "go.mongodb.org/mongo-driver/v2/bson" mongodriver "go.mongodb.org/mongo-driver/v2/mongo" ) const noticesCollectionName = "notices" func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) { now := time.Now().UTC() endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) startDay := endDay.AddDate(0, 0, -(days - 1)) scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{ "source": notice.TenderSourceTEDScraper, "processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0}, }, "processing_metadata.scraped_at") if err != nil { return nil, fmt.Errorf("scraped ted per day: %w", err) } scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix) if err != nil { return nil, fmt.Errorf("scraped documents per day: %w", err) } translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay) if err != nil { return nil, fmt.Errorf("translated notices per day: %w", err) } totalDocuments, err := r.totalScrapedDocuments(ctx) if err != nil { return nil, fmt.Errorf("total scraped documents: %w", err) } totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey) if err != nil { return nil, fmt.Errorf("total translated tenders: %w", err) } return &StatisticsReportResponse{ Days: days, GeneratedAt: now.Unix(), Daily: StatisticsDailySeries{ ScrapedTED: fillTrendSeries(days, scrapedTED), ScrapedDocuments: fillTrendSeries(days, scrapedDocuments), TranslatedNotices: fillTrendSeries(days, translatedNotices), }, Totals: StatisticsLifetimeTotals{ ScrapedDocuments: totalDocuments, TranslatedTenders: totalTranslated, }, }, nil } func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) { pipeline := mongodriver.Pipeline{ {{Key: "$match", Value: match}}, {{Key: "$addFields", Value: bson.M{ "metric_ts": normalizeTimestampExpr(timestampField), }}}, {{Key: "$group", Value: bson.M{ "_id": bson.M{ "$dateToString": bson.M{ "format": "%Y-%m-%d", "date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}}, "timezone": "UTC", }, }, "count": bson.M{"$sum": 1}, }}}, } cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline) if err != nil { return nil, err } defer cursor.Close(ctx) return decodeDailyCounts(ctx, cursor) } func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) { pipeline := mongodriver.Pipeline{ {{Key: "$match", Value: bson.M{ "processing_metadata.documents_scraped": true, "scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}}, }}}, {{Key: "$unwind", Value: "$scraped_documents"}}, {{Key: "$addFields", Value: bson.M{ "metric_ts": bson.M{ "$cond": bson.A{ bson.M{"$gt": bson.A{"$scraped_documents.scraped_at", 0}}, "$scraped_documents.scraped_at", "$processing_metadata.documents_scraped_at", }, }, }}}, {{Key: "$match", Value: bson.M{ "metric_ts": bson.M{"$gte": startUnix, "$gt": 0}, }}}, {{Key: "$addFields", Value: bson.M{ "metric_ts": normalizeTimestampExpr("metric_ts"), }}}, {{Key: "$group", Value: bson.M{ "_id": bson.M{ "$dateToString": bson.M{ "format": "%Y-%m-%d", "date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}}, "timezone": "UTC", }, }, "count": bson.M{"$sum": 1}, }}}, } cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline) if err != nil { return nil, err } defer cursor.Close(ctx) return decodeDailyCounts(ctx, cursor) } func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) { return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay) } func (r *repository) totalScrapedDocuments(ctx context.Context) (int64, error) { pipeline := mongodriver.Pipeline{ {{Key: "$match", Value: bson.M{ "processing_metadata.documents_scraped": true, "scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}}, }}}, {{Key: "$project", Value: bson.M{ "doc_count": bson.M{"$size": "$scraped_documents"}, }}}, {{Key: "$group", Value: bson.M{ "_id": nil, "total": bson.M{"$sum": "$doc_count"}, }}}, } cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline) if err != nil { return 0, err } defer cursor.Close(ctx) if !cursor.Next(ctx) { return 0, nil } var row bson.M if err := cursor.Decode(&row); err != nil { return 0, err } return aggregateCount(row, "total"), nil } func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) { counts := make(map[string]int64) for cursor.Next(ctx) { var row bson.M if err := cursor.Decode(&row); err != nil { return nil, err } date, ok := row["_id"].(string) if !ok || date == "" { continue } counts[date] = aggregateCount(row, "count") } if err := cursor.Err(); err != nil { return nil, err } return counts, nil }