package dashboard import ( "context" "fmt" "strings" "time" "tm/internal/notice" orm "tm/pkg/mongo" "go.mongodb.org/mongo-driver/v2/bson" mongodriver "go.mongodb.org/mongo-driver/v2/mongo" "golang.org/x/sync/errgroup" ) const noticesCollectionName = "notices" func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) { now := time.Now().UTC() endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) startDay := endDay.AddDate(0, 0, -(days - 1)) var ( scrapedTED map[string]int64 scrapedDocuments map[string]int64 translatedNotices map[string]int64 totalDocuments int64 totalTranslated int64 ) g, gctx := errgroup.WithContext(ctx) g.Go(func() error { // Use created_at (first ingest time). processing_metadata.scraped_at is reset on // notice refresh merges, which would collapse historical scrapes onto the latest day. counts, err := r.dailyCountByTimestamp(gctx, noticesCollectionName, bson.M{ "source": notice.TenderSourceTEDScraper, "created_at": bson.M{"$gte": startUnix, "$gt": 0}, }, "created_at") if err != nil { return fmt.Errorf("scraped ted per day: %w", err) } scrapedTED = counts return nil }) g.Go(func() error { counts, err := r.scrapedDocumentsPerDay(gctx, startUnix) if err != nil { return fmt.Errorf("scraped documents per day: %w", err) } scrapedDocuments = counts return nil }) g.Go(func() error { counts, err := r.translatedNoticesPerDay(gctx, startDay, endDay) if err != nil { return fmt.Errorf("translated notices per day: %w", err) } translatedNotices = counts return nil }) g.Go(func() error { total, err := r.totalScrapedTenders(gctx) if err != nil { return fmt.Errorf("total scraped tenders: %w", err) } totalDocuments = total return nil }) g.Go(func() error { total, err := r.metricsCounter.Get(gctx, orm.AITranslationSuccessCounterKey) if err != nil { return fmt.Errorf("total translated tenders: %w", err) } totalTranslated = total return nil }) if err := g.Wait(); err != nil { return nil, err } return &StatisticsReportResponse{ Days: days, GeneratedAt: now.Unix(), Daily: StatisticsDailySeries{ ScrapedTED: fillTrendSeries(days, scrapedTED), ScrapedDocuments: fillTrendSeries(days, scrapedDocuments), TranslatedNotices: fillTrendSeries(days, translatedNotices), }, Totals: StatisticsLifetimeTotals{ ScrapedDocuments: totalDocuments, TranslatedTenders: totalTranslated, }, }, nil } func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) { pipeline := mongodriver.Pipeline{ {{Key: "$match", Value: match}}, {{Key: "$addFields", Value: bson.M{ "metric_ts": normalizeTimestampExpr(timestampField), }}}, {{Key: "$group", Value: bson.M{ "_id": bson.M{ "$dateToString": bson.M{ "format": "%Y-%m-%d", "date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}}, "timezone": "UTC", }, }, "count": bson.M{"$sum": 1}, }}}, } cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline) if err != nil { return nil, err } defer cursor.Close(ctx) return decodeDailyCounts(ctx, cursor) } func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) { match, err := r.scrapedTendersMatchFilter(ctx) if err != nil { return nil, err } if len(match) == 0 { return map[string]int64{}, nil } pipeline := mongodriver.Pipeline{ {{Key: "$match", Value: match}}, {{Key: "$addFields", Value: bson.M{ "metric_ts": normalizeTimestampExpr("processing_metadata.documents_scraped_at"), }}}, {{Key: "$match", Value: bson.M{ "metric_ts": bson.M{"$gte": startUnix, "$gt": 0}, }}}, {{Key: "$group", Value: bson.M{ "_id": bson.M{ "$dateToString": bson.M{ "format": "%Y-%m-%d", "date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}}, "timezone": "UTC", }, }, "count": bson.M{"$sum": 1}, }}}, } cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline) if err != nil { return nil, err } defer cursor.Close(ctx) return decodeDailyCounts(ctx, cursor) } func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) { return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay) } func (r *repository) totalScrapedTenders(ctx context.Context) (int64, error) { match, err := r.scrapedTendersMatchFilter(ctx) if err != nil { return 0, err } if len(match) == 0 { return 0, nil } return r.ormRepo.Count(ctx, match) } func (r *repository) scrapedTendersMatchFilter(ctx context.Context) (bson.M, error) { folderIDs, err := r.scrapedDocumentsFolderIDs(ctx) if err != nil { return nil, err } if len(folderIDs) > 0 { return bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}, nil } if r.procedureLister != nil { return bson.M{}, nil } return bson.M{ "processing_metadata.documents_scraped": true, "scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}}, }, nil } func (r *repository) scrapedDocumentsFolderIDs(ctx context.Context) ([]string, error) { if r.procedureLister == nil { return nil, nil } procedures, err := r.procedureLister.ListProceduresWithDocuments(ctx) if err != nil { return nil, err } folderIDs := make([]string, 0, len(procedures)) seen := make(map[string]struct{}, len(procedures)) for _, proc := range procedures { if proc.DocumentCount <= 0 { continue } id := strings.TrimSpace(proc.ContractFolderID) if id == "" { continue } if _, ok := seen[id]; ok { continue } seen[id] = struct{}{} folderIDs = append(folderIDs, id) } return folderIDs, nil } func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) { counts := make(map[string]int64) for cursor.Next(ctx) { var row bson.M if err := cursor.Decode(&row); err != nil { return nil, err } date, ok := row["_id"].(string) if !ok || date == "" { continue } counts[date] = aggregateCount(row, "count") } if err := cursor.Err(); err != nil { return nil, err } return counts, nil }