diff --git a/internal/dashboard/statistics_repository.go b/internal/dashboard/statistics_repository.go index 9c1a34d..a63cb42 100644 --- a/internal/dashboard/statistics_repository.go +++ b/internal/dashboard/statistics_repository.go @@ -16,11 +16,23 @@ import ( const noticesCollectionName = "notices" +// scrapedTendersScope is the resolved Mongo filter for tenders with scraped documents. +// zero is set when MinIO reports no procedure folders; callers must return zero without querying. +type scrapedTendersScope struct { + match bson.M + zero bool +} + func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) { now := time.Now().UTC() endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) startDay := endDay.AddDate(0, 0, -(days - 1)) + scrapedScope, err := r.resolveScrapedTendersScope(ctx) + if err != nil { + return nil, err + } + var ( scrapedTED map[string]int64 scrapedDocuments map[string]int64 @@ -46,7 +58,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) }) g.Go(func() error { - counts, err := r.scrapedDocumentsPerDay(gctx, startUnix) + counts, err := r.scrapedDocumentsPerDay(gctx, startUnix, scrapedScope) if err != nil { return fmt.Errorf("scraped documents per day: %w", err) } @@ -64,7 +76,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) }) g.Go(func() error { - total, err := r.totalScrapedTenders(gctx) + total, err := r.totalScrapedTenders(gctx, scrapedScope) if err != nil { return fmt.Errorf("total scraped tenders: %w", err) } @@ -127,17 +139,15 @@ func (r *repository) dailyCountByTimestamp(ctx context.Context, collection strin return decodeDailyCounts(ctx, cursor) } -func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) { - match, err := r.scrapedTendersMatchFilter(ctx) - if err != nil { - return nil, err - } - if len(match) == 0 { +func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64, scope scrapedTendersScope) (map[string]int64, error) { + if scope.zero { return map[string]int64{}, nil } + // Daily buckets use documents_scraped_at only. Tenders present in MinIO but not yet + // synced to Mongo are included in the lifetime total but omitted from the daily series. pipeline := mongodriver.Pipeline{ - {{Key: "$match", Value: match}}, + {{Key: "$match", Value: scope.match}}, {{Key: "$addFields", Value: bson.M{ "metric_ts": normalizeTimestampExpr("processing_metadata.documents_scraped_at"), }}}, @@ -169,32 +179,32 @@ func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endD return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay) } -func (r *repository) totalScrapedTenders(ctx context.Context) (int64, error) { - match, err := r.scrapedTendersMatchFilter(ctx) - if err != nil { - return 0, err - } - if len(match) == 0 { +func (r *repository) totalScrapedTenders(ctx context.Context, scope scrapedTendersScope) (int64, error) { + if scope.zero { return 0, nil } - return r.ormRepo.Count(ctx, match) + return r.ormRepo.Count(ctx, scope.match) } -func (r *repository) scrapedTendersMatchFilter(ctx context.Context) (bson.M, error) { +func (r *repository) resolveScrapedTendersScope(ctx context.Context) (scrapedTendersScope, error) { folderIDs, err := r.scrapedDocumentsFolderIDs(ctx) if err != nil { - return nil, err + return scrapedTendersScope{}, err } if len(folderIDs) > 0 { - return bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}, nil + return scrapedTendersScope{ + match: bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}, + }, nil } if r.procedureLister != nil { - return bson.M{}, nil + return scrapedTendersScope{zero: true}, nil } - return bson.M{ - "processing_metadata.documents_scraped": true, - "scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}}, + return scrapedTendersScope{ + match: bson.M{ + "processing_metadata.documents_scraped": true, + "scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}}, + }, }, nil } diff --git a/internal/dashboard/statistics_repository_test.go b/internal/dashboard/statistics_repository_test.go index ed09c4e..ac9302f 100644 --- a/internal/dashboard/statistics_repository_test.go +++ b/internal/dashboard/statistics_repository_test.go @@ -6,6 +6,8 @@ import ( "testing" "tm/pkg/ai_summarizer" + + "go.mongodb.org/mongo-driver/v2/bson" ) type mockProcedureDocumentsLister struct { @@ -72,3 +74,58 @@ func TestScrapedDocumentsFolderIDsPropagatesError(t *testing.T) { t.Fatal("expected error") } } + +func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) { + repo := &repository{ + procedureLister: &mockProcedureDocumentsLister{ + procedures: []ai_summarizer.ProcedureDocumentsSummary{ + {ContractFolderID: "PROC-1", DocumentCount: 2}, + }, + }, + } + + scope, err := repo.resolveScrapedTendersScope(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if scope.zero { + t.Fatal("expected non-zero scope") + } + in, ok := scope.match["contract_folder_id"].(bson.M) + if !ok { + t.Fatalf("expected contract_folder_id filter, got %#v", scope.match) + } + ids, ok := in["$in"].([]string) + if !ok || len(ids) != 1 || ids[0] != "PROC-1" { + t.Fatalf("unexpected folder IDs: %#v", in["$in"]) + } +} + +func TestResolveScrapedTendersScopeZeroWhenMinIOEmpty(t *testing.T) { + repo := &repository{ + procedureLister: &mockProcedureDocumentsLister{}, + } + + scope, err := repo.resolveScrapedTendersScope(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !scope.zero { + t.Fatalf("expected zero scope, got %#v", scope) + } +} + +func TestResolveScrapedTendersScopeMongoFallbackWithoutLister(t *testing.T) { + repo := &repository{} + + scope, err := repo.resolveScrapedTendersScope(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if scope.zero { + t.Fatal("expected mongo fallback scope") + } + if scope.match["processing_metadata.documents_scraped"] != true { + t.Fatalf("expected mongo fallback filter, got %#v", scope.match) + } +}