Enhance dashboard statistics repository with scraped tenders scope resolution

- Introduced a new `scrapedTendersScope` type to encapsulate the MongoDB filter for tenders with scraped documents, improving clarity and maintainability.
- Updated the `Statistics` method to utilize the new scope resolution, allowing for more accurate data retrieval based on the presence of scraped documents.
- Implemented multiple tests for the `resolveScrapedTendersScope` method, ensuring correct behavior for various scenarios, including empty and fallback cases.

This update enhances the dashboard's ability to manage scraped document statistics, improving overall data accuracy and system performance.

internal/dashboard/statistics_repository.go

@@ -16,11 +16,23 @@ import (
 
 const noticesCollectionName = "notices"
 
+// scrapedTendersScope is the resolved Mongo filter for tenders with scraped documents.
+// zero is set when MinIO reports no procedure folders; callers must return zero without querying.
+type scrapedTendersScope struct {
+	match bson.M
+	zero  bool
+}
+
 func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
 	now := time.Now().UTC()
 	endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
 	startDay := endDay.AddDate(0, 0, -(days - 1))
 
+	scrapedScope, err := r.resolveScrapedTendersScope(ctx)
+	if err != nil {
+		return nil, err
+	}
+
 	var (
 		scrapedTED        map[string]int64
 		scrapedDocuments  map[string]int64
@@ -46,7 +58,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 	})
 
 	g.Go(func() error {
-		counts, err := r.scrapedDocumentsPerDay(gctx, startUnix)
+		counts, err := r.scrapedDocumentsPerDay(gctx, startUnix, scrapedScope)
 		if err != nil {
 			return fmt.Errorf("scraped documents per day: %w", err)
 		}
@@ -64,7 +76,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 	})
 
 	g.Go(func() error {
-		total, err := r.totalScrapedTenders(gctx)
+		total, err := r.totalScrapedTenders(gctx, scrapedScope)
 		if err != nil {
 			return fmt.Errorf("total scraped tenders: %w", err)
 		}
@@ -127,17 +139,15 @@ func (r *repository) dailyCountByTimestamp(ctx context.Context, collection strin
 	return decodeDailyCounts(ctx, cursor)
 }
 
-func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) {
-	match, err := r.scrapedTendersMatchFilter(ctx)
-	if err != nil {
-		return nil, err
-	}
-	if len(match) == 0 {
+func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64, scope scrapedTendersScope) (map[string]int64, error) {
+	if scope.zero {
 		return map[string]int64{}, nil
 	}
 
+	// Daily buckets use documents_scraped_at only. Tenders present in MinIO but not yet
+	// synced to Mongo are included in the lifetime total but omitted from the daily series.
 	pipeline := mongodriver.Pipeline{
-		{{Key: "$match", Value: match}},
+		{{Key: "$match", Value: scope.match}},
 		{{Key: "$addFields", Value: bson.M{
 			"metric_ts": normalizeTimestampExpr("processing_metadata.documents_scraped_at"),
 		}}},
@@ -169,32 +179,32 @@ func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endD
 	return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
 }
 
-func (r *repository) totalScrapedTenders(ctx context.Context) (int64, error) {
-	match, err := r.scrapedTendersMatchFilter(ctx)
-	if err != nil {
-		return 0, err
-	}
-	if len(match) == 0 {
+func (r *repository) totalScrapedTenders(ctx context.Context, scope scrapedTendersScope) (int64, error) {
+	if scope.zero {
 		return 0, nil
 	}
 
-	return r.ormRepo.Count(ctx, match)
+	return r.ormRepo.Count(ctx, scope.match)
 }
 
-func (r *repository) scrapedTendersMatchFilter(ctx context.Context) (bson.M, error) {
+func (r *repository) resolveScrapedTendersScope(ctx context.Context) (scrapedTendersScope, error) {
 	folderIDs, err := r.scrapedDocumentsFolderIDs(ctx)
 	if err != nil {
-		return nil, err
+		return scrapedTendersScope{}, err
 	}
 	if len(folderIDs) > 0 {
-		return bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}, nil
+		return scrapedTendersScope{
+			match: bson.M{"contract_folder_id": bson.M{"$in": folderIDs}},
+		}, nil
 	}
 	if r.procedureLister != nil {
-		return bson.M{}, nil
+		return scrapedTendersScope{zero: true}, nil
 	}
-	return bson.M{
-		"processing_metadata.documents_scraped": true,
-		"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
+	return scrapedTendersScope{
+		match: bson.M{
+			"processing_metadata.documents_scraped": true,
+			"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
+		},
 	}, nil
 }
 

internal/dashboard/statistics_repository_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 
 	"tm/pkg/ai_summarizer"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
 )
 
 type mockProcedureDocumentsLister struct {
@@ -72,3 +74,58 @@ func TestScrapedDocumentsFolderIDsPropagatesError(t *testing.T) {
 		t.Fatal("expected error")
 	}
 }
+
+func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) {
+	repo := &repository{
+		procedureLister: &mockProcedureDocumentsLister{
+			procedures: []ai_summarizer.ProcedureDocumentsSummary{
+				{ContractFolderID: "PROC-1", DocumentCount: 2},
+			},
+		},
+	}
+
+	scope, err := repo.resolveScrapedTendersScope(context.Background())
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if scope.zero {
+		t.Fatal("expected non-zero scope")
+	}
+	in, ok := scope.match["contract_folder_id"].(bson.M)
+	if !ok {
+		t.Fatalf("expected contract_folder_id filter, got %#v", scope.match)
+	}
+	ids, ok := in["$in"].([]string)
+	if !ok || len(ids) != 1 || ids[0] != "PROC-1" {
+		t.Fatalf("unexpected folder IDs: %#v", in["$in"])
+	}
+}
+
+func TestResolveScrapedTendersScopeZeroWhenMinIOEmpty(t *testing.T) {
+	repo := &repository{
+		procedureLister: &mockProcedureDocumentsLister{},
+	}
+
+	scope, err := repo.resolveScrapedTendersScope(context.Background())
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !scope.zero {
+		t.Fatalf("expected zero scope, got %#v", scope)
+	}
+}
+
+func TestResolveScrapedTendersScopeMongoFallbackWithoutLister(t *testing.T) {
+	repo := &repository{}
+
+	scope, err := repo.resolveScrapedTendersScope(context.Background())
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if scope.zero {
+		t.Fatal("expected mongo fallback scope")
+	}
+	if scope.match["processing_metadata.documents_scraped"] != true {
+		t.Fatalf("expected mongo fallback filter, got %#v", scope.match)
+	}
+}