Refactor dashboard repository to integrate ProcedureDocumentsLister

- Introduced the ProcedureDocumentsLister interface to list contract folders with scraped documents, enhancing the accuracy of document-scrape statistics.
- Updated the dashboard repository to accept ProcedureDocumentsLister as a dependency, allowing for improved data retrieval.
- Implemented tests for the new functionality, ensuring proper handling of scraped document folder IDs and error propagation.

This update enhances the dashboard's capability to manage and report on scraped documents, improving overall system efficiency and data integrity.

internal/dashboard/statistics_repository.go

@@ -3,6 +3,7 @@ package dashboard
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"tm/internal/notice"
@@ -63,9 +64,9 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 	})
 
 	g.Go(func() error {
-		total, err := r.totalScrapedDocuments(gctx)
+		total, err := r.totalScrapedTenders(gctx)
 		if err != nil {
-			return fmt.Errorf("total scraped documents: %w", err)
+			return fmt.Errorf("total scraped tenders: %w", err)
 		}
 		totalDocuments = total
 		return nil
@@ -127,23 +128,18 @@ func (r *repository) dailyCountByTimestamp(ctx context.Context, collection strin
 }
 
 func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) {
+	match, err := r.scrapedTendersMatchFilter(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if len(match) == 0 {
+		return map[string]int64{}, nil
+	}
+
 	pipeline := mongodriver.Pipeline{
-		{{Key: "$match", Value: bson.M{
-			"processing_metadata.documents_scraped": true,
-			"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
-		}}},
-		{{Key: "$unwind", Value: "$scraped_documents"}},
+		{{Key: "$match", Value: match}},
 		{{Key: "$addFields", Value: bson.M{
-			"metric_ts": bson.M{
-				"$cond": bson.A{
-					bson.M{"$gt": bson.A{"$scraped_documents.scraped_at", 0}},
-					"$scraped_documents.scraped_at",
-					"$processing_metadata.documents_scraped_at",
-				},
-			},
-		}}},
-		{{Key: "$addFields", Value: bson.M{
-			"metric_ts": normalizeTimestampExpr("metric_ts"),
+			"metric_ts": normalizeTimestampExpr("processing_metadata.documents_scraped_at"),
 		}}},
 		{{Key: "$match", Value: bson.M{
 			"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
@@ -173,37 +169,63 @@ func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endD
 	return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
 }
 
-func (r *repository) totalScrapedDocuments(ctx context.Context) (int64, error) {
-	pipeline := mongodriver.Pipeline{
-		{{Key: "$match", Value: bson.M{
-			"processing_metadata.documents_scraped": true,
-			"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
-		}}},
-		{{Key: "$project", Value: bson.M{
-			"doc_count": bson.M{"$size": "$scraped_documents"},
-		}}},
-		{{Key: "$group", Value: bson.M{
-			"_id":   nil,
-			"total": bson.M{"$sum": "$doc_count"},
-		}}},
-	}
-
-	cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
+func (r *repository) totalScrapedTenders(ctx context.Context) (int64, error) {
+	match, err := r.scrapedTendersMatchFilter(ctx)
 	if err != nil {
 		return 0, err
 	}
-	defer cursor.Close(ctx)
-
-	if !cursor.Next(ctx) {
+	if len(match) == 0 {
 		return 0, nil
 	}
 
-	var row bson.M
-	if err := cursor.Decode(&row); err != nil {
-		return 0, err
+	return r.ormRepo.Count(ctx, match)
+}
+
+func (r *repository) scrapedTendersMatchFilter(ctx context.Context) (bson.M, error) {
+	folderIDs, err := r.scrapedDocumentsFolderIDs(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if len(folderIDs) > 0 {
+		return bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}, nil
+	}
+	if r.procedureLister != nil {
+		return bson.M{}, nil
+	}
+	return bson.M{
+		"processing_metadata.documents_scraped": true,
+		"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
+	}, nil
+}
+
+func (r *repository) scrapedDocumentsFolderIDs(ctx context.Context) ([]string, error) {
+	if r.procedureLister == nil {
+		return nil, nil
 	}
 
-	return aggregateCount(row, "total"), nil
+	procedures, err := r.procedureLister.ListProceduresWithDocuments(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	folderIDs := make([]string, 0, len(procedures))
+	seen := make(map[string]struct{}, len(procedures))
+	for _, proc := range procedures {
+		if proc.DocumentCount <= 0 {
+			continue
+		}
+		id := strings.TrimSpace(proc.ContractFolderID)
+		if id == "" {
+			continue
+		}
+		if _, ok := seen[id]; ok {
+			continue
+		}
+		seen[id] = struct{}{}
+		folderIDs = append(folderIDs, id)
+	}
+
+	return folderIDs, nil
 }
 
 func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) {