Update dashboard statistics caching and retrieval logic

- Increased the cache duration for dashboard statistics from 60 seconds to 5 minutes, improving data freshness and reducing load on the backend.
- Introduced a stale cache mechanism that allows retrieval of stale statistics while refreshing them in the background, enhancing user experience by providing quicker access to data.
- Updated the statistics repository to handle the new caching logic, ensuring accurate and timely statistics reporting.
- Added tests to validate the new caching behavior and ensure the integrity of statistics retrieval.

This update optimizes the dashboard's performance and responsiveness by improving the caching strategy for statistics.

internal/dashboard/statistics_repository.go

@@ -30,10 +30,11 @@ const (
 // scrapedTendersScope is the resolved Mongo filter for tenders with scraped documents.
 // zero is set when MinIO reports no procedure folders; callers must return zero without querying.
 type scrapedTendersScope struct {
-	match       bson.M
-	zero        bool
-	fromMinIO   bool
-	dailyCounts map[string]int64 // non-nil when scope was resolved from MinIO
+	match            bson.M
+	zero             bool
+	fromMinIO        bool
+	dailyCounts      map[string]int64 // non-nil when scope was resolved from MinIO
+	totalTenderCount int64            // >0 when MinIO procedure count should be used instead of Mongo Count
 }
 
 func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
@@ -44,18 +45,13 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 	endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
 	startDay := endDay.AddDate(0, 0, -(days - 1))
 
-	// Resolve scraped-document scope from MinIO (cached); falls back to Mongo flags on error.
-	scrapedScope, err := r.cachedScrapedTendersScope(queryCtx)
-	if err != nil {
-		return nil, err
-	}
-
 	var (
 		scrapedTED        map[string]int64
 		scrapedDocuments  map[string]int64
 		translatedNotices map[string]int64
 		totalDocuments    int64
 		totalTranslated   int64
+		scrapedScope      scrapedTendersScope
 	)
 
 	g, gctx := errgroup.WithContext(queryCtx)
@@ -74,15 +70,6 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		return nil
 	})
 
-	g.Go(func() error {
-		counts, err := r.scrapedDocumentsPerDay(gctx, startUnix, scrapedScope)
-		if err != nil {
-			return fmt.Errorf("scraped documents per day: %w", err)
-		}
-		scrapedDocuments = counts
-		return nil
-	})
-
 	g.Go(func() error {
 		counts, err := r.translatedNoticesPerDay(gctx, startDay, endDay)
 		if err != nil {
@@ -92,15 +79,6 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		return nil
 	})
 
-	g.Go(func() error {
-		total, err := r.totalScrapedTenders(gctx, scrapedScope)
-		if err != nil {
-			return fmt.Errorf("total scraped tenders: %w", err)
-		}
-		totalDocuments = total
-		return nil
-	})
-
 	g.Go(func() error {
 		total, err := r.metricsCounter.Get(gctx, orm.AITranslationSuccessCounterKey)
 		if err != nil {
@@ -110,10 +88,44 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		return nil
 	})
 
+	// Resolve scraped-document scope from MinIO (cached) in parallel with Mongo queries.
+	g.Go(func() error {
+		scope, err := r.cachedScrapedTendersScope(gctx)
+		if err != nil {
+			return fmt.Errorf("scraped tenders scope: %w", err)
+		}
+		scrapedScope = scope
+		return nil
+	})
+
 	if err := g.Wait(); err != nil {
 		return nil, err
 	}
 
+	g2, gctx2 := errgroup.WithContext(queryCtx)
+
+	g2.Go(func() error {
+		counts, err := r.scrapedDocumentsPerDay(gctx2, startUnix, scrapedScope)
+		if err != nil {
+			return fmt.Errorf("scraped documents per day: %w", err)
+		}
+		scrapedDocuments = counts
+		return nil
+	})
+
+	g2.Go(func() error {
+		total, err := r.totalScrapedTenders(gctx2, scrapedScope)
+		if err != nil {
+			return fmt.Errorf("total scraped tenders: %w", err)
+		}
+		totalDocuments = total
+		return nil
+	})
+
+	if err := g2.Wait(); err != nil {
+		return nil, err
+	}
+
 	return &StatisticsReportResponse{
 		Days:        days,
 		GeneratedAt: now.Unix(),
@@ -230,6 +242,9 @@ func (r *repository) totalScrapedTenders(ctx context.Context, scope scrapedTende
 	if scope.zero {
 		return 0, nil
 	}
+	if scope.totalTenderCount > 0 {
+		return scope.totalTenderCount, nil
+	}
 
 	return r.ormRepo.Count(ctx, scope.match)
 }
@@ -297,17 +312,23 @@ func (r *repository) resolveScrapedTendersScope(ctx context.Context) (scrapedTen
 
 	folderIDs := folderIDsFromProcedures(procedures)
 	if len(folderIDs) > 0 {
+		totalTenders := int64(len(folderIDs))
 		if len(folderIDs) > maxScrapedFolderInClause {
-			r.logger.Warn("Too many MinIO procedure folders for dashboard statistics, using Mongo fallback", map[string]interface{}{
+			r.logger.Warn("Too many MinIO procedure folders for dashboard statistics, using MinIO counts without Mongo $in filter", map[string]interface{}{
 				"folder_count":  len(folderIDs),
 				"max_in_clause": maxScrapedFolderInClause,
 			})
-			return mongoScrapedTendersScope(), nil
+			return scrapedTendersScope{
+				fromMinIO:        true,
+				dailyCounts:      dailyCounts,
+				totalTenderCount: totalTenders,
+			}, nil
 		}
 		return scrapedTendersScope{
-			match:       bson.M{"contract_folder_id": bson.M{"$in": folderIDs}},
-			fromMinIO:   true,
-			dailyCounts: dailyCounts,
+			match:            bson.M{"contract_folder_id": bson.M{"$in": folderIDs}},
+			fromMinIO:        true,
+			dailyCounts:      dailyCounts,
+			totalTenderCount: totalTenders,
 		}, nil
 	}
 	return mongoScrapedTendersScope(), nil