Update dashboard statistics caching and retrieval logic

- Increased the cache duration for dashboard statistics from 60 seconds to 5 minutes, improving data freshness and reducing load on the backend.
- Introduced a stale cache mechanism that allows retrieval of stale statistics while refreshing them in the background, enhancing user experience by providing quicker access to data.
- Updated the statistics repository to handle the new caching logic, ensuring accurate and timely statistics reporting.
- Added tests to validate the new caching behavior and ensure the integrity of statistics retrieval.

This update optimizes the dashboard's performance and responsiveness by improving the caching strategy for statistics.

internal/dashboard/handler.go

@@ -181,7 +181,7 @@ func (h *Handler) Statistics(c echo.Context) error {
 		return response.InternalServerError(c, "Failed to load dashboard statistics")
 	}
 
-	setPrivateCache(c, 60)
+	setPrivateCache(c, 300)
 	return response.Success(c, out, "Dashboard statistics retrieved successfully")
 }
 

internal/dashboard/service.go

@@ -20,7 +20,8 @@ const (
 	defaultListLimit          = 5
 	maxListLimit              = 20
 	summaryCacheTTL           = 60 * time.Second
-	statisticsCacheTTL        = 60 * time.Second
+	statisticsCacheTTL        = 5 * time.Minute
+	statisticsStaleGraceTTL   = 15 * time.Minute
 )
 
 // Service defines dashboard business operations.
@@ -36,6 +37,7 @@ type Service interface {
 
 type cacheEntry[T any] struct {
 	expiresAt  time.Time
+	staleUntil time.Time
 	value      T
 }
 
@@ -203,6 +205,19 @@ func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*Stati
 		return cached, nil
 	}
 
+	if stale, ok := s.staleStatistics(days); ok {
+		go s.refreshStatistics(days)
+		return stale, nil
+	}
+
+	return s.loadStatistics(ctx, days)
+}
+
+func (s *service) refreshStatistics(days int) {
+	_, _ = s.loadStatistics(context.Background(), days)
+}
+
+func (s *service) loadStatistics(ctx context.Context, days int) (*StatisticsReportResponse, error) {
 	cacheKey := strconv.Itoa(days)
 	out, err, _ := s.statisticsGroup.Do(cacheKey, func() (interface{}, error) {
 		if cached, ok := s.cachedStatistics(days); ok {
@@ -256,6 +271,7 @@ func (s *service) storeSummary(windowSec int64, value *SummaryResponse) {
 
 	s.summary[windowSec] = cacheEntry[*SummaryResponse]{
 		expiresAt:  time.Now().Add(summaryCacheTTL),
+		staleUntil: time.Now().Add(summaryCacheTTL),
 		value:      value,
 	}
 }
@@ -268,6 +284,20 @@ func (s *service) cachedStatistics(days int) (*StatisticsReportResponse, bool) {
 
 	entry, ok := s.statistics[days]
 	if !ok || now.After(entry.expiresAt) {
+		return nil, false
+	}
+
+	return entry.value, true
+}
+
+func (s *service) staleStatistics(days int) (*StatisticsReportResponse, bool) {
+	now := time.Now()
+
+	s.statisticsMu.Lock()
+	defer s.statisticsMu.Unlock()
+
+	entry, ok := s.statistics[days]
+	if !ok || now.After(entry.staleUntil) {
 		if ok {
 			delete(s.statistics, days)
 		}
@@ -278,11 +308,14 @@ func (s *service) cachedStatistics(days int) (*StatisticsReportResponse, bool) {
 }
 
 func (s *service) storeStatistics(days int, value *StatisticsReportResponse) {
+	now := time.Now()
+
 	s.statisticsMu.Lock()
 	defer s.statisticsMu.Unlock()
 
 	s.statistics[days] = cacheEntry[*StatisticsReportResponse]{
-		expiresAt: time.Now().Add(statisticsCacheTTL),
+		expiresAt:  now.Add(statisticsCacheTTL),
+		staleUntil: now.Add(statisticsCacheTTL + statisticsStaleGraceTTL),
 		value:      value,
 	}
 }

internal/dashboard/statistics_repository.go

@@ -34,6 +34,7 @@ type scrapedTendersScope struct {
 	zero             bool
 	fromMinIO        bool
 	dailyCounts      map[string]int64 // non-nil when scope was resolved from MinIO
+	totalTenderCount int64            // >0 when MinIO procedure count should be used instead of Mongo Count
 }
 
 func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
@@ -44,18 +45,13 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 	endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
 	startDay := endDay.AddDate(0, 0, -(days - 1))
 
-	// Resolve scraped-document scope from MinIO (cached); falls back to Mongo flags on error.
-	scrapedScope, err := r.cachedScrapedTendersScope(queryCtx)
-	if err != nil {
-		return nil, err
-	}
-
 	var (
 		scrapedTED        map[string]int64
 		scrapedDocuments  map[string]int64
 		translatedNotices map[string]int64
 		totalDocuments    int64
 		totalTranslated   int64
+		scrapedScope      scrapedTendersScope
 	)
 
 	g, gctx := errgroup.WithContext(queryCtx)
@@ -74,15 +70,6 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		return nil
 	})
 
-	g.Go(func() error {
-		counts, err := r.scrapedDocumentsPerDay(gctx, startUnix, scrapedScope)
-		if err != nil {
-			return fmt.Errorf("scraped documents per day: %w", err)
-		}
-		scrapedDocuments = counts
-		return nil
-	})
-
 	g.Go(func() error {
 		counts, err := r.translatedNoticesPerDay(gctx, startDay, endDay)
 		if err != nil {
@@ -92,15 +79,6 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		return nil
 	})
 
-	g.Go(func() error {
-		total, err := r.totalScrapedTenders(gctx, scrapedScope)
-		if err != nil {
-			return fmt.Errorf("total scraped tenders: %w", err)
-		}
-		totalDocuments = total
-		return nil
-	})
-
 	g.Go(func() error {
 		total, err := r.metricsCounter.Get(gctx, orm.AITranslationSuccessCounterKey)
 		if err != nil {
@@ -110,10 +88,44 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 		return nil
 	})
 
+	// Resolve scraped-document scope from MinIO (cached) in parallel with Mongo queries.
+	g.Go(func() error {
+		scope, err := r.cachedScrapedTendersScope(gctx)
+		if err != nil {
+			return fmt.Errorf("scraped tenders scope: %w", err)
+		}
+		scrapedScope = scope
+		return nil
+	})
+
 	if err := g.Wait(); err != nil {
 		return nil, err
 	}
 
+	g2, gctx2 := errgroup.WithContext(queryCtx)
+
+	g2.Go(func() error {
+		counts, err := r.scrapedDocumentsPerDay(gctx2, startUnix, scrapedScope)
+		if err != nil {
+			return fmt.Errorf("scraped documents per day: %w", err)
+		}
+		scrapedDocuments = counts
+		return nil
+	})
+
+	g2.Go(func() error {
+		total, err := r.totalScrapedTenders(gctx2, scrapedScope)
+		if err != nil {
+			return fmt.Errorf("total scraped tenders: %w", err)
+		}
+		totalDocuments = total
+		return nil
+	})
+
+	if err := g2.Wait(); err != nil {
+		return nil, err
+	}
+
 	return &StatisticsReportResponse{
 		Days:        days,
 		GeneratedAt: now.Unix(),
@@ -230,6 +242,9 @@ func (r *repository) totalScrapedTenders(ctx context.Context, scope scrapedTende
 	if scope.zero {
 		return 0, nil
 	}
+	if scope.totalTenderCount > 0 {
+		return scope.totalTenderCount, nil
+	}
 
 	return r.ormRepo.Count(ctx, scope.match)
 }
@@ -297,17 +312,23 @@ func (r *repository) resolveScrapedTendersScope(ctx context.Context) (scrapedTen
 
 	folderIDs := folderIDsFromProcedures(procedures)
 	if len(folderIDs) > 0 {
+		totalTenders := int64(len(folderIDs))
 		if len(folderIDs) > maxScrapedFolderInClause {
-			r.logger.Warn("Too many MinIO procedure folders for dashboard statistics, using Mongo fallback", map[string]interface{}{
+			r.logger.Warn("Too many MinIO procedure folders for dashboard statistics, using MinIO counts without Mongo $in filter", map[string]interface{}{
 				"folder_count":  len(folderIDs),
 				"max_in_clause": maxScrapedFolderInClause,
 			})
-			return mongoScrapedTendersScope(), nil
+			return scrapedTendersScope{
+				fromMinIO:        true,
+				dailyCounts:      dailyCounts,
+				totalTenderCount: totalTenders,
+			}, nil
 		}
 		return scrapedTendersScope{
 			match:            bson.M{"contract_folder_id": bson.M{"$in": folderIDs}},
 			fromMinIO:        true,
 			dailyCounts:      dailyCounts,
+			totalTenderCount: totalTenders,
 		}, nil
 	}
 	return mongoScrapedTendersScope(), nil

internal/dashboard/statistics_repository_test.go

@@ -3,14 +3,28 @@ package dashboard
 import (
 	"context"
 	"errors"
+	"fmt"
 	"testing"
 	"time"
 
 	"tm/pkg/ai_summarizer"
+	"tm/pkg/logger"
 
 	"go.mongodb.org/mongo-driver/v2/bson"
 )
 
+type noopTestLogger struct{}
+
+func (noopTestLogger) Debug(string, map[string]interface{}) {}
+func (noopTestLogger) Info(string, map[string]interface{})  {}
+func (noopTestLogger) Warn(string, map[string]interface{})  {}
+func (noopTestLogger) Error(string, map[string]interface{}) {}
+func (noopTestLogger) Fatal(string, map[string]interface{}) {}
+func (l noopTestLogger) WithFields(map[string]interface{}) logger.Logger {
+	return l
+}
+func (noopTestLogger) Sync() error { return nil }
+
 type mockProcedureDocumentsLister struct {
 	procedures  []ai_summarizer.ProcedureDocumentsSummary
 	dailyCounts map[string]int64
@@ -90,6 +104,7 @@ func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) {
 			procedures: []ai_summarizer.ProcedureDocumentsSummary{
 				{ContractFolderID: "PROC-1", DocumentCount: 2},
 			},
+			dailyCounts: map[string]int64{"2026-06-28": 2},
 		},
 	}
 
@@ -103,6 +118,12 @@ func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) {
 	if !scope.fromMinIO {
 		t.Fatal("expected MinIO-backed scope")
 	}
+	if scope.totalTenderCount != 1 {
+		t.Fatalf("expected total tender count 1, got %d", scope.totalTenderCount)
+	}
+	if scope.dailyCounts == nil {
+		t.Fatal("expected MinIO daily counts")
+	}
 	in, ok := scope.match["contract_folder_id"].(bson.M)
 	if !ok {
 		t.Fatalf("expected contract_folder_id filter, got %#v", scope.match)
@@ -113,6 +134,44 @@ func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) {
 	}
 }
 
+func TestResolveScrapedTendersScopeKeepsMinIOCountsWhenTooManyFolders(t *testing.T) {
+	procedures := make([]ai_summarizer.ProcedureDocumentsSummary, maxScrapedFolderInClause+1)
+	for i := range procedures {
+		procedures[i] = ai_summarizer.ProcedureDocumentsSummary{
+			ContractFolderID: fmt.Sprintf("PROC-%d", i),
+			DocumentCount:    1,
+		}
+	}
+
+	repo := &repository{
+		logger: noopTestLogger{},
+		procedureLister: &mockProcedureDocumentsLister{
+			procedures:  procedures,
+			dailyCounts: map[string]int64{"2026-06-28": 3},
+		},
+	}
+
+	scope, err := repo.resolveScrapedTendersScope(context.Background())
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if scope.zero {
+		t.Fatal("expected non-zero scope")
+	}
+	if !scope.fromMinIO {
+		t.Fatal("expected MinIO-backed scope")
+	}
+	if scope.totalTenderCount != int64(len(procedures)) {
+		t.Fatalf("expected total tender count %d, got %d", len(procedures), scope.totalTenderCount)
+	}
+	if scope.dailyCounts == nil {
+		t.Fatal("expected MinIO daily counts to be preserved")
+	}
+	if scope.match != nil {
+		t.Fatalf("expected no Mongo $in filter for large MinIO scope, got %#v", scope.match)
+	}
+}
+
 func TestResolveScrapedTendersScopeMongoFallbackWhenMinIOEmpty(t *testing.T) {
 	repo := &repository{
 		procedureLister: &mockProcedureDocumentsLister{},