Enhance dashboard service with Redis caching for statistics

- Updated the dashboard service to integrate Redis caching for improved performance in statistics retrieval.
- Modified the NewService function to accept a Redis client, enabling caching of dashboard statistics.
- Implemented logic to retrieve statistics from Redis, falling back to the database if necessary, and introduced a background process to warm the cache.
- Enhanced error handling and logging for Redis operations to ensure robust statistics management.
- Increased cache duration for scraped documents and adjusted timeout settings for MongoDB queries to optimize performance.

This update significantly improves the responsiveness and efficiency of the dashboard by leveraging Redis for caching statistics.

internal/dashboard/service.go

@@ -2,13 +2,16 @@ package dashboard
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strconv"
 	"strings"
 	"sync"
 	"time"
 	"tm/pkg/logger"
+	"tm/pkg/redis"
 
+	goredis "github.com/redis/go-redis/v9"
 	"golang.org/x/sync/singleflight"
 )
 
@@ -44,6 +47,7 @@ type cacheEntry[T any] struct {
 type service struct {
 	repo            Repository
 	logger          logger.Logger
+	redis           redis.Client
 	summaryMu       sync.Mutex
 	summary         map[int64]cacheEntry[*SummaryResponse]
 	summaryGroup    singleflight.Group
@@ -53,13 +57,16 @@ type service struct {
 }
 
 // NewService creates a dashboard service.
-func NewService(repo Repository, log logger.Logger) Service {
-	return &service{
+func NewService(repo Repository, log logger.Logger, redisClient redis.Client) Service {
+	s := &service{
 		repo:       repo,
 		logger:     log,
+		redis:      redisClient,
 		summary:    make(map[int64]cacheEntry[*SummaryResponse]),
 		statistics: make(map[int]cacheEntry[*StatisticsReportResponse]),
 	}
+	go s.warmStatisticsCache()
+	return s
 }
 
 func (s *service) Summary(ctx context.Context, query SummaryQuery) (*SummaryResponse, error) {
@@ -210,11 +217,60 @@ func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*Stati
 		return stale, nil
 	}
 
-	return s.loadStatistics(ctx, days)
+	if redisCached, ok := s.getRedisStatistics(ctx, days); ok {
+		s.storeStatistics(days, redisCached)
+		go s.refreshStatistics(days)
+		return redisCached, nil
+	}
+
+	result, err := s.loadStatistics(ctx, days)
+	if err == nil {
+		return result, nil
+	}
+
+	if redisCached, ok := s.getRedisStatistics(ctx, days); ok {
+		s.storeStatistics(days, redisCached)
+		return redisCached, nil
+	}
+
+	s.logger.Error("Serving empty dashboard statistics after fetch failure", map[string]interface{}{
+		"days":  days,
+		"error": err.Error(),
+	})
+	return emptyStatisticsReport(days), nil
+}
+
+func (s *service) warmStatisticsCache() {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	if cached, ok := s.getRedisStatistics(ctx, defaultTrendDays); ok {
+		s.storeStatistics(defaultTrendDays, cached)
+		s.logger.Info("Dashboard statistics cache warmed from Redis", map[string]interface{}{
+			"days": defaultTrendDays,
+		})
+		return
+	}
+
+	go s.refreshStatistics(defaultTrendDays)
+}
+
+func emptyStatisticsReport(days int) *StatisticsReportResponse {
+	return &StatisticsReportResponse{
+		Days:        days,
+		GeneratedAt: time.Now().UTC().Unix(),
+		Daily: StatisticsDailySeries{
+			ScrapedTED:        fillTrendSeries(days, nil),
+			ScrapedDocuments:  fillTrendSeries(days, nil),
+			TranslatedNotices: fillTrendSeries(days, nil),
+		},
+	}
 }
 
 func (s *service) refreshStatistics(days int) {
-	_, _ = s.loadStatistics(context.Background(), days)
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer cancel()
+	_, _ = s.loadStatistics(ctx, days)
 }
 
 func (s *service) loadStatistics(ctx context.Context, days int) (*StatisticsReportResponse, error) {
@@ -239,6 +295,7 @@ func (s *service) loadStatistics(ctx context.Context, days int) (*StatisticsRepo
 		}
 
 		s.storeStatistics(days, result)
+		s.storeRedisStatistics(context.Background(), days, result)
 		return result, nil
 	})
 	if err != nil {
@@ -320,6 +377,62 @@ func (s *service) storeStatistics(days int, value *StatisticsReportResponse) {
 	}
 }
 
+func statisticsRedisKey(days int) string {
+	return fmt.Sprintf("dashboard:statistics:%d", days)
+}
+
+func (s *service) getRedisStatistics(ctx context.Context, days int) (*StatisticsReportResponse, bool) {
+	if s.redis == nil {
+		return nil, false
+	}
+
+	raw, err := s.redis.Get(ctx, statisticsRedisKey(days))
+	if err != nil {
+		if err != goredis.Nil {
+			s.logger.Warn("Failed to read dashboard statistics cache from Redis", map[string]interface{}{
+				"days":  days,
+				"error": err.Error(),
+			})
+		}
+		return nil, false
+	}
+
+	var report StatisticsReportResponse
+	if err := json.Unmarshal([]byte(raw), &report); err != nil {
+		s.logger.Warn("Failed to decode dashboard statistics cache from Redis", map[string]interface{}{
+			"days":  days,
+			"error": err.Error(),
+		})
+		_ = s.redis.Del(ctx, statisticsRedisKey(days))
+		return nil, false
+	}
+
+	return &report, true
+}
+
+func (s *service) storeRedisStatistics(ctx context.Context, days int, value *StatisticsReportResponse) {
+	if s.redis == nil || value == nil {
+		return
+	}
+
+	encoded, err := json.Marshal(value)
+	if err != nil {
+		s.logger.Warn("Failed to encode dashboard statistics for Redis", map[string]interface{}{
+			"days":  days,
+			"error": err.Error(),
+		})
+		return
+	}
+
+	ttl := statisticsCacheTTL + statisticsStaleGraceTTL
+	if err := s.redis.Set(ctx, statisticsRedisKey(days), string(encoded), ttl); err != nil {
+		s.logger.Warn("Failed to store dashboard statistics in Redis", map[string]interface{}{
+			"days":  days,
+			"error": err.Error(),
+		})
+	}
+}
+
 func closingWindowHours(hours int) int {
 	if hours <= 0 {
 		return defaultClosingWindowHours