Update MinIO bucket configuration and enhance dashboard service caching

- Changed the default MinIO bucket name from "opplens-documents" to "opplens" across multiple configuration files.
- Introduced caching for dashboard statistics in the service layer to improve performance and reduce redundant data fetching.
- Implemented a mutex for thread-safe access to cached statistics, ensuring data integrity during concurrent requests.

This update streamlines the configuration for the AI summarizer and optimizes the dashboard service, enhancing overall system efficiency.

internal/dashboard/service.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 	"tm/pkg/logger"
 )
@@ -15,6 +16,7 @@ const (
 	defaultCountriesLimit     = 6
 	defaultListLimit          = 5
 	maxListLimit              = 20
+	statisticsCacheTTL        = 60 * time.Second
 )
 
 // Service defines dashboard business operations.
@@ -28,14 +30,25 @@ type Service interface {
 	Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error)
 }
 
+type statisticsCacheEntry struct {
+	expiresAt time.Time
+	value     *StatisticsReportResponse
+}
+
 type service struct {
-	repo   Repository
-	logger logger.Logger
+	repo         Repository
+	logger       logger.Logger
+	statisticsMu sync.Mutex
+	statistics   map[int]statisticsCacheEntry
 }
 
 // NewService creates a dashboard service.
 func NewService(repo Repository, log logger.Logger) Service {
-	return &service{repo: repo, logger: log}
+	return &service{
+		repo:       repo,
+		logger:     log,
+		statistics: make(map[int]statisticsCacheEntry),
+	}
 }
 
 func (s *service) Summary(ctx context.Context, query SummaryQuery) (*SummaryResponse, error) {
@@ -160,6 +173,10 @@ func (s *service) Recent(ctx context.Context, query RecentQuery) (*RecentRespons
 
 func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error) {
 	days := trendDays(query.Days)
+	if cached, ok := s.cachedStatistics(days); ok {
+		return cached, nil
+	}
+
 	startUnix := trendStartUnix(days)
 
 	s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
@@ -174,9 +191,37 @@ func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*Stati
 		return nil, fmt.Errorf("dashboard statistics: %w", err)
 	}
 
+	s.storeStatistics(days, out)
 	return out, nil
 }
 
+func (s *service) cachedStatistics(days int) (*StatisticsReportResponse, bool) {
+	now := time.Now()
+
+	s.statisticsMu.Lock()
+	defer s.statisticsMu.Unlock()
+
+	entry, ok := s.statistics[days]
+	if !ok || now.After(entry.expiresAt) {
+		if ok {
+			delete(s.statistics, days)
+		}
+		return nil, false
+	}
+
+	return entry.value, true
+}
+
+func (s *service) storeStatistics(days int, value *StatisticsReportResponse) {
+	s.statisticsMu.Lock()
+	defer s.statisticsMu.Unlock()
+
+	s.statistics[days] = statisticsCacheEntry{
+		expiresAt: time.Now().Add(statisticsCacheTTL),
+		value:     value,
+	}
+}
+
 func closingWindowHours(hours int) int {
 	if hours <= 0 {
 		return defaultClosingWindowHours

internal/dashboard/statistics_repository.go

@@ -10,6 +10,7 @@ import (
 
 	"go.mongodb.org/mongo-driver/v2/bson"
 	mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
+	"golang.org/x/sync/errgroup"
 )
 
 const noticesCollectionName = "notices"
@@ -19,32 +20,66 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 	endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
 	startDay := endDay.AddDate(0, 0, -(days - 1))
 
-	scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{
-		"source": notice.TenderSourceTEDScraper,
-		"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
-	}, "processing_metadata.scraped_at")
-	if err != nil {
-		return nil, fmt.Errorf("scraped ted per day: %w", err)
-	}
+	var (
+		scrapedTED        map[string]int64
+		scrapedDocuments  map[string]int64
+		translatedNotices map[string]int64
+		totalDocuments    int64
+		totalTranslated   int64
+	)
 
-	scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix)
-	if err != nil {
-		return nil, fmt.Errorf("scraped documents per day: %w", err)
-	}
+	g, gctx := errgroup.WithContext(ctx)
 
-	translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay)
-	if err != nil {
-		return nil, fmt.Errorf("translated notices per day: %w", err)
-	}
+	g.Go(func() error {
+		counts, err := r.dailyCountByTimestamp(gctx, noticesCollectionName, bson.M{
+			"source": notice.TenderSourceTEDScraper,
+			"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
+		}, "processing_metadata.scraped_at")
+		if err != nil {
+			return fmt.Errorf("scraped ted per day: %w", err)
+		}
+		scrapedTED = counts
+		return nil
+	})
 
-	totalDocuments, err := r.totalScrapedDocuments(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("total scraped documents: %w", err)
-	}
+	g.Go(func() error {
+		counts, err := r.scrapedDocumentsPerDay(gctx, startUnix)
+		if err != nil {
+			return fmt.Errorf("scraped documents per day: %w", err)
+		}
+		scrapedDocuments = counts
+		return nil
+	})
 
-	totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey)
-	if err != nil {
-		return nil, fmt.Errorf("total translated tenders: %w", err)
+	g.Go(func() error {
+		counts, err := r.translatedNoticesPerDay(gctx, startDay, endDay)
+		if err != nil {
+			return fmt.Errorf("translated notices per day: %w", err)
+		}
+		translatedNotices = counts
+		return nil
+	})
+
+	g.Go(func() error {
+		total, err := r.totalScrapedDocuments(gctx)
+		if err != nil {
+			return fmt.Errorf("total scraped documents: %w", err)
+		}
+		totalDocuments = total
+		return nil
+	})
+
+	g.Go(func() error {
+		total, err := r.metricsCounter.Get(gctx, orm.AITranslationSuccessCounterKey)
+		if err != nil {
+			return fmt.Errorf("total translated tenders: %w", err)
+		}
+		totalTranslated = total
+		return nil
+	})
+
+	if err := g.Wait(); err != nil {
+		return nil, err
 	}
 
 	return &StatisticsReportResponse{
@@ -94,6 +129,10 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
 		{{Key: "$match", Value: bson.M{
 			"processing_metadata.documents_scraped": true,
 			"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
+			"$or": []bson.M{
+				{"processing_metadata.documents_scraped_at": bson.M{"$gte": startUnix}},
+				{"scraped_documents.scraped_at": bson.M{"$gte": startUnix}},
+			},
 		}}},
 		{{Key: "$unwind", Value: "$scraped_documents"}},
 		{{Key: "$addFields", Value: bson.M{

internal/notice/repository.go

@@ -39,6 +39,10 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
 	noticeIndexes := []orm.Index{
 		// processing_metadata.processed
 		*orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}),
+		*orm.NewIndex("source_scraped_at_idx", bson.D{
+			{Key: "source", Value: 1},
+			{Key: "processing_metadata.scraped_at", Value: 1},
+		}),
 		*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
 		*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
 		// One row per TED ContractNoticeID when present (parallel scrape races).

internal/tender/repository.go

@@ -150,6 +150,13 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
 			{Key: "tender_deadline", Value: 1},
 			{Key: "created_at", Value: 1},
 		}),
+		*orm.NewIndex("documents_scraped_at_idx", bson.D{
+			{Key: "processing_metadata.documents_scraped", Value: 1},
+			{Key: "processing_metadata.documents_scraped_at", Value: -1},
+		}),
+		*orm.NewIndex("scraped_documents_scraped_at_idx", bson.D{
+			{Key: "scraped_documents.scraped_at", Value: 1},
+		}),
 		// Index for CPV matching queries
 		*orm.NewIndex("cpv_matching_idx", bson.D{
 			{Key: "status", Value: 1},