diff --git a/internal/dashboard/handler.go b/internal/dashboard/handler.go index 6a1b9ea..3eed78f 100644 --- a/internal/dashboard/handler.go +++ b/internal/dashboard/handler.go @@ -181,7 +181,7 @@ func (h *Handler) Statistics(c echo.Context) error { return response.InternalServerError(c, "Failed to load dashboard statistics") } - setPrivateCache(c, 60) + setPrivateCache(c, 300) return response.Success(c, out, "Dashboard statistics retrieved successfully") } diff --git a/internal/dashboard/service.go b/internal/dashboard/service.go index b461b2b..102a3e5 100644 --- a/internal/dashboard/service.go +++ b/internal/dashboard/service.go @@ -20,7 +20,8 @@ const ( defaultListLimit = 5 maxListLimit = 20 summaryCacheTTL = 60 * time.Second - statisticsCacheTTL = 60 * time.Second + statisticsCacheTTL = 5 * time.Minute + statisticsStaleGraceTTL = 15 * time.Minute ) // Service defines dashboard business operations. @@ -35,8 +36,9 @@ type Service interface { } type cacheEntry[T any] struct { - expiresAt time.Time - value T + expiresAt time.Time + staleUntil time.Time + value T } type service struct { @@ -203,6 +205,19 @@ func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*Stati return cached, nil } + if stale, ok := s.staleStatistics(days); ok { + go s.refreshStatistics(days) + return stale, nil + } + + return s.loadStatistics(ctx, days) +} + +func (s *service) refreshStatistics(days int) { + _, _ = s.loadStatistics(context.Background(), days) +} + +func (s *service) loadStatistics(ctx context.Context, days int) (*StatisticsReportResponse, error) { cacheKey := strconv.Itoa(days) out, err, _ := s.statisticsGroup.Do(cacheKey, func() (interface{}, error) { if cached, ok := s.cachedStatistics(days); ok { @@ -255,8 +270,9 @@ func (s *service) storeSummary(windowSec int64, value *SummaryResponse) { defer s.summaryMu.Unlock() s.summary[windowSec] = cacheEntry[*SummaryResponse]{ - expiresAt: time.Now().Add(summaryCacheTTL), - value: value, + expiresAt: time.Now().Add(summaryCacheTTL), + staleUntil: time.Now().Add(summaryCacheTTL), + value: value, } } @@ -268,6 +284,20 @@ func (s *service) cachedStatistics(days int) (*StatisticsReportResponse, bool) { entry, ok := s.statistics[days] if !ok || now.After(entry.expiresAt) { + return nil, false + } + + return entry.value, true +} + +func (s *service) staleStatistics(days int) (*StatisticsReportResponse, bool) { + now := time.Now() + + s.statisticsMu.Lock() + defer s.statisticsMu.Unlock() + + entry, ok := s.statistics[days] + if !ok || now.After(entry.staleUntil) { if ok { delete(s.statistics, days) } @@ -278,12 +308,15 @@ func (s *service) cachedStatistics(days int) (*StatisticsReportResponse, bool) { } func (s *service) storeStatistics(days int, value *StatisticsReportResponse) { + now := time.Now() + s.statisticsMu.Lock() defer s.statisticsMu.Unlock() s.statistics[days] = cacheEntry[*StatisticsReportResponse]{ - expiresAt: time.Now().Add(statisticsCacheTTL), - value: value, + expiresAt: now.Add(statisticsCacheTTL), + staleUntil: now.Add(statisticsCacheTTL + statisticsStaleGraceTTL), + value: value, } } diff --git a/internal/dashboard/statistics_repository.go b/internal/dashboard/statistics_repository.go index b5320bf..23997f8 100644 --- a/internal/dashboard/statistics_repository.go +++ b/internal/dashboard/statistics_repository.go @@ -30,10 +30,11 @@ const ( // scrapedTendersScope is the resolved Mongo filter for tenders with scraped documents. // zero is set when MinIO reports no procedure folders; callers must return zero without querying. type scrapedTendersScope struct { - match bson.M - zero bool - fromMinIO bool - dailyCounts map[string]int64 // non-nil when scope was resolved from MinIO + match bson.M + zero bool + fromMinIO bool + dailyCounts map[string]int64 // non-nil when scope was resolved from MinIO + totalTenderCount int64 // >0 when MinIO procedure count should be used instead of Mongo Count } func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) { @@ -44,18 +45,13 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) startDay := endDay.AddDate(0, 0, -(days - 1)) - // Resolve scraped-document scope from MinIO (cached); falls back to Mongo flags on error. - scrapedScope, err := r.cachedScrapedTendersScope(queryCtx) - if err != nil { - return nil, err - } - var ( scrapedTED map[string]int64 scrapedDocuments map[string]int64 translatedNotices map[string]int64 totalDocuments int64 totalTranslated int64 + scrapedScope scrapedTendersScope ) g, gctx := errgroup.WithContext(queryCtx) @@ -74,15 +70,6 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) return nil }) - g.Go(func() error { - counts, err := r.scrapedDocumentsPerDay(gctx, startUnix, scrapedScope) - if err != nil { - return fmt.Errorf("scraped documents per day: %w", err) - } - scrapedDocuments = counts - return nil - }) - g.Go(func() error { counts, err := r.translatedNoticesPerDay(gctx, startDay, endDay) if err != nil { @@ -92,15 +79,6 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) return nil }) - g.Go(func() error { - total, err := r.totalScrapedTenders(gctx, scrapedScope) - if err != nil { - return fmt.Errorf("total scraped tenders: %w", err) - } - totalDocuments = total - return nil - }) - g.Go(func() error { total, err := r.metricsCounter.Get(gctx, orm.AITranslationSuccessCounterKey) if err != nil { @@ -110,10 +88,44 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) return nil }) + // Resolve scraped-document scope from MinIO (cached) in parallel with Mongo queries. + g.Go(func() error { + scope, err := r.cachedScrapedTendersScope(gctx) + if err != nil { + return fmt.Errorf("scraped tenders scope: %w", err) + } + scrapedScope = scope + return nil + }) + if err := g.Wait(); err != nil { return nil, err } + g2, gctx2 := errgroup.WithContext(queryCtx) + + g2.Go(func() error { + counts, err := r.scrapedDocumentsPerDay(gctx2, startUnix, scrapedScope) + if err != nil { + return fmt.Errorf("scraped documents per day: %w", err) + } + scrapedDocuments = counts + return nil + }) + + g2.Go(func() error { + total, err := r.totalScrapedTenders(gctx2, scrapedScope) + if err != nil { + return fmt.Errorf("total scraped tenders: %w", err) + } + totalDocuments = total + return nil + }) + + if err := g2.Wait(); err != nil { + return nil, err + } + return &StatisticsReportResponse{ Days: days, GeneratedAt: now.Unix(), @@ -230,6 +242,9 @@ func (r *repository) totalScrapedTenders(ctx context.Context, scope scrapedTende if scope.zero { return 0, nil } + if scope.totalTenderCount > 0 { + return scope.totalTenderCount, nil + } return r.ormRepo.Count(ctx, scope.match) } @@ -297,17 +312,23 @@ func (r *repository) resolveScrapedTendersScope(ctx context.Context) (scrapedTen folderIDs := folderIDsFromProcedures(procedures) if len(folderIDs) > 0 { + totalTenders := int64(len(folderIDs)) if len(folderIDs) > maxScrapedFolderInClause { - r.logger.Warn("Too many MinIO procedure folders for dashboard statistics, using Mongo fallback", map[string]interface{}{ + r.logger.Warn("Too many MinIO procedure folders for dashboard statistics, using MinIO counts without Mongo $in filter", map[string]interface{}{ "folder_count": len(folderIDs), "max_in_clause": maxScrapedFolderInClause, }) - return mongoScrapedTendersScope(), nil + return scrapedTendersScope{ + fromMinIO: true, + dailyCounts: dailyCounts, + totalTenderCount: totalTenders, + }, nil } return scrapedTendersScope{ - match: bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}, - fromMinIO: true, - dailyCounts: dailyCounts, + match: bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}, + fromMinIO: true, + dailyCounts: dailyCounts, + totalTenderCount: totalTenders, }, nil } return mongoScrapedTendersScope(), nil diff --git a/internal/dashboard/statistics_repository_test.go b/internal/dashboard/statistics_repository_test.go index 77eef99..4b27ec4 100644 --- a/internal/dashboard/statistics_repository_test.go +++ b/internal/dashboard/statistics_repository_test.go @@ -3,14 +3,28 @@ package dashboard import ( "context" "errors" + "fmt" "testing" "time" "tm/pkg/ai_summarizer" + "tm/pkg/logger" "go.mongodb.org/mongo-driver/v2/bson" ) +type noopTestLogger struct{} + +func (noopTestLogger) Debug(string, map[string]interface{}) {} +func (noopTestLogger) Info(string, map[string]interface{}) {} +func (noopTestLogger) Warn(string, map[string]interface{}) {} +func (noopTestLogger) Error(string, map[string]interface{}) {} +func (noopTestLogger) Fatal(string, map[string]interface{}) {} +func (l noopTestLogger) WithFields(map[string]interface{}) logger.Logger { + return l +} +func (noopTestLogger) Sync() error { return nil } + type mockProcedureDocumentsLister struct { procedures []ai_summarizer.ProcedureDocumentsSummary dailyCounts map[string]int64 @@ -90,6 +104,7 @@ func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) { procedures: []ai_summarizer.ProcedureDocumentsSummary{ {ContractFolderID: "PROC-1", DocumentCount: 2}, }, + dailyCounts: map[string]int64{"2026-06-28": 2}, }, } @@ -103,6 +118,12 @@ func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) { if !scope.fromMinIO { t.Fatal("expected MinIO-backed scope") } + if scope.totalTenderCount != 1 { + t.Fatalf("expected total tender count 1, got %d", scope.totalTenderCount) + } + if scope.dailyCounts == nil { + t.Fatal("expected MinIO daily counts") + } in, ok := scope.match["contract_folder_id"].(bson.M) if !ok { t.Fatalf("expected contract_folder_id filter, got %#v", scope.match) @@ -113,6 +134,44 @@ func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) { } } +func TestResolveScrapedTendersScopeKeepsMinIOCountsWhenTooManyFolders(t *testing.T) { + procedures := make([]ai_summarizer.ProcedureDocumentsSummary, maxScrapedFolderInClause+1) + for i := range procedures { + procedures[i] = ai_summarizer.ProcedureDocumentsSummary{ + ContractFolderID: fmt.Sprintf("PROC-%d", i), + DocumentCount: 1, + } + } + + repo := &repository{ + logger: noopTestLogger{}, + procedureLister: &mockProcedureDocumentsLister{ + procedures: procedures, + dailyCounts: map[string]int64{"2026-06-28": 3}, + }, + } + + scope, err := repo.resolveScrapedTendersScope(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if scope.zero { + t.Fatal("expected non-zero scope") + } + if !scope.fromMinIO { + t.Fatal("expected MinIO-backed scope") + } + if scope.totalTenderCount != int64(len(procedures)) { + t.Fatalf("expected total tender count %d, got %d", len(procedures), scope.totalTenderCount) + } + if scope.dailyCounts == nil { + t.Fatal("expected MinIO daily counts to be preserved") + } + if scope.match != nil { + t.Fatalf("expected no Mongo $in filter for large MinIO scope, got %#v", scope.match) + } +} + func TestResolveScrapedTendersScopeMongoFallbackWhenMinIOEmpty(t *testing.T) { repo := &repository{ procedureLister: &mockProcedureDocumentsLister{},