diff --git a/internal/dashboard/repository.go b/internal/dashboard/repository.go index b9383db..6d17aff 100644 --- a/internal/dashboard/repository.go +++ b/internal/dashboard/repository.go @@ -27,6 +27,11 @@ type scrapedDocumentsScanner interface { ScanScrapedDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, map[string]int64, error) } +// translatedNoticesScanner optionally scans MinIO for per-day translated notice counts. +type translatedNoticesScanner interface { + ScanTranslatedNotices(ctx context.Context) (map[string]int64, int64, error) +} + const defaultValueCurrency = "EUR" // Repository defines dashboard data access. @@ -51,6 +56,11 @@ type repository struct { scrapedScopeExpiry time.Time scrapedScopeCached bool scrapedScopeGroup singleflight.Group + translatedScopeMu sync.Mutex + translatedScope translatedNoticesScope + translatedScopeExpiry time.Time + translatedScopeCached bool + translatedScopeGroup singleflight.Group } // NewRepository creates a dashboard repository backed by the tenders collection. diff --git a/internal/dashboard/statistics_repository.go b/internal/dashboard/statistics_repository.go index 0b38b9d..3a2eaeb 100644 --- a/internal/dashboard/statistics_repository.go +++ b/internal/dashboard/statistics_repository.go @@ -33,6 +33,13 @@ type scrapedTendersScope struct { totalTenderCount int64 // >0 when MinIO procedure count should be used instead of Mongo Count } +// translatedNoticesScope holds MinIO-backed translation statistics when available. +type translatedNoticesScope struct { + fromMinIO bool + dailyCounts map[string]int64 + totalCount int64 +} + func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) { _ = ctx @@ -47,6 +54,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) totalTranslated int64 totalScrapedTED int64 scrapedScope scrapedTendersScope + translatedScope translatedNoticesScope ) var wg sync.WaitGroup @@ -78,23 +86,26 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) wg.Add(1) go func() { defer wg.Done() + scope, err := r.cachedTranslatedNoticesScope(context.Background()) + if err != nil { + recordFailure("translated_scope", err) + return + } + translatedScope = scope + if scope.fromMinIO { + return + } + qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout) defer cancel() - counts, err := r.translatedNoticesPerDay(qctx, startDay, endDay) + counts, err := r.translatedNoticesPerDayFromMetrics(qctx, startDay, endDay) if err != nil { recordFailure("translated_notices", err) translatedNotices = map[string]int64{} - return + } else { + translatedNotices = counts } - translatedNotices = counts - }() - - wg.Add(1) - go func() { - defer wg.Done() - qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout) - defer cancel() total, err := r.metricsCounter.Get(qctx, orm.AITranslationSuccessCounterKey) if err != nil { @@ -118,6 +129,11 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) wg.Wait() + if translatedScope.fromMinIO { + translatedNotices = filterDailyCountsSince(translatedScope.dailyCounts, startUnix) + totalTranslated = translatedScope.totalCount + } + wg.Add(1) go func() { defer wg.Done() @@ -224,10 +240,79 @@ func (r *repository) scrapedDocumentsPerDayFromMongo(ctx context.Context, startU return decodeDailyCounts(ctx, cursor) } -func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) { +func (r *repository) translatedNoticesPerDayFromMetrics(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) { return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay) } +func (r *repository) cachedTranslatedNoticesScope(_ context.Context) (translatedNoticesScope, error) { + now := time.Now() + + r.translatedScopeMu.Lock() + if r.translatedScopeCached && now.Before(r.translatedScopeExpiry) { + scope := r.translatedScope + r.translatedScopeMu.Unlock() + return scope, nil + } + r.translatedScopeMu.Unlock() + + v, err, _ := r.translatedScopeGroup.Do("translated-scope", func() (interface{}, error) { + r.translatedScopeMu.Lock() + if r.translatedScopeCached && time.Now().Before(r.translatedScopeExpiry) { + scope := r.translatedScope + r.translatedScopeMu.Unlock() + return scope, nil + } + r.translatedScopeMu.Unlock() + + resolveCtx, cancel := context.WithTimeout(context.Background(), scrapedScopeResolveTimeout) + defer cancel() + + scope, resolveErr := r.resolveTranslatedNoticesScope(resolveCtx) + if resolveErr != nil { + if r.logger != nil { + r.logger.Warn("MinIO translation scope unavailable, using metrics counter fallback for dashboard statistics", map[string]interface{}{ + "error": resolveErr.Error(), + }) + } + return translatedNoticesScope{}, nil + } + + r.translatedScopeMu.Lock() + r.translatedScope = scope + r.translatedScopeExpiry = time.Now().Add(scrapedScopeCacheTTL) + r.translatedScopeCached = true + r.translatedScopeMu.Unlock() + + return scope, nil + }) + if err != nil { + return translatedNoticesScope{}, err + } + + return v.(translatedNoticesScope), nil +} + +func (r *repository) resolveTranslatedNoticesScope(ctx context.Context) (translatedNoticesScope, error) { + if r.procedureLister == nil { + return translatedNoticesScope{}, nil + } + scanner, ok := r.procedureLister.(translatedNoticesScanner) + if !ok { + return translatedNoticesScope{}, nil + } + + dailyCounts, total, err := scanner.ScanTranslatedNotices(ctx) + if err != nil { + return translatedNoticesScope{}, err + } + + return translatedNoticesScope{ + fromMinIO: true, + dailyCounts: dailyCounts, + totalCount: total, + }, nil +} + func (r *repository) totalScrapedTenders(ctx context.Context, scope scrapedTendersScope) (int64, error) { if scope.zero { return 0, nil diff --git a/internal/dashboard/statistics_repository_test.go b/internal/dashboard/statistics_repository_test.go index 4b27ec4..d24bb06 100644 --- a/internal/dashboard/statistics_repository_test.go +++ b/internal/dashboard/statistics_repository_test.go @@ -29,6 +29,10 @@ type mockProcedureDocumentsLister struct { procedures []ai_summarizer.ProcedureDocumentsSummary dailyCounts map[string]int64 err error + + translatedDailyCounts map[string]int64 + translatedTotal int64 + translatedErr error } func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) { @@ -45,6 +49,13 @@ func (m *mockProcedureDocumentsLister) ScanScrapedDocuments(context.Context) ([] return m.procedures, m.dailyCounts, nil } +func (m *mockProcedureDocumentsLister) ScanTranslatedNotices(context.Context) (map[string]int64, int64, error) { + if m.translatedErr != nil { + return nil, 0, m.translatedErr + } + return m.translatedDailyCounts, m.translatedTotal, nil +} + func TestScrapedDocumentsFolderIDsUsesMinIOProcedures(t *testing.T) { repo := &repository{ procedureLister: &mockProcedureDocumentsLister{ @@ -259,3 +270,41 @@ func TestMongoScrapedTendersScopeMatchesStatisticsFilter(t *testing.T) { t.Fatalf("expected documents_scraped filter, got %#v", scope.match) } } + +func TestResolveTranslatedNoticesScopeUsesMinIOCounts(t *testing.T) { + repo := &repository{ + procedureLister: &mockProcedureDocumentsLister{ + translatedDailyCounts: map[string]int64{ + "2026-07-01": 4, + "2026-07-09": 2, + }, + translatedTotal: 6, + }, + } + + scope, err := repo.resolveTranslatedNoticesScope(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !scope.fromMinIO { + t.Fatal("expected MinIO-backed translation scope") + } + if scope.totalCount != 6 { + t.Fatalf("expected total 6, got %d", scope.totalCount) + } + if scope.dailyCounts["2026-07-09"] != 2 { + t.Fatalf("expected 2 on 2026-07-09, got %d", scope.dailyCounts["2026-07-09"]) + } +} + +func TestResolveTranslatedNoticesScopeWithoutLister(t *testing.T) { + repo := &repository{} + + scope, err := repo.resolveTranslatedNoticesScope(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if scope.fromMinIO { + t.Fatal("expected metrics fallback without lister") + } +} diff --git a/pkg/ai_summarizer/entities.go b/pkg/ai_summarizer/entities.go index 3d1b5eb..a58ec98 100644 --- a/pkg/ai_summarizer/entities.go +++ b/pkg/ai_summarizer/entities.go @@ -389,3 +389,25 @@ func (t *TenderJSON) translationForLanguage(language string) (StoredTranslation, } return t.storedTranslationText(language) } + +// hasAnyTranslation reports whether tender.json contains at least one non-empty +// title translation (via translation_status.done or the translations map). +func (t *TenderJSON) hasAnyTranslation() bool { + if t == nil { + return false + } + for _, lang := range t.translationsDone() { + if _, ok := t.storedTranslationText(lang); ok { + return true + } + } + if t.Translations == nil { + return false + } + for _, entry := range t.Translations { + if strings.TrimSpace(entry.Title) != "" { + return true + } + } + return false +} diff --git a/pkg/ai_summarizer/storage.go b/pkg/ai_summarizer/storage.go index eeb6531..807ac20 100644 --- a/pkg/ai_summarizer/storage.go +++ b/pkg/ai_summarizer/storage.go @@ -654,6 +654,89 @@ func (s *StorageClient) ScanScrapedDocuments(ctx context.Context) ([]ProcedureDo return results, dailyCounts, nil } +// ScanTranslatedNotices scans MinIO for tender.json objects with stored translations +// and returns daily notice counts keyed by UTC date (YYYY-MM-DD) plus the total. +func (s *StorageClient) ScanTranslatedNotices(ctx context.Context) (map[string]int64, int64, error) { + s.logger.Debug("Scanning translated notices from MinIO", map[string]interface{}{ + "bucket": s.config.MinioBucket, + "prefix": procedurePrefix, + }) + + dailyCounts := make(map[string]int64) + var total int64 + seen := make(map[string]struct{}) + var fatalErr error + + appendFromPrefix := func(prefix string) bool { + objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) + for object := range objectCh { + if object.Err != nil { + fatalErr = s.logMinIOFailure("list_translated_notices", object.Err, map[string]interface{}{ + "prefix": prefix, + }) + return false + } + key := object.Key + if !strings.HasSuffix(key, "/tender.json") { + continue + } + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + + tenderJSON, err := s.readTenderJSONAtKey(ctx, key) + if err != nil { + if errors.Is(err, ErrObjectNotFound) { + continue + } + if errors.Is(err, ErrMinIOUnavailable) { + fatalErr = err + return false + } + s.logger.Debug("Skipping tender.json during translation scan", map[string]interface{}{ + "object_key": key, + "error": err.Error(), + }) + continue + } + if !tenderJSON.hasAnyTranslation() { + continue + } + + total++ + modified := normalizeUnixSeconds(object.LastModified.Unix()) + if modified > 0 { + date := time.Unix(modified, 0).UTC().Format("2006-01-02") + dailyCounts[date]++ + } + } + return true + } + + if !appendFromPrefix(procedurePrefix) { + return nil, 0, fatalErr + } + if len(seen) == 0 { + if !appendFromPrefix("") { + return nil, 0, fatalErr + } + } + if fatalErr != nil { + return nil, 0, fatalErr + } + + s.logger.Info("Scanned translated notices from storage", map[string]interface{}{ + "bucket": s.config.MinioBucket, + "total_count": total, + }) + + return dailyCounts, total, nil +} + func normalizeUnixSeconds(ts int64) int64 { if ts > 1_000_000_000_000 { return ts / 1000