From 1df44ec8ca1ae941657a032596af1c0662feae19 Mon Sep 17 00:00:00 2001 From: Mazyar Date: Fri, 10 Jul 2026 17:05:38 +0330 Subject: [PATCH] Add translated notices scanning functionality to dashboard repository - Introduced a new `translatedNoticesScanner` interface for scanning MinIO for daily translated notice counts. - Enhanced the `repository` struct to include fields for managing translated notice scope and caching. - Implemented the `ScanTranslatedNotices` method in the `StorageClient` to retrieve daily counts and total from MinIO. - Updated the `Statistics` method in the repository to utilize the new translated notices scope, improving data accuracy. - Added unit tests for the translated notices functionality, ensuring robust validation of the new features. This update significantly enhances the dashboard's ability to handle translated notice statistics, improving overall data management and reporting capabilities. --- internal/dashboard/repository.go | 10 ++ internal/dashboard/statistics_repository.go | 107 ++++++++++++++++-- .../dashboard/statistics_repository_test.go | 49 ++++++++ pkg/ai_summarizer/entities.go | 22 ++++ pkg/ai_summarizer/storage.go | 83 ++++++++++++++ 5 files changed, 260 insertions(+), 11 deletions(-) diff --git a/internal/dashboard/repository.go b/internal/dashboard/repository.go index b9383db..6d17aff 100644 --- a/internal/dashboard/repository.go +++ b/internal/dashboard/repository.go @@ -27,6 +27,11 @@ type scrapedDocumentsScanner interface { ScanScrapedDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, map[string]int64, error) } +// translatedNoticesScanner optionally scans MinIO for per-day translated notice counts. +type translatedNoticesScanner interface { + ScanTranslatedNotices(ctx context.Context) (map[string]int64, int64, error) +} + const defaultValueCurrency = "EUR" // Repository defines dashboard data access. @@ -51,6 +56,11 @@ type repository struct { scrapedScopeExpiry time.Time scrapedScopeCached bool scrapedScopeGroup singleflight.Group + translatedScopeMu sync.Mutex + translatedScope translatedNoticesScope + translatedScopeExpiry time.Time + translatedScopeCached bool + translatedScopeGroup singleflight.Group } // NewRepository creates a dashboard repository backed by the tenders collection. diff --git a/internal/dashboard/statistics_repository.go b/internal/dashboard/statistics_repository.go index 0b38b9d..3a2eaeb 100644 --- a/internal/dashboard/statistics_repository.go +++ b/internal/dashboard/statistics_repository.go @@ -33,6 +33,13 @@ type scrapedTendersScope struct { totalTenderCount int64 // >0 when MinIO procedure count should be used instead of Mongo Count } +// translatedNoticesScope holds MinIO-backed translation statistics when available. +type translatedNoticesScope struct { + fromMinIO bool + dailyCounts map[string]int64 + totalCount int64 +} + func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) { _ = ctx @@ -47,6 +54,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) totalTranslated int64 totalScrapedTED int64 scrapedScope scrapedTendersScope + translatedScope translatedNoticesScope ) var wg sync.WaitGroup @@ -78,23 +86,26 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) wg.Add(1) go func() { defer wg.Done() + scope, err := r.cachedTranslatedNoticesScope(context.Background()) + if err != nil { + recordFailure("translated_scope", err) + return + } + translatedScope = scope + if scope.fromMinIO { + return + } + qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout) defer cancel() - counts, err := r.translatedNoticesPerDay(qctx, startDay, endDay) + counts, err := r.translatedNoticesPerDayFromMetrics(qctx, startDay, endDay) if err != nil { recordFailure("translated_notices", err) translatedNotices = map[string]int64{} - return + } else { + translatedNotices = counts } - translatedNotices = counts - }() - - wg.Add(1) - go func() { - defer wg.Done() - qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout) - defer cancel() total, err := r.metricsCounter.Get(qctx, orm.AITranslationSuccessCounterKey) if err != nil { @@ -118,6 +129,11 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) wg.Wait() + if translatedScope.fromMinIO { + translatedNotices = filterDailyCountsSince(translatedScope.dailyCounts, startUnix) + totalTranslated = translatedScope.totalCount + } + wg.Add(1) go func() { defer wg.Done() @@ -224,10 +240,79 @@ func (r *repository) scrapedDocumentsPerDayFromMongo(ctx context.Context, startU return decodeDailyCounts(ctx, cursor) } -func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) { +func (r *repository) translatedNoticesPerDayFromMetrics(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) { return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay) } +func (r *repository) cachedTranslatedNoticesScope(_ context.Context) (translatedNoticesScope, error) { + now := time.Now() + + r.translatedScopeMu.Lock() + if r.translatedScopeCached && now.Before(r.translatedScopeExpiry) { + scope := r.translatedScope + r.translatedScopeMu.Unlock() + return scope, nil + } + r.translatedScopeMu.Unlock() + + v, err, _ := r.translatedScopeGroup.Do("translated-scope", func() (interface{}, error) { + r.translatedScopeMu.Lock() + if r.translatedScopeCached && time.Now().Before(r.translatedScopeExpiry) { + scope := r.translatedScope + r.translatedScopeMu.Unlock() + return scope, nil + } + r.translatedScopeMu.Unlock() + + resolveCtx, cancel := context.WithTimeout(context.Background(), scrapedScopeResolveTimeout) + defer cancel() + + scope, resolveErr := r.resolveTranslatedNoticesScope(resolveCtx) + if resolveErr != nil { + if r.logger != nil { + r.logger.Warn("MinIO translation scope unavailable, using metrics counter fallback for dashboard statistics", map[string]interface{}{ + "error": resolveErr.Error(), + }) + } + return translatedNoticesScope{}, nil + } + + r.translatedScopeMu.Lock() + r.translatedScope = scope + r.translatedScopeExpiry = time.Now().Add(scrapedScopeCacheTTL) + r.translatedScopeCached = true + r.translatedScopeMu.Unlock() + + return scope, nil + }) + if err != nil { + return translatedNoticesScope{}, err + } + + return v.(translatedNoticesScope), nil +} + +func (r *repository) resolveTranslatedNoticesScope(ctx context.Context) (translatedNoticesScope, error) { + if r.procedureLister == nil { + return translatedNoticesScope{}, nil + } + scanner, ok := r.procedureLister.(translatedNoticesScanner) + if !ok { + return translatedNoticesScope{}, nil + } + + dailyCounts, total, err := scanner.ScanTranslatedNotices(ctx) + if err != nil { + return translatedNoticesScope{}, err + } + + return translatedNoticesScope{ + fromMinIO: true, + dailyCounts: dailyCounts, + totalCount: total, + }, nil +} + func (r *repository) totalScrapedTenders(ctx context.Context, scope scrapedTendersScope) (int64, error) { if scope.zero { return 0, nil diff --git a/internal/dashboard/statistics_repository_test.go b/internal/dashboard/statistics_repository_test.go index 4b27ec4..d24bb06 100644 --- a/internal/dashboard/statistics_repository_test.go +++ b/internal/dashboard/statistics_repository_test.go @@ -29,6 +29,10 @@ type mockProcedureDocumentsLister struct { procedures []ai_summarizer.ProcedureDocumentsSummary dailyCounts map[string]int64 err error + + translatedDailyCounts map[string]int64 + translatedTotal int64 + translatedErr error } func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) { @@ -45,6 +49,13 @@ func (m *mockProcedureDocumentsLister) ScanScrapedDocuments(context.Context) ([] return m.procedures, m.dailyCounts, nil } +func (m *mockProcedureDocumentsLister) ScanTranslatedNotices(context.Context) (map[string]int64, int64, error) { + if m.translatedErr != nil { + return nil, 0, m.translatedErr + } + return m.translatedDailyCounts, m.translatedTotal, nil +} + func TestScrapedDocumentsFolderIDsUsesMinIOProcedures(t *testing.T) { repo := &repository{ procedureLister: &mockProcedureDocumentsLister{ @@ -259,3 +270,41 @@ func TestMongoScrapedTendersScopeMatchesStatisticsFilter(t *testing.T) { t.Fatalf("expected documents_scraped filter, got %#v", scope.match) } } + +func TestResolveTranslatedNoticesScopeUsesMinIOCounts(t *testing.T) { + repo := &repository{ + procedureLister: &mockProcedureDocumentsLister{ + translatedDailyCounts: map[string]int64{ + "2026-07-01": 4, + "2026-07-09": 2, + }, + translatedTotal: 6, + }, + } + + scope, err := repo.resolveTranslatedNoticesScope(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !scope.fromMinIO { + t.Fatal("expected MinIO-backed translation scope") + } + if scope.totalCount != 6 { + t.Fatalf("expected total 6, got %d", scope.totalCount) + } + if scope.dailyCounts["2026-07-09"] != 2 { + t.Fatalf("expected 2 on 2026-07-09, got %d", scope.dailyCounts["2026-07-09"]) + } +} + +func TestResolveTranslatedNoticesScopeWithoutLister(t *testing.T) { + repo := &repository{} + + scope, err := repo.resolveTranslatedNoticesScope(context.Background()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if scope.fromMinIO { + t.Fatal("expected metrics fallback without lister") + } +} diff --git a/pkg/ai_summarizer/entities.go b/pkg/ai_summarizer/entities.go index 3d1b5eb..a58ec98 100644 --- a/pkg/ai_summarizer/entities.go +++ b/pkg/ai_summarizer/entities.go @@ -389,3 +389,25 @@ func (t *TenderJSON) translationForLanguage(language string) (StoredTranslation, } return t.storedTranslationText(language) } + +// hasAnyTranslation reports whether tender.json contains at least one non-empty +// title translation (via translation_status.done or the translations map). +func (t *TenderJSON) hasAnyTranslation() bool { + if t == nil { + return false + } + for _, lang := range t.translationsDone() { + if _, ok := t.storedTranslationText(lang); ok { + return true + } + } + if t.Translations == nil { + return false + } + for _, entry := range t.Translations { + if strings.TrimSpace(entry.Title) != "" { + return true + } + } + return false +} diff --git a/pkg/ai_summarizer/storage.go b/pkg/ai_summarizer/storage.go index eeb6531..807ac20 100644 --- a/pkg/ai_summarizer/storage.go +++ b/pkg/ai_summarizer/storage.go @@ -654,6 +654,89 @@ func (s *StorageClient) ScanScrapedDocuments(ctx context.Context) ([]ProcedureDo return results, dailyCounts, nil } +// ScanTranslatedNotices scans MinIO for tender.json objects with stored translations +// and returns daily notice counts keyed by UTC date (YYYY-MM-DD) plus the total. +func (s *StorageClient) ScanTranslatedNotices(ctx context.Context) (map[string]int64, int64, error) { + s.logger.Debug("Scanning translated notices from MinIO", map[string]interface{}{ + "bucket": s.config.MinioBucket, + "prefix": procedurePrefix, + }) + + dailyCounts := make(map[string]int64) + var total int64 + seen := make(map[string]struct{}) + var fatalErr error + + appendFromPrefix := func(prefix string) bool { + objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) + for object := range objectCh { + if object.Err != nil { + fatalErr = s.logMinIOFailure("list_translated_notices", object.Err, map[string]interface{}{ + "prefix": prefix, + }) + return false + } + key := object.Key + if !strings.HasSuffix(key, "/tender.json") { + continue + } + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + + tenderJSON, err := s.readTenderJSONAtKey(ctx, key) + if err != nil { + if errors.Is(err, ErrObjectNotFound) { + continue + } + if errors.Is(err, ErrMinIOUnavailable) { + fatalErr = err + return false + } + s.logger.Debug("Skipping tender.json during translation scan", map[string]interface{}{ + "object_key": key, + "error": err.Error(), + }) + continue + } + if !tenderJSON.hasAnyTranslation() { + continue + } + + total++ + modified := normalizeUnixSeconds(object.LastModified.Unix()) + if modified > 0 { + date := time.Unix(modified, 0).UTC().Format("2006-01-02") + dailyCounts[date]++ + } + } + return true + } + + if !appendFromPrefix(procedurePrefix) { + return nil, 0, fatalErr + } + if len(seen) == 0 { + if !appendFromPrefix("") { + return nil, 0, fatalErr + } + } + if fatalErr != nil { + return nil, 0, fatalErr + } + + s.logger.Info("Scanned translated notices from storage", map[string]interface{}{ + "bucket": s.config.MinioBucket, + "total_count": total, + }) + + return dailyCounts, total, nil +} + func normalizeUnixSeconds(ts int64) int64 { if ts > 1_000_000_000_000 { return ts / 1000