From e31bccced6c76085012d317767728c12484f312b Mon Sep 17 00:00:00 2001 From: Mazyar Date: Sun, 21 Jun 2026 10:03:17 +0330 Subject: [PATCH] Add unit tests for dashboard repository and enhance BSON handling --- internal/dashboard/repository.go | 50 +++++++++++++++---- internal/dashboard/repository_test.go | 44 ++++++++++++++++ internal/dashboard/service.go | 4 +- internal/dashboard/statistics_repository.go | 35 ++++++++----- .../dashboard/statistics_repository_test.go | 9 ++-- 5 files changed, 114 insertions(+), 28 deletions(-) create mode 100644 internal/dashboard/repository_test.go diff --git a/internal/dashboard/repository.go b/internal/dashboard/repository.go index 0b05df0..bc7171b 100644 --- a/internal/dashboard/repository.go +++ b/internal/dashboard/repository.go @@ -149,24 +149,20 @@ func (r *repository) Summary(ctx context.Context, closingWindowSec int64) (*Summ } func facetCount(facet bson.M, key string) int64 { - rows, ok := facet[key].(bson.A) - if !ok || len(rows) == 0 { + rows := facetRows(facet, key) + if len(rows) == 0 { return 0 } - row, ok := rows[0].(bson.M) - if !ok { - return 0 - } - return aggregateCount(row, "count") + return aggregateCount(asBSONMap(rows[0]), "count") } func facetCurrencyValue(facet bson.M) (string, float64) { - rows, ok := facet["currency_value"].(bson.A) - if !ok || len(rows) == 0 { + rows := facetRows(facet, "currency_value") + if len(rows) == 0 { return defaultValueCurrency, 0 } - row, ok := rows[0].(bson.M) - if !ok { + row := asBSONMap(rows[0]) + if row == nil { return defaultValueCurrency, 0 } @@ -177,6 +173,38 @@ func facetCurrencyValue(facet bson.M) (string, float64) { return currency, toFloat64(row["total"]) } +func facetRows(facet bson.M, key string) []interface{} { + raw, ok := facet[key] + if !ok { + return nil + } + switch rows := raw.(type) { + case bson.A: + return []interface{}(rows) + case []interface{}: + return rows + default: + return nil + } +} + +func asBSONMap(v interface{}) bson.M { + switch doc := v.(type) { + case bson.M: + return doc + case bson.D: + out := make(bson.M, len(doc)) + for _, elem := range doc { + out[elem.Key] = elem.Value + } + return out + case map[string]interface{}: + return doc + default: + return nil + } +} + func (r *repository) Trend(ctx context.Context, days int, metric string, startUnix int64) (map[string]int64, error) { field := trendTimestampField(metric) diff --git a/internal/dashboard/repository_test.go b/internal/dashboard/repository_test.go new file mode 100644 index 0000000..222c1a1 --- /dev/null +++ b/internal/dashboard/repository_test.go @@ -0,0 +1,44 @@ +package dashboard + +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +func TestFacetCountDecodesBSOND(t *testing.T) { + facet := bson.M{ + "total": bson.A{ + bson.D{{Key: "count", Value: int32(42)}}, + }, + "active": bson.A{ + bson.D{{Key: "count", Value: int64(7)}}, + }, + } + + if got := facetCount(facet, "total"); got != 42 { + t.Fatalf("expected total 42, got %d", got) + } + if got := facetCount(facet, "active"); got != 7 { + t.Fatalf("expected active 7, got %d", got) + } +} + +func TestFacetCurrencyValueDecodesBSOND(t *testing.T) { + facet := bson.M{ + "currency_value": bson.A{ + bson.D{ + {Key: "_id", Value: "eur"}, + {Key: "total", Value: float64(12345.67)}, + }, + }, + } + + currency, total := facetCurrencyValue(facet) + if currency != "EUR" { + t.Fatalf("expected EUR, got %q", currency) + } + if total != 12345.67 { + t.Fatalf("expected 12345.67, got %f", total) + } +} diff --git a/internal/dashboard/service.go b/internal/dashboard/service.go index 0dc5bf6..b461b2b 100644 --- a/internal/dashboard/service.go +++ b/internal/dashboard/service.go @@ -78,7 +78,7 @@ func (s *service) Summary(ctx context.Context, query SummaryQuery) (*SummaryResp "closing_window_hours": windowHours, }) - result, err := s.repo.Summary(ctx, windowSec) + result, err := s.repo.Summary(context.WithoutCancel(ctx), windowSec) if err != nil { s.logger.Error("Failed to fetch dashboard summary", map[string]interface{}{ "error": err.Error(), @@ -215,7 +215,7 @@ func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*Stati "days": days, }) - result, err := s.repo.Statistics(ctx, days, startUnix) + result, err := s.repo.Statistics(context.WithoutCancel(ctx), days, startUnix) if err != nil { s.logger.Error("Failed to fetch dashboard statistics report", map[string]interface{}{ "error": err.Error(), diff --git a/internal/dashboard/statistics_repository.go b/internal/dashboard/statistics_repository.go index c03215f..764549d 100644 --- a/internal/dashboard/statistics_repository.go +++ b/internal/dashboard/statistics_repository.go @@ -20,6 +20,12 @@ const noticesCollectionName = "notices" // keeping counts aligned with the tender panel (MinIO is the source of truth for scraped docs). const scrapedScopeCacheTTL = 5 * time.Minute +const ( + scrapedScopeResolveTimeout = 2 * time.Minute + statisticsQueryTimeout = 2 * time.Minute + maxScrapedFolderInClause = 5000 +) + // scrapedTendersScope is the resolved Mongo filter for tenders with scraped documents. // zero is set when MinIO reports no procedure folders; callers must return zero without querying. type scrapedTendersScope struct { @@ -28,12 +34,15 @@ type scrapedTendersScope struct { } func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) { + queryCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), statisticsQueryTimeout) + defer cancel() + now := time.Now().UTC() endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC) startDay := endDay.AddDate(0, 0, -(days - 1)) // Resolve scraped-document scope from MinIO (cached); falls back to Mongo flags on error. - scrapedScope, err := r.cachedScrapedTendersScope(ctx) + scrapedScope, err := r.cachedScrapedTendersScope(queryCtx) if err != nil { return nil, err } @@ -46,7 +55,7 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) totalTranslated int64 ) - g, gctx := errgroup.WithContext(ctx) + g, gctx := errgroup.WithContext(queryCtx) g.Go(func() error { // Use created_at (first ingest time). processing_metadata.scraped_at is reset on @@ -221,7 +230,10 @@ func (r *repository) cachedScrapedTendersScope(ctx context.Context) (scrapedTend } r.scrapedScopeMu.Unlock() - scope, resolveErr := r.resolveScrapedTendersScope(ctx) + resolveCtx, cancel := context.WithTimeout(context.Background(), scrapedScopeResolveTimeout) + defer cancel() + + scope, resolveErr := r.resolveScrapedTendersScope(resolveCtx) if resolveErr != nil { r.logger.Warn("MinIO scraped scope unavailable, using Mongo fallback for dashboard statistics", map[string]interface{}{ "error": resolveErr.Error(), @@ -250,19 +262,18 @@ func (r *repository) resolveScrapedTendersScope(ctx context.Context) (scrapedTen return scrapedTendersScope{}, err } if len(folderIDs) > 0 { + if len(folderIDs) > maxScrapedFolderInClause { + r.logger.Warn("Too many MinIO procedure folders for dashboard statistics, using Mongo fallback", map[string]interface{}{ + "folder_count": len(folderIDs), + "max_in_clause": maxScrapedFolderInClause, + }) + return mongoScrapedTendersScope(), nil + } return scrapedTendersScope{ match: bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}, }, nil } - if r.procedureLister != nil { - return scrapedTendersScope{zero: true}, nil - } - return scrapedTendersScope{ - match: bson.M{ - "processing_metadata.documents_scraped": true, - "scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}}, - }, - }, nil + return mongoScrapedTendersScope(), nil } func (r *repository) scrapedDocumentsFolderIDs(ctx context.Context) ([]string, error) { diff --git a/internal/dashboard/statistics_repository_test.go b/internal/dashboard/statistics_repository_test.go index c30b715..f76ea3b 100644 --- a/internal/dashboard/statistics_repository_test.go +++ b/internal/dashboard/statistics_repository_test.go @@ -101,7 +101,7 @@ func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) { } } -func TestResolveScrapedTendersScopeZeroWhenMinIOEmpty(t *testing.T) { +func TestResolveScrapedTendersScopeMongoFallbackWhenMinIOEmpty(t *testing.T) { repo := &repository{ procedureLister: &mockProcedureDocumentsLister{}, } @@ -110,8 +110,11 @@ func TestResolveScrapedTendersScopeZeroWhenMinIOEmpty(t *testing.T) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if !scope.zero { - t.Fatalf("expected zero scope, got %#v", scope) + if scope.zero { + t.Fatal("expected mongo fallback scope, not zero scope") + } + if scope.match["processing_metadata.documents_scraped"] != true { + t.Fatalf("expected mongo fallback filter, got %#v", scope.match) } }