diff --git a/internal/tender/repository.go b/internal/tender/repository.go index e21699a..e7b3094 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -64,6 +64,13 @@ func tenderSearchListProjection() bson.M { "modifications": 0, "source_file_url": 0, "source_file_name": 0, + // Not used by list response mapping; can be large per tender. + "translations": 0, + "organizations": 0, + "review_organization": 0, + "processing_metadata.translated_data": 0, + "processing_metadata.parsing_errors": 0, + "processing_metadata.validation_errors": 0, } } diff --git a/pkg/mongo/cursor_test.go b/pkg/mongo/cursor_test.go index b776b6d..641bba3 100644 --- a/pkg/mongo/cursor_test.go +++ b/pkg/mongo/cursor_test.go @@ -44,6 +44,39 @@ func TestBuildListPaginationOffsetMode(t *testing.T) { if p.Skip != 20 || p.Limit != 10 || p.Cursor != "" { t.Fatalf("unexpected pagination: %+v", p) } + if p.SkipCount { + t.Fatal("expected count on offset pagination") + } +} + +func TestBuildListPaginationCursorSkipsCount(t *testing.T) { + encoded, err := encodePageCursor("created_at", -1, int64(1700000000), "507f1f77bcf86cd799439011", "abc123", 10) + if err != nil { + t.Fatalf("encode: %v", err) + } + + p, err := BuildListPagination(10, 0, encoded, "created_at", "desc", bson.M{}, ListPaginationOptions{}) + if err != nil { + t.Fatalf("build: %v", err) + } + if !p.SkipCount { + t.Fatal("expected SkipCount on cursor pagination") + } +} + +func TestBuildListPaginationCursorIncludeCount(t *testing.T) { + encoded, err := encodePageCursor("created_at", -1, int64(1700000000), "507f1f77bcf86cd799439011", "abc123", 10) + if err != nil { + t.Fatalf("encode: %v", err) + } + + p, err := BuildListPagination(10, 0, encoded, "created_at", "desc", bson.M{}, ListPaginationOptions{IncludeCount: true}) + if err != nil { + t.Fatalf("build: %v", err) + } + if p.SkipCount { + t.Fatal("expected count when IncludeCount is set") + } } func TestHashFilterStable(t *testing.T) { diff --git a/pkg/mongo/list_pagination.go b/pkg/mongo/list_pagination.go index d7c778f..ae677e5 100644 --- a/pkg/mongo/list_pagination.go +++ b/pkg/mongo/list_pagination.go @@ -15,7 +15,9 @@ const ( // ListPaginationOptions configures optional list pagination behavior. type ListPaginationOptions struct { - Projection bson.M + Projection bson.M + SkipCount bool + IncludeCount bool // when true, count even when using cursor pagination } // BuildListPagination converts API pagination parameters into MongoDB pagination options. @@ -55,6 +57,14 @@ func BuildListPagination( pb.Projection(opts.Projection) } + skipCount := opts.SkipCount + if cursor != "" && !opts.IncludeCount { + skipCount = true + } + if skipCount { + pb.SkipCount(true) + } + if cursor != "" { pageCursor, err := decodePageCursor(cursor) if err != nil { diff --git a/pkg/mongo/repository.go b/pkg/mongo/repository.go index 9a96086..8809b44 100644 --- a/pkg/mongo/repository.go +++ b/pkg/mongo/repository.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync" "time" "go.mongodb.org/mongo-driver/v2/bson" @@ -176,9 +177,21 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P findOptions.SetSkip(int64(pagination.Skip)) } - // Execute query + // Run count in parallel with the find so list latency is max(find, count), not sum. + var totalCount int64 = -1 + var countErr error + var countWG sync.WaitGroup + if !pagination.SkipCount { + countWG.Add(1) + go func() { + defer countWG.Done() + totalCount, countErr = r.Count(ctx, countFilter) + }() + } + cursor, err := r.collection.Find(ctx, filter, findOptions) if err != nil { + countWG.Wait() r.logger.Error("Failed to execute find query", map[string]interface{}{ "filter": filter, "pagination": pagination, @@ -188,27 +201,23 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P } defer cursor.Close(ctx) - // Decode results var items []T if err = cursor.All(ctx, &items); err != nil { + countWG.Wait() r.logger.Error("Failed to decode query results", map[string]interface{}{ "error": err.Error(), }) return nil, fmt.Errorf("failed to decode results: %w", err) } - // Check if there are more documents hasMore := false if len(items) > pagination.Limit { hasMore = true - items = items[:pagination.Limit] // Remove the extra item + items = items[:pagination.Limit] } - // Get total count for pagination metadata (never include cursor constraints). - var totalCount int64 = -1 + countWG.Wait() if !pagination.SkipCount { - var countErr error - totalCount, countErr = r.Count(ctx, countFilter) if countErr != nil { r.logger.Error("Failed to get total count", map[string]interface{}{ "error": countErr.Error(), @@ -365,9 +374,21 @@ func (r *repository[T]) Delete(ctx context.Context, id string) error { return nil } -// Count returns the number of documents matching the filter +// isEmptyFilter reports whether a MongoDB filter matches all documents. +func isEmptyFilter(filter bson.M) bool { + return len(filter) == 0 +} + +// Count returns the number of documents matching the filter. +// Unfiltered counts use EstimatedDocumentCount for O(1) metadata lookup on large collections. func (r *repository[T]) Count(ctx context.Context, filter bson.M) (int64, error) { - count, err := r.collection.CountDocuments(ctx, filter) + var count int64 + var err error + if isEmptyFilter(filter) { + count, err = r.collection.EstimatedDocumentCount(ctx) + } else { + count, err = r.collection.CountDocuments(ctx, filter) + } if err != nil { r.logger.Error("Failed to count documents", map[string]interface{}{ "filter": filter,