From 7aacb7dfc998f80728b5820dd4fc2ff52e1410dc Mon Sep 17 00:00:00 2001 From: Mazyar Date: Tue, 30 Jun 2026 18:35:34 +0330 Subject: [PATCH] Enhance pagination and repository functionality for MongoDB - Updated the ListPaginationOptions struct to include SkipCount and IncludeCount fields, allowing for more flexible pagination behavior. - Modified the BuildListPagination function to handle cursor pagination with count options, improving performance by running count queries in parallel with data retrieval. - Enhanced the FindAll method in the repository to support concurrent counting of documents, reducing overall latency for list operations. - Added tests for pagination behavior, ensuring accurate handling of count scenarios in both offset and cursor pagination. This update improves the efficiency and flexibility of pagination in the MongoDB repository, enhancing the overall performance of list operations. --- internal/tender/repository.go | 7 ++++++ pkg/mongo/cursor_test.go | 33 ++++++++++++++++++++++++++++ pkg/mongo/list_pagination.go | 12 +++++++++- pkg/mongo/repository.go | 41 ++++++++++++++++++++++++++--------- 4 files changed, 82 insertions(+), 11 deletions(-) diff --git a/internal/tender/repository.go b/internal/tender/repository.go index e21699a..e7b3094 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -64,6 +64,13 @@ func tenderSearchListProjection() bson.M { "modifications": 0, "source_file_url": 0, "source_file_name": 0, + // Not used by list response mapping; can be large per tender. + "translations": 0, + "organizations": 0, + "review_organization": 0, + "processing_metadata.translated_data": 0, + "processing_metadata.parsing_errors": 0, + "processing_metadata.validation_errors": 0, } } diff --git a/pkg/mongo/cursor_test.go b/pkg/mongo/cursor_test.go index b776b6d..641bba3 100644 --- a/pkg/mongo/cursor_test.go +++ b/pkg/mongo/cursor_test.go @@ -44,6 +44,39 @@ func TestBuildListPaginationOffsetMode(t *testing.T) { if p.Skip != 20 || p.Limit != 10 || p.Cursor != "" { t.Fatalf("unexpected pagination: %+v", p) } + if p.SkipCount { + t.Fatal("expected count on offset pagination") + } +} + +func TestBuildListPaginationCursorSkipsCount(t *testing.T) { + encoded, err := encodePageCursor("created_at", -1, int64(1700000000), "507f1f77bcf86cd799439011", "abc123", 10) + if err != nil { + t.Fatalf("encode: %v", err) + } + + p, err := BuildListPagination(10, 0, encoded, "created_at", "desc", bson.M{}, ListPaginationOptions{}) + if err != nil { + t.Fatalf("build: %v", err) + } + if !p.SkipCount { + t.Fatal("expected SkipCount on cursor pagination") + } +} + +func TestBuildListPaginationCursorIncludeCount(t *testing.T) { + encoded, err := encodePageCursor("created_at", -1, int64(1700000000), "507f1f77bcf86cd799439011", "abc123", 10) + if err != nil { + t.Fatalf("encode: %v", err) + } + + p, err := BuildListPagination(10, 0, encoded, "created_at", "desc", bson.M{}, ListPaginationOptions{IncludeCount: true}) + if err != nil { + t.Fatalf("build: %v", err) + } + if p.SkipCount { + t.Fatal("expected count when IncludeCount is set") + } } func TestHashFilterStable(t *testing.T) { diff --git a/pkg/mongo/list_pagination.go b/pkg/mongo/list_pagination.go index d7c778f..ae677e5 100644 --- a/pkg/mongo/list_pagination.go +++ b/pkg/mongo/list_pagination.go @@ -15,7 +15,9 @@ const ( // ListPaginationOptions configures optional list pagination behavior. type ListPaginationOptions struct { - Projection bson.M + Projection bson.M + SkipCount bool + IncludeCount bool // when true, count even when using cursor pagination } // BuildListPagination converts API pagination parameters into MongoDB pagination options. @@ -55,6 +57,14 @@ func BuildListPagination( pb.Projection(opts.Projection) } + skipCount := opts.SkipCount + if cursor != "" && !opts.IncludeCount { + skipCount = true + } + if skipCount { + pb.SkipCount(true) + } + if cursor != "" { pageCursor, err := decodePageCursor(cursor) if err != nil { diff --git a/pkg/mongo/repository.go b/pkg/mongo/repository.go index 9a96086..8809b44 100644 --- a/pkg/mongo/repository.go +++ b/pkg/mongo/repository.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync" "time" "go.mongodb.org/mongo-driver/v2/bson" @@ -176,9 +177,21 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P findOptions.SetSkip(int64(pagination.Skip)) } - // Execute query + // Run count in parallel with the find so list latency is max(find, count), not sum. + var totalCount int64 = -1 + var countErr error + var countWG sync.WaitGroup + if !pagination.SkipCount { + countWG.Add(1) + go func() { + defer countWG.Done() + totalCount, countErr = r.Count(ctx, countFilter) + }() + } + cursor, err := r.collection.Find(ctx, filter, findOptions) if err != nil { + countWG.Wait() r.logger.Error("Failed to execute find query", map[string]interface{}{ "filter": filter, "pagination": pagination, @@ -188,27 +201,23 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P } defer cursor.Close(ctx) - // Decode results var items []T if err = cursor.All(ctx, &items); err != nil { + countWG.Wait() r.logger.Error("Failed to decode query results", map[string]interface{}{ "error": err.Error(), }) return nil, fmt.Errorf("failed to decode results: %w", err) } - // Check if there are more documents hasMore := false if len(items) > pagination.Limit { hasMore = true - items = items[:pagination.Limit] // Remove the extra item + items = items[:pagination.Limit] } - // Get total count for pagination metadata (never include cursor constraints). - var totalCount int64 = -1 + countWG.Wait() if !pagination.SkipCount { - var countErr error - totalCount, countErr = r.Count(ctx, countFilter) if countErr != nil { r.logger.Error("Failed to get total count", map[string]interface{}{ "error": countErr.Error(), @@ -365,9 +374,21 @@ func (r *repository[T]) Delete(ctx context.Context, id string) error { return nil } -// Count returns the number of documents matching the filter +// isEmptyFilter reports whether a MongoDB filter matches all documents. +func isEmptyFilter(filter bson.M) bool { + return len(filter) == 0 +} + +// Count returns the number of documents matching the filter. +// Unfiltered counts use EstimatedDocumentCount for O(1) metadata lookup on large collections. func (r *repository[T]) Count(ctx context.Context, filter bson.M) (int64, error) { - count, err := r.collection.CountDocuments(ctx, filter) + var count int64 + var err error + if isEmptyFilter(filter) { + count, err = r.collection.EstimatedDocumentCount(ctx) + } else { + count, err = r.collection.CountDocuments(ctx, filter) + } if err != nil { r.logger.Error("Failed to count documents", map[string]interface{}{ "filter": filter,