Enhance pagination and repository functionality for MongoDB

- Updated the ListPaginationOptions struct to include SkipCount and IncludeCount fields, allowing for more flexible pagination behavior.
- Modified the BuildListPagination function to handle cursor pagination with count options, improving performance by running count queries in parallel with data retrieval.
- Enhanced the FindAll method in the repository to support concurrent counting of documents, reducing overall latency for list operations.
- Added tests for pagination behavior, ensuring accurate handling of count scenarios in both offset and cursor pagination.

This update improves the efficiency and flexibility of pagination in the MongoDB repository, enhancing the overall performance of list operations.

internal/tender/repository.go

@@ -64,6 +64,13 @@ func tenderSearchListProjection() bson.M {
 		"modifications":      0,
 		"source_file_url":    0,
 		"source_file_name":   0,
+		// Not used by list response mapping; can be large per tender.
+		"translations":       0,
+		"organizations":      0,
+		"review_organization": 0,
+		"processing_metadata.translated_data": 0,
+		"processing_metadata.parsing_errors":  0,
+		"processing_metadata.validation_errors": 0,
 	}
 }
 

pkg/mongo/cursor_test.go

@@ -44,6 +44,39 @@ func TestBuildListPaginationOffsetMode(t *testing.T) {
 	if p.Skip != 20 || p.Limit != 10 || p.Cursor != "" {
 		t.Fatalf("unexpected pagination: %+v", p)
 	}
+	if p.SkipCount {
+		t.Fatal("expected count on offset pagination")
+	}
+}
+
+func TestBuildListPaginationCursorSkipsCount(t *testing.T) {
+	encoded, err := encodePageCursor("created_at", -1, int64(1700000000), "507f1f77bcf86cd799439011", "abc123", 10)
+	if err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+
+	p, err := BuildListPagination(10, 0, encoded, "created_at", "desc", bson.M{}, ListPaginationOptions{})
+	if err != nil {
+		t.Fatalf("build: %v", err)
+	}
+	if !p.SkipCount {
+		t.Fatal("expected SkipCount on cursor pagination")
+	}
+}
+
+func TestBuildListPaginationCursorIncludeCount(t *testing.T) {
+	encoded, err := encodePageCursor("created_at", -1, int64(1700000000), "507f1f77bcf86cd799439011", "abc123", 10)
+	if err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+
+	p, err := BuildListPagination(10, 0, encoded, "created_at", "desc", bson.M{}, ListPaginationOptions{IncludeCount: true})
+	if err != nil {
+		t.Fatalf("build: %v", err)
+	}
+	if p.SkipCount {
+		t.Fatal("expected count when IncludeCount is set")
+	}
 }
 
 func TestHashFilterStable(t *testing.T) {

pkg/mongo/list_pagination.go

@@ -16,6 +16,8 @@ const (
 // ListPaginationOptions configures optional list pagination behavior.
 type ListPaginationOptions struct {
 	Projection   bson.M
+	SkipCount    bool
+	IncludeCount bool // when true, count even when using cursor pagination
 }
 
 // BuildListPagination converts API pagination parameters into MongoDB pagination options.
@@ -55,6 +57,14 @@ func BuildListPagination(
 		pb.Projection(opts.Projection)
 	}
 
+	skipCount := opts.SkipCount
+	if cursor != "" && !opts.IncludeCount {
+		skipCount = true
+	}
+	if skipCount {
+		pb.SkipCount(true)
+	}
+
 	if cursor != "" {
 		pageCursor, err := decodePageCursor(cursor)
 		if err != nil {

pkg/mongo/repository.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"sync"
 	"time"
 
 	"go.mongodb.org/mongo-driver/v2/bson"
@@ -176,9 +177,21 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
 		findOptions.SetSkip(int64(pagination.Skip))
 	}
 
-	// Execute query
+	// Run count in parallel with the find so list latency is max(find, count), not sum.
+	var totalCount int64 = -1
+	var countErr error
+	var countWG sync.WaitGroup
+	if !pagination.SkipCount {
+		countWG.Add(1)
+		go func() {
+			defer countWG.Done()
+			totalCount, countErr = r.Count(ctx, countFilter)
+		}()
+	}
+
 	cursor, err := r.collection.Find(ctx, filter, findOptions)
 	if err != nil {
+		countWG.Wait()
 		r.logger.Error("Failed to execute find query", map[string]interface{}{
 			"filter":     filter,
 			"pagination": pagination,
@@ -188,27 +201,23 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
 	}
 	defer cursor.Close(ctx)
 
-	// Decode results
 	var items []T
 	if err = cursor.All(ctx, &items); err != nil {
+		countWG.Wait()
 		r.logger.Error("Failed to decode query results", map[string]interface{}{
 			"error": err.Error(),
 		})
 		return nil, fmt.Errorf("failed to decode results: %w", err)
 	}
 
-	// Check if there are more documents
 	hasMore := false
 	if len(items) > pagination.Limit {
 		hasMore = true
-		items = items[:pagination.Limit] // Remove the extra item
+		items = items[:pagination.Limit]
 	}
 
-	// Get total count for pagination metadata (never include cursor constraints).
+	countWG.Wait()
-	var totalCount int64 = -1
 	if !pagination.SkipCount {
-		var countErr error
-		totalCount, countErr = r.Count(ctx, countFilter)
 		if countErr != nil {
 			r.logger.Error("Failed to get total count", map[string]interface{}{
 				"error": countErr.Error(),
@@ -365,9 +374,21 @@ func (r *repository[T]) Delete(ctx context.Context, id string) error {
 	return nil
 }
 
-// Count returns the number of documents matching the filter
+// isEmptyFilter reports whether a MongoDB filter matches all documents.
+func isEmptyFilter(filter bson.M) bool {
+	return len(filter) == 0
+}
+
+// Count returns the number of documents matching the filter.
+// Unfiltered counts use EstimatedDocumentCount for O(1) metadata lookup on large collections.
 func (r *repository[T]) Count(ctx context.Context, filter bson.M) (int64, error) {
-	count, err := r.collection.CountDocuments(ctx, filter)
+	var count int64
+	var err error
+	if isEmptyFilter(filter) {
+		count, err = r.collection.EstimatedDocumentCount(ctx)
+	} else {
+		count, err = r.collection.CountDocuments(ctx, filter)
+	}
 	if err != nil {
 		r.logger.Error("Failed to count documents", map[string]interface{}{
 			"filter": filter,