diff --git a/internal/tender/handler.go b/internal/tender/handler.go index e978418..0113cee 100644 --- a/internal/tender/handler.go +++ b/internal/tender/handler.go @@ -157,7 +157,8 @@ func (h *TenderHandler) Delete(c echo.Context) error { // @Param languages query []string false "Filter by languages (comma-separated)" // @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param limit query int false "Number of items per page (default: 20, max: 100)" -// @Param offset query int false "Number of items to skip (default: 0)" +// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages." +// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor" // @Param sort_by query string false "Sort by field" // @Param sort_order query string false "Sort order (asc or desc)" // @Success 200 {object} response.APIResponse{data=map[string]interface{}} @@ -179,6 +180,9 @@ func (h *TenderHandler) Search(c echo.Context) error { // Call service tenders, err := h.service.Search(c.Request().Context(), form, response.NewPagination(c)) if err != nil { + if isInvalidPaginationCursor(err) { + return response.ValidationError(c, "Invalid pagination cursor", err.Error()) + } return response.InternalServerError(c, "Failed to list tenders") } @@ -219,7 +223,8 @@ func (h *TenderHandler) Search(c echo.Context) error { // @Param languages query []string false "Filter by languages (comma-separated)" // @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param limit query int false "Number of items per page (default: 20, max: 100)" -// @Param offset query int false "Number of items to skip (default: 0)" +// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages." +// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor" // @Param sort_by query string false "Sort by field" // @Param sort_order query string false "Sort order (asc or desc)" // @Success 200 {object} response.APIResponse{data=SearchResponse} @@ -256,6 +261,9 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error { result, err := h.service.Search(c.Request().Context(), form, response.NewPagination(c)) if err != nil { + if isInvalidPaginationCursor(err) { + return response.ValidationError(c, "Invalid pagination cursor", err.Error()) + } return response.InternalServerError(c, "Failed to retrieve tenders") } @@ -292,7 +300,8 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error { // @Param languages query []string false "Filter by languages (comma-separated)" // @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param limit query int false "Number of items per page (default: 20, max: 100)" -// @Param offset query int false "Number of items to skip (default: 0)" +// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages." +// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor" // @Param sort_by query string false "Sort by field" // @Param sort_order query string false "Sort order (asc or desc)" // @Success 200 {object} response.APIResponse{data=SearchResponse} @@ -328,6 +337,9 @@ func (h *TenderHandler) RecommendTenders(c echo.Context) error { result, err := h.service.Recommend(c.Request().Context(), form, response.NewPagination(c)) if err != nil { + if isInvalidPaginationCursor(err) { + return response.ValidationError(c, "Invalid pagination cursor", err.Error()) + } return response.InternalServerError(c, "Failed to retrieve tenders") } @@ -651,3 +663,7 @@ func (h *TenderHandler) GetAllAISummaries(c echo.Context) error { return response.Success(c, summaries, "AI summaries retrieved successfully") } + +func isInvalidPaginationCursor(err error) bool { + return errors.Is(err, orm.ErrInvalidCursor) +} diff --git a/internal/tender/repository.go b/internal/tender/repository.go index 3e9b740..5e72f15 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -26,7 +26,7 @@ type TenderRepository interface { GetTenderCountByCountry(ctx context.Context) (map[string]int64, error) GetTenderCountByType(ctx context.Context) (map[string]int64, error) GetTenderCountByClassification(ctx context.Context) (map[string]int64, error) - Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) ([]Tender, int64, error) + Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error) GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) @@ -40,11 +40,27 @@ func collectionName() string { return "tenders" } +// deepOffsetSkipCountThreshold skips CountDocuments when offset pagination goes this deep. +const deepOffsetSkipCountThreshold = 1000 + +// SearchPageResult is the paginated outcome of a tender search/list query. +type SearchPageResult struct { + Items []Tender + TotalCount int64 // -1 when count was skipped + NextCursor string + HasMore bool +} + // tenderSearchListProjection excludes large fields not needed for list/search API payloads. func tenderSearchListProjection() bson.M { return bson.M{ - "content_xml": 0, - "document_summaries": 0, + "content_xml": 0, + "document_summaries": 0, + "scraped_documents": 0, + "selection_criteria": 0, + "modifications": 0, + "source_file_url": 0, + "source_file_name": 0, } } @@ -85,6 +101,26 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te *orm.NewIndex("tender_deadline_idx", bson.D{{Key: "tender_deadline", Value: 1}}), *orm.NewIndex("estimated_value_idx", bson.D{{Key: "estimated_value", Value: -1}}), *orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}), + // Stable keyset pagination for default admin list (created_at desc). + *orm.NewIndex("created_at_id_idx", bson.D{ + {Key: "created_at", Value: -1}, + {Key: "_id", Value: -1}, + }), + *orm.NewIndex("status_created_at_id_idx", bson.D{ + {Key: "status", Value: 1}, + {Key: "created_at", Value: -1}, + {Key: "_id", Value: -1}, + }), + *orm.NewIndex("country_code_created_at_id_idx", bson.D{ + {Key: "country_code", Value: 1}, + {Key: "created_at", Value: -1}, + {Key: "_id", Value: -1}, + }), + *orm.NewIndex("notice_type_code_created_at_id_idx", bson.D{ + {Key: "notice_type_code", Value: 1}, + {Key: "created_at", Value: -1}, + {Key: "_id", Value: -1}, + }), *orm.NewIndex("updated_at_idx", bson.D{{Key: "updated_at", Value: -1}}), // Compound indexes for common queries *orm.NewIndex("status_country_idx", bson.D{ @@ -321,10 +357,9 @@ func (r *tenderRepository) Delete(ctx context.Context, id string) error { } // Search searches tenders based on criteria -func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) ([]Tender, int64, error) { +func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) { filter := r.buildSearchFilter(form) - // Build sort sort := "created_at" if pagination.SortBy != "" { sort = pagination.SortBy @@ -335,23 +370,34 @@ func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, paginat sortOrder = pagination.SortOrder } - // Build pagination + skipCount := pagination.Cursor != "" || pagination.Offset >= deepOffsetSkipCountThreshold + paginationBuilder := orm.NewPaginationBuilder(). Limit(pagination.Limit). - Skip(pagination.Offset). SortBy(sort, sortOrder). Projection(tenderSearchListProjection()). - Build() + SkipCount(skipCount) - result, err := r.ormRepo.FindAll(ctx, filter, paginationBuilder) + if pagination.Cursor != "" { + paginationBuilder.Cursor(pagination.Cursor) + } else { + paginationBuilder.Skip(pagination.Offset) + } + + result, err := r.ormRepo.FindAll(ctx, filter, paginationBuilder.Build()) if err != nil { r.logger.Error("Failed to search tenders", map[string]interface{}{ "error": err.Error(), }) - return nil, 0, err + return nil, err } - return result.Items, result.TotalCount, nil + return &SearchPageResult{ + Items: result.Items, + TotalCount: result.TotalCount, + NextCursor: result.NextCursor, + HasMore: result.HasMore, + }, nil } // GetExpiredTenders retrieves tenders that are expired diff --git a/internal/tender/service.go b/internal/tender/service.go index 8be73ae..de42e7a 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -613,7 +613,7 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination } } - tenders, total, err := s.repository.Search(ctx, form, pagination) + page, err := s.repository.Search(ctx, form, pagination) if err != nil { s.logger.Error("Failed to list tenders", map[string]interface{}{ "error": err.Error(), @@ -623,14 +623,14 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination // If no company ID provided, return tenders without match percentage if form.CompanyID == nil || *form.CompanyID == "" { - tenderResponses := make([]TenderResponse, 0, len(tenders)) - for _, tender := range tenders { + tenderResponses := make([]TenderResponse, 0, len(page.Items)) + for _, tender := range page.Items { tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language))) } return &SearchResponse{ Tenders: tenderResponses, - Metadata: pagination.Response(total), + Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore), }, nil } @@ -642,14 +642,14 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination "error": err.Error(), }) // Continue without match percentage if company not found - tenderResponses := make([]TenderResponse, 0, len(tenders)) - for _, tender := range tenders { + tenderResponses := make([]TenderResponse, 0, len(page.Items)) + for _, tender := range page.Items { tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language))) } return &SearchResponse{ Tenders: tenderResponses, - Metadata: pagination.Response(total), + Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore), }, nil } @@ -662,17 +662,17 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination s.logger.Info("Tenders sorted by match percentage", map[string]interface{}{ "company_id": *form.CompanyID, "company_cpv_codes": len(companyCPVCodes), - "total_tenders": len(tenders), + "total_tenders": len(page.Items), }) - tenderResponses := make([]TenderResponse, 0, len(tenders)) - for _, tender := range tenders { + tenderResponses := make([]TenderResponse, 0, len(page.Items)) + for _, tender := range page.Items { tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language))) } return &SearchResponse{ Tenders: tenderResponses, - Metadata: pagination.Response(total), + Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore), }, nil } @@ -724,7 +724,7 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat } } - tenders, total, err := s.repository.Search(ctx, form, pagination) + page, err := s.repository.Search(ctx, form, pagination) if err != nil { s.logger.Error("Failed to list tenders", map[string]interface{}{ "error": err.Error(), @@ -732,14 +732,14 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat return nil, fmt.Errorf("failed to list tenders: %w", err) } - tenderResponses := make([]TenderResponse, 0, len(tenders)) - for _, tender := range tenders { + tenderResponses := make([]TenderResponse, 0, len(page.Items)) + for _, tender := range page.Items { tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language))) } return &SearchResponse{ Tenders: tenderResponses, - Metadata: pagination.Response(total), + Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore), }, nil } @@ -812,7 +812,7 @@ func (s *tenderService) CleanupOldData(ctx context.Context, olderThan time.Durat Status: []string{string(TenderStatusExpired), string(TenderStatusCancelled), string(TenderStatusAwarded)}, } - oldTenders, _, err := s.repository.Search(ctx, form, pagination) + oldPage, err := s.repository.Search(ctx, form, pagination) if err != nil { s.logger.Error("Failed to find old tenders for cleanup", map[string]interface{}{ "error": err.Error(), @@ -821,7 +821,7 @@ func (s *tenderService) CleanupOldData(ctx context.Context, olderThan time.Durat } cleanupCount := 0 - for _, tender := range oldTenders { + for _, tender := range oldPage.Items { if tender.CreatedAt < cutoffTime { cleanupCount++ // Here you would delete the tender if implementing actual cleanup diff --git a/pkg/mongo/cursor.go b/pkg/mongo/cursor.go new file mode 100644 index 0000000..6823576 --- /dev/null +++ b/pkg/mongo/cursor.go @@ -0,0 +1,234 @@ +package mongo + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strconv" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +const pageCursorVersion = 1 + +// pageCursor is an opaque pagination token for keyset (cursor) pagination. +type pageCursor struct { + V int `json:"v"` + SortField string `json:"sort_field"` + SortOrder int `json:"sort_order"` + ValueType string `json:"value_type"` // i64, f64, str, oid + SortValue string `json:"sort_value"` + ID string `json:"id"` +} + +func encodePageCursor(sortField string, sortOrder int, sortValue interface{}, id string) (string, error) { + if sortField == "" { + return "", fmt.Errorf("%w: sort field is required", ErrInvalidCursor) + } + if id == "" { + return "", fmt.Errorf("%w: document id is required", ErrInvalidCursor) + } + + valueType, sortValueStr, err := formatCursorSortValue(sortField, sortValue) + if err != nil { + return "", err + } + + payload, err := json.Marshal(pageCursor{ + V: pageCursorVersion, + SortField: sortField, + SortOrder: sortOrder, + ValueType: valueType, + SortValue: sortValueStr, + ID: id, + }) + if err != nil { + return "", fmt.Errorf("failed to marshal cursor: %w", err) + } + + return base64.StdEncoding.EncodeToString(payload), nil +} + +func decodePageCursor(encoded string) (*pageCursor, error) { + cursorBytes, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, fmt.Errorf("%w: invalid cursor encoding", ErrInvalidCursor) + } + + var cursor pageCursor + if err := json.Unmarshal(cursorBytes, &cursor); err != nil { + return nil, fmt.Errorf("%w: invalid cursor payload", ErrInvalidCursor) + } + if cursor.V != pageCursorVersion || cursor.SortField == "" || cursor.ID == "" { + return nil, fmt.Errorf("%w: unsupported or incomplete cursor", ErrInvalidCursor) + } + if cursor.SortOrder != 1 && cursor.SortOrder != -1 { + return nil, fmt.Errorf("%w: invalid sort order in cursor", ErrInvalidCursor) + } + + return &cursor, nil +} + +func (c *pageCursor) matches(sortField string, sortOrder int) bool { + return c.SortField == sortField && c.SortOrder == sortOrder +} + +func (c *pageCursor) mongoFilter() (bson.M, error) { + if c.SortField == "_id" { + objectID, err := bson.ObjectIDFromHex(c.ID) + if err != nil { + return nil, fmt.Errorf("%w: invalid ObjectID in cursor", ErrInvalidCursor) + } + operator := "$lt" + if c.SortOrder == 1 { + operator = "$gt" + } + return bson.M{"_id": bson.M{operator: objectID}}, nil + } + + sortValue, err := c.parseSortValue() + if err != nil { + return nil, err + } + + objectID, err := bson.ObjectIDFromHex(c.ID) + if err != nil { + return nil, fmt.Errorf("%w: invalid ObjectID in cursor", ErrInvalidCursor) + } + + primaryOp := "$lt" + secondaryOp := "$lt" + if c.SortOrder == 1 { + primaryOp = "$gt" + secondaryOp = "$gt" + } + + return bson.M{ + "$or": []bson.M{ + {c.SortField: bson.M{primaryOp: sortValue}}, + { + c.SortField: sortValue, + "_id": bson.M{secondaryOp: objectID}, + }, + }, + }, nil +} + +func (c *pageCursor) parseSortValue() (interface{}, error) { + switch c.ValueType { + case "i64": + v, err := strconv.ParseInt(c.SortValue, 10, 64) + if err != nil { + return nil, fmt.Errorf("%w: invalid int64 sort value", ErrInvalidCursor) + } + return v, nil + case "f64": + v, err := strconv.ParseFloat(c.SortValue, 64) + if err != nil { + return nil, fmt.Errorf("%w: invalid float sort value", ErrInvalidCursor) + } + return v, nil + case "str": + return c.SortValue, nil + case "oid": + return bson.ObjectIDFromHex(c.SortValue) + default: + return nil, fmt.Errorf("%w: unknown sort value type", ErrInvalidCursor) + } +} + +func formatCursorSortValue(sortField string, sortValue interface{}) (string, string, error) { + if sortField == "_id" { + switch v := sortValue.(type) { + case bson.ObjectID: + return "oid", v.Hex(), nil + case string: + if _, err := bson.ObjectIDFromHex(v); err != nil { + return "", "", fmt.Errorf("%w: invalid ObjectID sort value", ErrInvalidCursor) + } + return "oid", v, nil + default: + return "", "", fmt.Errorf("%w: unsupported _id sort value", ErrInvalidCursor) + } + } + + switch v := sortValue.(type) { + case int64: + return "i64", strconv.FormatInt(v, 10), nil + case int32: + return "i64", strconv.FormatInt(int64(v), 10), nil + case int: + return "i64", strconv.FormatInt(int64(v), 10), nil + case float64: + return "f64", strconv.FormatFloat(v, 'f', -1, 64), nil + case float32: + return "f64", strconv.FormatFloat(float64(v), 'f', -1, 64), nil + case string: + return "str", v, nil + case bson.ObjectID: + return "oid", v.Hex(), nil + default: + return "", "", fmt.Errorf("%w: unsupported sort value type for field %s", ErrInvalidCursor, sortField) + } +} + +func documentSortValue(docMap bson.M, sortField string) (interface{}, error) { + if sortField == "_id" { + id, exists := docMap["_id"] + if !exists { + return nil, fmt.Errorf("sort field _id not found in document") + } + return id, nil + } + + sortValue, exists := docMap[sortField] + if !exists { + return nil, fmt.Errorf("sort field %s not found in document", sortField) + } + return sortValue, nil +} + +func documentIDHex(docMap bson.M) (string, error) { + id, exists := docMap["_id"] + if !exists { + return "", fmt.Errorf("document _id not found") + } + switch v := id.(type) { + case bson.ObjectID: + return v.Hex(), nil + case string: + if _, err := bson.ObjectIDFromHex(v); err != nil { + return "", fmt.Errorf("invalid document _id") + } + return v, nil + default: + return "", fmt.Errorf("unsupported document _id type") + } +} + +func cloneFilter(filter bson.M) (bson.M, error) { + if len(filter) == 0 { + return bson.M{}, nil + } + raw, err := bson.Marshal(filter) + if err != nil { + return nil, err + } + var cloned bson.M + if err := bson.Unmarshal(raw, &cloned); err != nil { + return nil, err + } + return cloned, nil +} + +func mergeFilters(base, extra bson.M) bson.M { + if len(extra) == 0 { + return base + } + if len(base) == 0 { + return extra + } + return bson.M{ + "$and": []bson.M{base, extra}, + } +} diff --git a/pkg/mongo/repository.go b/pkg/mongo/repository.go index d5b124b..b9dffbf 100644 --- a/pkg/mongo/repository.go +++ b/pkg/mongo/repository.go @@ -2,7 +2,6 @@ package mongo import ( "context" - "encoding/base64" "errors" "fmt" "time" @@ -26,6 +25,8 @@ type Pagination struct { Cursor string `json:"cursor"` // Base64 encoded cursor for cursor-based pagination SortField string `json:"sortField"` // Field to sort by (default: "_id") SortOrder int `json:"sortOrder"` // Sort order: 1 for ascending, -1 for descending + // SkipCount skips CountDocuments (use with cursor or deep offset pagination). + SkipCount bool `json:"skipCount,omitempty"` // Projection limits fields returned from Find (e.g. exclude large blobs on list endpoints). Projection bson.M `json:"projection,omitempty"` } @@ -132,8 +133,16 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P pagination.SortOrder = -1 // Default to descending } - // Build sort options + // Build sort with _id tie-breaker for stable keyset pagination. sort := bson.D{{Key: pagination.SortField, Value: pagination.SortOrder}} + if pagination.SortField != "_id" { + sort = append(sort, bson.E{Key: "_id", Value: pagination.SortOrder}) + } + + countFilter, err := cloneFilter(filter) + if err != nil { + return nil, fmt.Errorf("failed to clone filter for count: %w", err) + } // Build find options findOptions := options.Find(). @@ -150,10 +159,7 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P if err != nil { return nil, err } - // Merge cursor filter with existing filter - for key, value := range cursorFilter { - filter[key] = value - } + filter = mergeFilters(filter, cursorFilter) } else if pagination.Skip > 0 { // Use offset-based pagination findOptions.SetSkip(int64(pagination.Skip)) @@ -187,21 +193,23 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P items = items[:pagination.Limit] // Remove the extra item } - // Get total count for pagination metadata - var totalCount int64 - var countErr error - totalCount, countErr = r.Count(ctx, filter) - if countErr != nil { - r.logger.Error("Failed to get total count", map[string]interface{}{ - "error": countErr.Error(), - }) - totalCount = 0 + // Get total count for pagination metadata (never include cursor constraints). + var totalCount int64 = -1 + if !pagination.SkipCount { + var countErr error + totalCount, countErr = r.Count(ctx, countFilter) + if countErr != nil { + r.logger.Error("Failed to get total count", map[string]interface{}{ + "error": countErr.Error(), + }) + totalCount = 0 + } } // Generate next cursor var nextCursor string if hasMore && len(items) > 0 { - nextCursor, err = r.generateCursor(items[len(items)-1], pagination.SortField) + nextCursor, err = r.generateCursor(items[len(items)-1], pagination.SortField, pagination.SortOrder) if err != nil { r.logger.Error("Failed to generate next cursor", map[string]interface{}{ "error": err.Error(), @@ -445,43 +453,20 @@ func (r *repository[T]) validatePagination(pagination Pagination) error { return nil } -// buildCursorFilter builds a filter for cursor-based pagination +// buildCursorFilter builds a filter for cursor-based pagination. func (r *repository[T]) buildCursorFilter(cursor, sortField string, sortOrder int) (bson.M, error) { - // Decode cursor - cursorBytes, err := base64.StdEncoding.DecodeString(cursor) + pageCursor, err := decodePageCursor(cursor) if err != nil { - return nil, fmt.Errorf("%w: invalid cursor encoding", ErrInvalidCursor) + return nil, err } - - var cursorValue interface{} - - // Try to parse as ObjectID first - if sortField == "_id" { - if objectID, err := bson.ObjectIDFromHex(string(cursorBytes)); err == nil { - cursorValue = objectID - } else { - return nil, fmt.Errorf("%w: invalid ObjectID in cursor", ErrInvalidCursor) - } - } else { - // For other fields, try to parse as appropriate type - // This is a simplified implementation - you might want to make this more sophisticated - cursorValue = string(cursorBytes) + if !pageCursor.matches(sortField, sortOrder) { + return nil, fmt.Errorf("%w: cursor does not match requested sort", ErrInvalidCursor) } - - // Build comparison operator based on sort order - var operator string - if sortOrder == 1 { - operator = "$gt" - } else { - operator = "$lt" - } - - return bson.M{sortField: bson.M{operator: cursorValue}}, nil + return pageCursor.mongoFilter() } -// generateCursor generates a cursor from a document -func (r *repository[T]) generateCursor(doc T, sortField string) (string, error) { - // Convert document to BSON +// generateCursor generates an opaque cursor from the last document in a page. +func (r *repository[T]) generateCursor(doc T, sortField string, sortOrder int) (string, error) { docBytes, err := bson.Marshal(doc) if err != nil { return "", fmt.Errorf("failed to marshal document: %w", err) @@ -492,29 +477,17 @@ func (r *repository[T]) generateCursor(doc T, sortField string) (string, error) return "", fmt.Errorf("failed to unmarshal document: %w", err) } - // Get the sort field value - sortValue, exists := docMap[sortField] - if !exists { - return "", fmt.Errorf("sort field %s not found in document", sortField) + sortValue, err := documentSortValue(docMap, sortField) + if err != nil { + return "", err } - // Convert to string for encoding - var cursorStr string - switch v := sortValue.(type) { - case bson.ObjectID: - cursorStr = v.Hex() - case string: - cursorStr = v - case int64: - cursorStr = fmt.Sprintf("%d", v) - case float64: - cursorStr = fmt.Sprintf("%f", v) - default: - cursorStr = fmt.Sprintf("%v", v) + idHex, err := documentIDHex(docMap) + if err != nil { + return "", err } - // Encode as base64 - return base64.StdEncoding.EncodeToString([]byte(cursorStr)), nil + return encodePageCursor(sortField, sortOrder, sortValue, idHex) } // getModelID extracts the ID from a model diff --git a/pkg/mongo/utils.go b/pkg/mongo/utils.go index 9557793..d52a62a 100644 --- a/pkg/mongo/utils.go +++ b/pkg/mongo/utils.go @@ -299,6 +299,12 @@ func (pb *PaginationBuilder) Projection(projection bson.M) *PaginationBuilder { return pb } +// SkipCount disables CountDocuments for this query. +func (pb *PaginationBuilder) SkipCount(skip bool) *PaginationBuilder { + pb.pagination.SkipCount = skip + return pb +} + // Build returns the final pagination options func (pb *PaginationBuilder) Build() Pagination { return pb.pagination diff --git a/pkg/response/response.go b/pkg/response/response.go index 12bee54..a058e50 100644 --- a/pkg/response/response.go +++ b/pkg/response/response.go @@ -26,16 +26,19 @@ type APIError struct { // Meta represents metadata for paginated responses type Meta struct { - Total int `json:"total"` - Limit int `json:"limit"` - Offset int `json:"offset"` - Page int `json:"page"` - Pages int `json:"pages"` + Total int `json:"total,omitempty"` + Limit int `json:"limit"` + Offset int `json:"offset"` + Page int `json:"page,omitempty"` + Pages int `json:"pages,omitempty"` + NextCursor string `json:"next_cursor,omitempty"` + HasMore bool `json:"has_more,omitempty"` } type Pagination struct { Limit int `query:"limit" valid:"optional,range(1|100)" default:"20"` Offset int `query:"offset" valid:"optional,range(0|1000000)" default:"0"` + Cursor string `query:"cursor" valid:"optional"` SortBy string `query:"sort_by" valid:"optional"` SortOrder string `query:"sort_order" valid:"optional,in(asc|desc)"` } @@ -210,22 +213,38 @@ func NewPagination(c echo.Context) *Pagination { sortOrder = "desc" } + cursor := c.QueryParam("cursor") + return &Pagination{ Limit: limit, Offset: offset, + Cursor: cursor, SortBy: sortBy, SortOrder: sortOrder, } } -// Pagination calculates pagination metadata +// Response calculates offset-based pagination metadata. func (p *Pagination) Response(total int64) *Meta { - pages := (total + int64(p.Limit) - 1) / int64(p.Limit) - return &Meta{ - Total: int(total), - Limit: p.Limit, - Offset: p.Offset, - Page: int(p.Offset/p.Limit) + 1, - Pages: int(pages), - } + return p.ListMeta(total, "", false) +} + +// ListMeta builds pagination metadata for list endpoints (offset and/or cursor). +// When total is negative, total/page/pages are omitted (count was skipped). +func (p *Pagination) ListMeta(total int64, nextCursor string, hasMore bool) *Meta { + meta := &Meta{ + Limit: p.Limit, + Offset: p.Offset, + NextCursor: nextCursor, + HasMore: hasMore, + } + if total >= 0 { + pages := (total + int64(p.Limit) - 1) / int64(p.Limit) + meta.Total = int(total) + if p.Limit > 0 { + meta.Page = p.Offset/p.Limit + 1 + } + meta.Pages = int(pages) + } + return meta }