cursor pagination

This commit is contained in:
Mazyar
2026-05-16 15:51:18 +03:30
parent bcb345103f
commit 70f581da09
7 changed files with 405 additions and 111 deletions
+19 -3
View File
@@ -157,7 +157,8 @@ func (h *TenderHandler) Delete(c echo.Context) error {
// @Param languages query []string false "Filter by languages (comma-separated)" // @Param languages query []string false "Filter by languages (comma-separated)"
// @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param buyer_organization_id query string false "Filter by buyer organization ID"
// @Param limit query int false "Number of items per page (default: 20, max: 100)" // @Param limit query int false "Number of items per page (default: 20, max: 100)"
// @Param offset query int false "Number of items to skip (default: 0)" // @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
// @Param sort_by query string false "Sort by field" // @Param sort_by query string false "Sort by field"
// @Param sort_order query string false "Sort order (asc or desc)" // @Param sort_order query string false "Sort order (asc or desc)"
// @Success 200 {object} response.APIResponse{data=map[string]interface{}} // @Success 200 {object} response.APIResponse{data=map[string]interface{}}
@@ -179,6 +180,9 @@ func (h *TenderHandler) Search(c echo.Context) error {
// Call service // Call service
tenders, err := h.service.Search(c.Request().Context(), form, response.NewPagination(c)) tenders, err := h.service.Search(c.Request().Context(), form, response.NewPagination(c))
if err != nil { if err != nil {
if isInvalidPaginationCursor(err) {
return response.ValidationError(c, "Invalid pagination cursor", err.Error())
}
return response.InternalServerError(c, "Failed to list tenders") return response.InternalServerError(c, "Failed to list tenders")
} }
@@ -219,7 +223,8 @@ func (h *TenderHandler) Search(c echo.Context) error {
// @Param languages query []string false "Filter by languages (comma-separated)" // @Param languages query []string false "Filter by languages (comma-separated)"
// @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param buyer_organization_id query string false "Filter by buyer organization ID"
// @Param limit query int false "Number of items per page (default: 20, max: 100)" // @Param limit query int false "Number of items per page (default: 20, max: 100)"
// @Param offset query int false "Number of items to skip (default: 0)" // @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
// @Param sort_by query string false "Sort by field" // @Param sort_by query string false "Sort by field"
// @Param sort_order query string false "Sort order (asc or desc)" // @Param sort_order query string false "Sort order (asc or desc)"
// @Success 200 {object} response.APIResponse{data=SearchResponse} // @Success 200 {object} response.APIResponse{data=SearchResponse}
@@ -256,6 +261,9 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error {
result, err := h.service.Search(c.Request().Context(), form, response.NewPagination(c)) result, err := h.service.Search(c.Request().Context(), form, response.NewPagination(c))
if err != nil { if err != nil {
if isInvalidPaginationCursor(err) {
return response.ValidationError(c, "Invalid pagination cursor", err.Error())
}
return response.InternalServerError(c, "Failed to retrieve tenders") return response.InternalServerError(c, "Failed to retrieve tenders")
} }
@@ -292,7 +300,8 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error {
// @Param languages query []string false "Filter by languages (comma-separated)" // @Param languages query []string false "Filter by languages (comma-separated)"
// @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param buyer_organization_id query string false "Filter by buyer organization ID"
// @Param limit query int false "Number of items per page (default: 20, max: 100)" // @Param limit query int false "Number of items per page (default: 20, max: 100)"
// @Param offset query int false "Number of items to skip (default: 0)" // @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
// @Param sort_by query string false "Sort by field" // @Param sort_by query string false "Sort by field"
// @Param sort_order query string false "Sort order (asc or desc)" // @Param sort_order query string false "Sort order (asc or desc)"
// @Success 200 {object} response.APIResponse{data=SearchResponse} // @Success 200 {object} response.APIResponse{data=SearchResponse}
@@ -328,6 +337,9 @@ func (h *TenderHandler) RecommendTenders(c echo.Context) error {
result, err := h.service.Recommend(c.Request().Context(), form, response.NewPagination(c)) result, err := h.service.Recommend(c.Request().Context(), form, response.NewPagination(c))
if err != nil { if err != nil {
if isInvalidPaginationCursor(err) {
return response.ValidationError(c, "Invalid pagination cursor", err.Error())
}
return response.InternalServerError(c, "Failed to retrieve tenders") return response.InternalServerError(c, "Failed to retrieve tenders")
} }
@@ -651,3 +663,7 @@ func (h *TenderHandler) GetAllAISummaries(c echo.Context) error {
return response.Success(c, summaries, "AI summaries retrieved successfully") return response.Success(c, summaries, "AI summaries retrieved successfully")
} }
func isInvalidPaginationCursor(err error) bool {
return errors.Is(err, orm.ErrInvalidCursor)
}
+57 -11
View File
@@ -26,7 +26,7 @@ type TenderRepository interface {
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error) GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
GetTenderCountByType(ctx context.Context) (map[string]int64, error) GetTenderCountByType(ctx context.Context) (map[string]int64, error)
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error) GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) ([]Tender, int64, error) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error)
GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error) GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
@@ -40,11 +40,27 @@ func collectionName() string {
return "tenders" return "tenders"
} }
// deepOffsetSkipCountThreshold skips CountDocuments when offset pagination goes this deep.
const deepOffsetSkipCountThreshold = 1000
// SearchPageResult is the paginated outcome of a tender search/list query.
type SearchPageResult struct {
Items []Tender
TotalCount int64 // -1 when count was skipped
NextCursor string
HasMore bool
}
// tenderSearchListProjection excludes large fields not needed for list/search API payloads. // tenderSearchListProjection excludes large fields not needed for list/search API payloads.
func tenderSearchListProjection() bson.M { func tenderSearchListProjection() bson.M {
return bson.M{ return bson.M{
"content_xml": 0, "content_xml": 0,
"document_summaries": 0, "document_summaries": 0,
"scraped_documents": 0,
"selection_criteria": 0,
"modifications": 0,
"source_file_url": 0,
"source_file_name": 0,
} }
} }
@@ -85,6 +101,26 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
*orm.NewIndex("tender_deadline_idx", bson.D{{Key: "tender_deadline", Value: 1}}), *orm.NewIndex("tender_deadline_idx", bson.D{{Key: "tender_deadline", Value: 1}}),
*orm.NewIndex("estimated_value_idx", bson.D{{Key: "estimated_value", Value: -1}}), *orm.NewIndex("estimated_value_idx", bson.D{{Key: "estimated_value", Value: -1}}),
*orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}), *orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}),
// Stable keyset pagination for default admin list (created_at desc).
*orm.NewIndex("created_at_id_idx", bson.D{
{Key: "created_at", Value: -1},
{Key: "_id", Value: -1},
}),
*orm.NewIndex("status_created_at_id_idx", bson.D{
{Key: "status", Value: 1},
{Key: "created_at", Value: -1},
{Key: "_id", Value: -1},
}),
*orm.NewIndex("country_code_created_at_id_idx", bson.D{
{Key: "country_code", Value: 1},
{Key: "created_at", Value: -1},
{Key: "_id", Value: -1},
}),
*orm.NewIndex("notice_type_code_created_at_id_idx", bson.D{
{Key: "notice_type_code", Value: 1},
{Key: "created_at", Value: -1},
{Key: "_id", Value: -1},
}),
*orm.NewIndex("updated_at_idx", bson.D{{Key: "updated_at", Value: -1}}), *orm.NewIndex("updated_at_idx", bson.D{{Key: "updated_at", Value: -1}}),
// Compound indexes for common queries // Compound indexes for common queries
*orm.NewIndex("status_country_idx", bson.D{ *orm.NewIndex("status_country_idx", bson.D{
@@ -321,10 +357,9 @@ func (r *tenderRepository) Delete(ctx context.Context, id string) error {
} }
// Search searches tenders based on criteria // Search searches tenders based on criteria
func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) ([]Tender, int64, error) { func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) {
filter := r.buildSearchFilter(form) filter := r.buildSearchFilter(form)
// Build sort
sort := "created_at" sort := "created_at"
if pagination.SortBy != "" { if pagination.SortBy != "" {
sort = pagination.SortBy sort = pagination.SortBy
@@ -335,23 +370,34 @@ func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, paginat
sortOrder = pagination.SortOrder sortOrder = pagination.SortOrder
} }
// Build pagination skipCount := pagination.Cursor != "" || pagination.Offset >= deepOffsetSkipCountThreshold
paginationBuilder := orm.NewPaginationBuilder(). paginationBuilder := orm.NewPaginationBuilder().
Limit(pagination.Limit). Limit(pagination.Limit).
Skip(pagination.Offset).
SortBy(sort, sortOrder). SortBy(sort, sortOrder).
Projection(tenderSearchListProjection()). Projection(tenderSearchListProjection()).
Build() SkipCount(skipCount)
result, err := r.ormRepo.FindAll(ctx, filter, paginationBuilder) if pagination.Cursor != "" {
paginationBuilder.Cursor(pagination.Cursor)
} else {
paginationBuilder.Skip(pagination.Offset)
}
result, err := r.ormRepo.FindAll(ctx, filter, paginationBuilder.Build())
if err != nil { if err != nil {
r.logger.Error("Failed to search tenders", map[string]interface{}{ r.logger.Error("Failed to search tenders", map[string]interface{}{
"error": err.Error(), "error": err.Error(),
}) })
return nil, 0, err return nil, err
} }
return result.Items, result.TotalCount, nil return &SearchPageResult{
Items: result.Items,
TotalCount: result.TotalCount,
NextCursor: result.NextCursor,
HasMore: result.HasMore,
}, nil
} }
// GetExpiredTenders retrieves tenders that are expired // GetExpiredTenders retrieves tenders that are expired
+17 -17
View File
@@ -613,7 +613,7 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
} }
} }
tenders, total, err := s.repository.Search(ctx, form, pagination) page, err := s.repository.Search(ctx, form, pagination)
if err != nil { if err != nil {
s.logger.Error("Failed to list tenders", map[string]interface{}{ s.logger.Error("Failed to list tenders", map[string]interface{}{
"error": err.Error(), "error": err.Error(),
@@ -623,14 +623,14 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
// If no company ID provided, return tenders without match percentage // If no company ID provided, return tenders without match percentage
if form.CompanyID == nil || *form.CompanyID == "" { if form.CompanyID == nil || *form.CompanyID == "" {
tenderResponses := make([]TenderResponse, 0, len(tenders)) tenderResponses := make([]TenderResponse, 0, len(page.Items))
for _, tender := range tenders { for _, tender := range page.Items {
tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language))) tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language)))
} }
return &SearchResponse{ return &SearchResponse{
Tenders: tenderResponses, Tenders: tenderResponses,
Metadata: pagination.Response(total), Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore),
}, nil }, nil
} }
@@ -642,14 +642,14 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
"error": err.Error(), "error": err.Error(),
}) })
// Continue without match percentage if company not found // Continue without match percentage if company not found
tenderResponses := make([]TenderResponse, 0, len(tenders)) tenderResponses := make([]TenderResponse, 0, len(page.Items))
for _, tender := range tenders { for _, tender := range page.Items {
tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language))) tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language)))
} }
return &SearchResponse{ return &SearchResponse{
Tenders: tenderResponses, Tenders: tenderResponses,
Metadata: pagination.Response(total), Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore),
}, },
nil nil
} }
@@ -662,17 +662,17 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
s.logger.Info("Tenders sorted by match percentage", map[string]interface{}{ s.logger.Info("Tenders sorted by match percentage", map[string]interface{}{
"company_id": *form.CompanyID, "company_id": *form.CompanyID,
"company_cpv_codes": len(companyCPVCodes), "company_cpv_codes": len(companyCPVCodes),
"total_tenders": len(tenders), "total_tenders": len(page.Items),
}) })
tenderResponses := make([]TenderResponse, 0, len(tenders)) tenderResponses := make([]TenderResponse, 0, len(page.Items))
for _, tender := range tenders { for _, tender := range page.Items {
tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language))) tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language)))
} }
return &SearchResponse{ return &SearchResponse{
Tenders: tenderResponses, Tenders: tenderResponses,
Metadata: pagination.Response(total), Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore),
}, },
nil nil
} }
@@ -724,7 +724,7 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat
} }
} }
tenders, total, err := s.repository.Search(ctx, form, pagination) page, err := s.repository.Search(ctx, form, pagination)
if err != nil { if err != nil {
s.logger.Error("Failed to list tenders", map[string]interface{}{ s.logger.Error("Failed to list tenders", map[string]interface{}{
"error": err.Error(), "error": err.Error(),
@@ -732,14 +732,14 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat
return nil, fmt.Errorf("failed to list tenders: %w", err) return nil, fmt.Errorf("failed to list tenders: %w", err)
} }
tenderResponses := make([]TenderResponse, 0, len(tenders)) tenderResponses := make([]TenderResponse, 0, len(page.Items))
for _, tender := range tenders { for _, tender := range page.Items {
tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language))) tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language)))
} }
return &SearchResponse{ return &SearchResponse{
Tenders: tenderResponses, Tenders: tenderResponses,
Metadata: pagination.Response(total), Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore),
}, nil }, nil
} }
@@ -812,7 +812,7 @@ func (s *tenderService) CleanupOldData(ctx context.Context, olderThan time.Durat
Status: []string{string(TenderStatusExpired), string(TenderStatusCancelled), string(TenderStatusAwarded)}, Status: []string{string(TenderStatusExpired), string(TenderStatusCancelled), string(TenderStatusAwarded)},
} }
oldTenders, _, err := s.repository.Search(ctx, form, pagination) oldPage, err := s.repository.Search(ctx, form, pagination)
if err != nil { if err != nil {
s.logger.Error("Failed to find old tenders for cleanup", map[string]interface{}{ s.logger.Error("Failed to find old tenders for cleanup", map[string]interface{}{
"error": err.Error(), "error": err.Error(),
@@ -821,7 +821,7 @@ func (s *tenderService) CleanupOldData(ctx context.Context, olderThan time.Durat
} }
cleanupCount := 0 cleanupCount := 0
for _, tender := range oldTenders { for _, tender := range oldPage.Items {
if tender.CreatedAt < cutoffTime { if tender.CreatedAt < cutoffTime {
cleanupCount++ cleanupCount++
// Here you would delete the tender if implementing actual cleanup // Here you would delete the tender if implementing actual cleanup
+234
View File
@@ -0,0 +1,234 @@
package mongo
import (
"encoding/base64"
"encoding/json"
"fmt"
"strconv"
"go.mongodb.org/mongo-driver/v2/bson"
)
const pageCursorVersion = 1
// pageCursor is an opaque pagination token for keyset (cursor) pagination.
type pageCursor struct {
V int `json:"v"`
SortField string `json:"sort_field"`
SortOrder int `json:"sort_order"`
ValueType string `json:"value_type"` // i64, f64, str, oid
SortValue string `json:"sort_value"`
ID string `json:"id"`
}
func encodePageCursor(sortField string, sortOrder int, sortValue interface{}, id string) (string, error) {
if sortField == "" {
return "", fmt.Errorf("%w: sort field is required", ErrInvalidCursor)
}
if id == "" {
return "", fmt.Errorf("%w: document id is required", ErrInvalidCursor)
}
valueType, sortValueStr, err := formatCursorSortValue(sortField, sortValue)
if err != nil {
return "", err
}
payload, err := json.Marshal(pageCursor{
V: pageCursorVersion,
SortField: sortField,
SortOrder: sortOrder,
ValueType: valueType,
SortValue: sortValueStr,
ID: id,
})
if err != nil {
return "", fmt.Errorf("failed to marshal cursor: %w", err)
}
return base64.StdEncoding.EncodeToString(payload), nil
}
func decodePageCursor(encoded string) (*pageCursor, error) {
cursorBytes, err := base64.StdEncoding.DecodeString(encoded)
if err != nil {
return nil, fmt.Errorf("%w: invalid cursor encoding", ErrInvalidCursor)
}
var cursor pageCursor
if err := json.Unmarshal(cursorBytes, &cursor); err != nil {
return nil, fmt.Errorf("%w: invalid cursor payload", ErrInvalidCursor)
}
if cursor.V != pageCursorVersion || cursor.SortField == "" || cursor.ID == "" {
return nil, fmt.Errorf("%w: unsupported or incomplete cursor", ErrInvalidCursor)
}
if cursor.SortOrder != 1 && cursor.SortOrder != -1 {
return nil, fmt.Errorf("%w: invalid sort order in cursor", ErrInvalidCursor)
}
return &cursor, nil
}
func (c *pageCursor) matches(sortField string, sortOrder int) bool {
return c.SortField == sortField && c.SortOrder == sortOrder
}
func (c *pageCursor) mongoFilter() (bson.M, error) {
if c.SortField == "_id" {
objectID, err := bson.ObjectIDFromHex(c.ID)
if err != nil {
return nil, fmt.Errorf("%w: invalid ObjectID in cursor", ErrInvalidCursor)
}
operator := "$lt"
if c.SortOrder == 1 {
operator = "$gt"
}
return bson.M{"_id": bson.M{operator: objectID}}, nil
}
sortValue, err := c.parseSortValue()
if err != nil {
return nil, err
}
objectID, err := bson.ObjectIDFromHex(c.ID)
if err != nil {
return nil, fmt.Errorf("%w: invalid ObjectID in cursor", ErrInvalidCursor)
}
primaryOp := "$lt"
secondaryOp := "$lt"
if c.SortOrder == 1 {
primaryOp = "$gt"
secondaryOp = "$gt"
}
return bson.M{
"$or": []bson.M{
{c.SortField: bson.M{primaryOp: sortValue}},
{
c.SortField: sortValue,
"_id": bson.M{secondaryOp: objectID},
},
},
}, nil
}
func (c *pageCursor) parseSortValue() (interface{}, error) {
switch c.ValueType {
case "i64":
v, err := strconv.ParseInt(c.SortValue, 10, 64)
if err != nil {
return nil, fmt.Errorf("%w: invalid int64 sort value", ErrInvalidCursor)
}
return v, nil
case "f64":
v, err := strconv.ParseFloat(c.SortValue, 64)
if err != nil {
return nil, fmt.Errorf("%w: invalid float sort value", ErrInvalidCursor)
}
return v, nil
case "str":
return c.SortValue, nil
case "oid":
return bson.ObjectIDFromHex(c.SortValue)
default:
return nil, fmt.Errorf("%w: unknown sort value type", ErrInvalidCursor)
}
}
func formatCursorSortValue(sortField string, sortValue interface{}) (string, string, error) {
if sortField == "_id" {
switch v := sortValue.(type) {
case bson.ObjectID:
return "oid", v.Hex(), nil
case string:
if _, err := bson.ObjectIDFromHex(v); err != nil {
return "", "", fmt.Errorf("%w: invalid ObjectID sort value", ErrInvalidCursor)
}
return "oid", v, nil
default:
return "", "", fmt.Errorf("%w: unsupported _id sort value", ErrInvalidCursor)
}
}
switch v := sortValue.(type) {
case int64:
return "i64", strconv.FormatInt(v, 10), nil
case int32:
return "i64", strconv.FormatInt(int64(v), 10), nil
case int:
return "i64", strconv.FormatInt(int64(v), 10), nil
case float64:
return "f64", strconv.FormatFloat(v, 'f', -1, 64), nil
case float32:
return "f64", strconv.FormatFloat(float64(v), 'f', -1, 64), nil
case string:
return "str", v, nil
case bson.ObjectID:
return "oid", v.Hex(), nil
default:
return "", "", fmt.Errorf("%w: unsupported sort value type for field %s", ErrInvalidCursor, sortField)
}
}
func documentSortValue(docMap bson.M, sortField string) (interface{}, error) {
if sortField == "_id" {
id, exists := docMap["_id"]
if !exists {
return nil, fmt.Errorf("sort field _id not found in document")
}
return id, nil
}
sortValue, exists := docMap[sortField]
if !exists {
return nil, fmt.Errorf("sort field %s not found in document", sortField)
}
return sortValue, nil
}
func documentIDHex(docMap bson.M) (string, error) {
id, exists := docMap["_id"]
if !exists {
return "", fmt.Errorf("document _id not found")
}
switch v := id.(type) {
case bson.ObjectID:
return v.Hex(), nil
case string:
if _, err := bson.ObjectIDFromHex(v); err != nil {
return "", fmt.Errorf("invalid document _id")
}
return v, nil
default:
return "", fmt.Errorf("unsupported document _id type")
}
}
func cloneFilter(filter bson.M) (bson.M, error) {
if len(filter) == 0 {
return bson.M{}, nil
}
raw, err := bson.Marshal(filter)
if err != nil {
return nil, err
}
var cloned bson.M
if err := bson.Unmarshal(raw, &cloned); err != nil {
return nil, err
}
return cloned, nil
}
func mergeFilters(base, extra bson.M) bson.M {
if len(extra) == 0 {
return base
}
if len(base) == 0 {
return extra
}
return bson.M{
"$and": []bson.M{base, extra},
}
}
+39 -66
View File
@@ -2,7 +2,6 @@ package mongo
import ( import (
"context" "context"
"encoding/base64"
"errors" "errors"
"fmt" "fmt"
"time" "time"
@@ -26,6 +25,8 @@ type Pagination struct {
Cursor string `json:"cursor"` // Base64 encoded cursor for cursor-based pagination Cursor string `json:"cursor"` // Base64 encoded cursor for cursor-based pagination
SortField string `json:"sortField"` // Field to sort by (default: "_id") SortField string `json:"sortField"` // Field to sort by (default: "_id")
SortOrder int `json:"sortOrder"` // Sort order: 1 for ascending, -1 for descending SortOrder int `json:"sortOrder"` // Sort order: 1 for ascending, -1 for descending
// SkipCount skips CountDocuments (use with cursor or deep offset pagination).
SkipCount bool `json:"skipCount,omitempty"`
// Projection limits fields returned from Find (e.g. exclude large blobs on list endpoints). // Projection limits fields returned from Find (e.g. exclude large blobs on list endpoints).
Projection bson.M `json:"projection,omitempty"` Projection bson.M `json:"projection,omitempty"`
} }
@@ -132,8 +133,16 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
pagination.SortOrder = -1 // Default to descending pagination.SortOrder = -1 // Default to descending
} }
// Build sort options // Build sort with _id tie-breaker for stable keyset pagination.
sort := bson.D{{Key: pagination.SortField, Value: pagination.SortOrder}} sort := bson.D{{Key: pagination.SortField, Value: pagination.SortOrder}}
if pagination.SortField != "_id" {
sort = append(sort, bson.E{Key: "_id", Value: pagination.SortOrder})
}
countFilter, err := cloneFilter(filter)
if err != nil {
return nil, fmt.Errorf("failed to clone filter for count: %w", err)
}
// Build find options // Build find options
findOptions := options.Find(). findOptions := options.Find().
@@ -150,10 +159,7 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Merge cursor filter with existing filter filter = mergeFilters(filter, cursorFilter)
for key, value := range cursorFilter {
filter[key] = value
}
} else if pagination.Skip > 0 { } else if pagination.Skip > 0 {
// Use offset-based pagination // Use offset-based pagination
findOptions.SetSkip(int64(pagination.Skip)) findOptions.SetSkip(int64(pagination.Skip))
@@ -187,21 +193,23 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
items = items[:pagination.Limit] // Remove the extra item items = items[:pagination.Limit] // Remove the extra item
} }
// Get total count for pagination metadata // Get total count for pagination metadata (never include cursor constraints).
var totalCount int64 var totalCount int64 = -1
var countErr error if !pagination.SkipCount {
totalCount, countErr = r.Count(ctx, filter) var countErr error
if countErr != nil { totalCount, countErr = r.Count(ctx, countFilter)
r.logger.Error("Failed to get total count", map[string]interface{}{ if countErr != nil {
"error": countErr.Error(), r.logger.Error("Failed to get total count", map[string]interface{}{
}) "error": countErr.Error(),
totalCount = 0 })
totalCount = 0
}
} }
// Generate next cursor // Generate next cursor
var nextCursor string var nextCursor string
if hasMore && len(items) > 0 { if hasMore && len(items) > 0 {
nextCursor, err = r.generateCursor(items[len(items)-1], pagination.SortField) nextCursor, err = r.generateCursor(items[len(items)-1], pagination.SortField, pagination.SortOrder)
if err != nil { if err != nil {
r.logger.Error("Failed to generate next cursor", map[string]interface{}{ r.logger.Error("Failed to generate next cursor", map[string]interface{}{
"error": err.Error(), "error": err.Error(),
@@ -445,43 +453,20 @@ func (r *repository[T]) validatePagination(pagination Pagination) error {
return nil return nil
} }
// buildCursorFilter builds a filter for cursor-based pagination // buildCursorFilter builds a filter for cursor-based pagination.
func (r *repository[T]) buildCursorFilter(cursor, sortField string, sortOrder int) (bson.M, error) { func (r *repository[T]) buildCursorFilter(cursor, sortField string, sortOrder int) (bson.M, error) {
// Decode cursor pageCursor, err := decodePageCursor(cursor)
cursorBytes, err := base64.StdEncoding.DecodeString(cursor)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: invalid cursor encoding", ErrInvalidCursor) return nil, err
} }
if !pageCursor.matches(sortField, sortOrder) {
var cursorValue interface{} return nil, fmt.Errorf("%w: cursor does not match requested sort", ErrInvalidCursor)
// Try to parse as ObjectID first
if sortField == "_id" {
if objectID, err := bson.ObjectIDFromHex(string(cursorBytes)); err == nil {
cursorValue = objectID
} else {
return nil, fmt.Errorf("%w: invalid ObjectID in cursor", ErrInvalidCursor)
}
} else {
// For other fields, try to parse as appropriate type
// This is a simplified implementation - you might want to make this more sophisticated
cursorValue = string(cursorBytes)
} }
return pageCursor.mongoFilter()
// Build comparison operator based on sort order
var operator string
if sortOrder == 1 {
operator = "$gt"
} else {
operator = "$lt"
}
return bson.M{sortField: bson.M{operator: cursorValue}}, nil
} }
// generateCursor generates a cursor from a document // generateCursor generates an opaque cursor from the last document in a page.
func (r *repository[T]) generateCursor(doc T, sortField string) (string, error) { func (r *repository[T]) generateCursor(doc T, sortField string, sortOrder int) (string, error) {
// Convert document to BSON
docBytes, err := bson.Marshal(doc) docBytes, err := bson.Marshal(doc)
if err != nil { if err != nil {
return "", fmt.Errorf("failed to marshal document: %w", err) return "", fmt.Errorf("failed to marshal document: %w", err)
@@ -492,29 +477,17 @@ func (r *repository[T]) generateCursor(doc T, sortField string) (string, error)
return "", fmt.Errorf("failed to unmarshal document: %w", err) return "", fmt.Errorf("failed to unmarshal document: %w", err)
} }
// Get the sort field value sortValue, err := documentSortValue(docMap, sortField)
sortValue, exists := docMap[sortField] if err != nil {
if !exists { return "", err
return "", fmt.Errorf("sort field %s not found in document", sortField)
} }
// Convert to string for encoding idHex, err := documentIDHex(docMap)
var cursorStr string if err != nil {
switch v := sortValue.(type) { return "", err
case bson.ObjectID:
cursorStr = v.Hex()
case string:
cursorStr = v
case int64:
cursorStr = fmt.Sprintf("%d", v)
case float64:
cursorStr = fmt.Sprintf("%f", v)
default:
cursorStr = fmt.Sprintf("%v", v)
} }
// Encode as base64 return encodePageCursor(sortField, sortOrder, sortValue, idHex)
return base64.StdEncoding.EncodeToString([]byte(cursorStr)), nil
} }
// getModelID extracts the ID from a model // getModelID extracts the ID from a model
+6
View File
@@ -299,6 +299,12 @@ func (pb *PaginationBuilder) Projection(projection bson.M) *PaginationBuilder {
return pb return pb
} }
// SkipCount disables CountDocuments for this query.
func (pb *PaginationBuilder) SkipCount(skip bool) *PaginationBuilder {
pb.pagination.SkipCount = skip
return pb
}
// Build returns the final pagination options // Build returns the final pagination options
func (pb *PaginationBuilder) Build() Pagination { func (pb *PaginationBuilder) Build() Pagination {
return pb.pagination return pb.pagination
+33 -14
View File
@@ -26,16 +26,19 @@ type APIError struct {
// Meta represents metadata for paginated responses // Meta represents metadata for paginated responses
type Meta struct { type Meta struct {
Total int `json:"total"` Total int `json:"total,omitempty"`
Limit int `json:"limit"` Limit int `json:"limit"`
Offset int `json:"offset"` Offset int `json:"offset"`
Page int `json:"page"` Page int `json:"page,omitempty"`
Pages int `json:"pages"` Pages int `json:"pages,omitempty"`
NextCursor string `json:"next_cursor,omitempty"`
HasMore bool `json:"has_more,omitempty"`
} }
type Pagination struct { type Pagination struct {
Limit int `query:"limit" valid:"optional,range(1|100)" default:"20"` Limit int `query:"limit" valid:"optional,range(1|100)" default:"20"`
Offset int `query:"offset" valid:"optional,range(0|1000000)" default:"0"` Offset int `query:"offset" valid:"optional,range(0|1000000)" default:"0"`
Cursor string `query:"cursor" valid:"optional"`
SortBy string `query:"sort_by" valid:"optional"` SortBy string `query:"sort_by" valid:"optional"`
SortOrder string `query:"sort_order" valid:"optional,in(asc|desc)"` SortOrder string `query:"sort_order" valid:"optional,in(asc|desc)"`
} }
@@ -210,22 +213,38 @@ func NewPagination(c echo.Context) *Pagination {
sortOrder = "desc" sortOrder = "desc"
} }
cursor := c.QueryParam("cursor")
return &Pagination{ return &Pagination{
Limit: limit, Limit: limit,
Offset: offset, Offset: offset,
Cursor: cursor,
SortBy: sortBy, SortBy: sortBy,
SortOrder: sortOrder, SortOrder: sortOrder,
} }
} }
// Pagination calculates pagination metadata // Response calculates offset-based pagination metadata.
func (p *Pagination) Response(total int64) *Meta { func (p *Pagination) Response(total int64) *Meta {
pages := (total + int64(p.Limit) - 1) / int64(p.Limit) return p.ListMeta(total, "", false)
return &Meta{ }
Total: int(total),
Limit: p.Limit, // ListMeta builds pagination metadata for list endpoints (offset and/or cursor).
Offset: p.Offset, // When total is negative, total/page/pages are omitted (count was skipped).
Page: int(p.Offset/p.Limit) + 1, func (p *Pagination) ListMeta(total int64, nextCursor string, hasMore bool) *Meta {
Pages: int(pages), meta := &Meta{
} Limit: p.Limit,
Offset: p.Offset,
NextCursor: nextCursor,
HasMore: hasMore,
}
if total >= 0 {
pages := (total + int64(p.Limit) - 1) / int64(p.Limit)
meta.Total = int(total)
if p.Limit > 0 {
meta.Page = p.Offset/p.Limit + 1
}
meta.Pages = int(pages)
}
return meta
} }