mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

hybrid pagination

Browse Source
This commit is contained in:
Mazyar
2026-05-17 17:21:36 +03:30
parent 42d0a47226
commit 6dac5b482a
39 changed files with 696 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/mongo/cursor.go
+23 -15
View File
@@ -13,15 +13,17 @@ const pageCursorVersion = 1
// pageCursor is an opaque pagination token for keyset (cursor) pagination.
type pageCursor struct {
V int `json:"v"`
SortField string `json:"sort_field"`
SortOrder int `json:"sort_order"`
ValueType string `json:"value_type"` // i64, f64, str, oid
SortValue string `json:"sort_value"`
ID string `json:"id"`
V int `json:"v"`
SortField string `json:"sort_field"`
SortOrder int `json:"sort_order"`
ValueType string `json:"value_type"` // i64, f64, str, oid
SortValue string `json:"sort_value"`
ID string `json:"id"`
FiltersHash string `json:"filters_hash,omitempty"`
PageOffset int `json:"page_offset,omitempty"`
}
func encodePageCursor(sortField string, sortOrder int, sortValue interface{}, id string) (string, error) {
func encodePageCursor(sortField string, sortOrder int, sortValue interface{}, id, filtersHash string, pageOffset int) (string, error) {
if sortField == "" {
return "", fmt.Errorf("%w: sort field is required", ErrInvalidCursor)
}
@@ -35,22 +37,28 @@ func encodePageCursor(sortField string, sortOrder int, sortValue interface{}, id
}
payload, err := json.Marshal(pageCursor{
V: pageCursorVersion,
SortField: sortField,
SortOrder: sortOrder,
ValueType: valueType,
SortValue: sortValueStr,
ID: id,
V: pageCursorVersion,
SortField: sortField,
SortOrder: sortOrder,
ValueType: valueType,
SortValue: sortValueStr,
ID: id,
FiltersHash: filtersHash,
PageOffset: pageOffset,
})
if err != nil {
return "", fmt.Errorf("failed to marshal cursor: %w", err)
}
return base64.StdEncoding.EncodeToString(payload), nil
return base64.RawURLEncoding.EncodeToString(payload), nil
}
func decodePageCursor(encoded string) (*pageCursor, error) {
cursorBytes, err := base64.StdEncoding.DecodeString(encoded)
cursorBytes, err := base64.RawURLEncoding.DecodeString(encoded)
if err != nil {
// Accept standard Base64 cursors issued before URL-safe encoding.
cursorBytes, err = base64.StdEncoding.DecodeString(encoded)
}
if err != nil {
return nil, fmt.Errorf("%w: invalid cursor encoding", ErrInvalidCursor)
}
pkg/mongo/cursor_test.go
+55
View File
@@ -0,0 +1,55 @@
package mongo
import (
"strings"
"testing"
"go.mongodb.org/mongo-driver/v2/bson"
)
func TestEncodeDecodePageCursor(t *testing.T) {
encoded, err := encodePageCursor("created_at", -1, int64(1700000000), "507f1f77bcf86cd799439011", "abc123", 10)
if err != nil {
t.Fatalf("encode: %v", err)
}
if strings.Contains(encoded, "+") || strings.Contains(encoded, "/") {
t.Fatalf("expected URL-safe cursor, got %q", encoded)
}
decoded, err := decodePageCursor(encoded)
if err != nil {
t.Fatalf("decode: %v", err)
}
if decoded.SortField != "created_at" || decoded.SortOrder != -1 {
t.Fatalf("unexpected sort: %+v", decoded)
}
if decoded.FiltersHash != "abc123" || decoded.PageOffset != 10 {
t.Fatalf("unexpected metadata: %+v", decoded)
}
}
func TestBuildListPaginationCursorAndOffsetConflict(t *testing.T) {
_, err := BuildListPagination(10, 5, "cursor-token", "created_at", "desc", bson.M{}, ListPaginationOptions{})
if err == nil {
t.Fatal("expected error when cursor and offset are both set")
}
}
func TestBuildListPaginationOffsetMode(t *testing.T) {
p, err := BuildListPagination(10, 20, "", "created_at", "desc", bson.M{"status": "active"}, ListPaginationOptions{})
if err != nil {
t.Fatalf("build: %v", err)
}
if p.Skip != 20 || p.Limit != 10 || p.Cursor != "" {
t.Fatalf("unexpected pagination: %+v", p)
}
}
func TestHashFilterStable(t *testing.T) {
a := HashFilter(bson.M{"status": "active"})
b := HashFilter(bson.M{"status": "active"})
if a == "" || a != b {
t.Fatalf("expected stable non-empty hash, got %q and %q", a, b)
}
}
pkg/mongo/filter_hash.go
+21
View File
@@ -0,0 +1,21 @@
package mongo
import (
"crypto/sha256"
"encoding/hex"
"go.mongodb.org/mongo-driver/v2/bson"
)
// HashFilter returns a stable SHA-256 hex digest of a MongoDB filter for cursor binding.
func HashFilter(filter bson.M) string {
if len(filter) == 0 {
return ""
}
raw, err := bson.Marshal(filter)
if err != nil {
return ""
}
sum := sha256.Sum256(raw)
return hex.EncodeToString(sum[:])
}
pkg/mongo/list_pagination.go
+78
View File
@@ -0,0 +1,78 @@
package mongo
import (
"fmt"
"go.mongodb.org/mongo-driver/v2/bson"
)
const (
// DefaultListLimit is the default page size for admin list endpoints.
DefaultListLimit = 10
// MaxListLimit is the maximum allowed page size.
MaxListLimit = 100
)
// ListPaginationOptions configures optional list pagination behavior.
type ListPaginationOptions struct {
Projection bson.M
}
// BuildListPagination converts API pagination parameters into MongoDB pagination options.
// filter is used to bind cursors to the active query filters via a hash.
func BuildListPagination(
limit, offset int,
cursor, sortField, sortOrder string,
filter bson.M,
opts ListPaginationOptions,
) (Pagination, error) {
if limit <= 0 {
limit = DefaultListLimit
}
if limit > MaxListLimit {
return Pagination{}, fmt.Errorf("%w: limit must be between 1 and %d", ErrInvalidPagination, MaxListLimit)
}
if sortField == "" {
sortField = "created_at"
}
if sortOrder == "" {
sortOrder = "desc"
}
sortOrderInt := -1
if sortOrder == "asc" {
sortOrderInt = 1
}
filtersHash := HashFilter(filter)
pb := NewPaginationBuilder().
Limit(limit).
SortBy(sortField, sortOrder)
if len(opts.Projection) > 0 {
pb.Projection(opts.Projection)
}
if cursor != "" {
pageCursor, err := decodePageCursor(cursor)
if err != nil {
return Pagination{}, err
}
if !pageCursor.matches(sortField, sortOrderInt) {
return Pagination{}, fmt.Errorf("%w: cursor does not match requested sort", ErrInvalidCursor)
}
if pageCursor.FiltersHash != filtersHash {
return Pagination{}, fmt.Errorf("%w: cursor does not match current filters", ErrInvalidCursor)
}
pb.Cursor(cursor)
} else {
if offset < 0 {
return Pagination{}, fmt.Errorf("%w: offset cannot be negative", ErrInvalidPagination)
}
pb.Skip(offset)
}
return pb.Build(), nil
}
pkg/mongo/repository.go
+22 -6
View File
@@ -37,6 +37,7 @@ type PaginatedResult[T any] struct {
TotalCount int64 `json:"totalCount"` // Total number of documents matching the filter
NextCursor string `json:"nextCursor"` // Base64 encoded cursor for next page
HasMore bool `json:"hasMore"` // Whether there are more documents
PageOffset int `json:"pageOffset"` // Zero-based offset of the first item in this page
}
// Repository defines the interface for MongoDB operations
@@ -144,6 +145,16 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
return nil, fmt.Errorf("failed to clone filter for count: %w", err)
}
filtersHash := HashFilter(filter)
pageOffset := pagination.Skip
if pagination.Cursor != "" {
pageCursor, err := decodePageCursor(pagination.Cursor)
if err != nil {
return nil, err
}
pageOffset = pageCursor.PageOffset
}
// Build find options
findOptions := options.Find().
SetSort(sort).
@@ -155,7 +166,7 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
// Handle cursor-based pagination
if pagination.Cursor != "" {
cursorFilter, err := r.buildCursorFilter(pagination.Cursor, pagination.SortField, pagination.SortOrder)
cursorFilter, err := r.buildCursorFilter(pagination.Cursor, pagination.SortField, pagination.SortOrder, filtersHash)
if err != nil {
return nil, err
}
@@ -209,12 +220,13 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
// Generate next cursor
var nextCursor string
if hasMore && len(items) > 0 {
nextCursor, err = r.generateCursor(items[len(items)-1], pagination.SortField, pagination.SortOrder)
nextPageOffset := pageOffset + pagination.Limit
nextCursor, err = r.generateCursor(items[len(items)-1], pagination.SortField, pagination.SortOrder, filtersHash, nextPageOffset)
if err != nil {
r.logger.Error("Failed to generate next cursor", map[string]interface{}{
"error": err.Error(),
})
// Don't fail the entire query, just leave cursor empty
return nil, fmt.Errorf("failed to generate pagination cursor: %w", err)
}
}
@@ -223,6 +235,7 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
TotalCount: totalCount,
NextCursor: nextCursor,
HasMore: hasMore,
PageOffset: pageOffset,
}, nil
}
@@ -454,7 +467,7 @@ func (r *repository[T]) validatePagination(pagination Pagination) error {
}
// buildCursorFilter builds a filter for cursor-based pagination.
func (r *repository[T]) buildCursorFilter(cursor, sortField string, sortOrder int) (bson.M, error) {
func (r *repository[T]) buildCursorFilter(cursor, sortField string, sortOrder int, filtersHash string) (bson.M, error) {
pageCursor, err := decodePageCursor(cursor)
if err != nil {
return nil, err
@@ -462,11 +475,14 @@ func (r *repository[T]) buildCursorFilter(cursor, sortField string, sortOrder in
if !pageCursor.matches(sortField, sortOrder) {
return nil, fmt.Errorf("%w: cursor does not match requested sort", ErrInvalidCursor)
}
if pageCursor.FiltersHash != filtersHash {
return nil, fmt.Errorf("%w: cursor does not match current filters", ErrInvalidCursor)
}
return pageCursor.mongoFilter()
}
// generateCursor generates an opaque cursor from the last document in a page.
func (r *repository[T]) generateCursor(doc T, sortField string, sortOrder int) (string, error) {
func (r *repository[T]) generateCursor(doc T, sortField string, sortOrder int, filtersHash string, pageOffset int) (string, error) {
docBytes, err := bson.Marshal(doc)
if err != nil {
return "", fmt.Errorf("failed to marshal document: %w", err)
@@ -487,7 +503,7 @@ func (r *repository[T]) generateCursor(doc T, sortField string, sortOrder int) (
return "", err
}
return encodePageCursor(sortField, sortOrder, sortValue, idHex)
return encodePageCursor(sortField, sortOrder, sortValue, idHex, filtersHash, pageOffset)
}
// getModelID extracts the ID from a model