hybrid pagination

pkg/mongo/cursor.go

@@ -13,15 +13,17 @@ const pageCursorVersion = 1
 
 // pageCursor is an opaque pagination token for keyset (cursor) pagination.
 type pageCursor struct {
-	V         int    `json:"v"`
-	SortField string `json:"sort_field"`
-	SortOrder int    `json:"sort_order"`
-	ValueType string `json:"value_type"` // i64, f64, str, oid
-	SortValue string `json:"sort_value"`
-	ID        string `json:"id"`
+	V           int    `json:"v"`
+	SortField   string `json:"sort_field"`
+	SortOrder   int    `json:"sort_order"`
+	ValueType   string `json:"value_type"` // i64, f64, str, oid
+	SortValue   string `json:"sort_value"`
+	ID          string `json:"id"`
+	FiltersHash string `json:"filters_hash,omitempty"`
+	PageOffset  int    `json:"page_offset,omitempty"`
 }
 
-func encodePageCursor(sortField string, sortOrder int, sortValue interface{}, id string) (string, error) {
+func encodePageCursor(sortField string, sortOrder int, sortValue interface{}, id, filtersHash string, pageOffset int) (string, error) {
 	if sortField == "" {
 		return "", fmt.Errorf("%w: sort field is required", ErrInvalidCursor)
 	}
@@ -35,22 +37,28 @@ func encodePageCursor(sortField string, sortOrder int, sortValue interface{}, id
 	}
 
 	payload, err := json.Marshal(pageCursor{
-		V:         pageCursorVersion,
-		SortField: sortField,
-		SortOrder: sortOrder,
-		ValueType: valueType,
-		SortValue: sortValueStr,
-		ID:        id,
+		V:           pageCursorVersion,
+		SortField:   sortField,
+		SortOrder:   sortOrder,
+		ValueType:   valueType,
+		SortValue:   sortValueStr,
+		ID:          id,
+		FiltersHash: filtersHash,
+		PageOffset:  pageOffset,
 	})
 	if err != nil {
 		return "", fmt.Errorf("failed to marshal cursor: %w", err)
 	}
 
-	return base64.StdEncoding.EncodeToString(payload), nil
+	return base64.RawURLEncoding.EncodeToString(payload), nil
 }
 
 func decodePageCursor(encoded string) (*pageCursor, error) {
-	cursorBytes, err := base64.StdEncoding.DecodeString(encoded)
+	cursorBytes, err := base64.RawURLEncoding.DecodeString(encoded)
+	if err != nil {
+		// Accept standard Base64 cursors issued before URL-safe encoding.
+		cursorBytes, err = base64.StdEncoding.DecodeString(encoded)
+	}
 	if err != nil {
 		return nil, fmt.Errorf("%w: invalid cursor encoding", ErrInvalidCursor)
 	}

pkg/mongo/cursor_test.go

@@ -0,0 +1,55 @@
+package mongo
+
+import (
+	"strings"
+	"testing"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+func TestEncodeDecodePageCursor(t *testing.T) {
+	encoded, err := encodePageCursor("created_at", -1, int64(1700000000), "507f1f77bcf86cd799439011", "abc123", 10)
+	if err != nil {
+		t.Fatalf("encode: %v", err)
+	}
+	if strings.Contains(encoded, "+") || strings.Contains(encoded, "/") {
+		t.Fatalf("expected URL-safe cursor, got %q", encoded)
+	}
+
+	decoded, err := decodePageCursor(encoded)
+	if err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+
+	if decoded.SortField != "created_at" || decoded.SortOrder != -1 {
+		t.Fatalf("unexpected sort: %+v", decoded)
+	}
+	if decoded.FiltersHash != "abc123" || decoded.PageOffset != 10 {
+		t.Fatalf("unexpected metadata: %+v", decoded)
+	}
+}
+
+func TestBuildListPaginationCursorAndOffsetConflict(t *testing.T) {
+	_, err := BuildListPagination(10, 5, "cursor-token", "created_at", "desc", bson.M{}, ListPaginationOptions{})
+	if err == nil {
+		t.Fatal("expected error when cursor and offset are both set")
+	}
+}
+
+func TestBuildListPaginationOffsetMode(t *testing.T) {
+	p, err := BuildListPagination(10, 20, "", "created_at", "desc", bson.M{"status": "active"}, ListPaginationOptions{})
+	if err != nil {
+		t.Fatalf("build: %v", err)
+	}
+	if p.Skip != 20 || p.Limit != 10 || p.Cursor != "" {
+		t.Fatalf("unexpected pagination: %+v", p)
+	}
+}
+
+func TestHashFilterStable(t *testing.T) {
+	a := HashFilter(bson.M{"status": "active"})
+	b := HashFilter(bson.M{"status": "active"})
+	if a == "" || a != b {
+		t.Fatalf("expected stable non-empty hash, got %q and %q", a, b)
+	}
+}

pkg/mongo/filter_hash.go

@@ -0,0 +1,21 @@
+package mongo
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+// HashFilter returns a stable SHA-256 hex digest of a MongoDB filter for cursor binding.
+func HashFilter(filter bson.M) string {
+	if len(filter) == 0 {
+		return ""
+	}
+	raw, err := bson.Marshal(filter)
+	if err != nil {
+		return ""
+	}
+	sum := sha256.Sum256(raw)
+	return hex.EncodeToString(sum[:])
+}

pkg/mongo/list_pagination.go

@@ -0,0 +1,78 @@
+package mongo
+
+import (
+	"fmt"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
+
+const (
+	// DefaultListLimit is the default page size for admin list endpoints.
+	DefaultListLimit = 10
+	// MaxListLimit is the maximum allowed page size.
+	MaxListLimit = 100
+)
+
+// ListPaginationOptions configures optional list pagination behavior.
+type ListPaginationOptions struct {
+	Projection bson.M
+}
+
+// BuildListPagination converts API pagination parameters into MongoDB pagination options.
+// filter is used to bind cursors to the active query filters via a hash.
+func BuildListPagination(
+	limit, offset int,
+	cursor, sortField, sortOrder string,
+	filter bson.M,
+	opts ListPaginationOptions,
+) (Pagination, error) {
+	if limit <= 0 {
+		limit = DefaultListLimit
+	}
+	if limit > MaxListLimit {
+		return Pagination{}, fmt.Errorf("%w: limit must be between 1 and %d", ErrInvalidPagination, MaxListLimit)
+	}
+
+	if sortField == "" {
+		sortField = "created_at"
+	}
+	if sortOrder == "" {
+		sortOrder = "desc"
+	}
+
+	sortOrderInt := -1
+	if sortOrder == "asc" {
+		sortOrderInt = 1
+	}
+
+	filtersHash := HashFilter(filter)
+
+	pb := NewPaginationBuilder().
+		Limit(limit).
+		SortBy(sortField, sortOrder)
+
+	if len(opts.Projection) > 0 {
+		pb.Projection(opts.Projection)
+	}
+
+	if cursor != "" {
+		pageCursor, err := decodePageCursor(cursor)
+		if err != nil {
+			return Pagination{}, err
+		}
+		if !pageCursor.matches(sortField, sortOrderInt) {
+			return Pagination{}, fmt.Errorf("%w: cursor does not match requested sort", ErrInvalidCursor)
+		}
+		if pageCursor.FiltersHash != filtersHash {
+			return Pagination{}, fmt.Errorf("%w: cursor does not match current filters", ErrInvalidCursor)
+		}
+		pb.Cursor(cursor)
+	} else {
+		if offset < 0 {
+			return Pagination{}, fmt.Errorf("%w: offset cannot be negative", ErrInvalidPagination)
+		}
+		pb.Skip(offset)
+	}
+
+	return pb.Build(), nil
+}

pkg/mongo/repository.go

@@ -37,6 +37,7 @@ type PaginatedResult[T any] struct {
 	TotalCount int64  `json:"totalCount"` // Total number of documents matching the filter
 	NextCursor string `json:"nextCursor"` // Base64 encoded cursor for next page
 	HasMore    bool   `json:"hasMore"`    // Whether there are more documents
+	PageOffset int    `json:"pageOffset"` // Zero-based offset of the first item in this page
 }
 
 // Repository defines the interface for MongoDB operations
@@ -144,6 +145,16 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
 		return nil, fmt.Errorf("failed to clone filter for count: %w", err)
 	}
 
+	filtersHash := HashFilter(filter)
+	pageOffset := pagination.Skip
+	if pagination.Cursor != "" {
+		pageCursor, err := decodePageCursor(pagination.Cursor)
+		if err != nil {
+			return nil, err
+		}
+		pageOffset = pageCursor.PageOffset
+	}
+
 	// Build find options
 	findOptions := options.Find().
 		SetSort(sort).
@@ -155,7 +166,7 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
 
 	// Handle cursor-based pagination
 	if pagination.Cursor != "" {
-		cursorFilter, err := r.buildCursorFilter(pagination.Cursor, pagination.SortField, pagination.SortOrder)
+		cursorFilter, err := r.buildCursorFilter(pagination.Cursor, pagination.SortField, pagination.SortOrder, filtersHash)
 		if err != nil {
 			return nil, err
 		}
@@ -209,12 +220,13 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
 	// Generate next cursor
 	var nextCursor string
 	if hasMore && len(items) > 0 {
-		nextCursor, err = r.generateCursor(items[len(items)-1], pagination.SortField, pagination.SortOrder)
+		nextPageOffset := pageOffset + pagination.Limit
+		nextCursor, err = r.generateCursor(items[len(items)-1], pagination.SortField, pagination.SortOrder, filtersHash, nextPageOffset)
 		if err != nil {
 			r.logger.Error("Failed to generate next cursor", map[string]interface{}{
 				"error": err.Error(),
 			})
-			// Don't fail the entire query, just leave cursor empty
+			return nil, fmt.Errorf("failed to generate pagination cursor: %w", err)
 		}
 	}
 
@@ -223,6 +235,7 @@ func (r *repository[T]) FindAll(ctx context.Context, filter bson.M, pagination P
 		TotalCount: totalCount,
 		NextCursor: nextCursor,
 		HasMore:    hasMore,
+		PageOffset: pageOffset,
 	}, nil
 }
 
@@ -454,7 +467,7 @@ func (r *repository[T]) validatePagination(pagination Pagination) error {
 }
 
 // buildCursorFilter builds a filter for cursor-based pagination.
-func (r *repository[T]) buildCursorFilter(cursor, sortField string, sortOrder int) (bson.M, error) {
+func (r *repository[T]) buildCursorFilter(cursor, sortField string, sortOrder int, filtersHash string) (bson.M, error) {
 	pageCursor, err := decodePageCursor(cursor)
 	if err != nil {
 		return nil, err
@@ -462,11 +475,14 @@ func (r *repository[T]) buildCursorFilter(cursor, sortField string, sortOrder in
 	if !pageCursor.matches(sortField, sortOrder) {
 		return nil, fmt.Errorf("%w: cursor does not match requested sort", ErrInvalidCursor)
 	}
+	if pageCursor.FiltersHash != filtersHash {
+		return nil, fmt.Errorf("%w: cursor does not match current filters", ErrInvalidCursor)
+	}
 	return pageCursor.mongoFilter()
 }
 
 // generateCursor generates an opaque cursor from the last document in a page.
-func (r *repository[T]) generateCursor(doc T, sortField string, sortOrder int) (string, error) {
+func (r *repository[T]) generateCursor(doc T, sortField string, sortOrder int, filtersHash string, pageOffset int) (string, error) {
 	docBytes, err := bson.Marshal(doc)
 	if err != nil {
 		return "", fmt.Errorf("failed to marshal document: %w", err)
@@ -487,7 +503,7 @@ func (r *repository[T]) generateCursor(doc T, sortField string, sortOrder int) (
 		return "", err
 	}
 
-	return encodePageCursor(sortField, sortOrder, sortValue, idHex)
+	return encodePageCursor(sortField, sortOrder, sortValue, idHex, filtersHash, pageOffset)
 }
 
 // getModelID extracts the ID from a model

pkg/response/pagination.go

@@ -0,0 +1,105 @@
+package response
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+
+	"github.com/labstack/echo/v4"
+)
+
+const (
+	// DefaultListLimit is the default page size for admin list endpoints.
+	DefaultListLimit = 10
+	// MaxListLimit is the maximum allowed page size.
+	MaxListLimit = 100
+)
+
+var (
+	// ErrInvalidPagination is returned when pagination query parameters are invalid.
+	ErrInvalidPagination = errors.New("invalid pagination parameters")
+	// ErrPaginationConflict is returned when cursor and offset are both supplied.
+	ErrPaginationConflict = errors.New("cursor and offset pagination cannot be used together")
+)
+
+// NewPagination parses and validates hybrid pagination query parameters from the request.
+func NewPagination(c echo.Context) (*Pagination, error) {
+	limit := DefaultListLimit
+	if limitStr := c.QueryParam("limit"); limitStr != "" {
+		parsed, err := strconv.Atoi(limitStr)
+		if err != nil {
+			return nil, fmt.Errorf("%w: limit must be an integer", ErrInvalidPagination)
+		}
+		limit = parsed
+	}
+	if limit < 1 || limit > MaxListLimit {
+		return nil, fmt.Errorf("%w: limit must be between 1 and %d", ErrInvalidPagination, MaxListLimit)
+	}
+
+	cursor := c.QueryParam("cursor")
+	offsetParam := c.QueryParam("offset")
+	if cursor != "" && offsetParam != "" {
+		return nil, ErrPaginationConflict
+	}
+
+	offset := 0
+	if offsetParam != "" {
+		parsed, err := strconv.Atoi(offsetParam)
+		if err != nil {
+			return nil, fmt.Errorf("%w: offset must be an integer", ErrInvalidPagination)
+		}
+		if parsed < 0 {
+			return nil, fmt.Errorf("%w: offset cannot be negative", ErrInvalidPagination)
+		}
+		offset = parsed
+	}
+
+	sortBy := c.QueryParam("sort_by")
+	if sortBy == "" {
+		sortBy = "created_at"
+	}
+
+	sortOrder := c.QueryParam("sort_order")
+	if sortOrder == "" {
+		sortOrder = "desc"
+	}
+
+	return &Pagination{
+		Limit:     limit,
+		Offset:    offset,
+		Cursor:    cursor,
+		SortBy:    sortBy,
+		SortOrder: sortOrder,
+	}, nil
+}
+
+// ListMeta builds pagination metadata for a list page.
+// pageOffset is the zero-based index of the first item in this page (from offset or cursor).
+func (p *Pagination) ListMeta(total int64, nextCursor string, hasMore bool, pageOffset int) *Meta {
+	meta := &Meta{
+		Limit:   p.Limit,
+		Offset:  pageOffset,
+		HasMore: hasMore,
+	}
+	if hasMore && nextCursor != "" {
+		meta.NextCursor = nextCursor
+	}
+	if total >= 0 {
+		meta.Total = int(total)
+		if p.Limit > 0 {
+			meta.Page = pageOffset/p.Limit + 1
+			meta.Pages = int((total + int64(p.Limit) - 1) / int64(p.Limit))
+		}
+	}
+	return meta
+}
+
+// ListMetaFromPage builds metadata from a paginated repository result.
+func (p *Pagination) ListMetaFromPage(total int64, nextCursor string, hasMore bool, pageOffset int) *Meta {
+	return p.ListMeta(total, nextCursor, hasMore, pageOffset)
+}
+
+// IsPaginationError reports whether err is a client-facing pagination error.
+func IsPaginationError(err error) bool {
+	return errors.Is(err, ErrInvalidPagination) || errors.Is(err, ErrPaginationConflict)
+}

pkg/response/pagination_test.go

@@ -0,0 +1,47 @@
+package response
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/labstack/echo/v4"
+)
+
+func TestNewPaginationDefaults(t *testing.T) {
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodGet, "/?limit=10&offset=0&sort_by=created_at&sort_order=desc", nil)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	p, err := NewPagination(c)
+	if err != nil {
+		t.Fatalf("NewPagination: %v", err)
+	}
+	if p.Limit != 10 || p.Offset != 0 || p.Cursor != "" {
+		t.Fatalf("unexpected pagination: %+v", p)
+	}
+}
+
+func TestNewPaginationCursorConflict(t *testing.T) {
+	e := echo.New()
+	req := httptest.NewRequest(http.MethodGet, "/?limit=10&offset=5&cursor=abc", nil)
+	rec := httptest.NewRecorder()
+	c := e.NewContext(req, rec)
+
+	_, err := NewPagination(c)
+	if !IsPaginationError(err) {
+		t.Fatalf("expected pagination conflict, got %v", err)
+	}
+}
+
+func TestListMetaOmitsNextCursorOnLastPage(t *testing.T) {
+	p := &Pagination{Limit: 10, Offset: 0}
+	meta := p.ListMeta(15, "should-not-appear", false, 0)
+	if meta.HasMore || meta.NextCursor != "" {
+		t.Fatalf("expected no next cursor on last page: %+v", meta)
+	}
+	if meta.Page != 1 || meta.Pages != 2 || meta.Total != 15 {
+		t.Fatalf("unexpected page meta: %+v", meta)
+	}
+}

pkg/response/response.go

@@ -1,11 +1,13 @@
 package response
 
 import (
+	"errors"
 	"net/http"
-	"strconv"
 
 	"github.com/asaskevich/govalidator"
 	"github.com/labstack/echo/v4"
+
+	orm "tm/pkg/mongo"
 )
 
 // APIResponse represents a standard API response structure
@@ -26,17 +28,17 @@ type APIError struct {
 
 // Meta represents metadata for paginated responses
 type Meta struct {
-	Total      int    `json:"total,omitempty"`
+	Total      int    `json:"total"`
 	Limit      int    `json:"limit"`
 	Offset     int    `json:"offset"`
-	Page       int    `json:"page,omitempty"`
-	Pages      int    `json:"pages,omitempty"`
+	Page       int    `json:"page"`
+	Pages      int    `json:"pages"`
+	HasMore    bool   `json:"has_more"`
 	NextCursor string `json:"next_cursor,omitempty"`
-	HasMore    bool   `json:"has_more,omitempty"`
 }
 
 type Pagination struct {
-	Limit     int    `query:"limit" valid:"optional,range(1|100)" default:"20"`
+	Limit     int    `query:"limit" valid:"optional,range(1|100)" default:"10"`
 	Offset    int    `query:"offset" valid:"optional,range(0|1000000)" default:"0"`
 	Cursor    string `query:"cursor" valid:"optional"`
 	SortBy    string `query:"sort_by" valid:"optional"`
@@ -192,59 +194,16 @@ func Parse[T any](c echo.Context) (*T, error) {
 	return &form, nil
 }
 
-func NewPagination(c echo.Context) *Pagination {
-	limit, err := strconv.Atoi(c.QueryParam("limit"))
-	if err != nil {
-		limit = 20
-	}
-
-	offset, err := strconv.Atoi(c.QueryParam("offset"))
-	if err != nil {
-		offset = 0
-	}
-
-	sortBy := c.QueryParam("sort_by")
-	if sortBy == "" {
-		sortBy = "created_at"
-	}
-
-	sortOrder := c.QueryParam("sort_order")
-	if sortOrder == "" {
-		sortOrder = "desc"
-	}
-
-	cursor := c.QueryParam("cursor")
-
-	return &Pagination{
-		Limit:     limit,
-		Offset:    offset,
-		Cursor:    cursor,
-		SortBy:    sortBy,
-		SortOrder: sortOrder,
+// PaginationBadRequest writes a 400 response for invalid list pagination parameters.
+func PaginationBadRequest(c echo.Context, err error) error {
+	msg := "Invalid pagination parameters"
+	if errors.Is(err, orm.ErrInvalidCursor) {
+		msg = "Invalid pagination cursor"
 	}
+	return BadRequest(c, msg, err.Error())
 }
 
-// Response calculates offset-based pagination metadata.
-func (p *Pagination) Response(total int64) *Meta {
-	return p.ListMeta(total, "", false)
-}
-
-// ListMeta builds pagination metadata for list endpoints (offset and/or cursor).
-// When total is negative, total/page/pages are omitted (count was skipped).
-func (p *Pagination) ListMeta(total int64, nextCursor string, hasMore bool) *Meta {
-	meta := &Meta{
-		Limit:      p.Limit,
-		Offset:     p.Offset,
-		NextCursor: nextCursor,
-		HasMore:    hasMore,
-	}
-	if total >= 0 {
-		pages := (total + int64(p.Limit) - 1) / int64(p.Limit)
-		meta.Total = int(total)
-		if p.Limit > 0 {
-			meta.Page = p.Offset/p.Limit + 1
-		}
-		meta.Pages = int(pages)
-	}
-	return meta
+// IsListPaginationError reports client-facing pagination errors from list queries.
+func IsListPaginationError(err error) bool {
+	return IsPaginationError(err) || errors.Is(err, orm.ErrInvalidCursor) || errors.Is(err, orm.ErrInvalidPagination)
 }