mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cursor pagination

Browse Source
This commit is contained in:
Mazyar
2026-05-16 15:51:18 +03:30
parent bcb345103f
commit 70f581da09
7 changed files with 405 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tender/handler.go
+19 -3
View File
@@ -157,7 +157,8 @@ func (h *TenderHandler) Delete(c echo.Context) error {
// @Param languages query []string false "Filter by languages (comma-separated)"
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
// @Param offset query int false "Number of items to skip (default: 0)"
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
// @Param sort_by query string false "Sort by field"
// @Param sort_order query string false "Sort order (asc or desc)"
// @Success 200 {object} response.APIResponse{data=map[string]interface{}}
@@ -179,6 +180,9 @@ func (h *TenderHandler) Search(c echo.Context) error {
// Call service
tenders, err := h.service.Search(c.Request().Context(), form, response.NewPagination(c))
if err != nil {
if isInvalidPaginationCursor(err) {
return response.ValidationError(c, "Invalid pagination cursor", err.Error())
}
return response.InternalServerError(c, "Failed to list tenders")
}
@@ -219,7 +223,8 @@ func (h *TenderHandler) Search(c echo.Context) error {
// @Param languages query []string false "Filter by languages (comma-separated)"
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
// @Param offset query int false "Number of items to skip (default: 0)"
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
// @Param sort_by query string false "Sort by field"
// @Param sort_order query string false "Sort order (asc or desc)"
// @Success 200 {object} response.APIResponse{data=SearchResponse}
@@ -256,6 +261,9 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error {
result, err := h.service.Search(c.Request().Context(), form, response.NewPagination(c))
if err != nil {
if isInvalidPaginationCursor(err) {
return response.ValidationError(c, "Invalid pagination cursor", err.Error())
}
return response.InternalServerError(c, "Failed to retrieve tenders")
}
@@ -292,7 +300,8 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error {
// @Param languages query []string false "Filter by languages (comma-separated)"
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
// @Param offset query int false "Number of items to skip (default: 0)"
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
// @Param sort_by query string false "Sort by field"
// @Param sort_order query string false "Sort order (asc or desc)"
// @Success 200 {object} response.APIResponse{data=SearchResponse}
@@ -328,6 +337,9 @@ func (h *TenderHandler) RecommendTenders(c echo.Context) error {
result, err := h.service.Recommend(c.Request().Context(), form, response.NewPagination(c))
if err != nil {
if isInvalidPaginationCursor(err) {
return response.ValidationError(c, "Invalid pagination cursor", err.Error())
}
return response.InternalServerError(c, "Failed to retrieve tenders")
}
@@ -651,3 +663,7 @@ func (h *TenderHandler) GetAllAISummaries(c echo.Context) error {
return response.Success(c, summaries, "AI summaries retrieved successfully")
}
func isInvalidPaginationCursor(err error) bool {
return errors.Is(err, orm.ErrInvalidCursor)
}
internal/tender/repository.go
+57 -11
View File
@@ -26,7 +26,7 @@ type TenderRepository interface {
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) ([]Tender, int64, error)
Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error)
GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
@@ -40,11 +40,27 @@ func collectionName() string {
return "tenders"
}
// deepOffsetSkipCountThreshold skips CountDocuments when offset pagination goes this deep.
const deepOffsetSkipCountThreshold = 1000
// SearchPageResult is the paginated outcome of a tender search/list query.
type SearchPageResult struct {
Items []Tender
TotalCount int64 // -1 when count was skipped
NextCursor string
HasMore bool
}
// tenderSearchListProjection excludes large fields not needed for list/search API payloads.
func tenderSearchListProjection() bson.M {
return bson.M{
"content_xml": 0,
"document_summaries": 0,
"content_xml": 0,
"document_summaries": 0,
"scraped_documents": 0,
"selection_criteria": 0,
"modifications": 0,
"source_file_url": 0,
"source_file_name": 0,
}
}
@@ -85,6 +101,26 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
*orm.NewIndex("tender_deadline_idx", bson.D{{Key: "tender_deadline", Value: 1}}),
*orm.NewIndex("estimated_value_idx", bson.D{{Key: "estimated_value", Value: -1}}),
*orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}),
// Stable keyset pagination for default admin list (created_at desc).
*orm.NewIndex("created_at_id_idx", bson.D{
{Key: "created_at", Value: -1},
{Key: "_id", Value: -1},
}),
*orm.NewIndex("status_created_at_id_idx", bson.D{
{Key: "status", Value: 1},
{Key: "created_at", Value: -1},
{Key: "_id", Value: -1},
}),
*orm.NewIndex("country_code_created_at_id_idx", bson.D{
{Key: "country_code", Value: 1},
{Key: "created_at", Value: -1},
{Key: "_id", Value: -1},
}),
*orm.NewIndex("notice_type_code_created_at_id_idx", bson.D{
{Key: "notice_type_code", Value: 1},
{Key: "created_at", Value: -1},
{Key: "_id", Value: -1},
}),
*orm.NewIndex("updated_at_idx", bson.D{{Key: "updated_at", Value: -1}}),
// Compound indexes for common queries
*orm.NewIndex("status_country_idx", bson.D{
@@ -321,10 +357,9 @@ func (r *tenderRepository) Delete(ctx context.Context, id string) error {
}
// Search searches tenders based on criteria
func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) ([]Tender, int64, error) {
func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) {
filter := r.buildSearchFilter(form)
// Build sort
sort := "created_at"
if pagination.SortBy != "" {
sort = pagination.SortBy
@@ -335,23 +370,34 @@ func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, paginat
sortOrder = pagination.SortOrder
}
// Build pagination
skipCount := pagination.Cursor != "" || pagination.Offset >= deepOffsetSkipCountThreshold
paginationBuilder := orm.NewPaginationBuilder().
Limit(pagination.Limit).
Skip(pagination.Offset).
SortBy(sort, sortOrder).
Projection(tenderSearchListProjection()).
Build()
SkipCount(skipCount)
result, err := r.ormRepo.FindAll(ctx, filter, paginationBuilder)
if pagination.Cursor != "" {
paginationBuilder.Cursor(pagination.Cursor)
} else {
paginationBuilder.Skip(pagination.Offset)
}
result, err := r.ormRepo.FindAll(ctx, filter, paginationBuilder.Build())
if err != nil {
r.logger.Error("Failed to search tenders", map[string]interface{}{
"error": err.Error(),
})
return nil, 0, err
return nil, err
}
return result.Items, result.TotalCount, nil
return &SearchPageResult{
Items: result.Items,
TotalCount: result.TotalCount,
NextCursor: result.NextCursor,
HasMore: result.HasMore,
}, nil
}
// GetExpiredTenders retrieves tenders that are expired
internal/tender/service.go
+17 -17
View File
@@ -613,7 +613,7 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
}
}
tenders, total, err := s.repository.Search(ctx, form, pagination)
page, err := s.repository.Search(ctx, form, pagination)
if err != nil {
s.logger.Error("Failed to list tenders", map[string]interface{}{
"error": err.Error(),
@@ -623,14 +623,14 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
// If no company ID provided, return tenders without match percentage
if form.CompanyID == nil || *form.CompanyID == "" {
tenderResponses := make([]TenderResponse, 0, len(tenders))
for _, tender := range tenders {
tenderResponses := make([]TenderResponse, 0, len(page.Items))
for _, tender := range page.Items {
tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language)))
}
return &SearchResponse{
Tenders: tenderResponses,
Metadata: pagination.Response(total),
Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore),
}, nil
}
@@ -642,14 +642,14 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
"error": err.Error(),
})
// Continue without match percentage if company not found
tenderResponses := make([]TenderResponse, 0, len(tenders))
for _, tender := range tenders {
tenderResponses := make([]TenderResponse, 0, len(page.Items))
for _, tender := range page.Items {
tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language)))
}
return &SearchResponse{
Tenders: tenderResponses,
Metadata: pagination.Response(total),
Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore),
},
nil
}
@@ -662,17 +662,17 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
s.logger.Info("Tenders sorted by match percentage", map[string]interface{}{
"company_id": *form.CompanyID,
"company_cpv_codes": len(companyCPVCodes),
"total_tenders": len(tenders),
"total_tenders": len(page.Items),
})
tenderResponses := make([]TenderResponse, 0, len(tenders))
for _, tender := range tenders {
tenderResponses := make([]TenderResponse, 0, len(page.Items))
for _, tender := range page.Items {
tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language)))
}
return &SearchResponse{
Tenders: tenderResponses,
Metadata: pagination.Response(total),
Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore),
},
nil
}
@@ -724,7 +724,7 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat
}
}
tenders, total, err := s.repository.Search(ctx, form, pagination)
page, err := s.repository.Search(ctx, form, pagination)
if err != nil {
s.logger.Error("Failed to list tenders", map[string]interface{}{
"error": err.Error(),
@@ -732,14 +732,14 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat
return nil, fmt.Errorf("failed to list tenders: %w", err)
}
tenderResponses := make([]TenderResponse, 0, len(tenders))
for _, tender := range tenders {
tenderResponses := make([]TenderResponse, 0, len(page.Items))
for _, tender := range page.Items {
tenderResponses = append(tenderResponses, *tender.ToResponseWithLanguage(s.pickResponseLanguage(form.Language)))
}
return &SearchResponse{
Tenders: tenderResponses,
Metadata: pagination.Response(total),
Metadata: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore),
}, nil
}
@@ -812,7 +812,7 @@ func (s *tenderService) CleanupOldData(ctx context.Context, olderThan time.Durat
Status: []string{string(TenderStatusExpired), string(TenderStatusCancelled), string(TenderStatusAwarded)},
}
oldTenders, _, err := s.repository.Search(ctx, form, pagination)
oldPage, err := s.repository.Search(ctx, form, pagination)
if err != nil {
s.logger.Error("Failed to find old tenders for cleanup", map[string]interface{}{
"error": err.Error(),
@@ -821,7 +821,7 @@ func (s *tenderService) CleanupOldData(ctx context.Context, olderThan time.Durat
}
cleanupCount := 0
for _, tender := range oldTenders {
for _, tender := range oldPage.Items {
if tender.CreatedAt < cutoffTime {
cleanupCount++
// Here you would delete the tender if implementing actual cleanup