From 843e1508dfad63b0e8a8538b6cc2219e35fbf0ce Mon Sep 17 00:00:00 2001 From: Mazyar Date: Fri, 10 Jul 2026 16:23:46 +0330 Subject: [PATCH] Enhance tender search functionality with pagination and filtering improvements - Updated the tender search handler to include a new `include_total` query parameter, allowing clients to request total count and pagination metadata for search results. - Refactored the `tenderSearchListProjection` to exclude heavy fields from the list response, optimizing data retrieval. - Modified the `buildSearchFilter` method to utilize the `processing_metadata.documents_scraped` flag for filtering, improving search accuracy. - Added unit tests for the new pagination features and search filter logic, ensuring robust validation of the search functionality. This update significantly enhances the tender search experience by providing more flexible pagination options and improving the efficiency of data handling. --- internal/tender/handler.go | 6 +++-- internal/tender/repository.go | 26 ++++++++++++++------ internal/tender/repository_search_test.go | 29 +++++++++++++++++++++++ internal/tender/service.go | 9 ------- pkg/response/pagination.go | 13 ++++++---- pkg/response/pagination_test.go | 17 ++++++++++++- pkg/response/response.go | 11 +++++---- 7 files changed, 82 insertions(+), 29 deletions(-) create mode 100644 internal/tender/repository_search_test.go diff --git a/internal/tender/handler.go b/internal/tender/handler.go index eca3083..e872be2 100644 --- a/internal/tender/handler.go +++ b/internal/tender/handler.go @@ -160,10 +160,11 @@ func (h *TenderHandler) Delete(c echo.Context) error { // @Param languages query []string false "Filter by languages (comma-separated)" // @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param notice_publication_id query string false "Filter by TED notice publication ID" -// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO" +// @Param documents_scraped query bool false "When true, only tenders with scraped documents (processing_metadata.documents_scraped)" // @Param limit query int false "Number of items per page (default: 20, max: 100)" // @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages." // @Param cursor query string false "Opaque cursor from previous page meta.next_cursor" +// @Param include_total query bool false "When true, include meta.total and meta.pages (slower on large collections)" // @Param sort_by query string false "Sort by field" // @Param sort_order query string false "Sort order (asc or desc)" // @Success 200 {object} response.APIResponse{data=map[string]interface{}} @@ -270,10 +271,11 @@ func (h *TenderHandler) AdminRecommendTenders(c echo.Context) error { // @Param languages query []string false "Filter by languages (comma-separated)" // @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param notice_publication_id query string false "Filter by TED notice publication ID" -// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO" +// @Param documents_scraped query bool false "When true, only tenders with scraped documents (processing_metadata.documents_scraped)" // @Param limit query int false "Number of items per page (default: 20, max: 100)" // @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages." // @Param cursor query string false "Opaque cursor from previous page meta.next_cursor" +// @Param include_total query bool false "When true, include meta.total and meta.pages (slower on large collections)" // @Param sort_by query string false "Sort by field" // @Param sort_order query string false "Sort order (asc or desc)" // @Success 200 {object} response.APIResponse{data=SearchResponse} diff --git a/internal/tender/repository.go b/internal/tender/repository.go index 1bedf77..308a009 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -64,10 +64,17 @@ func tenderSearchListProjection() bson.M { "modifications": 0, "source_file_url": 0, "source_file_name": 0, - // Not used by list response mapping; can be large per tender. + // Detail-only or bulky fields; list rows use title and summary metadata only. + "description": 0, + "ai_overall_summary": 0, + "procurement_lots": 0, + "awarded": 0, + "cancellation_reason": 0, + "suspension_reason": 0, "translations": 0, "organizations": 0, "review_organization": 0, + "processing_metadata.enrichment_data": 0, "processing_metadata.translated_data": 0, "processing_metadata.parsing_errors": 0, "processing_metadata.validation_errors": 0, @@ -726,6 +733,15 @@ func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, paginat sortOrder = pagination.SortOrder } + listOpts := orm.ListPaginationOptions{Projection: tenderSearchListProjection()} + if pagination.IncludeTotal { + if pagination.Cursor != "" { + listOpts.IncludeCount = true + } + } else { + listOpts.SkipCount = true + } + mongoPagination, err := orm.BuildListPagination( pagination.Limit, pagination.Offset, @@ -733,7 +749,7 @@ func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, paginat sort, sortOrder, filter, - orm.ListPaginationOptions{Projection: tenderSearchListProjection()}, + listOpts, ) if err != nil { return nil, err @@ -1370,11 +1386,7 @@ func (r *tenderRepository) buildSearchFilter(form *SearchForm) bson.M { } if form.DocumentsScraped { - folderIDs := dedupeNonEmptyStrings(form.ContractFolderIDsWithDocuments) - scrapedClause := bson.M{"contract_folder_id": bson.M{"$in": folderIDs}} - if len(folderIDs) == 0 { - scrapedClause = bson.M{"_id": bson.M{"$exists": false}} - } + scrapedClause := bson.M{"processing_metadata.documents_scraped": true} if len(filter) == 0 { return scrapedClause } diff --git a/internal/tender/repository_search_test.go b/internal/tender/repository_search_test.go new file mode 100644 index 0000000..2c46e1c --- /dev/null +++ b/internal/tender/repository_search_test.go @@ -0,0 +1,29 @@ +package tender + +import ( + "testing" +) + +func TestBuildSearchFilterDocumentsScrapedUsesMongoFlag(t *testing.T) { + repo := &tenderRepository{} + filter := repo.buildSearchFilter(&SearchForm{DocumentsScraped: true}) + + if filter["processing_metadata.documents_scraped"] != true { + t.Fatalf("expected mongo documents_scraped flag, got %#v", filter) + } +} + +func TestTenderSearchListProjectionExcludesHeavyFields(t *testing.T) { + projection := tenderSearchListProjection() + for _, field := range []string{ + "content_xml", + "description", + "ai_overall_summary", + "procurement_lots", + "document_summaries", + } { + if projection[field] != 0 { + t.Fatalf("expected %s to be excluded from list projection", field) + } + } +} diff --git a/internal/tender/service.go b/internal/tender/service.go index 0d4aa38..d97dbdf 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -924,15 +924,6 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination // Profile keywords are not used for tender search. - if form.DocumentsScraped { - if err := s.enrichDocumentsScrapedFilter(ctx, form); err != nil { - return nil, err - } - if len(form.ContractFolderIDsWithDocuments) == 0 { - return s.emptySearchResponse(pagination), nil - } - } - page, err := s.repository.Search(ctx, form, pagination) if err != nil { s.logger.Error("Failed to list tenders", map[string]interface{}{ diff --git a/pkg/response/pagination.go b/pkg/response/pagination.go index bf8b4f1..b728a1f 100644 --- a/pkg/response/pagination.go +++ b/pkg/response/pagination.go @@ -64,12 +64,15 @@ func NewPagination(c echo.Context) (*Pagination, error) { sortOrder = "desc" } + includeTotal := c.QueryParam("include_total") == "true" + return &Pagination{ - Limit: limit, - Offset: offset, - Cursor: cursor, - SortBy: sortBy, - SortOrder: sortOrder, + Limit: limit, + Offset: offset, + Cursor: cursor, + SortBy: sortBy, + SortOrder: sortOrder, + IncludeTotal: includeTotal, }, nil } diff --git a/pkg/response/pagination_test.go b/pkg/response/pagination_test.go index 1956ecb..832851c 100644 --- a/pkg/response/pagination_test.go +++ b/pkg/response/pagination_test.go @@ -18,11 +18,26 @@ func TestNewPaginationDefaults(t *testing.T) { if err != nil { t.Fatalf("NewPagination: %v", err) } - if p.Limit != 10 || p.Offset != 0 || p.Cursor != "" { + if p.Limit != 10 || p.Offset != 0 || p.Cursor != "" || p.IncludeTotal { t.Fatalf("unexpected pagination: %+v", p) } } +func TestNewPaginationIncludeTotal(t *testing.T) { + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/?include_total=true", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + p, err := NewPagination(c) + if err != nil { + t.Fatalf("NewPagination: %v", err) + } + if !p.IncludeTotal { + t.Fatal("expected include_total=true") + } +} + func TestNewPaginationCursorConflict(t *testing.T) { e := echo.New() req := httptest.NewRequest(http.MethodGet, "/?limit=10&offset=5&cursor=abc", nil) diff --git a/pkg/response/response.go b/pkg/response/response.go index 02a2fef..399cb2b 100644 --- a/pkg/response/response.go +++ b/pkg/response/response.go @@ -38,11 +38,12 @@ type Meta struct { } type Pagination struct { - Limit int `query:"limit" valid:"optional,range(1|100)" default:"10"` - Offset int `query:"offset" valid:"optional,range(0|1000000)" default:"0"` - Cursor string `query:"cursor" valid:"optional"` - SortBy string `query:"sort_by" valid:"optional"` - SortOrder string `query:"sort_order" valid:"optional,in(asc|desc)"` + Limit int `query:"limit" valid:"optional,range(1|100)" default:"10"` + Offset int `query:"offset" valid:"optional,range(0|1000000)" default:"0"` + Cursor string `query:"cursor" valid:"optional"` + SortBy string `query:"sort_by" valid:"optional"` + SortOrder string `query:"sort_order" valid:"optional,in(asc|desc)"` + IncludeTotal bool `query:"include_total" valid:"optional"` } // Success returns a successful response