diff --git a/internal/tender/handler.go b/internal/tender/handler.go index eca3083..e872be2 100644 --- a/internal/tender/handler.go +++ b/internal/tender/handler.go @@ -160,10 +160,11 @@ func (h *TenderHandler) Delete(c echo.Context) error { // @Param languages query []string false "Filter by languages (comma-separated)" // @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param notice_publication_id query string false "Filter by TED notice publication ID" -// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO" +// @Param documents_scraped query bool false "When true, only tenders with scraped documents (processing_metadata.documents_scraped)" // @Param limit query int false "Number of items per page (default: 20, max: 100)" // @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages." // @Param cursor query string false "Opaque cursor from previous page meta.next_cursor" +// @Param include_total query bool false "When true, include meta.total and meta.pages (slower on large collections)" // @Param sort_by query string false "Sort by field" // @Param sort_order query string false "Sort order (asc or desc)" // @Success 200 {object} response.APIResponse{data=map[string]interface{}} @@ -270,10 +271,11 @@ func (h *TenderHandler) AdminRecommendTenders(c echo.Context) error { // @Param languages query []string false "Filter by languages (comma-separated)" // @Param buyer_organization_id query string false "Filter by buyer organization ID" // @Param notice_publication_id query string false "Filter by TED notice publication ID" -// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO" +// @Param documents_scraped query bool false "When true, only tenders with scraped documents (processing_metadata.documents_scraped)" // @Param limit query int false "Number of items per page (default: 20, max: 100)" // @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages." // @Param cursor query string false "Opaque cursor from previous page meta.next_cursor" +// @Param include_total query bool false "When true, include meta.total and meta.pages (slower on large collections)" // @Param sort_by query string false "Sort by field" // @Param sort_order query string false "Sort order (asc or desc)" // @Success 200 {object} response.APIResponse{data=SearchResponse} diff --git a/internal/tender/repository.go b/internal/tender/repository.go index 1bedf77..308a009 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -64,10 +64,17 @@ func tenderSearchListProjection() bson.M { "modifications": 0, "source_file_url": 0, "source_file_name": 0, - // Not used by list response mapping; can be large per tender. + // Detail-only or bulky fields; list rows use title and summary metadata only. + "description": 0, + "ai_overall_summary": 0, + "procurement_lots": 0, + "awarded": 0, + "cancellation_reason": 0, + "suspension_reason": 0, "translations": 0, "organizations": 0, "review_organization": 0, + "processing_metadata.enrichment_data": 0, "processing_metadata.translated_data": 0, "processing_metadata.parsing_errors": 0, "processing_metadata.validation_errors": 0, @@ -726,6 +733,15 @@ func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, paginat sortOrder = pagination.SortOrder } + listOpts := orm.ListPaginationOptions{Projection: tenderSearchListProjection()} + if pagination.IncludeTotal { + if pagination.Cursor != "" { + listOpts.IncludeCount = true + } + } else { + listOpts.SkipCount = true + } + mongoPagination, err := orm.BuildListPagination( pagination.Limit, pagination.Offset, @@ -733,7 +749,7 @@ func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, paginat sort, sortOrder, filter, - orm.ListPaginationOptions{Projection: tenderSearchListProjection()}, + listOpts, ) if err != nil { return nil, err @@ -1370,11 +1386,7 @@ func (r *tenderRepository) buildSearchFilter(form *SearchForm) bson.M { } if form.DocumentsScraped { - folderIDs := dedupeNonEmptyStrings(form.ContractFolderIDsWithDocuments) - scrapedClause := bson.M{"contract_folder_id": bson.M{"$in": folderIDs}} - if len(folderIDs) == 0 { - scrapedClause = bson.M{"_id": bson.M{"$exists": false}} - } + scrapedClause := bson.M{"processing_metadata.documents_scraped": true} if len(filter) == 0 { return scrapedClause } diff --git a/internal/tender/repository_search_test.go b/internal/tender/repository_search_test.go new file mode 100644 index 0000000..2c46e1c --- /dev/null +++ b/internal/tender/repository_search_test.go @@ -0,0 +1,29 @@ +package tender + +import ( + "testing" +) + +func TestBuildSearchFilterDocumentsScrapedUsesMongoFlag(t *testing.T) { + repo := &tenderRepository{} + filter := repo.buildSearchFilter(&SearchForm{DocumentsScraped: true}) + + if filter["processing_metadata.documents_scraped"] != true { + t.Fatalf("expected mongo documents_scraped flag, got %#v", filter) + } +} + +func TestTenderSearchListProjectionExcludesHeavyFields(t *testing.T) { + projection := tenderSearchListProjection() + for _, field := range []string{ + "content_xml", + "description", + "ai_overall_summary", + "procurement_lots", + "document_summaries", + } { + if projection[field] != 0 { + t.Fatalf("expected %s to be excluded from list projection", field) + } + } +} diff --git a/internal/tender/service.go b/internal/tender/service.go index 0d4aa38..d97dbdf 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -924,15 +924,6 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination // Profile keywords are not used for tender search. - if form.DocumentsScraped { - if err := s.enrichDocumentsScrapedFilter(ctx, form); err != nil { - return nil, err - } - if len(form.ContractFolderIDsWithDocuments) == 0 { - return s.emptySearchResponse(pagination), nil - } - } - page, err := s.repository.Search(ctx, form, pagination) if err != nil { s.logger.Error("Failed to list tenders", map[string]interface{}{ diff --git a/pkg/response/pagination.go b/pkg/response/pagination.go index bf8b4f1..b728a1f 100644 --- a/pkg/response/pagination.go +++ b/pkg/response/pagination.go @@ -64,12 +64,15 @@ func NewPagination(c echo.Context) (*Pagination, error) { sortOrder = "desc" } + includeTotal := c.QueryParam("include_total") == "true" + return &Pagination{ - Limit: limit, - Offset: offset, - Cursor: cursor, - SortBy: sortBy, - SortOrder: sortOrder, + Limit: limit, + Offset: offset, + Cursor: cursor, + SortBy: sortBy, + SortOrder: sortOrder, + IncludeTotal: includeTotal, }, nil } diff --git a/pkg/response/pagination_test.go b/pkg/response/pagination_test.go index 1956ecb..832851c 100644 --- a/pkg/response/pagination_test.go +++ b/pkg/response/pagination_test.go @@ -18,11 +18,26 @@ func TestNewPaginationDefaults(t *testing.T) { if err != nil { t.Fatalf("NewPagination: %v", err) } - if p.Limit != 10 || p.Offset != 0 || p.Cursor != "" { + if p.Limit != 10 || p.Offset != 0 || p.Cursor != "" || p.IncludeTotal { t.Fatalf("unexpected pagination: %+v", p) } } +func TestNewPaginationIncludeTotal(t *testing.T) { + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/?include_total=true", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + + p, err := NewPagination(c) + if err != nil { + t.Fatalf("NewPagination: %v", err) + } + if !p.IncludeTotal { + t.Fatal("expected include_total=true") + } +} + func TestNewPaginationCursorConflict(t *testing.T) { e := echo.New() req := httptest.NewRequest(http.MethodGet, "/?limit=10&offset=5&cursor=abc", nil) diff --git a/pkg/response/response.go b/pkg/response/response.go index 02a2fef..399cb2b 100644 --- a/pkg/response/response.go +++ b/pkg/response/response.go @@ -38,11 +38,12 @@ type Meta struct { } type Pagination struct { - Limit int `query:"limit" valid:"optional,range(1|100)" default:"10"` - Offset int `query:"offset" valid:"optional,range(0|1000000)" default:"0"` - Cursor string `query:"cursor" valid:"optional"` - SortBy string `query:"sort_by" valid:"optional"` - SortOrder string `query:"sort_order" valid:"optional,in(asc|desc)"` + Limit int `query:"limit" valid:"optional,range(1|100)" default:"10"` + Offset int `query:"offset" valid:"optional,range(0|1000000)" default:"0"` + Cursor string `query:"cursor" valid:"optional"` + SortBy string `query:"sort_by" valid:"optional"` + SortOrder string `query:"sort_order" valid:"optional,in(asc|desc)"` + IncludeTotal bool `query:"include_total" valid:"optional"` } // Success returns a successful response