Enhance tender search functionality with pagination and filtering improvements

- Updated the tender search handler to include a new `include_total` query parameter, allowing clients to request total count and pagination metadata for search results.
- Refactored the `tenderSearchListProjection` to exclude heavy fields from the list response, optimizing data retrieval.
- Modified the `buildSearchFilter` method to utilize the `processing_metadata.documents_scraped` flag for filtering, improving search accuracy.
- Added unit tests for the new pagination features and search filter logic, ensuring robust validation of the search functionality.

This update significantly enhances the tender search experience by providing more flexible pagination options and improving the efficiency of data handling.

internal/tender/handler.go

@@ -160,10 +160,11 @@ func (h *TenderHandler) Delete(c echo.Context) error {
 // @Param languages query []string false "Filter by languages (comma-separated)"
 // @Param buyer_organization_id query string false "Filter by buyer organization ID"
 // @Param notice_publication_id query string false "Filter by TED notice publication ID"
-// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO"
+// @Param documents_scraped query bool false "When true, only tenders with scraped documents (processing_metadata.documents_scraped)"
 // @Param limit query int false "Number of items per page (default: 20, max: 100)"
 // @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
 // @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
+// @Param include_total query bool false "When true, include meta.total and meta.pages (slower on large collections)"
 // @Param sort_by query string false "Sort by field"
 // @Param sort_order query string false "Sort order (asc or desc)"
 // @Success 200 {object} response.APIResponse{data=map[string]interface{}}
@@ -270,10 +271,11 @@ func (h *TenderHandler) AdminRecommendTenders(c echo.Context) error {
 // @Param languages query []string false "Filter by languages (comma-separated)"
 // @Param buyer_organization_id query string false "Filter by buyer organization ID"
 // @Param notice_publication_id query string false "Filter by TED notice publication ID"
-// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO"
+// @Param documents_scraped query bool false "When true, only tenders with scraped documents (processing_metadata.documents_scraped)"
 // @Param limit query int false "Number of items per page (default: 20, max: 100)"
 // @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
 // @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
+// @Param include_total query bool false "When true, include meta.total and meta.pages (slower on large collections)"
 // @Param sort_by query string false "Sort by field"
 // @Param sort_order query string false "Sort order (asc or desc)"
 // @Success 200 {object} response.APIResponse{data=SearchResponse}

internal/tender/repository.go

@@ -64,10 +64,17 @@ func tenderSearchListProjection() bson.M {
 		"modifications":      0,
 		"source_file_url":    0,
 		"source_file_name":   0,
-		// Not used by list response mapping; can be large per tender.
+		// Detail-only or bulky fields; list rows use title and summary metadata only.
+		"description":                           0,
+		"ai_overall_summary":                    0,
+		"procurement_lots":                      0,
+		"awarded":                               0,
+		"cancellation_reason":                   0,
+		"suspension_reason":                     0,
 		"translations":                          0,
 		"organizations":                         0,
 		"review_organization":                   0,
+		"processing_metadata.enrichment_data":   0,
 		"processing_metadata.translated_data":   0,
 		"processing_metadata.parsing_errors":    0,
 		"processing_metadata.validation_errors": 0,
@@ -726,6 +733,15 @@ func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, paginat
 		sortOrder = pagination.SortOrder
 	}
 
+	listOpts := orm.ListPaginationOptions{Projection: tenderSearchListProjection()}
+	if pagination.IncludeTotal {
+		if pagination.Cursor != "" {
+			listOpts.IncludeCount = true
+		}
+	} else {
+		listOpts.SkipCount = true
+	}
+
 	mongoPagination, err := orm.BuildListPagination(
 		pagination.Limit,
 		pagination.Offset,
@@ -733,7 +749,7 @@ func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, paginat
 		sort,
 		sortOrder,
 		filter,
-		orm.ListPaginationOptions{Projection: tenderSearchListProjection()},
+		listOpts,
 	)
 	if err != nil {
 		return nil, err
@@ -1370,11 +1386,7 @@ func (r *tenderRepository) buildSearchFilter(form *SearchForm) bson.M {
 	}
 
 	if form.DocumentsScraped {
-		folderIDs := dedupeNonEmptyStrings(form.ContractFolderIDsWithDocuments)
-		scrapedClause := bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}
-		if len(folderIDs) == 0 {
-			scrapedClause = bson.M{"_id": bson.M{"$exists": false}}
-		}
+		scrapedClause := bson.M{"processing_metadata.documents_scraped": true}
 		if len(filter) == 0 {
 			return scrapedClause
 		}

internal/tender/repository_search_test.go

@@ -0,0 +1,29 @@
+package tender
+
+import (
+	"testing"
+)
+
+func TestBuildSearchFilterDocumentsScrapedUsesMongoFlag(t *testing.T) {
+	repo := &tenderRepository{}
+	filter := repo.buildSearchFilter(&SearchForm{DocumentsScraped: true})
+
+	if filter["processing_metadata.documents_scraped"] != true {
+		t.Fatalf("expected mongo documents_scraped flag, got %#v", filter)
+	}
+}
+
+func TestTenderSearchListProjectionExcludesHeavyFields(t *testing.T) {
+	projection := tenderSearchListProjection()
+	for _, field := range []string{
+		"content_xml",
+		"description",
+		"ai_overall_summary",
+		"procurement_lots",
+		"document_summaries",
+	} {
+		if projection[field] != 0 {
+			t.Fatalf("expected %s to be excluded from list projection", field)
+		}
+	}
+}

internal/tender/service.go

@@ -924,15 +924,6 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
 
 	// Profile keywords are not used for tender search.
 
-	if form.DocumentsScraped {
-		if err := s.enrichDocumentsScrapedFilter(ctx, form); err != nil {
-			return nil, err
-		}
-		if len(form.ContractFolderIDsWithDocuments) == 0 {
-			return s.emptySearchResponse(pagination), nil
-		}
-	}
-
 	page, err := s.repository.Search(ctx, form, pagination)
 	if err != nil {
 		s.logger.Error("Failed to list tenders", map[string]interface{}{