get pending tenders slow request fix

internal/tender/repository.go

@@ -29,6 +29,7 @@ type TenderRepository interface {
 	Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error)
 	GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
 	GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
+	GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error)
 	GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
 	GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error)
 	Update(ctx context.Context, tender *Tender) error
@@ -53,13 +54,25 @@ type SearchPageResult struct {
 // tenderSearchListProjection excludes large fields not needed for list/search API payloads.
 func tenderSearchListProjection() bson.M {
 	return bson.M{
-		"content_xml":         0,
-		"document_summaries":  0,
-		"scraped_documents":   0,
-		"selection_criteria":  0,
-		"modifications":       0,
-		"source_file_url":     0,
-		"source_file_name":    0,
+		"content_xml":        0,
+		"document_summaries": 0,
+		"scraped_documents":  0,
+		"selection_criteria": 0,
+		"modifications":      0,
+		"source_file_url":    0,
+		"source_file_name":   0,
+	}
+}
+
+// documentScraperListProjection loads only fields required by the document scraper API.
+func documentScraperListProjection() bson.M {
+	return bson.M{
+		"contract_folder_id":    1,
+		"notice_publication_id": 1,
+		"document_uri":          1,
+		"tender_url":            1,
+		"title":                 1,
+		"description":           1,
 	}
 }
 
@@ -130,6 +143,12 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
 			{Key: "notice_type_code", Value: 1},
 			{Key: "status", Value: 1},
 		}),
+		*orm.NewIndex("doc_scrape_pending_idx", bson.D{
+			{Key: "country_code", Value: 1},
+			{Key: "processing_metadata.documents_scraped", Value: 1},
+			{Key: "tender_deadline", Value: 1},
+			{Key: "created_at", Value: 1},
+		}),
 		// Index for CPV matching queries
 		*orm.NewIndex("cpv_matching_idx", bson.D{
 			{Key: "status", Value: 1},
@@ -461,6 +480,40 @@ func (r *tenderRepository) GetUnScrapedTenders(ctx context.Context, limit, skip
 	return result.Items, result.TotalCount, nil
 }
 
+// GetPendingDocumentScrapeTenders returns unscraped tenders for one country with an active deadline.
+func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error) {
+	filter := bson.M{
+		"country_code": countryCode,
+		"tender_deadline": bson.M{
+			"$gt": minDeadline,
+		},
+		"processing_metadata.documents_scraped": bson.M{"$ne": true},
+	}
+
+	pagination := orm.Pagination{
+		Limit:      limit,
+		Skip:       skip,
+		SortField:  "created_at",
+		SortOrder:  1,
+		SkipCount:  true,
+		Projection: documentScraperListProjection(),
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{
+			"country_code": countryCode,
+			"min_deadline": minDeadline,
+			"limit":        limit,
+			"skip":         skip,
+			"error":        err.Error(),
+		})
+		return nil, false, err
+	}
+
+	return result.Items, result.HasMore, nil
+}
+
 // GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized
 func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
 	filter := bson.M{