get pending tenders slow request fix

internal/document_scraper/form.go

@@ -11,78 +11,18 @@ type DocumentScraperGetRequest struct {
 	NoticeID string `param:"notice_id" valid:"required~Notice ID is required"`
 }
 
-// DocumentScraperBuyerResponse is buyer/contact and location data exposed to the document scraper.
+// DocumentScraperTenderResponse is the minimal payload for the document scraper pipeline.
-type DocumentScraperBuyerResponse struct {
-	BuyerName       string `json:"buyer_name"`
-	BuyerEmail      string `json:"buyer_email"`
-	BuyerCountry    string `json:"buyer_country"`
-	BuyerRegionNuts string `json:"buyer_region_nuts"`
-	BuyerCity       string `json:"buyer_city"`
-	BuyerProfileURL string `json:"buyer_profile_url"`
-}
-
-// DocumentScraperLotResponse is one procurement lot for the scraper task payload.
-type DocumentScraperLotResponse struct {
-	LotID                            string   `json:"lot_id,omitempty"`
-	LotTitle                         string   `json:"lot_title"`
-	LotDescription                   string   `json:"lot_description"`
-	LotMainNature                    string   `json:"lot_main_nature"`
-	LotMainClassificationCode        string   `json:"lot_main_classification_code"`
-	LotMainClassificationDescription string   `json:"lot_main_classification_description"`
-	LotMainClassificationDisplay     string   `json:"lot_main_classification_display,omitempty"`
-	AdditionalClassifications        []string `json:"additional_classifications,omitempty"`
-	LotDuration                      string   `json:"lot_duration"`
-	LotEstimatedValueExclVAT         float64  `json:"lot_estimated_value_excl_vat"`
-	LotEstimatedValueCurrency        string   `json:"lot_estimated_value_currency,omitempty"`
-}
-
-// DocumentScraperWinnerResponse is a contract winner / winning tenderer for result notices.
-type DocumentScraperWinnerResponse struct {
-	OfficialName           string `json:"official_name"`
-	RegistrationNumber     string `json:"registration_number"`
-	PostalAddress          string `json:"postal_address"`
-	Town                   string `json:"town"`
-	Postcode               string `json:"postcode"`
-	CountrySubdivisionNuts string `json:"country_subdivision_nuts"`
-	Country                string `json:"country"`
-	ContactPoint           string `json:"contact_point"`
-	Email                  string `json:"email"`
-	Telephone              string `json:"telephone"`
-	LotID                  string `json:"lot_id,omitempty"`
-}
-
-// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service.
 //
-// ContractFolderID is the TED procedure identifier (BT-04 / ContractFolderID),
+// ContractFolderID and NoticePublicationID are used to build MinIO paths:
-// included so the scraper can build the MinIO upload path for this notice:
 //
 //	PROC_<contract_folder_id>/<notice_publication_id>/documents/<file>
 //	PROC_<contract_folder_id>/<notice_publication_id>/tender.json
 type DocumentScraperTenderResponse struct {
-	ContractFolderID               string                          `json:"contract_folder_id"`
+	ContractFolderID    string `json:"contract_folder_id"`
-	NoticePublicationID            string                          `json:"notice_publication_id"`
+	NoticePublicationID string `json:"notice_publication_id"`
-	DocumentURL                    string                          `json:"document_url"`
+	DocumentURL         string `json:"document_url"`
-	Title                          string                          `json:"title"`
+	Title               string `json:"title"`
-	Description                    string                          `json:"description"`
+	Description         string `json:"description"`
-	NoticeIdentifier               string                          `json:"notice_identifier"`
-	NoticeVersion                  string                          `json:"notice_version"`
-	FormType                       string                          `json:"form_type"`
-	NoticeType                     string                          `json:"notice_type"`
-	NoticeSubtype                  string                          `json:"notice_subtype"`
-	PublicationDate                int64                           `json:"publication_date"`
-	NoticeDispatchDateWithTimezone string                          `json:"notice_dispatch_date_with_timezone"`
-	ProcurementTypeCode            string                          `json:"procurement_type_code"`
-	MainClassification             string                          `json:"main_classification"`
-	MainClassificationDescription  string                          `json:"main_classification_description"`
-	MainClassificationDisplay      string                          `json:"main_classification_display,omitempty"`
-	AdditionalClassifications      []string                        `json:"additional_classifications,omitempty"`
-	EstimatedValue                 float64                         `json:"estimated_value"`
-	EstimatedValueCurrency         string                          `json:"estimated_value_currency,omitempty"`
-	EstimatedValueVATBasis         string                          `json:"estimated_value_vat_basis"`
-	Status                         string                          `json:"status"`
-	Buyer                          DocumentScraperBuyerResponse    `json:"buyer"`
-	Lots                           []DocumentScraperLotResponse    `json:"lots"`
-	Winners                        []DocumentScraperWinnerResponse `json:"winners"`
 }
 
 // DocumentScraperListResponse represents the paginated list response

internal/document_scraper/handler.go

@@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler {
 
 // ListPendingTenders retrieves tenders that need document scraping
 // @Summary List pending tenders for document scraping
-// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes notice identifiers, dispatch time (RFC3339), procurement lots, buyer, winners (when present), CPV fields, and value metadata.
+// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description.
 // @Tags Document-Scraper
 // @Produce json
 // @Param limit query int false "Number of items per page (default: 20, max: 100)"
@@ -62,7 +62,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
 
 // GetTenderByNoticeID retrieves a specific tender by its notice publication ID
 // @Summary Get tender by notice ID for document scraping
-// @Description Retrieve a specific tender by its notice publication ID with full notice, lot, buyer, and winner fields for the document scraper pipeline.
+// @Description Retrieve a specific tender by its notice publication ID (contract_folder_id, notice_publication_id, document_url, title, description).
 // @Tags Document-Scraper
 // @Produce json
 // @Param notice_id path string true "Notice Publication ID"

internal/document_scraper/service.go

@@ -9,6 +9,8 @@ import (
 	"tm/pkg/response"
 )
 
+const documentScraperCountryCode = "SWE"
+
 // Service defines business logic for the document scraper API
 type Service interface {
 	// ListPendingTenders retrieves tenders that need document scraping
@@ -40,70 +42,35 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D
 
 	now := time.Now().Unix()
 
-	// Fetch a larger batch to account for filtering, then apply Sweden + active deadline filters
+	tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, documentScraperCountryCode, now, limit, offset)
-	// We need to over-fetch since the repository doesn't filter by country/deadline
+	if err != nil {
-	fetchLimit := limit * 5
+		s.logger.Error("Failed to list pending tenders", map[string]interface{}{
-	if fetchLimit > 500 {
+			"error": err.Error(),
-		fetchLimit = 500
+		})
+		return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
 	}
 
-	var filteredTenders []tender.Tender
-	currentOffset := offset
-	hasMore := true
-
-	for hasMore {
-		tenders, _, err := s.tenderRepo.GetUnScrapedTenders(ctx, fetchLimit, currentOffset)
-		if err != nil {
-			s.logger.Error("Failed to list pending tenders", map[string]interface{}{
-				"error": err.Error(),
-			})
-			return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
-		}
-
-		if len(tenders) == 0 {
-			hasMore = false
-			break
-		}
-
-		for i := range tenders {
-			if tenders[i].CountryCode == "SWE" && tenders[i].TenderDeadline > now {
-				filteredTenders = append(filteredTenders, tenders[i])
-			}
-			if len(filteredTenders) >= limit {
-				break
-			}
-		}
-
-		if len(filteredTenders) >= limit {
-			filteredTenders = filteredTenders[:limit]
-			hasMore = false
-		} else {
-			currentOffset += fetchLimit
-		}
-	}
-
-	// Convert to response
 	result := &DocumentScraperListResponse{
-		Tenders: make([]DocumentScraperTenderResponse, len(filteredTenders)),
+		Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
 	}
 
-	for i := range filteredTenders {
+	for i := range tenders {
-		result.Tenders[i] = *BuildDocumentScraperTenderResponse(&filteredTenders[i])
+		result.Tenders[i] = *BuildDocumentScraperTenderResponse(&tenders[i])
 	}
 
-	// Build pagination metadata
 	meta := &response.Meta{
-		Total:  len(filteredTenders),
+		Total:   len(tenders),
-		Limit:  limit,
+		Limit:   limit,
-		Offset: offset,
+		Offset:  offset,
+		HasMore: hasMore,
 	}
 	if meta.Limit > 0 {
 		meta.Page = (offset / meta.Limit) + 1
-		meta.Pages = 1
 	}
 
 	s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
-		"count": len(filteredTenders),
+		"count":    len(tenders),
+		"has_more": hasMore,
 	})
 
 	return result, meta, nil
@@ -129,9 +96,8 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do
 		return nil, fmt.Errorf("tender not found")
 	}
 
-	// Only return tenders from Sweden with active deadlines
 	now := time.Now().Unix()
-	if t.CountryCode != "SWE" || t.TenderDeadline <= now {
+	if t.CountryCode != documentScraperCountryCode || t.TenderDeadline <= now {
 		s.logger.Info("Tender filtered out: not Sweden or deadline passed", map[string]interface{}{
 			"notice_id":       noticeID,
 			"country_code":    t.CountryCode,

internal/document_scraper/tender_response.go

@@ -2,23 +2,14 @@ package document_scraper
 
 import (
 	"strings"
-	"time"
 
 	"tm/internal/tender"
 )
 
-const (
-	enrichmentMainCPVDescription = "main_classification_description"
-	vatBasisUnspecified          = "unspecified"
-)
-
 // BuildDocumentScraperTenderResponse maps a stored tender to the scraper API contract.
 func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTenderResponse {
 	if t == nil {
-		return &DocumentScraperTenderResponse{
+		return &DocumentScraperTenderResponse{}
-			Lots:    []DocumentScraperLotResponse{},
-			Winners: []DocumentScraperWinnerResponse{},
-		}
 	}
 
 	documentURL := strings.TrimSpace(t.DocumentURI)
@@ -26,208 +17,11 @@ func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTender
 		documentURL = strings.TrimSpace(t.TenderURL)
 	}
 
-	mainDesc := firstNonEmpty(
+	return &DocumentScraperTenderResponse{
-		strings.TrimSpace(t.ProcessingMetadata.EnrichmentData[enrichmentMainCPVDescription]),
+		ContractFolderID:    t.ContractFolderID,
-		strings.TrimSpace(t.MainClassificationDescription),
+		NoticePublicationID: t.NoticePublicationID,
-	)
+		DocumentURL:         documentURL,
-
+		Title:               t.Title,
-	resp := &DocumentScraperTenderResponse{
+		Description:         t.Description,
-		ContractFolderID:               t.ContractFolderID,
-		NoticePublicationID:            t.NoticePublicationID,
-		DocumentURL:                    documentURL,
-		Title:                          t.Title,
-		Description:                    t.Description,
-		NoticeIdentifier:               strings.TrimSpace(t.NoticeIdentifier),
-		NoticeVersion:                  strings.TrimSpace(t.NoticeVersion),
-		FormType:                       strings.TrimSpace(t.FormType),
-		NoticeType:                     strings.TrimSpace(t.NoticeTypeCode),
-		NoticeSubtype:                  strings.TrimSpace(t.NoticeSubTypeCode),
-		PublicationDate:                t.PublicationDate,
-		NoticeDispatchDateWithTimezone: formatNoticeDispatchRFC3339(t),
-		ProcurementTypeCode:            strings.TrimSpace(t.ProcurementTypeCode),
-		MainClassification:             strings.TrimSpace(t.MainClassification),
-		MainClassificationDescription:  mainDesc,
-		MainClassificationDisplay:      classificationDisplay(strings.TrimSpace(t.MainClassification), mainDesc),
-		AdditionalClassifications:      append([]string(nil), t.AdditionalClassifications...),
-		EstimatedValue:                 t.EstimatedValue,
-		EstimatedValueCurrency:         strings.TrimSpace(t.Currency),
-		EstimatedValueVATBasis:         vatBasisUnspecified,
-		Status:                         string(t.Status),
-		Buyer:                          buildBuyer(t),
-		Lots:                           buildLots(t, mainDesc),
-		Winners:                        buildWinners(t),
-	}
-
-	return resp
-}
-
-func formatNoticeDispatchRFC3339(t *tender.Tender) string {
-	sec := t.NoticeDispatchAt
-	if sec == 0 {
-		sec = t.ESenderDispatchAt
-	}
-	if sec == 0 {
-		return ""
-	}
-	return time.Unix(sec, 0).UTC().Format(time.RFC3339)
-}
-
-func buildBuyer(t *tender.Tender) DocumentScraperBuyerResponse {
-	if t.BuyerOrganization == nil {
-		return DocumentScraperBuyerResponse{}
-	}
-	b := t.BuyerOrganization
-	return DocumentScraperBuyerResponse{
-		BuyerName:       strings.TrimSpace(b.Name),
-		BuyerEmail:      strings.TrimSpace(b.ContactEmail),
-		BuyerCountry:    strings.TrimSpace(b.Address.CountryCode),
-		BuyerRegionNuts: strings.TrimSpace(b.Address.CountrySubentityCode),
-		BuyerCity:       strings.TrimSpace(b.Address.CityName),
-		BuyerProfileURL: strings.TrimSpace(firstNonEmpty(t.BuyerProfileURL, b.WebsiteURI)),
-	}
-}
-
-func firstNonEmpty(values ...string) string {
-	for _, v := range values {
-		if s := strings.TrimSpace(v); s != "" {
-			return s
-		}
-	}
-	return ""
-}
-
-func classificationDisplay(code, description string) string {
-	code = strings.TrimSpace(code)
-	description = strings.TrimSpace(description)
-	if code != "" && description != "" {
-		return code + " - " + description
-	}
-	if code != "" {
-		return code
-	}
-	return description
-}
-
-func buildLots(t *tender.Tender, globalMainDesc string) []DocumentScraperLotResponse {
-	if len(t.ProcurementLots) == 0 {
-		return []DocumentScraperLotResponse{}
-	}
-	out := make([]DocumentScraperLotResponse, 0, len(t.ProcurementLots))
-	for _, lot := range t.ProcurementLots {
-		lotCode := strings.TrimSpace(lot.MainClassification)
-		perLotDesc := strings.TrimSpace(lot.MainClassificationDescription)
-		if perLotDesc == "" && lotCode != "" && lotCode == strings.TrimSpace(t.MainClassification) {
-			perLotDesc = globalMainDesc
-		}
-		out = append(out, DocumentScraperLotResponse{
-			LotID:                            strings.TrimSpace(lot.LotID),
-			LotTitle:                         strings.TrimSpace(lot.Title),
-			LotDescription:                   strings.TrimSpace(lot.Description),
-			LotMainNature:                    strings.TrimSpace(lot.MainNatureOfContract),
-			LotMainClassificationCode:        lotCode,
-			LotMainClassificationDescription: perLotDesc,
-			LotMainClassificationDisplay:     classificationDisplay(lotCode, perLotDesc),
-			AdditionalClassifications:        append([]string(nil), lot.AdditionalClassifications...),
-			LotDuration:                      strings.TrimSpace(lot.Duration),
-			LotEstimatedValueExclVAT:         lot.EstimatedValue,
-			LotEstimatedValueCurrency:        strings.TrimSpace(lot.Currency),
-		})
-	}
-	return out
-}
-
-func buildWinners(t *tender.Tender) []DocumentScraperWinnerResponse {
-	if len(t.AwardedEntities) > 0 {
-		out := make([]DocumentScraperWinnerResponse, 0, len(t.AwardedEntities))
-		for _, a := range t.AwardedEntities {
-			w := DocumentScraperWinnerResponse{
-				OfficialName:           strings.TrimSpace(a.Name),
-				RegistrationNumber:     strings.TrimSpace(a.CompanyID),
-				PostalAddress:          strings.TrimSpace(a.Address),
-				Country:                strings.TrimSpace(a.Country),
-				LotID:                  strings.TrimSpace(a.LotID),
-				Town:                   "",
-				Postcode:               "",
-				CountrySubdivisionNuts: "",
-				ContactPoint:           "",
-				Email:                  "",
-				Telephone:              "",
-			}
-			if org := orgByID(t, a.OrganizationID); org != nil {
-				mergeWinnerFromOrganization(&w, org)
-			}
-			out = append(out, w)
-		}
-		return out
-	}
-
-	if t.WinningTenderer == nil {
-		return []DocumentScraperWinnerResponse{}
-	}
-	w := t.WinningTenderer
-	if strings.TrimSpace(w.Name) == "" {
-		return []DocumentScraperWinnerResponse{}
-	}
-	addr := w.Address
-	return []DocumentScraperWinnerResponse{{
-		OfficialName:           strings.TrimSpace(w.Name),
-		RegistrationNumber:     strings.TrimSpace(w.CompanyID),
-		PostalAddress:          strings.TrimSpace(addr.StreetName),
-		Town:                   strings.TrimSpace(addr.CityName),
-		Postcode:               strings.TrimSpace(addr.PostalZone),
-		CountrySubdivisionNuts: strings.TrimSpace(addr.CountrySubentityCode),
-		Country:                strings.TrimSpace(addr.CountryCode),
-		ContactPoint:           strings.TrimSpace(w.ContactName),
-		Email:                  strings.TrimSpace(w.ContactEmail),
-		Telephone:              strings.TrimSpace(w.ContactTelephone),
-	}}
-}
-
-func orgByID(t *tender.Tender, id string) *tender.Organization {
-	id = strings.TrimSpace(id)
-	if id == "" || t == nil {
-		return nil
-	}
-	for i := range t.Organizations {
-		if strings.TrimSpace(t.Organizations[i].ID) == id {
-			return &t.Organizations[i]
-		}
-	}
-	return nil
-}
-
-func mergeWinnerFromOrganization(w *DocumentScraperWinnerResponse, org *tender.Organization) {
-	if w == nil || org == nil {
-		return
-	}
-	if w.OfficialName == "" {
-		w.OfficialName = strings.TrimSpace(org.Name)
-	}
-	if w.RegistrationNumber == "" {
-		w.RegistrationNumber = strings.TrimSpace(org.CompanyID)
-	}
-	if w.PostalAddress == "" {
-		w.PostalAddress = strings.TrimSpace(org.Address.StreetName)
-	}
-	if w.Town == "" {
-		w.Town = strings.TrimSpace(org.Address.CityName)
-	}
-	if w.Postcode == "" {
-		w.Postcode = strings.TrimSpace(org.Address.PostalZone)
-	}
-	if w.CountrySubdivisionNuts == "" {
-		w.CountrySubdivisionNuts = strings.TrimSpace(org.Address.CountrySubentityCode)
-	}
-	if w.Country == "" {
-		w.Country = strings.TrimSpace(org.Address.CountryCode)
-	}
-	if w.ContactPoint == "" {
-		w.ContactPoint = strings.TrimSpace(org.ContactName)
-	}
-	if w.Email == "" {
-		w.Email = strings.TrimSpace(org.ContactEmail)
-	}
-	if w.Telephone == "" {
-		w.Telephone = strings.TrimSpace(org.ContactTelephone)
 	}
 }

internal/tender/repository.go

@@ -29,6 +29,7 @@ type TenderRepository interface {
 	Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error)
 	GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
 	GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
+	GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error)
 	GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
 	GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error)
 	Update(ctx context.Context, tender *Tender) error
@@ -53,13 +54,25 @@ type SearchPageResult struct {
 // tenderSearchListProjection excludes large fields not needed for list/search API payloads.
 func tenderSearchListProjection() bson.M {
 	return bson.M{
-		"content_xml":         0,
+		"content_xml":        0,
-		"document_summaries":  0,
+		"document_summaries": 0,
-		"scraped_documents":   0,
+		"scraped_documents":  0,
-		"selection_criteria":  0,
+		"selection_criteria": 0,
-		"modifications":       0,
+		"modifications":      0,
-		"source_file_url":     0,
+		"source_file_url":    0,
-		"source_file_name":    0,
+		"source_file_name":   0,
+	}
+}
+
+// documentScraperListProjection loads only fields required by the document scraper API.
+func documentScraperListProjection() bson.M {
+	return bson.M{
+		"contract_folder_id":    1,
+		"notice_publication_id": 1,
+		"document_uri":          1,
+		"tender_url":            1,
+		"title":                 1,
+		"description":           1,
 	}
 }
 
@@ -130,6 +143,12 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
 			{Key: "notice_type_code", Value: 1},
 			{Key: "status", Value: 1},
 		}),
+		*orm.NewIndex("doc_scrape_pending_idx", bson.D{
+			{Key: "country_code", Value: 1},
+			{Key: "processing_metadata.documents_scraped", Value: 1},
+			{Key: "tender_deadline", Value: 1},
+			{Key: "created_at", Value: 1},
+		}),
 		// Index for CPV matching queries
 		*orm.NewIndex("cpv_matching_idx", bson.D{
 			{Key: "status", Value: 1},
@@ -461,6 +480,40 @@ func (r *tenderRepository) GetUnScrapedTenders(ctx context.Context, limit, skip
 	return result.Items, result.TotalCount, nil
 }
 
+// GetPendingDocumentScrapeTenders returns unscraped tenders for one country with an active deadline.
+func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error) {
+	filter := bson.M{
+		"country_code": countryCode,
+		"tender_deadline": bson.M{
+			"$gt": minDeadline,
+		},
+		"processing_metadata.documents_scraped": bson.M{"$ne": true},
+	}
+
+	pagination := orm.Pagination{
+		Limit:      limit,
+		Skip:       skip,
+		SortField:  "created_at",
+		SortOrder:  1,
+		SkipCount:  true,
+		Projection: documentScraperListProjection(),
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{
+			"country_code": countryCode,
+			"min_deadline": minDeadline,
+			"limit":        limit,
+			"skip":         skip,
+			"error":        err.Error(),
+		})
+		return nil, false, err
+	}
+
+	return result.Items, result.HasMore, nil
+}
+
 // GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized
 func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
 	filter := bson.M{