diff --git a/internal/document_scraper/form.go b/internal/document_scraper/form.go index 65c91bc..8c0aad6 100644 --- a/internal/document_scraper/form.go +++ b/internal/document_scraper/form.go @@ -11,78 +11,18 @@ type DocumentScraperGetRequest struct { NoticeID string `param:"notice_id" valid:"required~Notice ID is required"` } -// DocumentScraperBuyerResponse is buyer/contact and location data exposed to the document scraper. -type DocumentScraperBuyerResponse struct { - BuyerName string `json:"buyer_name"` - BuyerEmail string `json:"buyer_email"` - BuyerCountry string `json:"buyer_country"` - BuyerRegionNuts string `json:"buyer_region_nuts"` - BuyerCity string `json:"buyer_city"` - BuyerProfileURL string `json:"buyer_profile_url"` -} - -// DocumentScraperLotResponse is one procurement lot for the scraper task payload. -type DocumentScraperLotResponse struct { - LotID string `json:"lot_id,omitempty"` - LotTitle string `json:"lot_title"` - LotDescription string `json:"lot_description"` - LotMainNature string `json:"lot_main_nature"` - LotMainClassificationCode string `json:"lot_main_classification_code"` - LotMainClassificationDescription string `json:"lot_main_classification_description"` - LotMainClassificationDisplay string `json:"lot_main_classification_display,omitempty"` - AdditionalClassifications []string `json:"additional_classifications,omitempty"` - LotDuration string `json:"lot_duration"` - LotEstimatedValueExclVAT float64 `json:"lot_estimated_value_excl_vat"` - LotEstimatedValueCurrency string `json:"lot_estimated_value_currency,omitempty"` -} - -// DocumentScraperWinnerResponse is a contract winner / winning tenderer for result notices. -type DocumentScraperWinnerResponse struct { - OfficialName string `json:"official_name"` - RegistrationNumber string `json:"registration_number"` - PostalAddress string `json:"postal_address"` - Town string `json:"town"` - Postcode string `json:"postcode"` - CountrySubdivisionNuts string `json:"country_subdivision_nuts"` - Country string `json:"country"` - ContactPoint string `json:"contact_point"` - Email string `json:"email"` - Telephone string `json:"telephone"` - LotID string `json:"lot_id,omitempty"` -} - -// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service. +// DocumentScraperTenderResponse is the minimal payload for the document scraper pipeline. // -// ContractFolderID is the TED procedure identifier (BT-04 / ContractFolderID), -// included so the scraper can build the MinIO upload path for this notice: +// ContractFolderID and NoticePublicationID are used to build MinIO paths: // // PROC_//documents/ // PROC_//tender.json type DocumentScraperTenderResponse struct { - ContractFolderID string `json:"contract_folder_id"` - NoticePublicationID string `json:"notice_publication_id"` - DocumentURL string `json:"document_url"` - Title string `json:"title"` - Description string `json:"description"` - NoticeIdentifier string `json:"notice_identifier"` - NoticeVersion string `json:"notice_version"` - FormType string `json:"form_type"` - NoticeType string `json:"notice_type"` - NoticeSubtype string `json:"notice_subtype"` - PublicationDate int64 `json:"publication_date"` - NoticeDispatchDateWithTimezone string `json:"notice_dispatch_date_with_timezone"` - ProcurementTypeCode string `json:"procurement_type_code"` - MainClassification string `json:"main_classification"` - MainClassificationDescription string `json:"main_classification_description"` - MainClassificationDisplay string `json:"main_classification_display,omitempty"` - AdditionalClassifications []string `json:"additional_classifications,omitempty"` - EstimatedValue float64 `json:"estimated_value"` - EstimatedValueCurrency string `json:"estimated_value_currency,omitempty"` - EstimatedValueVATBasis string `json:"estimated_value_vat_basis"` - Status string `json:"status"` - Buyer DocumentScraperBuyerResponse `json:"buyer"` - Lots []DocumentScraperLotResponse `json:"lots"` - Winners []DocumentScraperWinnerResponse `json:"winners"` + ContractFolderID string `json:"contract_folder_id"` + NoticePublicationID string `json:"notice_publication_id"` + DocumentURL string `json:"document_url"` + Title string `json:"title"` + Description string `json:"description"` } // DocumentScraperListResponse represents the paginated list response diff --git a/internal/document_scraper/handler.go b/internal/document_scraper/handler.go index dd60ea1..c5fa551 100644 --- a/internal/document_scraper/handler.go +++ b/internal/document_scraper/handler.go @@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler { // ListPendingTenders retrieves tenders that need document scraping // @Summary List pending tenders for document scraping -// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes notice identifiers, dispatch time (RFC3339), procurement lots, buyer, winners (when present), CPV fields, and value metadata. +// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. // @Tags Document-Scraper // @Produce json // @Param limit query int false "Number of items per page (default: 20, max: 100)" @@ -62,7 +62,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error { // GetTenderByNoticeID retrieves a specific tender by its notice publication ID // @Summary Get tender by notice ID for document scraping -// @Description Retrieve a specific tender by its notice publication ID with full notice, lot, buyer, and winner fields for the document scraper pipeline. +// @Description Retrieve a specific tender by its notice publication ID (contract_folder_id, notice_publication_id, document_url, title, description). // @Tags Document-Scraper // @Produce json // @Param notice_id path string true "Notice Publication ID" diff --git a/internal/document_scraper/service.go b/internal/document_scraper/service.go index bed1383..4f13216 100644 --- a/internal/document_scraper/service.go +++ b/internal/document_scraper/service.go @@ -9,6 +9,8 @@ import ( "tm/pkg/response" ) +const documentScraperCountryCode = "SWE" + // Service defines business logic for the document scraper API type Service interface { // ListPendingTenders retrieves tenders that need document scraping @@ -40,70 +42,35 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D now := time.Now().Unix() - // Fetch a larger batch to account for filtering, then apply Sweden + active deadline filters - // We need to over-fetch since the repository doesn't filter by country/deadline - fetchLimit := limit * 5 - if fetchLimit > 500 { - fetchLimit = 500 + tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, documentScraperCountryCode, now, limit, offset) + if err != nil { + s.logger.Error("Failed to list pending tenders", map[string]interface{}{ + "error": err.Error(), + }) + return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err) } - var filteredTenders []tender.Tender - currentOffset := offset - hasMore := true - - for hasMore { - tenders, _, err := s.tenderRepo.GetUnScrapedTenders(ctx, fetchLimit, currentOffset) - if err != nil { - s.logger.Error("Failed to list pending tenders", map[string]interface{}{ - "error": err.Error(), - }) - return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err) - } - - if len(tenders) == 0 { - hasMore = false - break - } - - for i := range tenders { - if tenders[i].CountryCode == "SWE" && tenders[i].TenderDeadline > now { - filteredTenders = append(filteredTenders, tenders[i]) - } - if len(filteredTenders) >= limit { - break - } - } - - if len(filteredTenders) >= limit { - filteredTenders = filteredTenders[:limit] - hasMore = false - } else { - currentOffset += fetchLimit - } - } - - // Convert to response result := &DocumentScraperListResponse{ - Tenders: make([]DocumentScraperTenderResponse, len(filteredTenders)), + Tenders: make([]DocumentScraperTenderResponse, len(tenders)), } - for i := range filteredTenders { - result.Tenders[i] = *BuildDocumentScraperTenderResponse(&filteredTenders[i]) + for i := range tenders { + result.Tenders[i] = *BuildDocumentScraperTenderResponse(&tenders[i]) } - // Build pagination metadata meta := &response.Meta{ - Total: len(filteredTenders), - Limit: limit, - Offset: offset, + Total: len(tenders), + Limit: limit, + Offset: offset, + HasMore: hasMore, } if meta.Limit > 0 { meta.Page = (offset / meta.Limit) + 1 - meta.Pages = 1 } s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{ - "count": len(filteredTenders), + "count": len(tenders), + "has_more": hasMore, }) return result, meta, nil @@ -129,9 +96,8 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do return nil, fmt.Errorf("tender not found") } - // Only return tenders from Sweden with active deadlines now := time.Now().Unix() - if t.CountryCode != "SWE" || t.TenderDeadline <= now { + if t.CountryCode != documentScraperCountryCode || t.TenderDeadline <= now { s.logger.Info("Tender filtered out: not Sweden or deadline passed", map[string]interface{}{ "notice_id": noticeID, "country_code": t.CountryCode, diff --git a/internal/document_scraper/tender_response.go b/internal/document_scraper/tender_response.go index 5288609..c44e19d 100644 --- a/internal/document_scraper/tender_response.go +++ b/internal/document_scraper/tender_response.go @@ -2,23 +2,14 @@ package document_scraper import ( "strings" - "time" "tm/internal/tender" ) -const ( - enrichmentMainCPVDescription = "main_classification_description" - vatBasisUnspecified = "unspecified" -) - // BuildDocumentScraperTenderResponse maps a stored tender to the scraper API contract. func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTenderResponse { if t == nil { - return &DocumentScraperTenderResponse{ - Lots: []DocumentScraperLotResponse{}, - Winners: []DocumentScraperWinnerResponse{}, - } + return &DocumentScraperTenderResponse{} } documentURL := strings.TrimSpace(t.DocumentURI) @@ -26,208 +17,11 @@ func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTender documentURL = strings.TrimSpace(t.TenderURL) } - mainDesc := firstNonEmpty( - strings.TrimSpace(t.ProcessingMetadata.EnrichmentData[enrichmentMainCPVDescription]), - strings.TrimSpace(t.MainClassificationDescription), - ) - - resp := &DocumentScraperTenderResponse{ - ContractFolderID: t.ContractFolderID, - NoticePublicationID: t.NoticePublicationID, - DocumentURL: documentURL, - Title: t.Title, - Description: t.Description, - NoticeIdentifier: strings.TrimSpace(t.NoticeIdentifier), - NoticeVersion: strings.TrimSpace(t.NoticeVersion), - FormType: strings.TrimSpace(t.FormType), - NoticeType: strings.TrimSpace(t.NoticeTypeCode), - NoticeSubtype: strings.TrimSpace(t.NoticeSubTypeCode), - PublicationDate: t.PublicationDate, - NoticeDispatchDateWithTimezone: formatNoticeDispatchRFC3339(t), - ProcurementTypeCode: strings.TrimSpace(t.ProcurementTypeCode), - MainClassification: strings.TrimSpace(t.MainClassification), - MainClassificationDescription: mainDesc, - MainClassificationDisplay: classificationDisplay(strings.TrimSpace(t.MainClassification), mainDesc), - AdditionalClassifications: append([]string(nil), t.AdditionalClassifications...), - EstimatedValue: t.EstimatedValue, - EstimatedValueCurrency: strings.TrimSpace(t.Currency), - EstimatedValueVATBasis: vatBasisUnspecified, - Status: string(t.Status), - Buyer: buildBuyer(t), - Lots: buildLots(t, mainDesc), - Winners: buildWinners(t), - } - - return resp -} - -func formatNoticeDispatchRFC3339(t *tender.Tender) string { - sec := t.NoticeDispatchAt - if sec == 0 { - sec = t.ESenderDispatchAt - } - if sec == 0 { - return "" - } - return time.Unix(sec, 0).UTC().Format(time.RFC3339) -} - -func buildBuyer(t *tender.Tender) DocumentScraperBuyerResponse { - if t.BuyerOrganization == nil { - return DocumentScraperBuyerResponse{} - } - b := t.BuyerOrganization - return DocumentScraperBuyerResponse{ - BuyerName: strings.TrimSpace(b.Name), - BuyerEmail: strings.TrimSpace(b.ContactEmail), - BuyerCountry: strings.TrimSpace(b.Address.CountryCode), - BuyerRegionNuts: strings.TrimSpace(b.Address.CountrySubentityCode), - BuyerCity: strings.TrimSpace(b.Address.CityName), - BuyerProfileURL: strings.TrimSpace(firstNonEmpty(t.BuyerProfileURL, b.WebsiteURI)), - } -} - -func firstNonEmpty(values ...string) string { - for _, v := range values { - if s := strings.TrimSpace(v); s != "" { - return s - } - } - return "" -} - -func classificationDisplay(code, description string) string { - code = strings.TrimSpace(code) - description = strings.TrimSpace(description) - if code != "" && description != "" { - return code + " - " + description - } - if code != "" { - return code - } - return description -} - -func buildLots(t *tender.Tender, globalMainDesc string) []DocumentScraperLotResponse { - if len(t.ProcurementLots) == 0 { - return []DocumentScraperLotResponse{} - } - out := make([]DocumentScraperLotResponse, 0, len(t.ProcurementLots)) - for _, lot := range t.ProcurementLots { - lotCode := strings.TrimSpace(lot.MainClassification) - perLotDesc := strings.TrimSpace(lot.MainClassificationDescription) - if perLotDesc == "" && lotCode != "" && lotCode == strings.TrimSpace(t.MainClassification) { - perLotDesc = globalMainDesc - } - out = append(out, DocumentScraperLotResponse{ - LotID: strings.TrimSpace(lot.LotID), - LotTitle: strings.TrimSpace(lot.Title), - LotDescription: strings.TrimSpace(lot.Description), - LotMainNature: strings.TrimSpace(lot.MainNatureOfContract), - LotMainClassificationCode: lotCode, - LotMainClassificationDescription: perLotDesc, - LotMainClassificationDisplay: classificationDisplay(lotCode, perLotDesc), - AdditionalClassifications: append([]string(nil), lot.AdditionalClassifications...), - LotDuration: strings.TrimSpace(lot.Duration), - LotEstimatedValueExclVAT: lot.EstimatedValue, - LotEstimatedValueCurrency: strings.TrimSpace(lot.Currency), - }) - } - return out -} - -func buildWinners(t *tender.Tender) []DocumentScraperWinnerResponse { - if len(t.AwardedEntities) > 0 { - out := make([]DocumentScraperWinnerResponse, 0, len(t.AwardedEntities)) - for _, a := range t.AwardedEntities { - w := DocumentScraperWinnerResponse{ - OfficialName: strings.TrimSpace(a.Name), - RegistrationNumber: strings.TrimSpace(a.CompanyID), - PostalAddress: strings.TrimSpace(a.Address), - Country: strings.TrimSpace(a.Country), - LotID: strings.TrimSpace(a.LotID), - Town: "", - Postcode: "", - CountrySubdivisionNuts: "", - ContactPoint: "", - Email: "", - Telephone: "", - } - if org := orgByID(t, a.OrganizationID); org != nil { - mergeWinnerFromOrganization(&w, org) - } - out = append(out, w) - } - return out - } - - if t.WinningTenderer == nil { - return []DocumentScraperWinnerResponse{} - } - w := t.WinningTenderer - if strings.TrimSpace(w.Name) == "" { - return []DocumentScraperWinnerResponse{} - } - addr := w.Address - return []DocumentScraperWinnerResponse{{ - OfficialName: strings.TrimSpace(w.Name), - RegistrationNumber: strings.TrimSpace(w.CompanyID), - PostalAddress: strings.TrimSpace(addr.StreetName), - Town: strings.TrimSpace(addr.CityName), - Postcode: strings.TrimSpace(addr.PostalZone), - CountrySubdivisionNuts: strings.TrimSpace(addr.CountrySubentityCode), - Country: strings.TrimSpace(addr.CountryCode), - ContactPoint: strings.TrimSpace(w.ContactName), - Email: strings.TrimSpace(w.ContactEmail), - Telephone: strings.TrimSpace(w.ContactTelephone), - }} -} - -func orgByID(t *tender.Tender, id string) *tender.Organization { - id = strings.TrimSpace(id) - if id == "" || t == nil { - return nil - } - for i := range t.Organizations { - if strings.TrimSpace(t.Organizations[i].ID) == id { - return &t.Organizations[i] - } - } - return nil -} - -func mergeWinnerFromOrganization(w *DocumentScraperWinnerResponse, org *tender.Organization) { - if w == nil || org == nil { - return - } - if w.OfficialName == "" { - w.OfficialName = strings.TrimSpace(org.Name) - } - if w.RegistrationNumber == "" { - w.RegistrationNumber = strings.TrimSpace(org.CompanyID) - } - if w.PostalAddress == "" { - w.PostalAddress = strings.TrimSpace(org.Address.StreetName) - } - if w.Town == "" { - w.Town = strings.TrimSpace(org.Address.CityName) - } - if w.Postcode == "" { - w.Postcode = strings.TrimSpace(org.Address.PostalZone) - } - if w.CountrySubdivisionNuts == "" { - w.CountrySubdivisionNuts = strings.TrimSpace(org.Address.CountrySubentityCode) - } - if w.Country == "" { - w.Country = strings.TrimSpace(org.Address.CountryCode) - } - if w.ContactPoint == "" { - w.ContactPoint = strings.TrimSpace(org.ContactName) - } - if w.Email == "" { - w.Email = strings.TrimSpace(org.ContactEmail) - } - if w.Telephone == "" { - w.Telephone = strings.TrimSpace(org.ContactTelephone) + return &DocumentScraperTenderResponse{ + ContractFolderID: t.ContractFolderID, + NoticePublicationID: t.NoticePublicationID, + DocumentURL: documentURL, + Title: t.Title, + Description: t.Description, } } diff --git a/internal/tender/repository.go b/internal/tender/repository.go index 386f585..a76dd83 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -29,6 +29,7 @@ type TenderRepository interface { Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error) GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) + GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) Update(ctx context.Context, tender *Tender) error @@ -53,13 +54,25 @@ type SearchPageResult struct { // tenderSearchListProjection excludes large fields not needed for list/search API payloads. func tenderSearchListProjection() bson.M { return bson.M{ - "content_xml": 0, - "document_summaries": 0, - "scraped_documents": 0, - "selection_criteria": 0, - "modifications": 0, - "source_file_url": 0, - "source_file_name": 0, + "content_xml": 0, + "document_summaries": 0, + "scraped_documents": 0, + "selection_criteria": 0, + "modifications": 0, + "source_file_url": 0, + "source_file_name": 0, + } +} + +// documentScraperListProjection loads only fields required by the document scraper API. +func documentScraperListProjection() bson.M { + return bson.M{ + "contract_folder_id": 1, + "notice_publication_id": 1, + "document_uri": 1, + "tender_url": 1, + "title": 1, + "description": 1, } } @@ -130,6 +143,12 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te {Key: "notice_type_code", Value: 1}, {Key: "status", Value: 1}, }), + *orm.NewIndex("doc_scrape_pending_idx", bson.D{ + {Key: "country_code", Value: 1}, + {Key: "processing_metadata.documents_scraped", Value: 1}, + {Key: "tender_deadline", Value: 1}, + {Key: "created_at", Value: 1}, + }), // Index for CPV matching queries *orm.NewIndex("cpv_matching_idx", bson.D{ {Key: "status", Value: 1}, @@ -461,6 +480,40 @@ func (r *tenderRepository) GetUnScrapedTenders(ctx context.Context, limit, skip return result.Items, result.TotalCount, nil } +// GetPendingDocumentScrapeTenders returns unscraped tenders for one country with an active deadline. +func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error) { + filter := bson.M{ + "country_code": countryCode, + "tender_deadline": bson.M{ + "$gt": minDeadline, + }, + "processing_metadata.documents_scraped": bson.M{"$ne": true}, + } + + pagination := orm.Pagination{ + Limit: limit, + Skip: skip, + SortField: "created_at", + SortOrder: 1, + SkipCount: true, + Projection: documentScraperListProjection(), + } + + result, err := r.ormRepo.FindAll(ctx, filter, pagination) + if err != nil { + r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{ + "country_code": countryCode, + "min_deadline": minDeadline, + "limit": limit, + "skip": skip, + "error": err.Error(), + }) + return nil, false, err + } + + return result.Items, result.HasMore, nil +} + // GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) { filter := bson.M{