diff --git a/cmd/worker/workers/notice.go b/cmd/worker/workers/notice.go index 6846017..c8e6c32 100644 --- a/cmd/worker/workers/notice.go +++ b/cmd/worker/workers/notice.go @@ -445,15 +445,16 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { procurementLots := make([]tender.ProcurementLot, len(n.ProcurementLots)) for i, l := range n.ProcurementLots { procurementLots[i] = tender.ProcurementLot{ - LotID: l.LotID, - Title: l.Title, - Description: l.Description, - MainNatureOfContract: l.MainNatureOfContract, - MainClassification: l.MainClassification, - AdditionalClassifications: append([]string(nil), l.AdditionalClassifications...), - Duration: l.Duration, - EstimatedValue: l.EstimatedValue, - Currency: l.Currency, + LotID: l.LotID, + Title: l.Title, + Description: l.Description, + MainNatureOfContract: l.MainNatureOfContract, + MainClassification: l.MainClassification, + MainClassificationDescription: l.MainClassificationDescription, + AdditionalClassifications: append([]string(nil), l.AdditionalClassifications...), + Duration: l.Duration, + EstimatedValue: l.EstimatedValue, + Currency: l.Currency, } } @@ -528,6 +529,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { t.BuyerProfileURL = n.BuyerProfileURL t.TenderURL = n.TenderURL t.MainClassification = n.MainClassification + t.MainClassificationDescription = n.MainClassificationDescription t.AdditionalClassifications = n.AdditionalClassifications t.OfficialLanguages = n.OfficialLanguages t.Status = chooseTenderStatus(previousStatus, tender.TenderStatus(n.Status)) diff --git a/internal/document_scraper/form.go b/internal/document_scraper/form.go index fa23eb1..23ee630 100644 --- a/internal/document_scraper/form.go +++ b/internal/document_scraper/form.go @@ -11,6 +11,46 @@ type DocumentScraperGetRequest struct { NoticeID string `param:"notice_id" valid:"required~Notice ID is required"` } +// DocumentScraperBuyerResponse is buyer/contact and location data exposed to the document scraper. +type DocumentScraperBuyerResponse struct { + BuyerName string `json:"buyer_name"` + BuyerEmail string `json:"buyer_email"` + BuyerCountry string `json:"buyer_country"` + BuyerRegionNuts string `json:"buyer_region_nuts"` + BuyerCity string `json:"buyer_city"` + BuyerProfileURL string `json:"buyer_profile_url"` +} + +// DocumentScraperLotResponse is one procurement lot for the scraper task payload. +type DocumentScraperLotResponse struct { + LotID string `json:"lot_id,omitempty"` + LotTitle string `json:"lot_title"` + LotDescription string `json:"lot_description"` + LotMainNature string `json:"lot_main_nature"` + LotMainClassificationCode string `json:"lot_main_classification_code"` + LotMainClassificationDescription string `json:"lot_main_classification_description"` + LotMainClassificationDisplay string `json:"lot_main_classification_display,omitempty"` + AdditionalClassifications []string `json:"additional_classifications,omitempty"` + LotDuration string `json:"lot_duration"` + LotEstimatedValueExclVAT float64 `json:"lot_estimated_value_excl_vat"` + LotEstimatedValueCurrency string `json:"lot_estimated_value_currency,omitempty"` +} + +// DocumentScraperWinnerResponse is a contract winner / winning tenderer for result notices. +type DocumentScraperWinnerResponse struct { + OfficialName string `json:"official_name"` + RegistrationNumber string `json:"registration_number"` + PostalAddress string `json:"postal_address"` + Town string `json:"town"` + Postcode string `json:"postcode"` + CountrySubdivisionNuts string `json:"country_subdivision_nuts"` + Country string `json:"country"` + ContactPoint string `json:"contact_point"` + Email string `json:"email"` + Telephone string `json:"telephone"` + LotID string `json:"lot_id,omitempty"` +} + // DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service. // // ContractFolderID is the TED procedure identifier (BT-04 / ContractFolderID), @@ -18,11 +58,30 @@ type DocumentScraperGetRequest struct { // // PROC_// type DocumentScraperTenderResponse struct { - ContractFolderID string `json:"contract_folder_id"` - NoticePublicationID string `json:"notice_publication_id"` - DocumentURL string `json:"document_url"` - Title string `json:"title"` - Description string `json:"description"` + ContractFolderID string `json:"contract_folder_id"` + NoticePublicationID string `json:"notice_publication_id"` + DocumentURL string `json:"document_url"` + Title string `json:"title"` + Description string `json:"description"` + NoticeIdentifier string `json:"notice_identifier"` + NoticeVersion string `json:"notice_version"` + FormType string `json:"form_type"` + NoticeType string `json:"notice_type"` + NoticeSubtype string `json:"notice_subtype"` + PublicationDate int64 `json:"publication_date"` + NoticeDispatchDateWithTimezone string `json:"notice_dispatch_date_with_timezone"` + ProcurementTypeCode string `json:"procurement_type_code"` + MainClassification string `json:"main_classification"` + MainClassificationDescription string `json:"main_classification_description"` + MainClassificationDisplay string `json:"main_classification_display,omitempty"` + AdditionalClassifications []string `json:"additional_classifications,omitempty"` + EstimatedValue float64 `json:"estimated_value"` + EstimatedValueCurrency string `json:"estimated_value_currency,omitempty"` + EstimatedValueVATBasis string `json:"estimated_value_vat_basis"` + Status string `json:"status"` + Buyer DocumentScraperBuyerResponse `json:"buyer"` + Lots []DocumentScraperLotResponse `json:"lots"` + Winners []DocumentScraperWinnerResponse `json:"winners"` } // DocumentScraperListResponse represents the paginated list response diff --git a/internal/document_scraper/handler.go b/internal/document_scraper/handler.go index d7630da..dd60ea1 100644 --- a/internal/document_scraper/handler.go +++ b/internal/document_scraper/handler.go @@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler { // ListPendingTenders retrieves tenders that need document scraping // @Summary List pending tenders for document scraping -// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description +// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes notice identifiers, dispatch time (RFC3339), procurement lots, buyer, winners (when present), CPV fields, and value metadata. // @Tags Document-Scraper // @Produce json // @Param limit query int false "Number of items per page (default: 20, max: 100)" @@ -62,7 +62,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error { // GetTenderByNoticeID retrieves a specific tender by its notice publication ID // @Summary Get tender by notice ID for document scraping -// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description +// @Description Retrieve a specific tender by its notice publication ID with full notice, lot, buyer, and winner fields for the document scraper pipeline. // @Tags Document-Scraper // @Produce json // @Param notice_id path string true "Notice Publication ID" diff --git a/internal/document_scraper/service.go b/internal/document_scraper/service.go index 60adf5b..bed1383 100644 --- a/internal/document_scraper/service.go +++ b/internal/document_scraper/service.go @@ -7,7 +7,6 @@ import ( "tm/internal/tender" "tm/pkg/logger" "tm/pkg/response" - "tm/ted" ) // Service defines business logic for the document scraper API @@ -20,7 +19,6 @@ type Service interface { type service struct { tenderRepo tender.TenderRepository - tedParser *ted.TEDParser logger logger.Logger } @@ -28,27 +26,10 @@ type service struct { func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Service { return &service{ tenderRepo: tenderRepo, - tedParser: ted.NewTEDParser(), logger: logger, } } -// buildTenderResponse creates a DocumentScraperTenderResponse from a tender -func (s *service) buildTenderResponse(_ context.Context, t *tender.Tender) *DocumentScraperTenderResponse { - documentURL := t.DocumentURI - if documentURL == "" { - documentURL = t.TenderURL - } - - return &DocumentScraperTenderResponse{ - ContractFolderID: t.ContractFolderID, - NoticePublicationID: t.NoticePublicationID, - DocumentURL: documentURL, - Title: t.Title, - Description: t.Description, - } -} - // ListPendingTenders retrieves tenders that have not yet been scraped for documents. // Only returns tenders from Sweden with active deadlines (deadline not yet reached). func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) { @@ -107,7 +88,7 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D } for i := range filteredTenders { - result.Tenders[i] = *s.buildTenderResponse(ctx, &filteredTenders[i]) + result.Tenders[i] = *BuildDocumentScraperTenderResponse(&filteredTenders[i]) } // Build pagination metadata @@ -160,7 +141,7 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do return nil, fmt.Errorf("tender not found") } - result := s.buildTenderResponse(ctx, t) + result := BuildDocumentScraperTenderResponse(t) s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{ "notice_id": noticeID, diff --git a/internal/document_scraper/tender_response.go b/internal/document_scraper/tender_response.go new file mode 100644 index 0000000..5288609 --- /dev/null +++ b/internal/document_scraper/tender_response.go @@ -0,0 +1,233 @@ +package document_scraper + +import ( + "strings" + "time" + + "tm/internal/tender" +) + +const ( + enrichmentMainCPVDescription = "main_classification_description" + vatBasisUnspecified = "unspecified" +) + +// BuildDocumentScraperTenderResponse maps a stored tender to the scraper API contract. +func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTenderResponse { + if t == nil { + return &DocumentScraperTenderResponse{ + Lots: []DocumentScraperLotResponse{}, + Winners: []DocumentScraperWinnerResponse{}, + } + } + + documentURL := strings.TrimSpace(t.DocumentURI) + if documentURL == "" { + documentURL = strings.TrimSpace(t.TenderURL) + } + + mainDesc := firstNonEmpty( + strings.TrimSpace(t.ProcessingMetadata.EnrichmentData[enrichmentMainCPVDescription]), + strings.TrimSpace(t.MainClassificationDescription), + ) + + resp := &DocumentScraperTenderResponse{ + ContractFolderID: t.ContractFolderID, + NoticePublicationID: t.NoticePublicationID, + DocumentURL: documentURL, + Title: t.Title, + Description: t.Description, + NoticeIdentifier: strings.TrimSpace(t.NoticeIdentifier), + NoticeVersion: strings.TrimSpace(t.NoticeVersion), + FormType: strings.TrimSpace(t.FormType), + NoticeType: strings.TrimSpace(t.NoticeTypeCode), + NoticeSubtype: strings.TrimSpace(t.NoticeSubTypeCode), + PublicationDate: t.PublicationDate, + NoticeDispatchDateWithTimezone: formatNoticeDispatchRFC3339(t), + ProcurementTypeCode: strings.TrimSpace(t.ProcurementTypeCode), + MainClassification: strings.TrimSpace(t.MainClassification), + MainClassificationDescription: mainDesc, + MainClassificationDisplay: classificationDisplay(strings.TrimSpace(t.MainClassification), mainDesc), + AdditionalClassifications: append([]string(nil), t.AdditionalClassifications...), + EstimatedValue: t.EstimatedValue, + EstimatedValueCurrency: strings.TrimSpace(t.Currency), + EstimatedValueVATBasis: vatBasisUnspecified, + Status: string(t.Status), + Buyer: buildBuyer(t), + Lots: buildLots(t, mainDesc), + Winners: buildWinners(t), + } + + return resp +} + +func formatNoticeDispatchRFC3339(t *tender.Tender) string { + sec := t.NoticeDispatchAt + if sec == 0 { + sec = t.ESenderDispatchAt + } + if sec == 0 { + return "" + } + return time.Unix(sec, 0).UTC().Format(time.RFC3339) +} + +func buildBuyer(t *tender.Tender) DocumentScraperBuyerResponse { + if t.BuyerOrganization == nil { + return DocumentScraperBuyerResponse{} + } + b := t.BuyerOrganization + return DocumentScraperBuyerResponse{ + BuyerName: strings.TrimSpace(b.Name), + BuyerEmail: strings.TrimSpace(b.ContactEmail), + BuyerCountry: strings.TrimSpace(b.Address.CountryCode), + BuyerRegionNuts: strings.TrimSpace(b.Address.CountrySubentityCode), + BuyerCity: strings.TrimSpace(b.Address.CityName), + BuyerProfileURL: strings.TrimSpace(firstNonEmpty(t.BuyerProfileURL, b.WebsiteURI)), + } +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if s := strings.TrimSpace(v); s != "" { + return s + } + } + return "" +} + +func classificationDisplay(code, description string) string { + code = strings.TrimSpace(code) + description = strings.TrimSpace(description) + if code != "" && description != "" { + return code + " - " + description + } + if code != "" { + return code + } + return description +} + +func buildLots(t *tender.Tender, globalMainDesc string) []DocumentScraperLotResponse { + if len(t.ProcurementLots) == 0 { + return []DocumentScraperLotResponse{} + } + out := make([]DocumentScraperLotResponse, 0, len(t.ProcurementLots)) + for _, lot := range t.ProcurementLots { + lotCode := strings.TrimSpace(lot.MainClassification) + perLotDesc := strings.TrimSpace(lot.MainClassificationDescription) + if perLotDesc == "" && lotCode != "" && lotCode == strings.TrimSpace(t.MainClassification) { + perLotDesc = globalMainDesc + } + out = append(out, DocumentScraperLotResponse{ + LotID: strings.TrimSpace(lot.LotID), + LotTitle: strings.TrimSpace(lot.Title), + LotDescription: strings.TrimSpace(lot.Description), + LotMainNature: strings.TrimSpace(lot.MainNatureOfContract), + LotMainClassificationCode: lotCode, + LotMainClassificationDescription: perLotDesc, + LotMainClassificationDisplay: classificationDisplay(lotCode, perLotDesc), + AdditionalClassifications: append([]string(nil), lot.AdditionalClassifications...), + LotDuration: strings.TrimSpace(lot.Duration), + LotEstimatedValueExclVAT: lot.EstimatedValue, + LotEstimatedValueCurrency: strings.TrimSpace(lot.Currency), + }) + } + return out +} + +func buildWinners(t *tender.Tender) []DocumentScraperWinnerResponse { + if len(t.AwardedEntities) > 0 { + out := make([]DocumentScraperWinnerResponse, 0, len(t.AwardedEntities)) + for _, a := range t.AwardedEntities { + w := DocumentScraperWinnerResponse{ + OfficialName: strings.TrimSpace(a.Name), + RegistrationNumber: strings.TrimSpace(a.CompanyID), + PostalAddress: strings.TrimSpace(a.Address), + Country: strings.TrimSpace(a.Country), + LotID: strings.TrimSpace(a.LotID), + Town: "", + Postcode: "", + CountrySubdivisionNuts: "", + ContactPoint: "", + Email: "", + Telephone: "", + } + if org := orgByID(t, a.OrganizationID); org != nil { + mergeWinnerFromOrganization(&w, org) + } + out = append(out, w) + } + return out + } + + if t.WinningTenderer == nil { + return []DocumentScraperWinnerResponse{} + } + w := t.WinningTenderer + if strings.TrimSpace(w.Name) == "" { + return []DocumentScraperWinnerResponse{} + } + addr := w.Address + return []DocumentScraperWinnerResponse{{ + OfficialName: strings.TrimSpace(w.Name), + RegistrationNumber: strings.TrimSpace(w.CompanyID), + PostalAddress: strings.TrimSpace(addr.StreetName), + Town: strings.TrimSpace(addr.CityName), + Postcode: strings.TrimSpace(addr.PostalZone), + CountrySubdivisionNuts: strings.TrimSpace(addr.CountrySubentityCode), + Country: strings.TrimSpace(addr.CountryCode), + ContactPoint: strings.TrimSpace(w.ContactName), + Email: strings.TrimSpace(w.ContactEmail), + Telephone: strings.TrimSpace(w.ContactTelephone), + }} +} + +func orgByID(t *tender.Tender, id string) *tender.Organization { + id = strings.TrimSpace(id) + if id == "" || t == nil { + return nil + } + for i := range t.Organizations { + if strings.TrimSpace(t.Organizations[i].ID) == id { + return &t.Organizations[i] + } + } + return nil +} + +func mergeWinnerFromOrganization(w *DocumentScraperWinnerResponse, org *tender.Organization) { + if w == nil || org == nil { + return + } + if w.OfficialName == "" { + w.OfficialName = strings.TrimSpace(org.Name) + } + if w.RegistrationNumber == "" { + w.RegistrationNumber = strings.TrimSpace(org.CompanyID) + } + if w.PostalAddress == "" { + w.PostalAddress = strings.TrimSpace(org.Address.StreetName) + } + if w.Town == "" { + w.Town = strings.TrimSpace(org.Address.CityName) + } + if w.Postcode == "" { + w.Postcode = strings.TrimSpace(org.Address.PostalZone) + } + if w.CountrySubdivisionNuts == "" { + w.CountrySubdivisionNuts = strings.TrimSpace(org.Address.CountrySubentityCode) + } + if w.Country == "" { + w.Country = strings.TrimSpace(org.Address.CountryCode) + } + if w.ContactPoint == "" { + w.ContactPoint = strings.TrimSpace(org.ContactName) + } + if w.Email == "" { + w.Email = strings.TrimSpace(org.ContactEmail) + } + if w.Telephone == "" { + w.Telephone = strings.TrimSpace(org.ContactTelephone) + } +} diff --git a/internal/notice/entity.go b/internal/notice/entity.go index eb2fef8..0033519 100644 --- a/internal/notice/entity.go +++ b/internal/notice/entity.go @@ -10,58 +10,59 @@ import ( // Tender represents a tender/contract notice entity type Notice struct { - mongo.Model `bson:",inline"` - ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"` - NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"` - ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"` + mongo.Model `bson:",inline"` + ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"` + NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"` + ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"` // ProcurementProjectID is UBL ProcurementProject/ID — stable across per-lot notices for one procedure. - ProcurementProjectID string `bson:"procurement_project_id,omitempty" json:"procurement_project_id,omitempty"` - NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"` - FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"` - NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"` - NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"` - NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"` - NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"` - ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"` - NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"` - IssueDate int64 `bson:"issue_date" json:"issue_date"` - IssueTime int64 `bson:"issue_time" json:"issue_time"` - TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` - PublicationDate int64 `bson:"publication_date" json:"publication_date"` - SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` - ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` - GazetteID string `bson:"gazette_id" json:"gazette_id"` - Title string `bson:"title" json:"title"` - Description string `bson:"description" json:"description"` - ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"` - ProcedureCode string `bson:"procedure_code" json:"procedure_code"` - MainClassification string `bson:"main_classification" json:"main_classification"` - AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"` - EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"` - Currency string `bson:"currency" json:"currency"` - Duration string `bson:"duration" json:"duration"` - DurationUnit string `bson:"duration_unit" json:"duration_unit"` - PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"` - CountryCode string `bson:"country_code" json:"country_code"` - RegionCode string `bson:"region_code" json:"region_code"` - CityName string `bson:"city_name" json:"city_name"` - PostalCode string `bson:"postal_code" json:"postal_code"` - DocumentURI string `bson:"document_uri" json:"document_uri"` - BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"` - ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"` - TenderURL string `bson:"tender_url" json:"tender_url"` - SubmissionURL string `bson:"submission_url" json:"submission_url"` - BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"` - ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"` - Organizations []Organization `bson:"organizations" json:"organizations"` - SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"` - OfficialLanguages []string `bson:"official_languages" json:"official_languages"` - Status TenderStatus `bson:"status" json:"status"` - Source TenderSource `bson:"source" json:"source"` - SourceFileURL string `bson:"source_file_url" json:"source_file_url"` - SourceFileName string `bson:"source_file_name" json:"source_file_name"` - ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` - ContentXML string `bson:"content_xml" json:"content_xml"` + ProcurementProjectID string `bson:"procurement_project_id,omitempty" json:"procurement_project_id,omitempty"` + NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"` + FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"` + NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"` + NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"` + NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"` + NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"` + ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"` + NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"` + IssueDate int64 `bson:"issue_date" json:"issue_date"` + IssueTime int64 `bson:"issue_time" json:"issue_time"` + TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` + PublicationDate int64 `bson:"publication_date" json:"publication_date"` + SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` + ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` + GazetteID string `bson:"gazette_id" json:"gazette_id"` + Title string `bson:"title" json:"title"` + Description string `bson:"description" json:"description"` + ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"` + ProcedureCode string `bson:"procedure_code" json:"procedure_code"` + MainClassification string `bson:"main_classification" json:"main_classification"` + MainClassificationDescription string `bson:"main_classification_description,omitempty" json:"main_classification_description,omitempty"` + AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"` + EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"` + Currency string `bson:"currency" json:"currency"` + Duration string `bson:"duration" json:"duration"` + DurationUnit string `bson:"duration_unit" json:"duration_unit"` + PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"` + CountryCode string `bson:"country_code" json:"country_code"` + RegionCode string `bson:"region_code" json:"region_code"` + CityName string `bson:"city_name" json:"city_name"` + PostalCode string `bson:"postal_code" json:"postal_code"` + DocumentURI string `bson:"document_uri" json:"document_uri"` + BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"` + ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"` + TenderURL string `bson:"tender_url" json:"tender_url"` + SubmissionURL string `bson:"submission_url" json:"submission_url"` + BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"` + ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"` + Organizations []Organization `bson:"organizations" json:"organizations"` + SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"` + OfficialLanguages []string `bson:"official_languages" json:"official_languages"` + Status TenderStatus `bson:"status" json:"status"` + Source TenderSource `bson:"source" json:"source"` + SourceFileURL string `bson:"source_file_url" json:"source_file_url"` + SourceFileName string `bson:"source_file_name" json:"source_file_name"` + ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` + ContentXML string `bson:"content_xml" json:"content_xml"` // Status-specific fields CancellationReason string `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"` @@ -110,15 +111,16 @@ type SelectionCriterion struct { // ProcurementLot captures per-lot procurement fields from TED notices. type ProcurementLot struct { - LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` - Title string `bson:"title,omitempty" json:"title,omitempty"` - Description string `bson:"description,omitempty" json:"description,omitempty"` - MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"` - MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"` - AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"` - Duration string `bson:"duration,omitempty" json:"duration,omitempty"` - EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"` - Currency string `bson:"currency,omitempty" json:"currency,omitempty"` + LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` + Title string `bson:"title,omitempty" json:"title,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"` + MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"` + MainClassificationDescription string `bson:"main_classification_description,omitempty" json:"main_classification_description,omitempty"` + AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"` + Duration string `bson:"duration,omitempty" json:"duration,omitempty"` + EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"` + Currency string `bson:"currency,omitempty" json:"currency,omitempty"` } // ProcessingMetadata contains metadata about how the tender was processed diff --git a/internal/tender/entity.go b/internal/tender/entity.go index acd9e78..10a259a 100644 --- a/internal/tender/entity.go +++ b/internal/tender/entity.go @@ -37,56 +37,57 @@ type Tender struct { NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"` // RelatedNoticePublicationIDs lists every TED notice publication id that has contributed to this tender // (same procedure / contract_folder_id). Ordered oldest-first; the latest is also reflected in NoticePublicationID. - RelatedNoticePublicationIDs []string `bson:"related_notice_publication_ids,omitempty" json:"related_notice_publication_ids,omitempty"` - ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"` + RelatedNoticePublicationIDs []string `bson:"related_notice_publication_ids,omitempty" json:"related_notice_publication_ids,omitempty"` + ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"` // ProcurementProjectID is UBL ProcurementProject/ID — merges per-lot CAN/CN notices into one tender. - ProcurementProjectID string `bson:"procurement_project_id,omitempty" json:"procurement_project_id,omitempty"` - NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"` - FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"` - NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"` - NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"` - NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"` - NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"` - ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"` - NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"` - IssueDate int64 `bson:"issue_date" json:"issue_date"` - IssueTime int64 `bson:"issue_time" json:"issue_time"` - TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` - PublicationDate int64 `bson:"publication_date" json:"publication_date"` - SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` - ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` - GazetteID string `bson:"gazette_id" json:"gazette_id"` - Title string `bson:"title" json:"title"` - Description string `bson:"description" json:"description"` - ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"` - ProcedureCode string `bson:"procedure_code" json:"procedure_code"` - MainClassification string `bson:"main_classification" json:"main_classification"` - AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"` - EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"` - Currency string `bson:"currency" json:"currency"` - Duration string `bson:"duration" json:"duration"` - DurationUnit string `bson:"duration_unit" json:"duration_unit"` - PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"` - CountryCode string `bson:"country_code" json:"country_code"` - RegionCode string `bson:"region_code" json:"region_code"` - CityName string `bson:"city_name" json:"city_name"` - PostalCode string `bson:"postal_code" json:"postal_code"` - DocumentURI string `bson:"document_uri" json:"document_uri"` - BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"` - ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"` - TenderURL string `bson:"tender_url" json:"tender_url"` - SubmissionURL string `bson:"submission_url" json:"submission_url"` - BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"` - ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"` - Organizations []Organization `bson:"organizations" json:"organizations"` - SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"` - OfficialLanguages []string `bson:"official_languages" json:"official_languages"` - Status TenderStatus `bson:"status" json:"status"` - Source TenderSource `bson:"source" json:"source"` - SourceFileURL string `bson:"source_file_url" json:"source_file_url"` - SourceFileName string `bson:"source_file_name" json:"source_file_name"` - ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"` - ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` + ProcurementProjectID string `bson:"procurement_project_id,omitempty" json:"procurement_project_id,omitempty"` + NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"` + FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"` + NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"` + NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"` + NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"` + NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"` + ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"` + NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"` + IssueDate int64 `bson:"issue_date" json:"issue_date"` + IssueTime int64 `bson:"issue_time" json:"issue_time"` + TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` + PublicationDate int64 `bson:"publication_date" json:"publication_date"` + SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` + ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` + GazetteID string `bson:"gazette_id" json:"gazette_id"` + Title string `bson:"title" json:"title"` + Description string `bson:"description" json:"description"` + ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"` + ProcedureCode string `bson:"procedure_code" json:"procedure_code"` + MainClassification string `bson:"main_classification" json:"main_classification"` + MainClassificationDescription string `bson:"main_classification_description,omitempty" json:"main_classification_description,omitempty"` + AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"` + EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"` + Currency string `bson:"currency" json:"currency"` + Duration string `bson:"duration" json:"duration"` + DurationUnit string `bson:"duration_unit" json:"duration_unit"` + PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"` + CountryCode string `bson:"country_code" json:"country_code"` + RegionCode string `bson:"region_code" json:"region_code"` + CityName string `bson:"city_name" json:"city_name"` + PostalCode string `bson:"postal_code" json:"postal_code"` + DocumentURI string `bson:"document_uri" json:"document_uri"` + BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"` + ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"` + TenderURL string `bson:"tender_url" json:"tender_url"` + SubmissionURL string `bson:"submission_url" json:"submission_url"` + BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"` + ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"` + Organizations []Organization `bson:"organizations" json:"organizations"` + SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"` + OfficialLanguages []string `bson:"official_languages" json:"official_languages"` + Status TenderStatus `bson:"status" json:"status"` + Source TenderSource `bson:"source" json:"source"` + SourceFileURL string `bson:"source_file_url" json:"source_file_url"` + SourceFileName string `bson:"source_file_name" json:"source_file_name"` + ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"` + ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` // AIOverallSummary optional denormalized overall summary for search/list (e.g. synced from AI pipeline storage). AIOverallSummary string `bson:"ai_overall_summary,omitempty" json:"ai_overall_summary,omitempty"` TenderID string `bson:"tender_id" json:"tender_id"` // PBL Project Number (SCDYYNNN format) @@ -150,15 +151,16 @@ type SelectionCriterion struct { // ProcurementLot captures per-lot procurement fields from TED notices. type ProcurementLot struct { - LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` - Title string `bson:"title,omitempty" json:"title,omitempty"` - Description string `bson:"description,omitempty" json:"description,omitempty"` - MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"` - MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"` - AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"` - Duration string `bson:"duration,omitempty" json:"duration,omitempty"` - EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"` - Currency string `bson:"currency,omitempty" json:"currency,omitempty"` + LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` + Title string `bson:"title,omitempty" json:"title,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"` + MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"` + MainClassificationDescription string `bson:"main_classification_description,omitempty" json:"main_classification_description,omitempty"` + AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"` + Duration string `bson:"duration,omitempty" json:"duration,omitempty"` + EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"` + Currency string `bson:"currency,omitempty" json:"currency,omitempty"` } // ProcessingMetadata contains metadata about how the tender was processed diff --git a/internal/tender/form.go b/internal/tender/form.go index ae75722..faf2a5b 100644 --- a/internal/tender/form.go +++ b/internal/tender/form.go @@ -1,6 +1,11 @@ package tender -import "tm/pkg/response" +import ( + "strings" + "time" + + "tm/pkg/response" +) // UpdateTenderRequest represents a request to update an existing tender type UpdateTenderRequest struct { @@ -64,33 +69,135 @@ type SearchResponse struct { Metadata *response.Meta `json:"-"` } -// TenderResponse represents a tender in API responses +// AddressResponse is postal / NUTS address data for API consumers. +type AddressResponse struct { + StreetName string `json:"street_name,omitempty"` + CityName string `json:"city_name,omitempty"` + PostalZone string `json:"postal_zone,omitempty"` + CountrySubentityCode string `json:"country_subentity_code,omitempty"` + Department string `json:"department,omitempty"` + Region string `json:"region,omitempty"` + CountryCode string `json:"country_code"` +} + +// OrganizationResponse is an organization (buyer, winner, etc.) in tender API payloads. +type OrganizationResponse struct { + ID string `json:"id,omitempty"` + Name string `json:"name"` + CompanyID string `json:"company_id,omitempty"` + WebsiteURI string `json:"website_uri,omitempty"` + ContactName string `json:"contact_name,omitempty"` + ContactTelephone string `json:"contact_telephone,omitempty"` + ContactEmail string `json:"contact_email,omitempty"` + Address AddressResponse `json:"address,omitempty"` +} + +// TenderLotResponse is one procurement lot for list/detail tender APIs. +type TenderLotResponse struct { + LotID string `json:"lot_id,omitempty"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + MainNatureOfContract string `json:"main_nature_of_contract,omitempty"` + MainClassification string `json:"main_classification,omitempty"` + MainClassificationDescription string `json:"main_classification_description,omitempty"` + MainClassificationDisplay string `json:"main_classification_display,omitempty"` + AdditionalClassifications []string `json:"additional_classifications,omitempty"` + Duration string `json:"duration,omitempty"` + EstimatedValue float64 `json:"estimated_value,omitempty"` + Currency string `json:"currency,omitempty"` +} + +// TenderWinnerResponse is a contract winner for awarded tenders. +type TenderWinnerResponse struct { + OfficialName string `json:"official_name"` + RegistrationNumber string `json:"registration_number,omitempty"` + PostalAddress string `json:"postal_address,omitempty"` + Town string `json:"town,omitempty"` + Postcode string `json:"postcode,omitempty"` + CountrySubdivisionNuts string `json:"country_subdivision_nuts,omitempty"` + Country string `json:"country,omitempty"` + ContactPoint string `json:"contact_point,omitempty"` + Email string `json:"email,omitempty"` + Telephone string `json:"telephone,omitempty"` + LotID string `json:"lot_id,omitempty"` + Amount float64 `json:"amount,omitempty"` + Currency string `json:"currency,omitempty"` +} + +// TenderResponse represents a tender in API responses (search, list, and detail). type TenderResponse struct { - ID string `json:"id"` - NoticePublicationID string `json:"notice_publication_id"` - RelatedNoticePublicationIDs []string `json:"related_notice_publication_ids,omitempty"` - NoticeTypeCode string `json:"notice_type_code"` - Title string `json:"title"` - Description string `json:"description"` - ProcurementTypeCode string `json:"procurement_type_code"` - ProcedureCode string `json:"procedure_code"` - MainClassification string `json:"main_classification"` - EstimatedValue float64 `json:"estimated_value"` - Currency string `json:"currency"` - Duration string `json:"duration"` - DurationUnit string `json:"duration_unit"` - PublicationDate int64 `json:"publication_date"` - TenderDeadline int64 `json:"tender_deadline"` - SubmissionDeadline int64 `json:"submission_deadline"` - ApplicationDeadline int64 `json:"application_deadline"` - SubmissionURL string `json:"submission_url"` - CountryCode string `json:"country_code"` - BuyerOrganization *OrganizationResponse `json:"buyer_organization"` - Status TenderStatus `json:"status"` - TenderID string `json:"tender_id"` - CreatedAt int64 `json:"created_at"` - OverallSummary string `json:"overall_summary,omitempty"` // AI-generated summary from the pipeline - Language string `json:"language,omitempty"` + ID string `json:"id"` + NoticePublicationID string `json:"notice_publication_id"` + RelatedNoticePublicationIDs []string `json:"related_notice_publication_ids,omitempty"` + ContractFolderID string `json:"contract_folder_id,omitempty"` + ProcurementProjectID string `json:"procurement_project_id,omitempty"` + ContractNoticeID string `json:"contract_notice_id,omitempty"` + + NoticeTypeCode string `json:"notice_type_code"` + FormType string `json:"form_type,omitempty"` + NoticeSubTypeCode string `json:"notice_sub_type_code,omitempty"` + NoticeIdentifier string `json:"notice_identifier,omitempty"` + NoticeVersion string `json:"notice_version,omitempty"` + NoticeLanguageCode string `json:"notice_language_code,omitempty"` + GazetteID string `json:"gazette_id,omitempty"` + + NoticeDispatchAt int64 `json:"notice_dispatch_at,omitempty"` + NoticeDispatchDateWithTimezone string `json:"notice_dispatch_date_with_timezone,omitempty"` + ESenderDispatchAt int64 `json:"esender_dispatch_at,omitempty"` + IssueDate int64 `json:"issue_date,omitempty"` + IssueTime int64 `json:"issue_time,omitempty"` + + Title string `json:"title"` + Description string `json:"description"` + + ProcurementTypeCode string `json:"procurement_type_code"` + ProcedureCode string `json:"procedure_code"` + MainClassification string `json:"main_classification"` + MainClassificationDescription string `json:"main_classification_description,omitempty"` + MainClassificationDisplay string `json:"main_classification_display,omitempty"` + AdditionalClassifications []string `json:"additional_classifications,omitempty"` + EstimatedValue float64 `json:"estimated_value"` + Currency string `json:"currency"` + EstimatedValueVATBasis string `json:"estimated_value_vat_basis,omitempty"` + + Duration string `json:"duration"` + DurationUnit string `json:"duration_unit"` + + PublicationDate int64 `json:"publication_date"` + TenderDeadline int64 `json:"tender_deadline"` + SubmissionDeadline int64 `json:"submission_deadline"` + ApplicationDeadline int64 `json:"application_deadline"` + SubmissionURL string `json:"submission_url"` + + PlaceOfPerformance string `json:"place_of_performance,omitempty"` + CountryCode string `json:"country_code"` + RegionCode string `json:"region_code,omitempty"` + CityName string `json:"city_name,omitempty"` + PostalCode string `json:"postal_code,omitempty"` + + DocumentURI string `json:"document_uri,omitempty"` + DocumentURL string `json:"document_url,omitempty"` + TenderURL string `json:"tender_url,omitempty"` + BuyerProfileURL string `json:"buyer_profile_url,omitempty"` + + BuyerOrganization *OrganizationResponse `json:"buyer_organization,omitempty"` + WinningTenderer *OrganizationResponse `json:"winning_tenderer,omitempty"` + Lots []TenderLotResponse `json:"lots,omitempty"` + Winners []TenderWinnerResponse `json:"winners,omitempty"` + OfficialLanguages []string `json:"official_languages,omitempty"` + + AwardDate int64 `json:"award_date,omitempty"` + AwardedValue float64 `json:"awarded_value,omitempty"` + ContractNumber string `json:"contract_number,omitempty"` + + Status TenderStatus `json:"status"` + Source TenderSource `json:"source,omitempty"` + TenderID string `json:"tender_id"` + ProjectName string `json:"project_name,omitempty"` + CreatedAt int64 `json:"created_at"` + + OverallSummary string `json:"overall_summary,omitempty"` + Language string `json:"language,omitempty"` } // TenderDocumentResponse represents a scraped tender document available for download. @@ -107,9 +214,10 @@ type TenderDocumentsResponse struct { Documents []TenderDocumentResponse `json:"documents"` } -type OrganizationResponse struct { - Name string `json:"name"` -} +const ( + enrichmentMainCPVDescriptionKey = "main_classification_description" + vatBasisUnspecified = "unspecified" +) // ToTenderResponse converts a Tender entity to TenderResponse func (t *Tender) ToResponse() *TenderResponse { @@ -119,11 +227,6 @@ func (t *Tender) ToResponse() *TenderResponse { // ToResponseWithLanguage converts a Tender entity to TenderResponse using a preferred language. // When language is provided and translation exists, translated title/description are returned. func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse { - - org := &OrganizationResponse{} - if t.BuyerOrganization != nil { - org.Name = t.BuyerOrganization.Name - } title := t.Title description := t.Description usedLanguage := "" @@ -135,36 +238,278 @@ func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse { } } + mainDesc := firstNonEmpty( + strings.TrimSpace(t.ProcessingMetadata.EnrichmentData[enrichmentMainCPVDescriptionKey]), + strings.TrimSpace(t.MainClassificationDescription), + ) + mainCode := strings.TrimSpace(t.MainClassification) + + documentURL := strings.TrimSpace(t.DocumentURI) + if documentURL == "" { + documentURL = strings.TrimSpace(t.TenderURL) + } + response := &TenderResponse{ ID: t.ID.Hex(), NoticePublicationID: t.NoticePublicationID, - RelatedNoticePublicationIDs: t.RelatedNoticePublicationIDs, - NoticeTypeCode: t.NoticeTypeCode, - Title: title, - Description: description, - ProcurementTypeCode: t.ProcurementTypeCode, - ProcedureCode: t.ProcedureCode, - MainClassification: t.MainClassification, - EstimatedValue: t.EstimatedValue, - Currency: t.Currency, - Duration: t.Duration, - DurationUnit: t.DurationUnit, - PublicationDate: t.PublicationDate, - TenderDeadline: t.TenderDeadline, - SubmissionDeadline: t.SubmissionDeadline, - ApplicationDeadline: t.ApplicationDeadline, - CountryCode: t.CountryCode, - SubmissionURL: t.SubmissionURL, - BuyerOrganization: org, - Status: t.Status, - TenderID: t.TenderID, - CreatedAt: t.CreatedAt, - Language: usedLanguage, + RelatedNoticePublicationIDs: append([]string(nil), t.RelatedNoticePublicationIDs...), + ContractFolderID: strings.TrimSpace(t.ContractFolderID), + ProcurementProjectID: strings.TrimSpace(t.ProcurementProjectID), + ContractNoticeID: strings.TrimSpace(t.ContractNoticeID), + + NoticeTypeCode: strings.TrimSpace(t.NoticeTypeCode), + FormType: strings.TrimSpace(t.FormType), + NoticeSubTypeCode: strings.TrimSpace(t.NoticeSubTypeCode), + NoticeIdentifier: strings.TrimSpace(t.NoticeIdentifier), + NoticeVersion: strings.TrimSpace(t.NoticeVersion), + NoticeLanguageCode: strings.TrimSpace(t.NoticeLanguageCode), + GazetteID: strings.TrimSpace(t.GazetteID), + + NoticeDispatchAt: t.NoticeDispatchAt, + NoticeDispatchDateWithTimezone: formatNoticeDispatchRFC3339(t), + ESenderDispatchAt: t.ESenderDispatchAt, + IssueDate: t.IssueDate, + IssueTime: t.IssueTime, + + Title: title, + Description: description, + ProcurementTypeCode: strings.TrimSpace(t.ProcurementTypeCode), + ProcedureCode: strings.TrimSpace(t.ProcedureCode), + MainClassification: mainCode, + MainClassificationDescription: mainDesc, + MainClassificationDisplay: classificationDisplay(mainCode, mainDesc), + AdditionalClassifications: append([]string(nil), t.AdditionalClassifications...), + EstimatedValue: t.EstimatedValue, + Currency: strings.TrimSpace(t.Currency), + EstimatedValueVATBasis: vatBasisUnspecified, + + Duration: t.Duration, + DurationUnit: t.DurationUnit, + + PublicationDate: t.PublicationDate, + TenderDeadline: t.TenderDeadline, + SubmissionDeadline: t.SubmissionDeadline, + ApplicationDeadline: t.ApplicationDeadline, + SubmissionURL: strings.TrimSpace(t.SubmissionURL), + + PlaceOfPerformance: strings.TrimSpace(t.PlaceOfPerformance), + CountryCode: strings.TrimSpace(t.CountryCode), + RegionCode: strings.TrimSpace(t.RegionCode), + CityName: strings.TrimSpace(t.CityName), + PostalCode: strings.TrimSpace(t.PostalCode), + + DocumentURI: strings.TrimSpace(t.DocumentURI), + DocumentURL: documentURL, + TenderURL: strings.TrimSpace(t.TenderURL), + BuyerProfileURL: strings.TrimSpace(t.BuyerProfileURL), + + BuyerOrganization: organizationToResponse(t.BuyerOrganization), + WinningTenderer: organizationToResponse(t.WinningTenderer), + Lots: buildTenderLotsResponse(t, mainDesc), + Winners: buildTenderWinnersResponse(t), + OfficialLanguages: append([]string(nil), t.OfficialLanguages...), + + AwardDate: t.AwardDate, + AwardedValue: t.AwardedValue, + ContractNumber: strings.TrimSpace(t.ContractNumber), + + Status: t.Status, + Source: t.Source, + TenderID: t.TenderID, + ProjectName: strings.TrimSpace(t.ProjectName), + CreatedAt: t.CreatedAt, + Language: usedLanguage, } return response } +func organizationToResponse(o *Organization) *OrganizationResponse { + if o == nil { + return nil + } + return &OrganizationResponse{ + ID: strings.TrimSpace(o.ID), + Name: strings.TrimSpace(o.Name), + CompanyID: strings.TrimSpace(o.CompanyID), + WebsiteURI: strings.TrimSpace(o.WebsiteURI), + ContactName: strings.TrimSpace(o.ContactName), + ContactTelephone: strings.TrimSpace(o.ContactTelephone), + ContactEmail: strings.TrimSpace(o.ContactEmail), + Address: AddressResponse{ + StreetName: strings.TrimSpace(o.Address.StreetName), + CityName: strings.TrimSpace(o.Address.CityName), + PostalZone: strings.TrimSpace(o.Address.PostalZone), + CountrySubentityCode: strings.TrimSpace(o.Address.CountrySubentityCode), + Department: strings.TrimSpace(o.Address.Department), + Region: strings.TrimSpace(o.Address.Region), + CountryCode: strings.TrimSpace(o.Address.CountryCode), + }, + } +} + +func formatNoticeDispatchRFC3339(t *Tender) string { + if t == nil { + return "" + } + sec := t.NoticeDispatchAt + if sec == 0 { + sec = t.ESenderDispatchAt + } + if sec == 0 { + return "" + } + return time.Unix(sec, 0).UTC().Format(time.RFC3339) +} + +func firstNonEmpty(values ...string) string { + for _, v := range values { + if s := strings.TrimSpace(v); s != "" { + return s + } + } + return "" +} + +func classificationDisplay(code, description string) string { + code = strings.TrimSpace(code) + description = strings.TrimSpace(description) + if code != "" && description != "" { + return code + " - " + description + } + if code != "" { + return code + } + return description +} + +func buildTenderLotsResponse(t *Tender, globalMainDesc string) []TenderLotResponse { + if t == nil || len(t.ProcurementLots) == 0 { + return nil + } + out := make([]TenderLotResponse, 0, len(t.ProcurementLots)) + mainCode := strings.TrimSpace(t.MainClassification) + for i := range t.ProcurementLots { + lot := t.ProcurementLots[i] + lotCode := strings.TrimSpace(lot.MainClassification) + perLotDesc := strings.TrimSpace(lot.MainClassificationDescription) + if perLotDesc == "" && lotCode != "" && lotCode == mainCode { + perLotDesc = globalMainDesc + } + out = append(out, TenderLotResponse{ + LotID: strings.TrimSpace(lot.LotID), + Title: strings.TrimSpace(lot.Title), + Description: strings.TrimSpace(lot.Description), + MainNatureOfContract: strings.TrimSpace(lot.MainNatureOfContract), + MainClassification: lotCode, + MainClassificationDescription: perLotDesc, + MainClassificationDisplay: classificationDisplay(lotCode, perLotDesc), + AdditionalClassifications: append([]string(nil), lot.AdditionalClassifications...), + Duration: strings.TrimSpace(lot.Duration), + EstimatedValue: lot.EstimatedValue, + Currency: strings.TrimSpace(lot.Currency), + }) + } + return out +} + +func buildTenderWinnersResponse(t *Tender) []TenderWinnerResponse { + if t == nil { + return nil + } + if len(t.AwardedEntities) > 0 { + out := make([]TenderWinnerResponse, 0, len(t.AwardedEntities)) + for _, a := range t.AwardedEntities { + w := TenderWinnerResponse{ + OfficialName: strings.TrimSpace(a.Name), + RegistrationNumber: strings.TrimSpace(a.CompanyID), + PostalAddress: strings.TrimSpace(a.Address), + Country: strings.TrimSpace(a.Country), + LotID: strings.TrimSpace(a.LotID), + Amount: a.Amount, + Currency: strings.TrimSpace(a.Currency), + Town: "", + Postcode: "", + CountrySubdivisionNuts: "", + ContactPoint: "", + Email: "", + Telephone: "", + } + if org := organizationByID(t, a.OrganizationID); org != nil { + mergeWinnerFromOrganization(&w, org) + } + out = append(out, w) + } + return out + } + if t.WinningTenderer == nil || strings.TrimSpace(t.WinningTenderer.Name) == "" { + return nil + } + w := t.WinningTenderer + addr := w.Address + return []TenderWinnerResponse{{ + OfficialName: strings.TrimSpace(w.Name), + RegistrationNumber: strings.TrimSpace(w.CompanyID), + PostalAddress: strings.TrimSpace(addr.StreetName), + Town: strings.TrimSpace(addr.CityName), + Postcode: strings.TrimSpace(addr.PostalZone), + CountrySubdivisionNuts: strings.TrimSpace(addr.CountrySubentityCode), + Country: strings.TrimSpace(addr.CountryCode), + ContactPoint: strings.TrimSpace(w.ContactName), + Email: strings.TrimSpace(w.ContactEmail), + Telephone: strings.TrimSpace(w.ContactTelephone), + }} +} + +func organizationByID(t *Tender, id string) *Organization { + id = strings.TrimSpace(id) + if id == "" || t == nil { + return nil + } + for i := range t.Organizations { + if strings.TrimSpace(t.Organizations[i].ID) == id { + return &t.Organizations[i] + } + } + return nil +} + +func mergeWinnerFromOrganization(w *TenderWinnerResponse, org *Organization) { + if w == nil || org == nil { + return + } + if w.OfficialName == "" { + w.OfficialName = strings.TrimSpace(org.Name) + } + if w.RegistrationNumber == "" { + w.RegistrationNumber = strings.TrimSpace(org.CompanyID) + } + if w.PostalAddress == "" { + w.PostalAddress = strings.TrimSpace(org.Address.StreetName) + } + if w.Town == "" { + w.Town = strings.TrimSpace(org.Address.CityName) + } + if w.Postcode == "" { + w.Postcode = strings.TrimSpace(org.Address.PostalZone) + } + if w.CountrySubdivisionNuts == "" { + w.CountrySubdivisionNuts = strings.TrimSpace(org.Address.CountrySubentityCode) + } + if w.Country == "" { + w.Country = strings.TrimSpace(org.Address.CountryCode) + } + if w.ContactPoint == "" { + w.ContactPoint = strings.TrimSpace(org.ContactName) + } + if w.Email == "" { + w.Email = strings.TrimSpace(org.ContactEmail) + } + if w.Telephone == "" { + w.Telephone = strings.TrimSpace(org.ContactTelephone) + } +} + // AITranslateRequest represents a request for on-demand AI translation. type AITranslateRequest struct { Language string `json:"language" valid:"required,stringlength(2|5)~Language must be between 2 and 5 characters"` diff --git a/internal/tender/service.go b/internal/tender/service.go index eb118cd..9e7ef0d 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -113,12 +113,7 @@ func (s *tenderService) GetByID(ctx context.Context, id string) (*TenderResponse return nil, fmt.Errorf("failed to retrieve tender: %w", err) } - resp := tender.ToResponse() - if tr, ok := tender.Translations[s.defaultLanguage]; ok { - resp.Title = tr.Title - resp.Description = tr.Description - resp.Language = s.defaultLanguage - } + resp := tender.ToResponseWithLanguage(s.defaultLanguage) // Enrich with AI summary from MinIO storage (best-effort, non-blocking) s.enrichWithAISummary(ctx, tender, resp) @@ -284,7 +279,7 @@ func (s *tenderService) Update(ctx context.Context, req UpdateTenderRequest) (*T "tender_id": tender.ID, }) - return tender.ToResponse(), nil + return tender.ToResponseWithLanguage(s.defaultLanguage), nil } // Delete deletes a tender diff --git a/ted/mapper.go b/ted/mapper.go index e3e76f4..2af0bb6 100644 --- a/ted/mapper.go +++ b/ted/mapper.go @@ -19,7 +19,9 @@ func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot out.Description = pp.Description out.MainNatureOfContract = pp.ProcurementTypeCode if pp.MainCommodityClassification != nil { - out.MainClassification = strings.TrimSpace(pp.MainCommodityClassification.ItemClassificationCode) + mc := pp.MainCommodityClassification + out.MainClassification = strings.TrimSpace(mc.ItemClassificationCode) + out.MainClassificationDescription = mainCommodityClassificationLabel(mc) } for _, ac := range pp.AdditionalCommodityClassification { c := strings.TrimSpace(ac.ItemClassificationCode) @@ -86,7 +88,7 @@ func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceF }) return nil } - return s.mapGenericNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName) + return s.mapPriorInformationNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName) default: s.logger.Error("Unknown document type for mapping", map[string]interface{}{ "document_type": string(parsedDoc.Type), @@ -95,34 +97,41 @@ func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceF } } -// mapGenericNoticeToTender maps any TED document type to tender entity using the common interface -func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *notice.Notice { - if doc == nil { - s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{}) +// mapPriorInformationNoticeToTender maps a TED PriorInformationNotice to the same notice shape as a contract notice, +// so procedure keys (ContractFolderID, NoticePublicationID, ProcurementProjectID) merge into one tender. +func (s *TEDScraper) mapPriorInformationNoticeToTender(pin *PriorInformationNotice, sourceFileURL, sourceFileName string) *notice.Notice { + if pin == nil { return nil } - now := time.Now().Unix() t := ¬ice.Notice{ - ContractNoticeID: doc.GetID(), - ContractFolderID: "", // Not available in generic interface - NoticeTypeCode: doc.GetNoticeType(), - NoticeSubTypeCode: "", // Not available in generic interface - NoticeLanguageCode: doc.GetLanguage(), - IssueDate: now, // TODO: Parse actual date from document - IssueTime: now, // TODO: Parse actual time from document - GazetteID: "", // Not available in generic interface - NoticePublicationID: "", // Not available in generic interface - PublicationDate: now, // TODO: Parse actual publication date - - // Basic tender information - Title: "Generic Notice " + doc.GetNoticeType(), - Description: "Processed from " + doc.GetNoticeType() + " document", - Status: notice.TenderStatusActive, - Source: notice.TenderSourceTEDScraper, - SourceFileURL: sourceFileURL, - SourceFileName: sourceFileName, + ContractNoticeID: pin.ID, + ContractFolderID: pin.ContractFolderID, + ProcurementProjectID: procurementProjectIDFromPriorInformationNotice(pin), + NoticeTypeCode: strings.TrimSpace(pin.GetNoticeType()), + FormType: pin.GetNoticeFormType(), + NoticeSubTypeCode: pin.GetNoticeSubType(), + NoticeIdentifier: pin.ID, + NoticeVersion: pin.VersionID, + NoticeDispatchAt: ParseIssueDateTimeToUnix(pin.IssueDate, pin.IssueTime), + ESenderDispatchAt: TransmissionDispatchUnix(pin.Extensions), + NoticeLanguageCode: pin.NoticeLanguageCode, + IssueDate: ParseDateToUnix(pin.IssueDate), + IssueTime: ParseTimeToUnix(pin.IssueTime), + GazetteID: pin.GetOJSID(), + Title: pin.GetProcurementTitle(), + Description: pin.GetProcurementDescription(), + ProcurementTypeCode: pin.GetContractNature(), + ProcedureCode: pin.GetProcedureCode(), + MainClassification: pin.GetMainClassification(), + MainClassificationDescription: pin.GetMainClassificationDescription(), + PlaceOfPerformance: pin.GetPlaceOfPerformance(), + DocumentURI: pin.GetDocumentURI(), + Status: notice.TenderStatusActive, + Source: notice.TenderSourceTEDScraper, + SourceFileURL: sourceFileURL, + SourceFileName: sourceFileName, ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, @@ -130,39 +139,180 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so }, } + if pubInfo := pin.GetPublicationInfo(); pubInfo != nil { + t.NoticePublicationID = pubInfo.NoticePublicationID + t.PublicationDate = ParseDateToUnix(pubInfo.PublicationDate) + t.TenderURL = GenerateTenderURL(pubInfo.NoticePublicationID) + } + + if pin.ProcurementProject != nil && pin.ProcurementProject.AdditionalCommodityClassification != nil { + for _, additionalClass := range pin.ProcurementProject.AdditionalCommodityClassification { + if additionalClass.ItemClassificationCode != "" { + t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode) + } + } + } + + if amount, currency := pin.GetBudget(); amount != "" { + if value, ok := ParseEuropeanAmount(amount); ok { + t.EstimatedValue = value + t.Currency = currency + } + } + + if pin.ContractingParty != nil { + t.BuyerProfileURL = pin.ContractingParty.BuyerProfileURI + } + + for i := range pin.ProcurementProjectLot { + t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&pin.ProcurementProjectLot[i])) + } + + if duration, unit := pin.GetDuration(); duration != "" { + t.Duration = duration + t.DurationUnit = unit + } + + if deadline := pin.GetTenderDeadline(); deadline != "" { + t.TenderDeadline = ParseDateToUnix(deadline) + if deadlineTime, err := ParseDate(deadline); err == nil { + applicationDeadline := CalculateApplicationDeadline(deadlineTime) + t.ApplicationDeadline = applicationDeadline.Unix() + } + } + + if pubDate, err := ParseDate(pin.GetPublicationDate()); err == nil { + submissionDeadline := CalculateSubmissionDeadline(pubDate) + t.SubmissionDeadline = submissionDeadline.Unix() + } + + if lot := pin.firstLot(); lot != nil && lot.TenderingTerms != nil { + if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil { + if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" { + if strings.HasPrefix(endpointID, "http") { + t.SubmissionURL = endpointID + } else { + t.SubmissionURL = "https://" + endpointID + } + } + } + } + + if pin.ProcurementProject != nil && pin.ProcurementProject.RealizedLocation != nil { + if addr := pin.ProcurementProject.RealizedLocation.Address; addr != nil { + t.CityName = addr.CityName + t.PostalCode = addr.PostalZone + t.RegionCode = addr.CountrySubentityCode + if addr.Country != nil { + t.CountryCode = addr.Country.IdentificationCode + } + } + } else if lot := pin.firstLot(); lot != nil && lot.ProcurementProject != nil && + lot.ProcurementProject.RealizedLocation != nil { + if addr := lot.ProcurementProject.RealizedLocation.Address; addr != nil { + t.CityName = addr.CityName + t.PostalCode = addr.PostalZone + t.RegionCode = addr.CountrySubentityCode + if addr.Country != nil { + t.CountryCode = addr.Country.IdentificationCode + } + } + } + + buyerID := pin.GetBuyerID() + if buyerID != "" { + if buyerOrg := pin.GetOrganizationByID(buyerID); buyerOrg != nil { + t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer") + } + } + + reviewID := pin.GetReviewOrganizationID() + if reviewID != "" { + if reviewOrg := pin.GetOrganizationByID(reviewID); reviewOrg != nil { + t.ReviewOrganization = s.mapOrganization(reviewOrg, "review") + } + } + + if orgs := pin.GetOrganizations(); orgs != nil { + for _, org := range orgs.Organization { + mappedOrg := s.mapOrganization(&org, "") + if mappedOrg != nil { + t.Organizations = append(t.Organizations, *mappedOrg) + } + } + } + + if criteria := pin.GetSelectionCriteria(); criteria != nil { + for _, criterion := range criteria { + t.SelectionCriteria = append(t.SelectionCriteria, notice.SelectionCriterion{ + TypeCode: criterion.CriterionTypeCode, + Description: criterion.Description, + LanguageID: criterion.LanguageID, + }) + } + } + + if languages := pin.GetOfficialLanguages(); languages != nil { + t.OfficialLanguages = languages + } + + if strings.TrimSpace(t.Title) == "" { + if np := strings.TrimSpace(t.NoticePublicationID); np != "" { + t.Title = "Prior information notice " + np + } else if cf := strings.TrimSpace(t.ContractFolderID); cf != "" { + t.Title = "Prior information notice — procedure " + cf + } + } + return t } +func procurementProjectIDFromPriorInformationNotice(pin *PriorInformationNotice) string { + if pin == nil { + return "" + } + if pin.ProcurementProject != nil { + if id := strings.TrimSpace(pin.ProcurementProject.ID); id != "" { + return id + } + } + if lot := pin.firstLot(); lot != nil && lot.ProcurementProject != nil { + return strings.TrimSpace(lot.ProcurementProject.ID) + } + return "" +} + // mapToTender maps a TED ContractNotice to tender entity func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *notice.Notice { now := time.Now().Unix() t := ¬ice.Notice{ - ContractNoticeID: cn.ID, - ContractFolderID: cn.ContractFolderID, - ProcurementProjectID: procurementProjectIDFromContractNotice(cn), - NoticeTypeCode: strings.TrimSpace(cn.GetNoticeType()), - FormType: cn.GetNoticeFormType(), - NoticeSubTypeCode: cn.GetNoticeSubType(), - NoticeIdentifier: cn.ID, - NoticeVersion: cn.VersionID, - NoticeDispatchAt: ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime), - ESenderDispatchAt: TransmissionDispatchUnix(cn.Extensions), - NoticeLanguageCode: cn.NoticeLanguageCode, - IssueDate: ParseDateToUnix(cn.IssueDate), - IssueTime: ParseTimeToUnix(cn.IssueTime), - GazetteID: cn.GetOJSID(), - Title: cn.GetProcurementTitle(), - Description: cn.GetProcurementDescription(), - ProcurementTypeCode: cn.GetContractNature(), - ProcedureCode: cn.GetProcedureCode(), - MainClassification: cn.GetMainClassification(), - PlaceOfPerformance: cn.GetPlaceOfPerformance(), - DocumentURI: cn.GetDocumentURI(), - Status: notice.TenderStatusActive, - Source: notice.TenderSourceTEDScraper, - SourceFileURL: sourceFileURL, - SourceFileName: sourceFileName, + ContractNoticeID: cn.ID, + ContractFolderID: cn.ContractFolderID, + ProcurementProjectID: procurementProjectIDFromContractNotice(cn), + NoticeTypeCode: strings.TrimSpace(cn.GetNoticeType()), + FormType: cn.GetNoticeFormType(), + NoticeSubTypeCode: cn.GetNoticeSubType(), + NoticeIdentifier: cn.ID, + NoticeVersion: cn.VersionID, + NoticeDispatchAt: ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime), + ESenderDispatchAt: TransmissionDispatchUnix(cn.Extensions), + NoticeLanguageCode: cn.NoticeLanguageCode, + IssueDate: ParseDateToUnix(cn.IssueDate), + IssueTime: ParseTimeToUnix(cn.IssueTime), + GazetteID: cn.GetOJSID(), + Title: cn.GetProcurementTitle(), + Description: cn.GetProcurementDescription(), + ProcurementTypeCode: cn.GetContractNature(), + ProcedureCode: cn.GetProcedureCode(), + MainClassification: cn.GetMainClassification(), + MainClassificationDescription: cn.GetMainClassificationDescription(), + PlaceOfPerformance: cn.GetPlaceOfPerformance(), + DocumentURI: cn.GetDocumentURI(), + Status: notice.TenderStatusActive, + Source: notice.TenderSourceTEDScraper, + SourceFileURL: sourceFileURL, + SourceFileName: sourceFileName, ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, @@ -298,29 +448,30 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so now := time.Now().Unix() t := ¬ice.Notice{ - ContractNoticeID: can.ID, - ContractFolderID: can.ContractFolderID, - ProcurementProjectID: procurementProjectIDFromContractAwardNotice(can), - NoticeTypeCode: strings.TrimSpace(can.GetNoticeType()), - FormType: can.GetNoticeFormType(), - NoticeSubTypeCode: can.GetNoticeSubType(), - NoticeIdentifier: can.ID, - NoticeVersion: can.VersionID, - NoticeDispatchAt: ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime), - ESenderDispatchAt: TransmissionDispatchUnix(can.Extensions), - NoticeLanguageCode: can.NoticeLanguageCode, - IssueDate: ParseDateToUnix(can.IssueDate), - IssueTime: ParseTimeToUnix(can.IssueTime), - GazetteID: can.GetOJSID(), - Title: can.GetProcurementTitle(), - Description: can.GetProcurementDescription(), - ProcurementTypeCode: can.GetContractNature(), - ProcedureCode: can.GetProcedureCode(), - MainClassification: can.GetMainClassification(), - Status: notice.TenderStatusAwarded, // Award notices are for completed tenders - Source: notice.TenderSourceTEDScraper, - SourceFileURL: sourceFileURL, - SourceFileName: sourceFileName, + ContractNoticeID: can.ID, + ContractFolderID: can.ContractFolderID, + ProcurementProjectID: procurementProjectIDFromContractAwardNotice(can), + NoticeTypeCode: strings.TrimSpace(can.GetNoticeType()), + FormType: can.GetNoticeFormType(), + NoticeSubTypeCode: can.GetNoticeSubType(), + NoticeIdentifier: can.ID, + NoticeVersion: can.VersionID, + NoticeDispatchAt: ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime), + ESenderDispatchAt: TransmissionDispatchUnix(can.Extensions), + NoticeLanguageCode: can.NoticeLanguageCode, + IssueDate: ParseDateToUnix(can.IssueDate), + IssueTime: ParseTimeToUnix(can.IssueTime), + GazetteID: can.GetOJSID(), + Title: can.GetProcurementTitle(), + Description: can.GetProcurementDescription(), + ProcurementTypeCode: can.GetContractNature(), + ProcedureCode: can.GetProcedureCode(), + MainClassification: can.GetMainClassification(), + MainClassificationDescription: can.GetMainClassificationDescription(), + Status: notice.TenderStatusAwarded, // Award notices are for completed tenders + Source: notice.TenderSourceTEDScraper, + SourceFileURL: sourceFileURL, + SourceFileName: sourceFileName, ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, diff --git a/ted/model.go b/ted/model.go index 16987aa..2bfd7cc 100644 --- a/ted/model.go +++ b/ted/model.go @@ -55,6 +55,14 @@ type TEDDocument interface { Validate() error } +// ublEformsExtension returns embedded eForms extension data from UBL extensions (shared across notice root types). +func ublEformsExtension(ext *UBLExtensions) *EformsExtension { + if ext == nil || ext.UBLExtension == nil || ext.UBLExtension.ExtensionContent == nil { + return nil + } + return ext.UBLExtension.ExtensionContent.EformsExtension +} + // PriorInformationNotice represents a prior information notice type PriorInformationNotice struct { XMLName xml.Name `xml:"PriorInformationNotice"` @@ -73,7 +81,7 @@ type PriorInformationNotice struct { IssueTime string `xml:"IssueTime"` VersionID string `xml:"VersionID"` RegulatoryDomain string `xml:"RegulatoryDomain"` - NoticeTypeCode string `xml:"NoticeTypeCode"` + NoticeTypeCode NoticeTypeCodeField `xml:"NoticeTypeCode,omitempty"` NoticeLanguageCode string `xml:"NoticeLanguageCode"` Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"` ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"` @@ -90,7 +98,274 @@ func (pin PriorInformationNotice) GetID() string { } func (pin PriorInformationNotice) GetNoticeType() string { - return pin.NoticeTypeCode + return strings.TrimSpace(pin.NoticeTypeCode.Text) +} + +// GetNoticeFormType returns the form type from listName on cbc:NoticeTypeCode (BT-03), e.g. "planning". +func (pin PriorInformationNotice) GetNoticeFormType() string { + return strings.TrimSpace(pin.NoticeTypeCode.ListName) +} + +func (pin PriorInformationNotice) firstLot() *ProcurementProjectLot { + if len(pin.ProcurementProjectLot) == 0 { + return nil + } + return &pin.ProcurementProjectLot[0] +} + +// GetPublicationInfo returns publication information if available. +func (pin PriorInformationNotice) GetPublicationInfo() *Publication { + if ef := ublEformsExtension(pin.Extensions); ef != nil { + return ef.Publication + } + return nil +} + +// GetOrganizations returns organizations from extensions. +func (pin PriorInformationNotice) GetOrganizations() *Organizations { + if ef := ublEformsExtension(pin.Extensions); ef != nil { + return ef.Organizations + } + return nil +} + +// GetNoticeSubType returns the notice subtype code from extensions. +func (pin PriorInformationNotice) GetNoticeSubType() string { + if ef := ublEformsExtension(pin.Extensions); ef != nil && ef.NoticeSubType != nil { + return ef.NoticeSubType.SubTypeCode + } + return "" +} + +// GetNoticePublicationID returns the notice publication ID from extensions. +func (pin PriorInformationNotice) GetNoticePublicationID() string { + pub := pin.GetPublicationInfo() + if pub != nil { + return pub.NoticePublicationID + } + return "" +} + +// GetOJSID returns the OJS gazette ID from extensions. +func (pin PriorInformationNotice) GetOJSID() string { + pub := pin.GetPublicationInfo() + if pub != nil { + return pub.GazetteID + } + return "" +} + +// GetPublicationDate returns the publication date from extensions. +func (pin PriorInformationNotice) GetPublicationDate() string { + pub := pin.GetPublicationInfo() + if pub != nil { + return pub.PublicationDate + } + return "" +} + +// GetOrganizationByID returns an organization by its ID. +func (pin PriorInformationNotice) GetOrganizationByID(orgID string) *Organization { + orgs := pin.GetOrganizations() + if orgs == nil { + return nil + } + for _, org := range orgs.Organization { + if org.Company != nil && + org.Company.PartyIdentification != nil && + org.Company.PartyIdentification.ID == orgID { + return &org + } + } + return nil +} + +// GetBuyerID returns the buyer organization ID from contracting party. +func (pin PriorInformationNotice) GetBuyerID() string { + if pin.ContractingParty != nil && + pin.ContractingParty.Party != nil && + pin.ContractingParty.Party.PartyIdentification != nil { + return pin.ContractingParty.Party.PartyIdentification.ID + } + return "" +} + +// GetReviewOrganizationID returns the review organization ID from appeal terms. +func (pin PriorInformationNotice) GetReviewOrganizationID() string { + if pin.firstLot() != nil && + pin.firstLot().TenderingTerms != nil && + pin.firstLot().TenderingTerms.AppealTerms != nil && + pin.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty != nil && + pin.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil { + return pin.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID + } + return "" +} + +// GetProcedureCode returns the type of procedure. +func (pin PriorInformationNotice) GetProcedureCode() string { + if pin.TenderingProcess != nil { + return pin.TenderingProcess.ProcedureCode + } + return "" +} + +// GetProcurementTitle returns the procurement project title. +func (pin PriorInformationNotice) GetProcurementTitle() string { + if pin.ProcurementProject != nil && strings.TrimSpace(pin.ProcurementProject.Name) != "" { + return pin.ProcurementProject.Name + } + if pin.firstLot() != nil && pin.firstLot().ProcurementProject != nil { + return pin.firstLot().ProcurementProject.Name + } + return "" +} + +// GetProcurementDescription returns the procurement project description. +func (pin PriorInformationNotice) GetProcurementDescription() string { + if pin.ProcurementProject != nil && strings.TrimSpace(pin.ProcurementProject.Description) != "" { + return pin.ProcurementProject.Description + } + if pin.firstLot() != nil && pin.firstLot().ProcurementProject != nil { + return pin.firstLot().ProcurementProject.Description + } + return "" +} + +// GetContractNature returns the main nature of the contract. +func (pin PriorInformationNotice) GetContractNature() string { + if pin.ProcurementProject != nil && strings.TrimSpace(pin.ProcurementProject.ProcurementTypeCode) != "" { + return pin.ProcurementProject.ProcurementTypeCode + } + if pin.firstLot() != nil && pin.firstLot().ProcurementProject != nil { + return pin.firstLot().ProcurementProject.ProcurementTypeCode + } + return "" +} + +// GetMainClassification returns the main CPV classification code. +func (pin PriorInformationNotice) GetMainClassification() string { + if pin.ProcurementProject != nil && + pin.ProcurementProject.MainCommodityClassification != nil { + return pin.ProcurementProject.MainCommodityClassification.ItemClassificationCode + } + if lot := pin.firstLot(); lot != nil && lot.ProcurementProject != nil && + lot.ProcurementProject.MainCommodityClassification != nil { + return lot.ProcurementProject.MainCommodityClassification.ItemClassificationCode + } + return "" +} + +// GetMainClassificationDescription returns the CPV / item classification description when present in the notice XML. +func (pin PriorInformationNotice) GetMainClassificationDescription() string { + if pin.ProcurementProject != nil { + if s := mainCommodityClassificationLabel(pin.ProcurementProject.MainCommodityClassification); s != "" { + return s + } + } + if lot := pin.firstLot(); lot != nil && lot.ProcurementProject != nil { + return mainCommodityClassificationLabel(lot.ProcurementProject.MainCommodityClassification) + } + return "" +} + +// GetPlaceOfPerformance returns the place of performance. +func (pin PriorInformationNotice) GetPlaceOfPerformance() string { + if pin.ProcurementProject != nil && + pin.ProcurementProject.RealizedLocation != nil && + pin.ProcurementProject.RealizedLocation.Address != nil { + return pin.ProcurementProject.RealizedLocation.Address.Region + } + if lot := pin.firstLot(); lot != nil && lot.ProcurementProject != nil && + lot.ProcurementProject.RealizedLocation != nil && + lot.ProcurementProject.RealizedLocation.Address != nil { + return lot.ProcurementProject.RealizedLocation.Address.Region + } + return "" +} + +// GetSelectionCriteria returns selection criteria from extensions. +func (pin PriorInformationNotice) GetSelectionCriteria() []SelectionCriteria { + if ef := ublEformsExtension(pin.Extensions); ef != nil { + return ef.SelectionCriteria + } + return nil +} + +// GetOfficialLanguages returns official languages from tender terms extensions. +func (pin PriorInformationNotice) GetOfficialLanguages() []string { + if pin.firstLot() != nil && + pin.firstLot().TenderingTerms != nil && + len(pin.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 && + pin.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil && + pin.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil && + pin.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil && + pin.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil && + pin.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil { + + langs := pin.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language + languages := make([]string, len(langs)) + for i, lang := range langs { + languages[i] = lang.ID + } + return languages + } + return nil +} + +// GetDocumentURI returns the document URI from call for tenders document reference. +func (pin PriorInformationNotice) GetDocumentURI() string { + if pin.firstLot() != nil && + pin.firstLot().TenderingTerms != nil && + len(pin.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 && + pin.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil && + pin.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil { + return pin.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI + } + return "" +} + +// GetTenderDeadline returns the tender submission deadline. +func (pin PriorInformationNotice) GetTenderDeadline() string { + if pin.firstLot() != nil && + pin.firstLot().TenderingProcess != nil && + pin.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil { + return pin.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate + } + return "" +} + +// GetBudget returns the estimated contract amount and currency. +func (pin PriorInformationNotice) GetBudget() (amount string, currency string) { + if pin.firstLot() != nil && + pin.firstLot().ProcurementProject != nil && + pin.firstLot().ProcurementProject.RequestedTenderTotal != nil { + contractAmount := pin.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount + return contractAmount.Text, contractAmount.CurrencyID + } + if pin.ProcurementProject != nil && pin.ProcurementProject.RequestedTenderTotal != nil { + a := pin.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount + return a.Text, a.CurrencyID + } + return "", "" +} + +// GetDuration returns the contract duration and unit. +func (pin PriorInformationNotice) GetDuration() (duration string, unit string) { + if pin.firstLot() != nil && + pin.firstLot().ProcurementProject != nil && + pin.firstLot().ProcurementProject.PlannedPeriod != nil && + pin.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil { + durationMeasure := pin.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure + return durationMeasure.Text, durationMeasure.UnitCode + } + if pin.ProcurementProject != nil && + pin.ProcurementProject.PlannedPeriod != nil && + pin.ProcurementProject.PlannedPeriod.DurationMeasure != nil { + dm := pin.ProcurementProject.PlannedPeriod.DurationMeasure + return dm.Text, dm.UnitCode + } + return "", "" } func (pin PriorInformationNotice) GetLanguage() string { @@ -1288,6 +1563,9 @@ type CurrencyText struct { type MainCommodityClassification struct { XMLName xml.Name `xml:"MainCommodityClassification"` ItemClassificationCode string `xml:"ItemClassificationCode"` + ItemClassificationName string `xml:"ItemClassificationName,omitempty"` + ItemClassificationDesc string `xml:"ItemClassificationDescription,omitempty"` + Name string `xml:"Name,omitempty"` } // AdditionalCommodityClassification contains additional commodity classification @@ -1451,34 +1729,24 @@ func (cn ContractNotice) GetLanguage() string { // GetPublicationInfo returns publication information if available func (cn ContractNotice) GetPublicationInfo() *Publication { - if cn.Extensions != nil && - cn.Extensions.UBLExtension != nil && - cn.Extensions.UBLExtension.ExtensionContent != nil && - cn.Extensions.UBLExtension.ExtensionContent.EformsExtension != nil { - return cn.Extensions.UBLExtension.ExtensionContent.EformsExtension.Publication + if ef := ublEformsExtension(cn.Extensions); ef != nil { + return ef.Publication } return nil } // GetOrganizations returns organizations from extensions func (cn ContractNotice) GetOrganizations() *Organizations { - if cn.Extensions != nil && - cn.Extensions.UBLExtension != nil && - cn.Extensions.UBLExtension.ExtensionContent != nil && - cn.Extensions.UBLExtension.ExtensionContent.EformsExtension != nil { - return cn.Extensions.UBLExtension.ExtensionContent.EformsExtension.Organizations + if ef := ublEformsExtension(cn.Extensions); ef != nil { + return ef.Organizations } return nil } // GetNoticeSubType returns the notice subtype code from extensions func (cn ContractNotice) GetNoticeSubType() string { - if cn.Extensions != nil && - cn.Extensions.UBLExtension != nil && - cn.Extensions.UBLExtension.ExtensionContent != nil && - cn.Extensions.UBLExtension.ExtensionContent.EformsExtension != nil && - cn.Extensions.UBLExtension.ExtensionContent.EformsExtension.NoticeSubType != nil { - return cn.Extensions.UBLExtension.ExtensionContent.EformsExtension.NoticeSubType.SubTypeCode + if ef := ublEformsExtension(cn.Extensions); ef != nil && ef.NoticeSubType != nil { + return ef.NoticeSubType.SubTypeCode } return "" } @@ -1592,6 +1860,26 @@ func (cn ContractNotice) GetMainClassification() string { return "" } +func mainCommodityClassificationLabel(m *MainCommodityClassification) string { + if m == nil { + return "" + } + for _, s := range []string{m.ItemClassificationName, m.ItemClassificationDesc, m.Name} { + if v := strings.TrimSpace(s); v != "" { + return v + } + } + return "" +} + +// GetMainClassificationDescription returns the CPV / item classification description when present in the notice XML. +func (cn ContractNotice) GetMainClassificationDescription() string { + if cn.ProcurementProject == nil { + return "" + } + return mainCommodityClassificationLabel(cn.ProcurementProject.MainCommodityClassification) +} + // GetPlaceOfPerformance returns the place of performance func (cn ContractNotice) GetPlaceOfPerformance() string { if cn.ProcurementProject != nil && @@ -1612,11 +1900,8 @@ func (cn ContractNotice) GetLotID() string { // GetSelectionCriteria returns selection criteria from extensions func (cn ContractNotice) GetSelectionCriteria() []SelectionCriteria { - if cn.Extensions != nil && - cn.Extensions.UBLExtension != nil && - cn.Extensions.UBLExtension.ExtensionContent != nil && - cn.Extensions.UBLExtension.ExtensionContent.EformsExtension != nil { - return cn.Extensions.UBLExtension.ExtensionContent.EformsExtension.SelectionCriteria + if ef := ublEformsExtension(cn.Extensions); ef != nil { + return ef.SelectionCriteria } return nil } @@ -1913,6 +2198,19 @@ func (can ContractAwardNotice) GetMainClassification() string { return "" } +// GetMainClassificationDescription returns the CPV / item classification description when present in the notice XML. +func (can ContractAwardNotice) GetMainClassificationDescription() string { + if can.ProcurementProject != nil { + if s := mainCommodityClassificationLabel(can.ProcurementProject.MainCommodityClassification); s != "" { + return s + } + } + if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil { + return mainCommodityClassificationLabel(lot.ProcurementProject.MainCommodityClassification) + } + return "" +} + // GetTotalAwardValue returns the total award amount and currency func (can ContractAwardNotice) GetTotalAwardValue() (amount string, currency string) { noticeResult := can.GetNoticeResult()