package ted import ( "strings" "time" "tm/internal/notice" ) func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot { if lot == nil { return notice.ProcurementLot{} } out := notice.ProcurementLot{LotID: lot.ID} if lot.ProcurementProject == nil { return out } pp := lot.ProcurementProject out.Title = pp.Name out.Description = pp.Description out.MainNatureOfContract = pp.ProcurementTypeCode if pp.MainCommodityClassification != nil { mc := pp.MainCommodityClassification out.MainClassification = strings.TrimSpace(mc.ItemClassificationCode) out.MainClassificationDescription = mainCommodityClassificationLabel(mc) } for _, ac := range pp.AdditionalCommodityClassification { c := strings.TrimSpace(ac.ItemClassificationCode) if c != "" { out.AdditionalClassifications = append(out.AdditionalClassifications, c) } } if pp.RequestedTenderTotal != nil { amount, currencyID := pp.RequestedTenderTotal.AmountAndCurrency() if v, ok := ParseEuropeanAmount(amount); ok { out.EstimatedValue = v } out.Currency = currencyID } if pp.PlannedPeriod != nil && pp.PlannedPeriod.DurationMeasure != nil { dm := pp.PlannedPeriod.DurationMeasure txt := strings.TrimSpace(dm.Text) unit := strings.TrimSpace(dm.UnitCode) switch unit { case "MONTH": out.Duration = txt + " Months" case "DAY": out.Duration = txt + " Days" case "WEEK": out.Duration = txt + " Weeks" case "YEAR": out.Duration = txt + " Years" default: if unit != "" && txt != "" { out.Duration = txt + " " + unit } else { out.Duration = txt } } } return out } // mapToTenderFromParsedDoc maps a parsed TED document to tender entity func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice { switch parsedDoc.Type { case DocumentTypeContractNotice: if parsedDoc.ContractNotice == nil { s.logger.Error("ContractNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapToTender(parsedDoc.ContractNotice, sourceFileURL, sourceFileName) case DocumentTypeContractAwardNotice: if parsedDoc.ContractAwardNotice == nil { s.logger.Error("ContractAwardNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapContractAwardNoticeToTender(parsedDoc.ContractAwardNotice, sourceFileURL, sourceFileName) case DocumentTypePriorInformationNotice: if parsedDoc.PriorInformationNotice == nil { s.logger.Error("PriorInformationNotice is nil", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } return s.mapPriorInformationNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName) default: s.logger.Error("Unknown document type for mapping", map[string]interface{}{ "document_type": string(parsedDoc.Type), }) return nil } } // mapPriorInformationNoticeToTender maps a TED PriorInformationNotice to the same notice shape as a contract notice, // so procedure keys (ContractFolderID, NoticePublicationID, ProcurementProjectID) merge into one tender. func (s *TEDScraper) mapPriorInformationNoticeToTender(pin *PriorInformationNotice, sourceFileURL, sourceFileName string) *notice.Notice { if pin == nil { return nil } now := time.Now().Unix() t := ¬ice.Notice{ ContractNoticeID: pin.ID, ContractFolderID: pin.ContractFolderID, ProcurementProjectID: procurementProjectIDFromPriorInformationNotice(pin), NoticeTypeCode: strings.TrimSpace(pin.GetNoticeType()), FormType: pin.GetNoticeFormType(), NoticeSubTypeCode: pin.GetNoticeSubType(), NoticeIdentifier: pin.ID, NoticeVersion: pin.VersionID, NoticeDispatchAt: ParseIssueDateTimeToUnix(pin.IssueDate, pin.IssueTime), ESenderDispatchAt: TransmissionDispatchUnix(pin.Extensions), NoticeLanguageCode: pin.NoticeLanguageCode, IssueDate: ParseDateToUnix(pin.IssueDate), IssueTime: ParseTimeToUnix(pin.IssueTime), GazetteID: pin.GetOJSID(), Title: pin.GetProcurementTitle(), Description: pin.GetProcurementDescription(), ProcurementTypeCode: pin.GetContractNature(), ProcedureCode: pin.GetProcedureCode(), MainClassification: pin.GetMainClassification(), MainClassificationDescription: pin.GetMainClassificationDescription(), PlaceOfPerformance: pin.GetPlaceOfPerformance(), DocumentURI: pin.GetDocumentURI(), Status: notice.TenderStatusActive, Source: notice.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", }, } if pubInfo := pin.GetPublicationInfo(); pubInfo != nil { t.NoticePublicationID = pubInfo.NoticePublicationID t.PublicationDate = ParseDateToUnix(pubInfo.PublicationDate) t.TenderURL = GenerateTenderURL(pubInfo.NoticePublicationID) } if pin.ProcurementProject != nil && pin.ProcurementProject.AdditionalCommodityClassification != nil { for _, additionalClass := range pin.ProcurementProject.AdditionalCommodityClassification { if additionalClass.ItemClassificationCode != "" { t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode) } } } if amount, currency := pin.GetBudget(); amount != "" { if value, ok := ParseEuropeanAmount(amount); ok { t.EstimatedValue = value t.Currency = currency } } if pin.ContractingParty != nil { t.BuyerProfileURL = pin.ContractingParty.BuyerProfileURI } for i := range pin.ProcurementProjectLot { t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&pin.ProcurementProjectLot[i])) } backfillNoticeEstimatedValueFromLots(t) if duration, unit := pin.GetDuration(); duration != "" { t.Duration = duration t.DurationUnit = unit } if deadline := pin.GetTenderDeadline(); deadline != "" { t.TenderDeadline = ParseDateToUnix(deadline) if deadlineTime, err := ParseDate(deadline); err == nil { applicationDeadline := CalculateApplicationDeadline(deadlineTime) t.ApplicationDeadline = applicationDeadline.Unix() } } if pubDate, err := ParseDate(pin.GetPublicationDate()); err == nil { submissionDeadline := CalculateSubmissionDeadline(pubDate) t.SubmissionDeadline = submissionDeadline.Unix() } if lot := pin.firstLot(); lot != nil && lot.TenderingTerms != nil { if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil { if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" { if strings.HasPrefix(endpointID, "http") { t.SubmissionURL = endpointID } else { t.SubmissionURL = "https://" + endpointID } } } } if pin.ProcurementProject != nil && pin.ProcurementProject.RealizedLocation != nil { if addr := pin.ProcurementProject.RealizedLocation.Address; addr != nil { t.CityName = addr.CityName t.PostalCode = addr.PostalZone t.RegionCode = addr.CountrySubentityCode if addr.Country != nil { t.CountryCode = addr.Country.IdentificationCode } } } else if lot := pin.firstLot(); lot != nil && lot.ProcurementProject != nil && lot.ProcurementProject.RealizedLocation != nil { if addr := lot.ProcurementProject.RealizedLocation.Address; addr != nil { t.CityName = addr.CityName t.PostalCode = addr.PostalZone t.RegionCode = addr.CountrySubentityCode if addr.Country != nil { t.CountryCode = addr.Country.IdentificationCode } } } buyerID := pin.GetBuyerID() if buyerID != "" { if buyerOrg := pin.GetOrganizationByID(buyerID); buyerOrg != nil { t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer") } } reviewID := pin.GetReviewOrganizationID() if reviewID != "" { if reviewOrg := pin.GetOrganizationByID(reviewID); reviewOrg != nil { t.ReviewOrganization = s.mapOrganization(reviewOrg, "review") } } if orgs := pin.GetOrganizations(); orgs != nil { for _, org := range orgs.Organization { mappedOrg := s.mapOrganization(&org, "") if mappedOrg != nil { t.Organizations = append(t.Organizations, *mappedOrg) } } } if criteria := pin.GetSelectionCriteria(); criteria != nil { for _, criterion := range criteria { t.SelectionCriteria = append(t.SelectionCriteria, notice.SelectionCriterion{ TypeCode: criterion.CriterionTypeCode, Description: criterion.Description, LanguageID: criterion.LanguageID, }) } } if languages := pin.GetOfficialLanguages(); languages != nil { t.OfficialLanguages = languages } if strings.TrimSpace(t.Title) == "" { if np := strings.TrimSpace(t.NoticePublicationID); np != "" { t.Title = "Prior information notice " + np } else if cf := strings.TrimSpace(t.ContractFolderID); cf != "" { t.Title = "Prior information notice — procedure " + cf } } return t } func procurementProjectIDFromPriorInformationNotice(pin *PriorInformationNotice) string { if pin == nil { return "" } if pin.ProcurementProject != nil { if id := strings.TrimSpace(pin.ProcurementProject.ID); id != "" { return id } } if lot := pin.firstLot(); lot != nil && lot.ProcurementProject != nil { return strings.TrimSpace(lot.ProcurementProject.ID) } return "" } // mapToTender maps a TED ContractNotice to tender entity func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *notice.Notice { now := time.Now().Unix() t := ¬ice.Notice{ ContractNoticeID: cn.ID, ContractFolderID: cn.ContractFolderID, ProcurementProjectID: procurementProjectIDFromContractNotice(cn), NoticeTypeCode: strings.TrimSpace(cn.GetNoticeType()), FormType: cn.GetNoticeFormType(), NoticeSubTypeCode: cn.GetNoticeSubType(), NoticeIdentifier: cn.ID, NoticeVersion: cn.VersionID, NoticeDispatchAt: ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime), ESenderDispatchAt: TransmissionDispatchUnix(cn.Extensions), NoticeLanguageCode: cn.NoticeLanguageCode, IssueDate: ParseDateToUnix(cn.IssueDate), IssueTime: ParseTimeToUnix(cn.IssueTime), GazetteID: cn.GetOJSID(), Title: cn.GetProcurementTitle(), Description: cn.GetProcurementDescription(), ProcurementTypeCode: cn.GetContractNature(), ProcedureCode: cn.GetProcedureCode(), MainClassification: cn.GetMainClassification(), MainClassificationDescription: cn.GetMainClassificationDescription(), PlaceOfPerformance: cn.GetPlaceOfPerformance(), DocumentURI: cn.GetDocumentURI(), Status: notice.TenderStatusActive, Source: notice.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", }, } // Set publication information if pubInfo := cn.GetPublicationInfo(); pubInfo != nil { t.NoticePublicationID = pubInfo.NoticePublicationID t.PublicationDate = ParseDateToUnix(pubInfo.PublicationDate) t.TenderURL = GenerateTenderURL(pubInfo.NoticePublicationID) } // Set additional classifications if cn.ProcurementProject != nil && cn.ProcurementProject.AdditionalCommodityClassification != nil { for _, additionalClass := range cn.ProcurementProject.AdditionalCommodityClassification { if additionalClass.ItemClassificationCode != "" { t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode) } } } // Set estimated value and currency if amount, currency := cn.GetBudget(); amount != "" { if value, ok := ParseEuropeanAmount(amount); ok { t.EstimatedValue = value t.Currency = currency } } if cn.ContractingParty != nil { t.BuyerProfileURL = cn.ContractingParty.BuyerProfileURI } for i := range cn.ProcurementProjectLots { t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&cn.ProcurementProjectLots[i])) } backfillNoticeEstimatedValueFromLots(t) // Set duration information if duration, unit := cn.GetDuration(); duration != "" { t.Duration = duration t.DurationUnit = unit } // Set tender deadline if deadline := cn.GetTenderDeadline(); deadline != "" { t.TenderDeadline = ParseDateToUnix(deadline) if deadlineTime, err := ParseDate(deadline); err == nil { applicationDeadline := CalculateApplicationDeadline(deadlineTime) t.ApplicationDeadline = applicationDeadline.Unix() } } if pubDate, err := ParseDate(cn.GetPublicationDate()); err == nil { submissionDeadline := CalculateSubmissionDeadline(pubDate) t.SubmissionDeadline = submissionDeadline.Unix() } // Set SubmissionURL from TenderingTerms if lot := cn.firstLot(); lot != nil && lot.TenderingTerms != nil { if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil { if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" { if strings.HasPrefix(endpointID, "http") { t.SubmissionURL = endpointID } else { t.SubmissionURL = "https://" + endpointID } } } } // Set location information if cn.ProcurementProject != nil && cn.ProcurementProject.RealizedLocation != nil { if addr := cn.ProcurementProject.RealizedLocation.Address; addr != nil { t.CityName = addr.CityName t.PostalCode = addr.PostalZone t.RegionCode = addr.CountrySubentityCode if addr.Country != nil { t.CountryCode = addr.Country.IdentificationCode } } } // Set buyer organization buyerID := cn.GetBuyerID() if buyerID != "" { if buyerOrg := cn.GetOrganizationByID(buyerID); buyerOrg != nil { t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer") } } // Set review organization reviewID := cn.GetReviewOrganizationID() if reviewID != "" { if reviewOrg := cn.GetOrganizationByID(reviewID); reviewOrg != nil { t.ReviewOrganization = s.mapOrganization(reviewOrg, "review") } } // Set all organizations if orgs := cn.GetOrganizations(); orgs != nil { for _, org := range orgs.Organization { mappedOrg := s.mapOrganization(&org, "") if mappedOrg != nil { t.Organizations = append(t.Organizations, *mappedOrg) } } } // Set selection criteria if criteria := cn.GetSelectionCriteria(); criteria != nil { for _, criterion := range criteria { t.SelectionCriteria = append(t.SelectionCriteria, notice.SelectionCriterion{ TypeCode: criterion.CriterionTypeCode, Description: criterion.Description, LanguageID: criterion.LanguageID, }) } } // Set official languages if languages := cn.GetOfficialLanguages(); languages != nil { t.OfficialLanguages = languages } return t } // mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *notice.Notice { now := time.Now().Unix() t := ¬ice.Notice{ ContractNoticeID: can.ID, ContractFolderID: can.ContractFolderID, ProcurementProjectID: procurementProjectIDFromContractAwardNotice(can), NoticeTypeCode: strings.TrimSpace(can.GetNoticeType()), FormType: can.GetNoticeFormType(), NoticeSubTypeCode: can.GetNoticeSubType(), NoticeIdentifier: can.ID, NoticeVersion: can.VersionID, NoticeDispatchAt: ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime), ESenderDispatchAt: TransmissionDispatchUnix(can.Extensions), NoticeLanguageCode: can.NoticeLanguageCode, IssueDate: ParseDateToUnix(can.IssueDate), IssueTime: ParseTimeToUnix(can.IssueTime), GazetteID: can.GetOJSID(), Title: can.GetProcurementTitle(), Description: can.GetProcurementDescription(), ProcurementTypeCode: can.GetContractNature(), ProcedureCode: can.GetProcedureCode(), MainClassification: can.GetMainClassification(), MainClassificationDescription: can.GetMainClassificationDescription(), Status: notice.TenderStatusAwarded, // Award notices are for completed tenders Source: notice.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", }, } // Set publication information if pubInfo := can.GetPublicationInfo(); pubInfo != nil { t.NoticePublicationID = pubInfo.NoticePublicationID t.PublicationDate = ParseDateToUnix(pubInfo.PublicationDate) t.TenderURL = GenerateTenderURL(pubInfo.NoticePublicationID) } // Set total award value if amount, currency := can.GetTotalAwardValue(); amount != "" { if parsedAmount, ok := ParseEuropeanAmount(amount); ok { t.EstimatedValue = parsedAmount t.AwardedValue = parsedAmount } t.Currency = currency } if ad := can.GetAwardDate(); ad != "" { t.AwardDate = ParseDateToUnix(ad) } if can.ContractingParty != nil { t.BuyerProfileURL = can.ContractingParty.BuyerProfileURI } for i := range can.ProcurementProjectLot { t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&can.ProcurementProjectLot[i])) } backfillNoticeEstimatedValueFromLots(t) // Set location information from main procurement project or first lot var realizedAddr *Address if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil { realizedAddr = can.ProcurementProject.RealizedLocation.Address } else if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil && lot.ProcurementProject.RealizedLocation != nil { realizedAddr = lot.ProcurementProject.RealizedLocation.Address } if realizedAddr != nil { t.CityName = realizedAddr.CityName t.PostalCode = realizedAddr.PostalZone t.RegionCode = realizedAddr.CountrySubentityCode t.PlaceOfPerformance = realizedAddr.Region if realizedAddr.Country != nil { t.CountryCode = realizedAddr.Country.IdentificationCode } } if wOrg := can.GetWinningTenderOrganization(); wOrg != nil { t.WinningTenderer = s.mapOrganization(wOrg, "winner") } if nr := can.GetNoticeResult(); nr != nil { for _, lr := range nr.LotResult { winner := pickWinningLotTender(lr.LotTender) if winner == nil || winner.TenderingParty == nil { continue } org := can.GetOrganizationByID(winner.TenderingParty.ID) aw := notice.Awarded{ LotID: lr.ID, OrganizationID: winner.TenderingParty.ID, AwardDate: t.AwardDate, } if org != nil && org.Company != nil { if org.Company.PartyName != nil { aw.Name = org.Company.PartyName.Name } if org.Company.PartyLegalEntity != nil { aw.CompanyID = org.Company.PartyLegalEntity.CompanyID } } if winner.LegalMonetaryTotal != nil { pa := winner.LegalMonetaryTotal.PayableAmount if v, ok := ParseEuropeanAmount(pa.Text); ok { aw.Amount = v } aw.Currency = pa.CurrencyID } t.AwardedEntities = append(t.AwardedEntities, aw) } } // Set buyer organization buyerID := can.GetBuyerID() if buyerID != "" { if buyerOrg := can.GetOrganizationByID(buyerID); buyerOrg != nil { t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer") } } // Set all organizations if orgs := can.GetOrganizations(); orgs != nil { for _, org := range orgs.Organization { mappedOrg := s.mapOrganization(&org, "") if mappedOrg != nil { t.Organizations = append(t.Organizations, *mappedOrg) } } } return t } func procurementProjectIDFromContractNotice(cn *ContractNotice) string { if cn == nil { return "" } if cn.ProcurementProject != nil { if id := strings.TrimSpace(cn.ProcurementProject.ID); id != "" { return id } } if lot := cn.firstLot(); lot != nil && lot.ProcurementProject != nil { return strings.TrimSpace(lot.ProcurementProject.ID) } return "" } func procurementProjectIDFromContractAwardNotice(can *ContractAwardNotice) string { if can == nil { return "" } if can.ProcurementProject != nil { if id := strings.TrimSpace(can.ProcurementProject.ID); id != "" { return id } } if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil { return strings.TrimSpace(lot.ProcurementProject.ID) } return "" } // mapOrganization maps a TED Organization to tender Organization func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.Organization { if tedOrg == nil || tedOrg.Company == nil { return nil } company := tedOrg.Company org := ¬ice.Organization{ Role: role, } // Set organization ID and name if company.PartyIdentification != nil { org.ID = company.PartyIdentification.ID } if company.PartyName != nil { org.Name = company.PartyName.Name } // Set company ID if company.PartyLegalEntity != nil { org.CompanyID = company.PartyLegalEntity.CompanyID } // Set website org.WebsiteURI = company.WebsiteURI // Set contact information if company.Contact != nil { org.ContactName = company.Contact.Name org.ContactTelephone = company.Contact.Telephone org.ContactEmail = company.Contact.ElectronicMail org.ContactFax = company.Contact.Telefax } // Set address if company.PostalAddress != nil { org.Address = notice.Address{ StreetName: company.PostalAddress.StreetName, CityName: company.PostalAddress.CityName, PostalZone: company.PostalAddress.PostalZone, CountrySubentityCode: company.PostalAddress.CountrySubentityCode, Department: company.PostalAddress.Department, Region: strings.TrimSpace(company.PostalAddress.CountrySubentity), } if company.PostalAddress.Country != nil { org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode } } return org } func backfillNoticeEstimatedValueFromLots(t *notice.Notice) { if t == nil || t.EstimatedValue > 0 { return } amounts := make([]valuedAmount, 0, len(t.ProcurementLots)) for _, lot := range t.ProcurementLots { if lot.EstimatedValue <= 0 { continue } amounts = append(amounts, valuedAmount{ value: lot.EstimatedValue, currency: lot.Currency, }) } total, currency := aggregateSameCurrencyAmounts(amounts) if total <= 0 { return } t.EstimatedValue = total if strings.TrimSpace(t.Currency) == "" { t.Currency = currency } }