Mazyar
483 lines
15 KiB
Go
483 lines
15 KiB
Go
package ted
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
"tm/internal/notice"
|
|
)
|
|
|
|
func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot {
|
|
if lot == nil {
|
|
return notice.ProcurementLot{}
|
|
}
|
|
out := notice.ProcurementLot{LotID: lot.ID}
|
|
if lot.ProcurementProject == nil {
|
|
return out
|
|
}
|
|
pp := lot.ProcurementProject
|
|
out.Title = pp.Name
|
|
out.Description = pp.Description
|
|
out.MainNatureOfContract = pp.ProcurementTypeCode
|
|
if pp.MainCommodityClassification != nil {
|
|
out.MainClassification = strings.TrimSpace(pp.MainCommodityClassification.ItemClassificationCode)
|
|
}
|
|
for _, ac := range pp.AdditionalCommodityClassification {
|
|
c := strings.TrimSpace(ac.ItemClassificationCode)
|
|
if c != "" {
|
|
out.AdditionalClassifications = append(out.AdditionalClassifications, c)
|
|
}
|
|
}
|
|
if pp.RequestedTenderTotal != nil {
|
|
a := pp.RequestedTenderTotal.EstimatedOverallContractAmount
|
|
if v, ok := ParseEuropeanAmount(a.Text); ok {
|
|
out.EstimatedValue = v
|
|
}
|
|
out.Currency = a.CurrencyID
|
|
}
|
|
if pp.PlannedPeriod != nil && pp.PlannedPeriod.DurationMeasure != nil {
|
|
dm := pp.PlannedPeriod.DurationMeasure
|
|
txt := strings.TrimSpace(dm.Text)
|
|
unit := strings.TrimSpace(dm.UnitCode)
|
|
switch unit {
|
|
case "MONTH":
|
|
out.Duration = txt + " Months"
|
|
case "DAY":
|
|
out.Duration = txt + " Days"
|
|
case "WEEK":
|
|
out.Duration = txt + " Weeks"
|
|
case "YEAR":
|
|
out.Duration = txt + " Years"
|
|
default:
|
|
if unit != "" && txt != "" {
|
|
out.Duration = txt + " " + unit
|
|
} else {
|
|
out.Duration = txt
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// mapToTenderFromParsedDoc maps a parsed TED document to tender entity
|
|
func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice {
|
|
switch parsedDoc.Type {
|
|
case DocumentTypeContractNotice:
|
|
if parsedDoc.ContractNotice == nil {
|
|
s.logger.Error("ContractNotice is nil", map[string]interface{}{
|
|
"document_type": string(parsedDoc.Type),
|
|
})
|
|
return nil
|
|
}
|
|
return s.mapToTender(parsedDoc.ContractNotice, sourceFileURL, sourceFileName)
|
|
|
|
case DocumentTypeContractAwardNotice:
|
|
if parsedDoc.ContractAwardNotice == nil {
|
|
s.logger.Error("ContractAwardNotice is nil", map[string]interface{}{
|
|
"document_type": string(parsedDoc.Type),
|
|
})
|
|
return nil
|
|
}
|
|
return s.mapContractAwardNoticeToTender(parsedDoc.ContractAwardNotice, sourceFileURL, sourceFileName)
|
|
|
|
case DocumentTypePriorInformationNotice:
|
|
if parsedDoc.PriorInformationNotice == nil {
|
|
s.logger.Error("PriorInformationNotice is nil", map[string]interface{}{
|
|
"document_type": string(parsedDoc.Type),
|
|
})
|
|
return nil
|
|
}
|
|
return s.mapGenericNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName)
|
|
default:
|
|
s.logger.Error("Unknown document type for mapping", map[string]interface{}{
|
|
"document_type": string(parsedDoc.Type),
|
|
})
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// mapGenericNoticeToTender maps any TED document type to tender entity using the common interface
|
|
func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *notice.Notice {
|
|
if doc == nil {
|
|
s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{})
|
|
return nil
|
|
}
|
|
|
|
now := time.Now().Unix()
|
|
|
|
t := ¬ice.Notice{
|
|
ContractNoticeID: doc.GetID(),
|
|
ContractFolderID: "", // Not available in generic interface
|
|
NoticeTypeCode: doc.GetNoticeType(),
|
|
NoticeSubTypeCode: "", // Not available in generic interface
|
|
NoticeLanguageCode: doc.GetLanguage(),
|
|
IssueDate: now, // TODO: Parse actual date from document
|
|
IssueTime: now, // TODO: Parse actual time from document
|
|
GazetteID: "", // Not available in generic interface
|
|
NoticePublicationID: "", // Not available in generic interface
|
|
PublicationDate: now, // TODO: Parse actual publication date
|
|
|
|
// Basic tender information
|
|
Title: "Generic Notice " + doc.GetNoticeType(),
|
|
Description: "Processed from " + doc.GetNoticeType() + " document",
|
|
Status: notice.TenderStatusActive,
|
|
Source: notice.TenderSourceTEDScraper,
|
|
SourceFileURL: sourceFileURL,
|
|
SourceFileName: sourceFileName,
|
|
ProcessingMetadata: notice.ProcessingMetadata{
|
|
ScrapedAt: now,
|
|
ProcessedAt: now,
|
|
ProcessingVersion: "1.0",
|
|
},
|
|
}
|
|
|
|
return t
|
|
}
|
|
|
|
// mapToTender maps a TED ContractNotice to tender entity
|
|
func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *notice.Notice {
|
|
now := time.Now().Unix()
|
|
|
|
t := ¬ice.Notice{
|
|
ContractNoticeID: cn.ID,
|
|
ContractFolderID: cn.ContractFolderID,
|
|
NoticeTypeCode: strings.TrimSpace(cn.GetNoticeType()),
|
|
FormType: cn.GetNoticeFormType(),
|
|
NoticeSubTypeCode: cn.GetNoticeSubType(),
|
|
NoticeIdentifier: cn.ID,
|
|
NoticeVersion: cn.VersionID,
|
|
NoticeDispatchAt: ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime),
|
|
ESenderDispatchAt: TransmissionDispatchUnix(cn.Extensions),
|
|
NoticeLanguageCode: cn.NoticeLanguageCode,
|
|
IssueDate: ParseDateToUnix(cn.IssueDate),
|
|
IssueTime: ParseTimeToUnix(cn.IssueTime),
|
|
GazetteID: cn.GetOJSID(),
|
|
Title: cn.GetProcurementTitle(),
|
|
Description: cn.GetProcurementDescription(),
|
|
ProcurementTypeCode: cn.GetContractNature(),
|
|
ProcedureCode: cn.GetProcedureCode(),
|
|
MainClassification: cn.GetMainClassification(),
|
|
PlaceOfPerformance: cn.GetPlaceOfPerformance(),
|
|
DocumentURI: cn.GetDocumentURI(),
|
|
Status: notice.TenderStatusActive,
|
|
Source: notice.TenderSourceTEDScraper,
|
|
SourceFileURL: sourceFileURL,
|
|
SourceFileName: sourceFileName,
|
|
ProcessingMetadata: notice.ProcessingMetadata{
|
|
ScrapedAt: now,
|
|
ProcessedAt: now,
|
|
ProcessingVersion: "1.0",
|
|
},
|
|
}
|
|
|
|
// Set publication information
|
|
if pubInfo := cn.GetPublicationInfo(); pubInfo != nil {
|
|
t.NoticePublicationID = pubInfo.NoticePublicationID
|
|
t.PublicationDate = ParseDateToUnix(pubInfo.PublicationDate)
|
|
t.TenderURL = GenerateTenderURL(pubInfo.NoticePublicationID)
|
|
}
|
|
|
|
// Set additional classifications
|
|
if cn.ProcurementProject != nil && cn.ProcurementProject.AdditionalCommodityClassification != nil {
|
|
for _, additionalClass := range cn.ProcurementProject.AdditionalCommodityClassification {
|
|
if additionalClass.ItemClassificationCode != "" {
|
|
t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Set estimated value and currency
|
|
if amount, currency := cn.GetBudget(); amount != "" {
|
|
if value, ok := ParseEuropeanAmount(amount); ok {
|
|
t.EstimatedValue = value
|
|
t.Currency = currency
|
|
}
|
|
}
|
|
|
|
if cn.ContractingParty != nil {
|
|
t.BuyerProfileURL = cn.ContractingParty.BuyerProfileURI
|
|
}
|
|
|
|
for i := range cn.ProcurementProjectLots {
|
|
t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&cn.ProcurementProjectLots[i]))
|
|
}
|
|
|
|
// Set duration information
|
|
if duration, unit := cn.GetDuration(); duration != "" {
|
|
t.Duration = duration
|
|
t.DurationUnit = unit
|
|
}
|
|
|
|
// Set tender deadline
|
|
if deadline := cn.GetTenderDeadline(); deadline != "" {
|
|
t.TenderDeadline = ParseDateToUnix(deadline)
|
|
|
|
if deadlineTime, err := ParseDate(deadline); err == nil {
|
|
applicationDeadline := CalculateApplicationDeadline(deadlineTime)
|
|
t.ApplicationDeadline = applicationDeadline.Unix()
|
|
}
|
|
}
|
|
|
|
if pubDate, err := ParseDate(cn.GetPublicationDate()); err == nil {
|
|
submissionDeadline := CalculateSubmissionDeadline(pubDate)
|
|
t.SubmissionDeadline = submissionDeadline.Unix()
|
|
}
|
|
|
|
// Set SubmissionURL from TenderingTerms
|
|
if lot := cn.firstLot(); lot != nil && lot.TenderingTerms != nil {
|
|
if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
|
|
if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
|
|
if strings.HasPrefix(endpointID, "http") {
|
|
t.SubmissionURL = endpointID
|
|
} else {
|
|
t.SubmissionURL = "https://" + endpointID
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Set location information
|
|
if cn.ProcurementProject != nil && cn.ProcurementProject.RealizedLocation != nil {
|
|
if addr := cn.ProcurementProject.RealizedLocation.Address; addr != nil {
|
|
t.CityName = addr.CityName
|
|
t.PostalCode = addr.PostalZone
|
|
t.RegionCode = addr.CountrySubentityCode
|
|
if addr.Country != nil {
|
|
t.CountryCode = addr.Country.IdentificationCode
|
|
}
|
|
}
|
|
}
|
|
|
|
// Set buyer organization
|
|
buyerID := cn.GetBuyerID()
|
|
if buyerID != "" {
|
|
if buyerOrg := cn.GetOrganizationByID(buyerID); buyerOrg != nil {
|
|
t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
|
|
}
|
|
}
|
|
|
|
// Set review organization
|
|
reviewID := cn.GetReviewOrganizationID()
|
|
if reviewID != "" {
|
|
if reviewOrg := cn.GetOrganizationByID(reviewID); reviewOrg != nil {
|
|
t.ReviewOrganization = s.mapOrganization(reviewOrg, "review")
|
|
}
|
|
}
|
|
|
|
// Set all organizations
|
|
if orgs := cn.GetOrganizations(); orgs != nil {
|
|
for _, org := range orgs.Organization {
|
|
mappedOrg := s.mapOrganization(&org, "")
|
|
if mappedOrg != nil {
|
|
t.Organizations = append(t.Organizations, *mappedOrg)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Set selection criteria
|
|
if criteria := cn.GetSelectionCriteria(); criteria != nil {
|
|
for _, criterion := range criteria {
|
|
t.SelectionCriteria = append(t.SelectionCriteria, notice.SelectionCriterion{
|
|
TypeCode: criterion.CriterionTypeCode,
|
|
Description: criterion.Description,
|
|
LanguageID: criterion.LanguageID,
|
|
})
|
|
}
|
|
}
|
|
|
|
// Set official languages
|
|
if languages := cn.GetOfficialLanguages(); languages != nil {
|
|
t.OfficialLanguages = languages
|
|
}
|
|
|
|
return t
|
|
}
|
|
|
|
// mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity
|
|
func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *notice.Notice {
|
|
now := time.Now().Unix()
|
|
|
|
t := ¬ice.Notice{
|
|
ContractNoticeID: can.ID,
|
|
ContractFolderID: can.ContractFolderID,
|
|
NoticeTypeCode: strings.TrimSpace(can.GetNoticeType()),
|
|
FormType: can.GetNoticeFormType(),
|
|
NoticeSubTypeCode: can.GetNoticeSubType(),
|
|
NoticeIdentifier: can.ID,
|
|
NoticeVersion: can.VersionID,
|
|
NoticeDispatchAt: ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime),
|
|
ESenderDispatchAt: TransmissionDispatchUnix(can.Extensions),
|
|
NoticeLanguageCode: can.NoticeLanguageCode,
|
|
IssueDate: ParseDateToUnix(can.IssueDate),
|
|
IssueTime: ParseTimeToUnix(can.IssueTime),
|
|
GazetteID: can.GetOJSID(),
|
|
Title: can.GetProcurementTitle(),
|
|
Description: can.GetProcurementDescription(),
|
|
ProcurementTypeCode: can.GetContractNature(),
|
|
ProcedureCode: can.GetProcedureCode(),
|
|
MainClassification: can.GetMainClassification(),
|
|
Status: notice.TenderStatusAwarded, // Award notices are for completed tenders
|
|
Source: notice.TenderSourceTEDScraper,
|
|
SourceFileURL: sourceFileURL,
|
|
SourceFileName: sourceFileName,
|
|
ProcessingMetadata: notice.ProcessingMetadata{
|
|
ScrapedAt: now,
|
|
ProcessedAt: now,
|
|
ProcessingVersion: "1.0",
|
|
},
|
|
}
|
|
|
|
// Set publication information
|
|
if pubInfo := can.GetPublicationInfo(); pubInfo != nil {
|
|
t.NoticePublicationID = pubInfo.NoticePublicationID
|
|
t.PublicationDate = ParseDateToUnix(pubInfo.PublicationDate)
|
|
t.TenderURL = GenerateTenderURL(pubInfo.NoticePublicationID)
|
|
}
|
|
|
|
// Set total award value
|
|
if amount, currency := can.GetTotalAwardValue(); amount != "" {
|
|
if parsedAmount, ok := ParseEuropeanAmount(amount); ok {
|
|
t.EstimatedValue = parsedAmount
|
|
t.AwardedValue = parsedAmount
|
|
}
|
|
t.Currency = currency
|
|
}
|
|
|
|
if ad := can.GetAwardDate(); ad != "" {
|
|
t.AwardDate = ParseDateToUnix(ad)
|
|
}
|
|
|
|
if can.ContractingParty != nil {
|
|
t.BuyerProfileURL = can.ContractingParty.BuyerProfileURI
|
|
}
|
|
|
|
for i := range can.ProcurementProjectLot {
|
|
t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&can.ProcurementProjectLot[i]))
|
|
}
|
|
|
|
// Set location information from main procurement project or first lot
|
|
var realizedAddr *Address
|
|
if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
|
|
realizedAddr = can.ProcurementProject.RealizedLocation.Address
|
|
} else if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil &&
|
|
lot.ProcurementProject.RealizedLocation != nil {
|
|
realizedAddr = lot.ProcurementProject.RealizedLocation.Address
|
|
}
|
|
if realizedAddr != nil {
|
|
t.CityName = realizedAddr.CityName
|
|
t.PostalCode = realizedAddr.PostalZone
|
|
t.RegionCode = realizedAddr.CountrySubentityCode
|
|
t.PlaceOfPerformance = realizedAddr.Region
|
|
if realizedAddr.Country != nil {
|
|
t.CountryCode = realizedAddr.Country.IdentificationCode
|
|
}
|
|
}
|
|
|
|
if wOrg := can.GetWinningTenderOrganization(); wOrg != nil {
|
|
t.WinningTenderer = s.mapOrganization(wOrg, "winner")
|
|
}
|
|
|
|
if nr := can.GetNoticeResult(); nr != nil {
|
|
for _, lr := range nr.LotResult {
|
|
winner := pickWinningLotTender(lr.LotTender)
|
|
if winner == nil || winner.TenderingParty == nil {
|
|
continue
|
|
}
|
|
org := can.GetOrganizationByID(winner.TenderingParty.ID)
|
|
aw := notice.Awarded{
|
|
LotID: lr.ID,
|
|
OrganizationID: winner.TenderingParty.ID,
|
|
AwardDate: t.AwardDate,
|
|
}
|
|
if org != nil && org.Company != nil {
|
|
if org.Company.PartyName != nil {
|
|
aw.Name = org.Company.PartyName.Name
|
|
}
|
|
if org.Company.PartyLegalEntity != nil {
|
|
aw.CompanyID = org.Company.PartyLegalEntity.CompanyID
|
|
}
|
|
}
|
|
if winner.LegalMonetaryTotal != nil {
|
|
pa := winner.LegalMonetaryTotal.PayableAmount
|
|
if v, ok := ParseEuropeanAmount(pa.Text); ok {
|
|
aw.Amount = v
|
|
}
|
|
aw.Currency = pa.CurrencyID
|
|
}
|
|
t.AwardedEntities = append(t.AwardedEntities, aw)
|
|
}
|
|
}
|
|
|
|
// Set buyer organization
|
|
buyerID := can.GetBuyerID()
|
|
if buyerID != "" {
|
|
if buyerOrg := can.GetOrganizationByID(buyerID); buyerOrg != nil {
|
|
t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
|
|
}
|
|
}
|
|
|
|
// Set all organizations
|
|
if orgs := can.GetOrganizations(); orgs != nil {
|
|
for _, org := range orgs.Organization {
|
|
mappedOrg := s.mapOrganization(&org, "")
|
|
if mappedOrg != nil {
|
|
t.Organizations = append(t.Organizations, *mappedOrg)
|
|
}
|
|
}
|
|
}
|
|
|
|
return t
|
|
}
|
|
|
|
// mapOrganization maps a TED Organization to tender Organization
|
|
func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.Organization {
|
|
if tedOrg == nil || tedOrg.Company == nil {
|
|
return nil
|
|
}
|
|
|
|
company := tedOrg.Company
|
|
org := ¬ice.Organization{
|
|
Role: role,
|
|
}
|
|
|
|
// Set organization ID and name
|
|
if company.PartyIdentification != nil {
|
|
org.ID = company.PartyIdentification.ID
|
|
}
|
|
if company.PartyName != nil {
|
|
org.Name = company.PartyName.Name
|
|
}
|
|
|
|
// Set company ID
|
|
if company.PartyLegalEntity != nil {
|
|
org.CompanyID = company.PartyLegalEntity.CompanyID
|
|
}
|
|
|
|
// Set website
|
|
org.WebsiteURI = company.WebsiteURI
|
|
|
|
// Set contact information
|
|
if company.Contact != nil {
|
|
org.ContactName = company.Contact.Name
|
|
org.ContactTelephone = company.Contact.Telephone
|
|
org.ContactEmail = company.Contact.ElectronicMail
|
|
org.ContactFax = company.Contact.Telefax
|
|
}
|
|
|
|
// Set address
|
|
if company.PostalAddress != nil {
|
|
org.Address = notice.Address{
|
|
StreetName: company.PostalAddress.StreetName,
|
|
CityName: company.PostalAddress.CityName,
|
|
PostalZone: company.PostalAddress.PostalZone,
|
|
CountrySubentityCode: company.PostalAddress.CountrySubentityCode,
|
|
Department: company.PostalAddress.Department,
|
|
Region: strings.TrimSpace(company.PostalAddress.CountrySubentity),
|
|
}
|
|
if company.PostalAddress.Country != nil {
|
|
org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode
|
|
}
|
|
}
|
|
|
|
return org
|
|
}
|