added missing tender data to response
This commit is contained in:
Mazyar
+224
-73
@@ -19,7 +19,9 @@ func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot
|
||||
out.Description = pp.Description
|
||||
out.MainNatureOfContract = pp.ProcurementTypeCode
|
||||
if pp.MainCommodityClassification != nil {
|
||||
out.MainClassification = strings.TrimSpace(pp.MainCommodityClassification.ItemClassificationCode)
|
||||
mc := pp.MainCommodityClassification
|
||||
out.MainClassification = strings.TrimSpace(mc.ItemClassificationCode)
|
||||
out.MainClassificationDescription = mainCommodityClassificationLabel(mc)
|
||||
}
|
||||
for _, ac := range pp.AdditionalCommodityClassification {
|
||||
c := strings.TrimSpace(ac.ItemClassificationCode)
|
||||
@@ -86,7 +88,7 @@ func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceF
|
||||
})
|
||||
return nil
|
||||
}
|
||||
return s.mapGenericNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName)
|
||||
return s.mapPriorInformationNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName)
|
||||
default:
|
||||
s.logger.Error("Unknown document type for mapping", map[string]interface{}{
|
||||
"document_type": string(parsedDoc.Type),
|
||||
@@ -95,34 +97,41 @@ func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceF
|
||||
}
|
||||
}
|
||||
|
||||
// mapGenericNoticeToTender maps any TED document type to tender entity using the common interface
|
||||
func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *notice.Notice {
|
||||
if doc == nil {
|
||||
s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{})
|
||||
// mapPriorInformationNoticeToTender maps a TED PriorInformationNotice to the same notice shape as a contract notice,
|
||||
// so procedure keys (ContractFolderID, NoticePublicationID, ProcurementProjectID) merge into one tender.
|
||||
func (s *TEDScraper) mapPriorInformationNoticeToTender(pin *PriorInformationNotice, sourceFileURL, sourceFileName string) *notice.Notice {
|
||||
if pin == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
now := time.Now().Unix()
|
||||
|
||||
t := ¬ice.Notice{
|
||||
ContractNoticeID: doc.GetID(),
|
||||
ContractFolderID: "", // Not available in generic interface
|
||||
NoticeTypeCode: doc.GetNoticeType(),
|
||||
NoticeSubTypeCode: "", // Not available in generic interface
|
||||
NoticeLanguageCode: doc.GetLanguage(),
|
||||
IssueDate: now, // TODO: Parse actual date from document
|
||||
IssueTime: now, // TODO: Parse actual time from document
|
||||
GazetteID: "", // Not available in generic interface
|
||||
NoticePublicationID: "", // Not available in generic interface
|
||||
PublicationDate: now, // TODO: Parse actual publication date
|
||||
|
||||
// Basic tender information
|
||||
Title: "Generic Notice " + doc.GetNoticeType(),
|
||||
Description: "Processed from " + doc.GetNoticeType() + " document",
|
||||
Status: notice.TenderStatusActive,
|
||||
Source: notice.TenderSourceTEDScraper,
|
||||
SourceFileURL: sourceFileURL,
|
||||
SourceFileName: sourceFileName,
|
||||
ContractNoticeID: pin.ID,
|
||||
ContractFolderID: pin.ContractFolderID,
|
||||
ProcurementProjectID: procurementProjectIDFromPriorInformationNotice(pin),
|
||||
NoticeTypeCode: strings.TrimSpace(pin.GetNoticeType()),
|
||||
FormType: pin.GetNoticeFormType(),
|
||||
NoticeSubTypeCode: pin.GetNoticeSubType(),
|
||||
NoticeIdentifier: pin.ID,
|
||||
NoticeVersion: pin.VersionID,
|
||||
NoticeDispatchAt: ParseIssueDateTimeToUnix(pin.IssueDate, pin.IssueTime),
|
||||
ESenderDispatchAt: TransmissionDispatchUnix(pin.Extensions),
|
||||
NoticeLanguageCode: pin.NoticeLanguageCode,
|
||||
IssueDate: ParseDateToUnix(pin.IssueDate),
|
||||
IssueTime: ParseTimeToUnix(pin.IssueTime),
|
||||
GazetteID: pin.GetOJSID(),
|
||||
Title: pin.GetProcurementTitle(),
|
||||
Description: pin.GetProcurementDescription(),
|
||||
ProcurementTypeCode: pin.GetContractNature(),
|
||||
ProcedureCode: pin.GetProcedureCode(),
|
||||
MainClassification: pin.GetMainClassification(),
|
||||
MainClassificationDescription: pin.GetMainClassificationDescription(),
|
||||
PlaceOfPerformance: pin.GetPlaceOfPerformance(),
|
||||
DocumentURI: pin.GetDocumentURI(),
|
||||
Status: notice.TenderStatusActive,
|
||||
Source: notice.TenderSourceTEDScraper,
|
||||
SourceFileURL: sourceFileURL,
|
||||
SourceFileName: sourceFileName,
|
||||
ProcessingMetadata: notice.ProcessingMetadata{
|
||||
ScrapedAt: now,
|
||||
ProcessedAt: now,
|
||||
@@ -130,39 +139,180 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so
|
||||
},
|
||||
}
|
||||
|
||||
if pubInfo := pin.GetPublicationInfo(); pubInfo != nil {
|
||||
t.NoticePublicationID = pubInfo.NoticePublicationID
|
||||
t.PublicationDate = ParseDateToUnix(pubInfo.PublicationDate)
|
||||
t.TenderURL = GenerateTenderURL(pubInfo.NoticePublicationID)
|
||||
}
|
||||
|
||||
if pin.ProcurementProject != nil && pin.ProcurementProject.AdditionalCommodityClassification != nil {
|
||||
for _, additionalClass := range pin.ProcurementProject.AdditionalCommodityClassification {
|
||||
if additionalClass.ItemClassificationCode != "" {
|
||||
t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if amount, currency := pin.GetBudget(); amount != "" {
|
||||
if value, ok := ParseEuropeanAmount(amount); ok {
|
||||
t.EstimatedValue = value
|
||||
t.Currency = currency
|
||||
}
|
||||
}
|
||||
|
||||
if pin.ContractingParty != nil {
|
||||
t.BuyerProfileURL = pin.ContractingParty.BuyerProfileURI
|
||||
}
|
||||
|
||||
for i := range pin.ProcurementProjectLot {
|
||||
t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&pin.ProcurementProjectLot[i]))
|
||||
}
|
||||
|
||||
if duration, unit := pin.GetDuration(); duration != "" {
|
||||
t.Duration = duration
|
||||
t.DurationUnit = unit
|
||||
}
|
||||
|
||||
if deadline := pin.GetTenderDeadline(); deadline != "" {
|
||||
t.TenderDeadline = ParseDateToUnix(deadline)
|
||||
if deadlineTime, err := ParseDate(deadline); err == nil {
|
||||
applicationDeadline := CalculateApplicationDeadline(deadlineTime)
|
||||
t.ApplicationDeadline = applicationDeadline.Unix()
|
||||
}
|
||||
}
|
||||
|
||||
if pubDate, err := ParseDate(pin.GetPublicationDate()); err == nil {
|
||||
submissionDeadline := CalculateSubmissionDeadline(pubDate)
|
||||
t.SubmissionDeadline = submissionDeadline.Unix()
|
||||
}
|
||||
|
||||
if lot := pin.firstLot(); lot != nil && lot.TenderingTerms != nil {
|
||||
if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
|
||||
if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
|
||||
if strings.HasPrefix(endpointID, "http") {
|
||||
t.SubmissionURL = endpointID
|
||||
} else {
|
||||
t.SubmissionURL = "https://" + endpointID
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if pin.ProcurementProject != nil && pin.ProcurementProject.RealizedLocation != nil {
|
||||
if addr := pin.ProcurementProject.RealizedLocation.Address; addr != nil {
|
||||
t.CityName = addr.CityName
|
||||
t.PostalCode = addr.PostalZone
|
||||
t.RegionCode = addr.CountrySubentityCode
|
||||
if addr.Country != nil {
|
||||
t.CountryCode = addr.Country.IdentificationCode
|
||||
}
|
||||
}
|
||||
} else if lot := pin.firstLot(); lot != nil && lot.ProcurementProject != nil &&
|
||||
lot.ProcurementProject.RealizedLocation != nil {
|
||||
if addr := lot.ProcurementProject.RealizedLocation.Address; addr != nil {
|
||||
t.CityName = addr.CityName
|
||||
t.PostalCode = addr.PostalZone
|
||||
t.RegionCode = addr.CountrySubentityCode
|
||||
if addr.Country != nil {
|
||||
t.CountryCode = addr.Country.IdentificationCode
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buyerID := pin.GetBuyerID()
|
||||
if buyerID != "" {
|
||||
if buyerOrg := pin.GetOrganizationByID(buyerID); buyerOrg != nil {
|
||||
t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
|
||||
}
|
||||
}
|
||||
|
||||
reviewID := pin.GetReviewOrganizationID()
|
||||
if reviewID != "" {
|
||||
if reviewOrg := pin.GetOrganizationByID(reviewID); reviewOrg != nil {
|
||||
t.ReviewOrganization = s.mapOrganization(reviewOrg, "review")
|
||||
}
|
||||
}
|
||||
|
||||
if orgs := pin.GetOrganizations(); orgs != nil {
|
||||
for _, org := range orgs.Organization {
|
||||
mappedOrg := s.mapOrganization(&org, "")
|
||||
if mappedOrg != nil {
|
||||
t.Organizations = append(t.Organizations, *mappedOrg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if criteria := pin.GetSelectionCriteria(); criteria != nil {
|
||||
for _, criterion := range criteria {
|
||||
t.SelectionCriteria = append(t.SelectionCriteria, notice.SelectionCriterion{
|
||||
TypeCode: criterion.CriterionTypeCode,
|
||||
Description: criterion.Description,
|
||||
LanguageID: criterion.LanguageID,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if languages := pin.GetOfficialLanguages(); languages != nil {
|
||||
t.OfficialLanguages = languages
|
||||
}
|
||||
|
||||
if strings.TrimSpace(t.Title) == "" {
|
||||
if np := strings.TrimSpace(t.NoticePublicationID); np != "" {
|
||||
t.Title = "Prior information notice " + np
|
||||
} else if cf := strings.TrimSpace(t.ContractFolderID); cf != "" {
|
||||
t.Title = "Prior information notice — procedure " + cf
|
||||
}
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
|
||||
func procurementProjectIDFromPriorInformationNotice(pin *PriorInformationNotice) string {
|
||||
if pin == nil {
|
||||
return ""
|
||||
}
|
||||
if pin.ProcurementProject != nil {
|
||||
if id := strings.TrimSpace(pin.ProcurementProject.ID); id != "" {
|
||||
return id
|
||||
}
|
||||
}
|
||||
if lot := pin.firstLot(); lot != nil && lot.ProcurementProject != nil {
|
||||
return strings.TrimSpace(lot.ProcurementProject.ID)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// mapToTender maps a TED ContractNotice to tender entity
|
||||
func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *notice.Notice {
|
||||
now := time.Now().Unix()
|
||||
|
||||
t := ¬ice.Notice{
|
||||
ContractNoticeID: cn.ID,
|
||||
ContractFolderID: cn.ContractFolderID,
|
||||
ProcurementProjectID: procurementProjectIDFromContractNotice(cn),
|
||||
NoticeTypeCode: strings.TrimSpace(cn.GetNoticeType()),
|
||||
FormType: cn.GetNoticeFormType(),
|
||||
NoticeSubTypeCode: cn.GetNoticeSubType(),
|
||||
NoticeIdentifier: cn.ID,
|
||||
NoticeVersion: cn.VersionID,
|
||||
NoticeDispatchAt: ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime),
|
||||
ESenderDispatchAt: TransmissionDispatchUnix(cn.Extensions),
|
||||
NoticeLanguageCode: cn.NoticeLanguageCode,
|
||||
IssueDate: ParseDateToUnix(cn.IssueDate),
|
||||
IssueTime: ParseTimeToUnix(cn.IssueTime),
|
||||
GazetteID: cn.GetOJSID(),
|
||||
Title: cn.GetProcurementTitle(),
|
||||
Description: cn.GetProcurementDescription(),
|
||||
ProcurementTypeCode: cn.GetContractNature(),
|
||||
ProcedureCode: cn.GetProcedureCode(),
|
||||
MainClassification: cn.GetMainClassification(),
|
||||
PlaceOfPerformance: cn.GetPlaceOfPerformance(),
|
||||
DocumentURI: cn.GetDocumentURI(),
|
||||
Status: notice.TenderStatusActive,
|
||||
Source: notice.TenderSourceTEDScraper,
|
||||
SourceFileURL: sourceFileURL,
|
||||
SourceFileName: sourceFileName,
|
||||
ContractNoticeID: cn.ID,
|
||||
ContractFolderID: cn.ContractFolderID,
|
||||
ProcurementProjectID: procurementProjectIDFromContractNotice(cn),
|
||||
NoticeTypeCode: strings.TrimSpace(cn.GetNoticeType()),
|
||||
FormType: cn.GetNoticeFormType(),
|
||||
NoticeSubTypeCode: cn.GetNoticeSubType(),
|
||||
NoticeIdentifier: cn.ID,
|
||||
NoticeVersion: cn.VersionID,
|
||||
NoticeDispatchAt: ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime),
|
||||
ESenderDispatchAt: TransmissionDispatchUnix(cn.Extensions),
|
||||
NoticeLanguageCode: cn.NoticeLanguageCode,
|
||||
IssueDate: ParseDateToUnix(cn.IssueDate),
|
||||
IssueTime: ParseTimeToUnix(cn.IssueTime),
|
||||
GazetteID: cn.GetOJSID(),
|
||||
Title: cn.GetProcurementTitle(),
|
||||
Description: cn.GetProcurementDescription(),
|
||||
ProcurementTypeCode: cn.GetContractNature(),
|
||||
ProcedureCode: cn.GetProcedureCode(),
|
||||
MainClassification: cn.GetMainClassification(),
|
||||
MainClassificationDescription: cn.GetMainClassificationDescription(),
|
||||
PlaceOfPerformance: cn.GetPlaceOfPerformance(),
|
||||
DocumentURI: cn.GetDocumentURI(),
|
||||
Status: notice.TenderStatusActive,
|
||||
Source: notice.TenderSourceTEDScraper,
|
||||
SourceFileURL: sourceFileURL,
|
||||
SourceFileName: sourceFileName,
|
||||
ProcessingMetadata: notice.ProcessingMetadata{
|
||||
ScrapedAt: now,
|
||||
ProcessedAt: now,
|
||||
@@ -298,29 +448,30 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
|
||||
now := time.Now().Unix()
|
||||
|
||||
t := ¬ice.Notice{
|
||||
ContractNoticeID: can.ID,
|
||||
ContractFolderID: can.ContractFolderID,
|
||||
ProcurementProjectID: procurementProjectIDFromContractAwardNotice(can),
|
||||
NoticeTypeCode: strings.TrimSpace(can.GetNoticeType()),
|
||||
FormType: can.GetNoticeFormType(),
|
||||
NoticeSubTypeCode: can.GetNoticeSubType(),
|
||||
NoticeIdentifier: can.ID,
|
||||
NoticeVersion: can.VersionID,
|
||||
NoticeDispatchAt: ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime),
|
||||
ESenderDispatchAt: TransmissionDispatchUnix(can.Extensions),
|
||||
NoticeLanguageCode: can.NoticeLanguageCode,
|
||||
IssueDate: ParseDateToUnix(can.IssueDate),
|
||||
IssueTime: ParseTimeToUnix(can.IssueTime),
|
||||
GazetteID: can.GetOJSID(),
|
||||
Title: can.GetProcurementTitle(),
|
||||
Description: can.GetProcurementDescription(),
|
||||
ProcurementTypeCode: can.GetContractNature(),
|
||||
ProcedureCode: can.GetProcedureCode(),
|
||||
MainClassification: can.GetMainClassification(),
|
||||
Status: notice.TenderStatusAwarded, // Award notices are for completed tenders
|
||||
Source: notice.TenderSourceTEDScraper,
|
||||
SourceFileURL: sourceFileURL,
|
||||
SourceFileName: sourceFileName,
|
||||
ContractNoticeID: can.ID,
|
||||
ContractFolderID: can.ContractFolderID,
|
||||
ProcurementProjectID: procurementProjectIDFromContractAwardNotice(can),
|
||||
NoticeTypeCode: strings.TrimSpace(can.GetNoticeType()),
|
||||
FormType: can.GetNoticeFormType(),
|
||||
NoticeSubTypeCode: can.GetNoticeSubType(),
|
||||
NoticeIdentifier: can.ID,
|
||||
NoticeVersion: can.VersionID,
|
||||
NoticeDispatchAt: ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime),
|
||||
ESenderDispatchAt: TransmissionDispatchUnix(can.Extensions),
|
||||
NoticeLanguageCode: can.NoticeLanguageCode,
|
||||
IssueDate: ParseDateToUnix(can.IssueDate),
|
||||
IssueTime: ParseTimeToUnix(can.IssueTime),
|
||||
GazetteID: can.GetOJSID(),
|
||||
Title: can.GetProcurementTitle(),
|
||||
Description: can.GetProcurementDescription(),
|
||||
ProcurementTypeCode: can.GetContractNature(),
|
||||
ProcedureCode: can.GetProcedureCode(),
|
||||
MainClassification: can.GetMainClassification(),
|
||||
MainClassificationDescription: can.GetMainClassificationDescription(),
|
||||
Status: notice.TenderStatusAwarded, // Award notices are for completed tenders
|
||||
Source: notice.TenderSourceTEDScraper,
|
||||
SourceFileURL: sourceFileURL,
|
||||
SourceFileName: sourceFileName,
|
||||
ProcessingMetadata: notice.ProcessingMetadata{
|
||||
ScrapedAt: now,
|
||||
ProcessedAt: now,
|
||||
|
||||
Reference in New Issue
Block a user