Added missing fields - scraper and worker fixes

This commit is contained in:
Mazyar
2026-05-11 00:09:37 +03:30
parent e136f0eaa7
commit eb6706e061
12 changed files with 667 additions and 174 deletions
+182 -94
View File
@@ -3,9 +3,16 @@ package ted
import (
"encoding/xml"
"fmt"
"strings"
"time"
)
// NoticeTypeCodeField captures cbc:NoticeTypeCode: inner text (notice type, BT-02) and listName (form type, BT-03).
type NoticeTypeCodeField struct {
ListName string `xml:"listName,attr,omitempty"`
Text string `xml:",chardata"`
}
// DocumentType represents the type of TED document
type DocumentType string
@@ -634,30 +641,37 @@ type SubcontractingTerm struct {
// ContractNotice represents the main TED XML contract notice structure
type ContractNotice struct {
XMLName xml.Name `xml:"ContractNotice"`
Xmlns string `xml:"xmlns,attr,omitempty"`
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
XMLName xml.Name `xml:"ContractNotice"`
Xmlns string `xml:"xmlns,attr,omitempty"`
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode NoticeTypeCodeField `xml:"NoticeTypeCode,omitempty"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
ProcurementProjectLots []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
}
func (cn ContractNotice) firstLot() *ProcurementProjectLot {
if len(cn.ProcurementProjectLots) == 0 {
return nil
}
return &cn.ProcurementProjectLots[0]
}
// ContractAwardNotice represents the main TED XML contract award notice structure
@@ -679,7 +693,7 @@ type ContractAwardNotice struct {
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeTypeCode NoticeTypeCodeField `xml:"NoticeTypeCode,omitempty"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
@@ -688,6 +702,25 @@ type ContractAwardNotice struct {
TenderResult *TenderResult `xml:"TenderResult,omitempty"`
}
func (can ContractAwardNotice) firstAwardLot() *ProcurementProjectLot {
if len(can.ProcurementProjectLot) == 0 {
return nil
}
return &can.ProcurementProjectLot[0]
}
func pickWinningLotTender(lotTenders []LotTender) *LotTender {
if len(lotTenders) == 0 {
return nil
}
for i := range lotTenders {
if strings.TrimSpace(lotTenders[i].RankCode) == "1" {
return &lotTenders[i]
}
}
return &lotTenders[0]
}
// NoticeResult contains notice result information from extensions
type NoticeResult struct {
XMLName xml.Name `xml:"NoticeResult"`
@@ -819,8 +852,10 @@ type UBLExtension struct {
// ExtensionContent contains eForms extension data
type ExtensionContent struct {
XMLName xml.Name `xml:"ExtensionContent"`
EformsExtension *EformsExtension `xml:"EformsExtension,omitempty"`
XMLName xml.Name `xml:"ExtensionContent"`
EformsExtension *EformsExtension `xml:"EformsExtension,omitempty"`
TransmissionDate string `xml:"TransmissionDate,omitempty"`
TransmissionTime string `xml:"TransmissionTime,omitempty"`
}
// EformsExtension contains eForms specific data
@@ -835,6 +870,8 @@ type EformsExtension struct {
OfficialLanguages *OfficialLanguages `xml:"OfficialLanguages,omitempty"`
AwardCriterionParameter *AwardCriterionParameter `xml:"AwardCriterionParameter,omitempty"`
AccessToolName string `xml:"AccessToolName,omitempty"`
TransmissionDate string `xml:"TransmissionDate,omitempty"`
TransmissionTime string `xml:"TransmissionTime,omitempty"`
}
// NoticeSubType contains notice subtype information
@@ -908,6 +945,7 @@ type PostalAddress struct {
StreetName string `xml:"StreetName,omitempty"`
CityName string `xml:"CityName,omitempty"`
PostalZone string `xml:"PostalZone,omitempty"`
CountrySubentity string `xml:"CountrySubentity,omitempty"`
CountrySubentityCode string `xml:"CountrySubentityCode,omitempty"`
Department string `xml:"Department,omitempty"`
Country *Country `xml:"Country,omitempty"`
@@ -1338,11 +1376,11 @@ func (cn ContractNotice) Validate() error {
}
// Validate tender submission deadline format if present
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingProcess != nil &&
cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
if cn.firstLot() != nil &&
cn.firstLot().TenderingProcess != nil &&
cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
deadline := cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
deadline := cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
if deadline != "" {
var validDeadline bool
for _, format := range validFormats {
@@ -1358,23 +1396,23 @@ func (cn ContractNotice) Validate() error {
}
// Validate currency code if provided in budget
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
currency := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID
currency := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID
if currency != "" && len(currency) != 3 {
return fmt.Errorf("invalid currency code: %s (expected 3-letter ISO code)", currency)
}
}
// Validate duration unit code if provided
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil {
unitCode := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode
unitCode := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode
if unitCode != "" {
validUnits := map[string]bool{
"MONTH": true,
@@ -1396,9 +1434,14 @@ func (cn ContractNotice) GetID() string {
return cn.ID
}
// GetNoticeType returns the notice type
// GetNoticeType returns the notice type code (BT-02, cbc:NoticeTypeCode text).
func (cn ContractNotice) GetNoticeType() string {
return cn.NoticeTypeCode
return strings.TrimSpace(cn.NoticeTypeCode.Text)
}
// GetNoticeFormType returns the form type from listName on cbc:NoticeTypeCode (BT-03), e.g. "competition".
func (cn ContractNotice) GetNoticeFormType() string {
return strings.TrimSpace(cn.NoticeTypeCode.ListName)
}
// GetLanguage returns the notice language
@@ -1496,12 +1539,12 @@ func (cn ContractNotice) GetBuyerID() string {
// GetReviewOrganizationID returns the review organization ID from appeal terms
func (cn ContractNotice) GetReviewOrganizationID() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil &&
cn.ProcurementProjectLot.TenderingTerms.AppealTerms != nil &&
cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty != nil &&
cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil {
return cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID
if cn.firstLot() != nil &&
cn.firstLot().TenderingTerms != nil &&
cn.firstLot().TenderingTerms.AppealTerms != nil &&
cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty != nil &&
cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil {
return cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID
}
return ""
}
@@ -1516,18 +1559,18 @@ func (cn ContractNotice) GetProcedureCode() string {
// GetProcurementTitle returns the procurement project title
func (cn ContractNotice) GetProcurementTitle() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil {
return cn.ProcurementProjectLot.ProcurementProject.Name
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil {
return cn.firstLot().ProcurementProject.Name
}
return ""
}
// GetProcurementDescription returns the procurement project description
func (cn ContractNotice) GetProcurementDescription() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil {
return cn.ProcurementProjectLot.ProcurementProject.Description
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil {
return cn.firstLot().ProcurementProject.Description
}
return ""
}
@@ -1561,8 +1604,8 @@ func (cn ContractNotice) GetPlaceOfPerformance() string {
// GetLotID returns the procurement lot ID
func (cn ContractNotice) GetLotID() string {
if cn.ProcurementProjectLot != nil {
return cn.ProcurementProjectLot.ID
if cn.firstLot() != nil {
return cn.firstLot().ID
}
return ""
}
@@ -1580,17 +1623,17 @@ func (cn ContractNotice) GetSelectionCriteria() []SelectionCriteria {
// GetOfficialLanguages returns official languages from tender terms extensions
func (cn ContractNotice) GetOfficialLanguages() []string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil &&
len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil {
if cn.firstLot() != nil &&
cn.firstLot().TenderingTerms != nil &&
len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil {
languages := make([]string, len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language))
for i, lang := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language {
languages := make([]string, len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language))
for i, lang := range cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language {
languages[i] = lang.ID
}
return languages
@@ -1600,12 +1643,12 @@ func (cn ContractNotice) GetOfficialLanguages() []string {
// GetDocumentURI returns the document URI from call for tenders document reference
func (cn ContractNotice) GetDocumentURI() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil &&
len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil {
return cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI
if cn.firstLot() != nil &&
cn.firstLot().TenderingTerms != nil &&
len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil {
return cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI
}
return ""
}
@@ -1613,10 +1656,12 @@ func (cn ContractNotice) GetDocumentURI() string {
// GetAllDocumentURIs returns all document URIs from call for tenders document references
func (cn ContractNotice) GetAllDocumentURIs() []string {
var uris []string
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil {
for _, docRef := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference {
for i := range cn.ProcurementProjectLots {
lot := &cn.ProcurementProjectLots[i]
if lot.TenderingTerms == nil {
continue
}
for _, docRef := range lot.TenderingTerms.CallForTendersDocumentReference {
if docRef.Attachment != nil &&
docRef.Attachment.ExternalReference != nil &&
docRef.Attachment.ExternalReference.URI != "" {
@@ -1629,21 +1674,21 @@ func (cn ContractNotice) GetAllDocumentURIs() []string {
// GetTenderDeadline returns the tender submission deadline
func (cn ContractNotice) GetTenderDeadline() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingProcess != nil &&
cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
return cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
if cn.firstLot() != nil &&
cn.firstLot().TenderingProcess != nil &&
cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
return cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
}
return ""
}
// GetBudget returns the estimated contract amount and currency
func (cn ContractNotice) GetBudget() (amount string, currency string) {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
contractAmount := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
contractAmount := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
return contractAmount.Text, contractAmount.CurrencyID
}
return "", ""
@@ -1651,12 +1696,12 @@ func (cn ContractNotice) GetBudget() (amount string, currency string) {
// GetDuration returns the contract duration and unit
func (cn ContractNotice) GetDuration() (duration string, unit string) {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil {
durationMeasure := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure
durationMeasure := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure
return durationMeasure.Text, durationMeasure.UnitCode
}
return "", ""
@@ -1716,9 +1761,14 @@ func (can ContractAwardNotice) GetID() string {
return can.ID
}
// GetNoticeType returns the notice type
// GetNoticeType returns the notice type code (BT-02).
func (can ContractAwardNotice) GetNoticeType() string {
return can.NoticeTypeCode
return strings.TrimSpace(can.NoticeTypeCode.Text)
}
// GetNoticeFormType returns the form type from listName (BT-03).
func (can ContractAwardNotice) GetNoticeFormType() string {
return strings.TrimSpace(can.NoticeTypeCode.ListName)
}
// GetLanguage returns the notice language
@@ -1818,25 +1868,34 @@ func (can ContractAwardNotice) GetProcedureCode() string {
// GetProcurementTitle returns the procurement project title
func (can ContractAwardNotice) GetProcurementTitle() string {
if can.ProcurementProject != nil {
if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Name) != "" {
return can.ProcurementProject.Name
}
if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
return lot.ProcurementProject.Name
}
return ""
}
// GetProcurementDescription returns the procurement project description
func (can ContractAwardNotice) GetProcurementDescription() string {
if can.ProcurementProject != nil {
if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Description) != "" {
return can.ProcurementProject.Description
}
if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
return lot.ProcurementProject.Description
}
return ""
}
// GetContractNature returns the main nature of the contract
func (can ContractAwardNotice) GetContractNature() string {
if can.ProcurementProject != nil {
if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.ProcurementTypeCode) != "" {
return can.ProcurementProject.ProcurementTypeCode
}
if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
return lot.ProcurementProject.ProcurementTypeCode
}
return ""
}
@@ -1846,6 +1905,11 @@ func (can ContractAwardNotice) GetMainClassification() string {
can.ProcurementProject.MainCommodityClassification != nil {
return can.ProcurementProject.MainCommodityClassification.ItemClassificationCode
}
if lot := can.firstAwardLot(); lot != nil &&
lot.ProcurementProject != nil &&
lot.ProcurementProject.MainCommodityClassification != nil {
return lot.ProcurementProject.MainCommodityClassification.ItemClassificationCode
}
return ""
}
@@ -1882,3 +1946,27 @@ func (can ContractAwardNotice) GetOrganizationByID(orgID string) *Organization {
}
return nil
}
// GetWinningTenderPartyID resolves the tendering party reference for the primary winning bid from NoticeResult.
func (can ContractAwardNotice) GetWinningTenderPartyID() string {
nr := can.GetNoticeResult()
if nr == nil {
return ""
}
for _, lr := range nr.LotResult {
winner := pickWinningLotTender(lr.LotTender)
if winner != nil && winner.TenderingParty != nil && winner.TenderingParty.ID != "" {
return winner.TenderingParty.ID
}
}
return ""
}
// GetWinningTenderOrganization resolves the winning tenderer organization from NoticeResult via Organizations.
func (can ContractAwardNotice) GetWinningTenderOrganization() *Organization {
id := can.GetWinningTenderPartyID()
if id == "" {
return nil
}
return can.GetOrganizationByID(id)
}