mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added missing fields - scraper and worker fixes

Browse Source
This commit is contained in:
Mazyar
2026-05-11 00:09:37 +03:30
parent e136f0eaa7
commit eb6706e061
12 changed files with 667 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ted/mapper.go
+138 -14
View File
@@ -1,12 +1,63 @@
package ted
import (
"strconv"
"strings"
"time"
"tm/internal/notice"
)
func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot {
if lot == nil {
return notice.ProcurementLot{}
}
out := notice.ProcurementLot{LotID: lot.ID}
if lot.ProcurementProject == nil {
return out
}
pp := lot.ProcurementProject
out.Title = pp.Name
out.Description = pp.Description
out.MainNatureOfContract = pp.ProcurementTypeCode
if pp.MainCommodityClassification != nil {
out.MainClassification = strings.TrimSpace(pp.MainCommodityClassification.ItemClassificationCode)
}
for _, ac := range pp.AdditionalCommodityClassification {
c := strings.TrimSpace(ac.ItemClassificationCode)
if c != "" {
out.AdditionalClassifications = append(out.AdditionalClassifications, c)
}
}
if pp.RequestedTenderTotal != nil {
a := pp.RequestedTenderTotal.EstimatedOverallContractAmount
if v, ok := ParseEuropeanAmount(a.Text); ok {
out.EstimatedValue = v
}
out.Currency = a.CurrencyID
}
if pp.PlannedPeriod != nil && pp.PlannedPeriod.DurationMeasure != nil {
dm := pp.PlannedPeriod.DurationMeasure
txt := strings.TrimSpace(dm.Text)
unit := strings.TrimSpace(dm.UnitCode)
switch unit {
case "MONTH":
out.Duration = txt + " Months"
case "DAY":
out.Duration = txt + " Days"
case "WEEK":
out.Duration = txt + " Weeks"
case "YEAR":
out.Duration = txt + " Years"
default:
if unit != "" && txt != "" {
out.Duration = txt + " " + unit
} else {
out.Duration = txt
}
}
}
return out
}
// mapToTenderFromParsedDoc maps a parsed TED document to tender entity
func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice {
switch parsedDoc.Type {
@@ -89,8 +140,13 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
t := &notice.Notice{
ContractNoticeID: cn.ID,
ContractFolderID: cn.ContractFolderID,
NoticeTypeCode: cn.GetNoticeSubType(),
NoticeTypeCode: strings.TrimSpace(cn.GetNoticeType()),
FormType: cn.GetNoticeFormType(),
NoticeSubTypeCode: cn.GetNoticeSubType(),
NoticeIdentifier: cn.ID,
NoticeVersion: cn.VersionID,
NoticeDispatchAt: ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime),
ESenderDispatchAt: TransmissionDispatchUnix(cn.Extensions),
NoticeLanguageCode: cn.NoticeLanguageCode,
IssueDate: ParseDateToUnix(cn.IssueDate),
IssueTime: ParseTimeToUnix(cn.IssueTime),
@@ -131,12 +187,20 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
// Set estimated value and currency
if amount, currency := cn.GetBudget(); amount != "" {
if value, err := strconv.ParseFloat(amount, 64); err == nil {
if value, ok := ParseEuropeanAmount(amount); ok {
t.EstimatedValue = value
t.Currency = currency
}
}
if cn.ContractingParty != nil {
t.BuyerProfileURL = cn.ContractingParty.BuyerProfileURI
}
for i := range cn.ProcurementProjectLots {
t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&cn.ProcurementProjectLots[i]))
}
// Set duration information
if duration, unit := cn.GetDuration(); duration != "" {
t.Duration = duration
@@ -159,8 +223,8 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
}
// Set SubmissionURL from TenderingTerms
if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil {
if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
if lot := cn.firstLot(); lot != nil && lot.TenderingTerms != nil {
if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
if strings.HasPrefix(endpointID, "http") {
t.SubmissionURL = endpointID
@@ -235,8 +299,13 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
t := &notice.Notice{
ContractNoticeID: can.ID,
ContractFolderID: can.ContractFolderID,
NoticeTypeCode: can.NoticeTypeCode,
NoticeTypeCode: strings.TrimSpace(can.GetNoticeType()),
FormType: can.GetNoticeFormType(),
NoticeSubTypeCode: can.GetNoticeSubType(),
NoticeIdentifier: can.ID,
NoticeVersion: can.VersionID,
NoticeDispatchAt: ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime),
ESenderDispatchAt: TransmissionDispatchUnix(can.Extensions),
NoticeLanguageCode: can.NoticeLanguageCode,
IssueDate: ParseDateToUnix(can.IssueDate),
IssueTime: ParseTimeToUnix(can.IssueTime),
@@ -266,21 +335,75 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
// Set total award value
if amount, currency := can.GetTotalAwardValue(); amount != "" {
if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
if parsedAmount, ok := ParseEuropeanAmount(amount); ok {
t.EstimatedValue = parsedAmount
t.AwardedValue = parsedAmount
}
t.Currency = currency
}
// Set location information from main procurement project
if ad := can.GetAwardDate(); ad != "" {
t.AwardDate = ParseDateToUnix(ad)
}
if can.ContractingParty != nil {
t.BuyerProfileURL = can.ContractingParty.BuyerProfileURI
}
for i := range can.ProcurementProjectLot {
t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&can.ProcurementProjectLot[i]))
}
// Set location information from main procurement project or first lot
var realizedAddr *Address
if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
if addr := can.ProcurementProject.RealizedLocation.Address; addr != nil {
t.CityName = addr.CityName
t.PostalCode = addr.PostalZone
t.RegionCode = addr.CountrySubentityCode
if addr.Country != nil {
t.CountryCode = addr.Country.IdentificationCode
realizedAddr = can.ProcurementProject.RealizedLocation.Address
} else if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil &&
lot.ProcurementProject.RealizedLocation != nil {
realizedAddr = lot.ProcurementProject.RealizedLocation.Address
}
if realizedAddr != nil {
t.CityName = realizedAddr.CityName
t.PostalCode = realizedAddr.PostalZone
t.RegionCode = realizedAddr.CountrySubentityCode
t.PlaceOfPerformance = realizedAddr.Region
if realizedAddr.Country != nil {
t.CountryCode = realizedAddr.Country.IdentificationCode
}
}
if wOrg := can.GetWinningTenderOrganization(); wOrg != nil {
t.WinningTenderer = s.mapOrganization(wOrg, "winner")
}
if nr := can.GetNoticeResult(); nr != nil {
for _, lr := range nr.LotResult {
winner := pickWinningLotTender(lr.LotTender)
if winner == nil || winner.TenderingParty == nil {
continue
}
org := can.GetOrganizationByID(winner.TenderingParty.ID)
aw := notice.Awarded{
LotID: lr.ID,
OrganizationID: winner.TenderingParty.ID,
AwardDate: t.AwardDate,
}
if org != nil && org.Company != nil {
if org.Company.PartyName != nil {
aw.Name = org.Company.PartyName.Name
}
if org.Company.PartyLegalEntity != nil {
aw.CompanyID = org.Company.PartyLegalEntity.CompanyID
}
}
if winner.LegalMonetaryTotal != nil {
pa := winner.LegalMonetaryTotal.PayableAmount
if v, ok := ParseEuropeanAmount(pa.Text); ok {
aw.Amount = v
}
aw.Currency = pa.CurrencyID
}
t.AwardedEntities = append(t.AwardedEntities, aw)
}
}
@@ -348,6 +471,7 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.
PostalZone: company.PostalAddress.PostalZone,
CountrySubentityCode: company.PostalAddress.CountrySubentityCode,
Department: company.PostalAddress.Department,
Region: strings.TrimSpace(company.PostalAddress.CountrySubentity),
}
if company.PostalAddress.Country != nil {
org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode
ted/model.go
+182 -94
View File
@@ -3,9 +3,16 @@ package ted
import (
"encoding/xml"
"fmt"
"strings"
"time"
)
// NoticeTypeCodeField captures cbc:NoticeTypeCode: inner text (notice type, BT-02) and listName (form type, BT-03).
type NoticeTypeCodeField struct {
ListName string `xml:"listName,attr,omitempty"`
Text string `xml:",chardata"`
}
// DocumentType represents the type of TED document
type DocumentType string
@@ -634,30 +641,37 @@ type SubcontractingTerm struct {
// ContractNotice represents the main TED XML contract notice structure
type ContractNotice struct {
XMLName xml.Name `xml:"ContractNotice"`
Xmlns string `xml:"xmlns,attr,omitempty"`
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
XMLName xml.Name `xml:"ContractNotice"`
Xmlns string `xml:"xmlns,attr,omitempty"`
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode NoticeTypeCodeField `xml:"NoticeTypeCode,omitempty"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
ProcurementProjectLots []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
}
func (cn ContractNotice) firstLot() *ProcurementProjectLot {
if len(cn.ProcurementProjectLots) == 0 {
return nil
}
return &cn.ProcurementProjectLots[0]
}
// ContractAwardNotice represents the main TED XML contract award notice structure
@@ -679,7 +693,7 @@ type ContractAwardNotice struct {
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeTypeCode NoticeTypeCodeField `xml:"NoticeTypeCode,omitempty"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
@@ -688,6 +702,25 @@ type ContractAwardNotice struct {
TenderResult *TenderResult `xml:"TenderResult,omitempty"`
}
func (can ContractAwardNotice) firstAwardLot() *ProcurementProjectLot {
if len(can.ProcurementProjectLot) == 0 {
return nil
}
return &can.ProcurementProjectLot[0]
}
func pickWinningLotTender(lotTenders []LotTender) *LotTender {
if len(lotTenders) == 0 {
return nil
}
for i := range lotTenders {
if strings.TrimSpace(lotTenders[i].RankCode) == "1" {
return &lotTenders[i]
}
}
return &lotTenders[0]
}
// NoticeResult contains notice result information from extensions
type NoticeResult struct {
XMLName xml.Name `xml:"NoticeResult"`
@@ -819,8 +852,10 @@ type UBLExtension struct {
// ExtensionContent contains eForms extension data
type ExtensionContent struct {
XMLName xml.Name `xml:"ExtensionContent"`
EformsExtension *EformsExtension `xml:"EformsExtension,omitempty"`
XMLName xml.Name `xml:"ExtensionContent"`
EformsExtension *EformsExtension `xml:"EformsExtension,omitempty"`
TransmissionDate string `xml:"TransmissionDate,omitempty"`
TransmissionTime string `xml:"TransmissionTime,omitempty"`
}
// EformsExtension contains eForms specific data
@@ -835,6 +870,8 @@ type EformsExtension struct {
OfficialLanguages *OfficialLanguages `xml:"OfficialLanguages,omitempty"`
AwardCriterionParameter *AwardCriterionParameter `xml:"AwardCriterionParameter,omitempty"`
AccessToolName string `xml:"AccessToolName,omitempty"`
TransmissionDate string `xml:"TransmissionDate,omitempty"`
TransmissionTime string `xml:"TransmissionTime,omitempty"`
}
// NoticeSubType contains notice subtype information
@@ -908,6 +945,7 @@ type PostalAddress struct {
StreetName string `xml:"StreetName,omitempty"`
CityName string `xml:"CityName,omitempty"`
PostalZone string `xml:"PostalZone,omitempty"`
CountrySubentity string `xml:"CountrySubentity,omitempty"`
CountrySubentityCode string `xml:"CountrySubentityCode,omitempty"`
Department string `xml:"Department,omitempty"`
Country *Country `xml:"Country,omitempty"`
@@ -1338,11 +1376,11 @@ func (cn ContractNotice) Validate() error {
}
// Validate tender submission deadline format if present
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingProcess != nil &&
cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
if cn.firstLot() != nil &&
cn.firstLot().TenderingProcess != nil &&
cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
deadline := cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
deadline := cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
if deadline != "" {
var validDeadline bool
for _, format := range validFormats {
@@ -1358,23 +1396,23 @@ func (cn ContractNotice) Validate() error {
}
// Validate currency code if provided in budget
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
currency := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID
currency := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID
if currency != "" && len(currency) != 3 {
return fmt.Errorf("invalid currency code: %s (expected 3-letter ISO code)", currency)
}
}
// Validate duration unit code if provided
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil {
unitCode := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode
unitCode := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode
if unitCode != "" {
validUnits := map[string]bool{
"MONTH": true,
@@ -1396,9 +1434,14 @@ func (cn ContractNotice) GetID() string {
return cn.ID
}
// GetNoticeType returns the notice type
// GetNoticeType returns the notice type code (BT-02, cbc:NoticeTypeCode text).
func (cn ContractNotice) GetNoticeType() string {
return cn.NoticeTypeCode
return strings.TrimSpace(cn.NoticeTypeCode.Text)
}
// GetNoticeFormType returns the form type from listName on cbc:NoticeTypeCode (BT-03), e.g. "competition".
func (cn ContractNotice) GetNoticeFormType() string {
return strings.TrimSpace(cn.NoticeTypeCode.ListName)
}
// GetLanguage returns the notice language
@@ -1496,12 +1539,12 @@ func (cn ContractNotice) GetBuyerID() string {
// GetReviewOrganizationID returns the review organization ID from appeal terms
func (cn ContractNotice) GetReviewOrganizationID() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil &&
cn.ProcurementProjectLot.TenderingTerms.AppealTerms != nil &&
cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty != nil &&
cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil {
return cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID
if cn.firstLot() != nil &&
cn.firstLot().TenderingTerms != nil &&
cn.firstLot().TenderingTerms.AppealTerms != nil &&
cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty != nil &&
cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil {
return cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID
}
return ""
}
@@ -1516,18 +1559,18 @@ func (cn ContractNotice) GetProcedureCode() string {
// GetProcurementTitle returns the procurement project title
func (cn ContractNotice) GetProcurementTitle() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil {
return cn.ProcurementProjectLot.ProcurementProject.Name
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil {
return cn.firstLot().ProcurementProject.Name
}
return ""
}
// GetProcurementDescription returns the procurement project description
func (cn ContractNotice) GetProcurementDescription() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil {
return cn.ProcurementProjectLot.ProcurementProject.Description
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil {
return cn.firstLot().ProcurementProject.Description
}
return ""
}
@@ -1561,8 +1604,8 @@ func (cn ContractNotice) GetPlaceOfPerformance() string {
// GetLotID returns the procurement lot ID
func (cn ContractNotice) GetLotID() string {
if cn.ProcurementProjectLot != nil {
return cn.ProcurementProjectLot.ID
if cn.firstLot() != nil {
return cn.firstLot().ID
}
return ""
}
@@ -1580,17 +1623,17 @@ func (cn ContractNotice) GetSelectionCriteria() []SelectionCriteria {
// GetOfficialLanguages returns official languages from tender terms extensions
func (cn ContractNotice) GetOfficialLanguages() []string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil &&
len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil {
if cn.firstLot() != nil &&
cn.firstLot().TenderingTerms != nil &&
len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil {
languages := make([]string, len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language))
for i, lang := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language {
languages := make([]string, len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language))
for i, lang := range cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language {
languages[i] = lang.ID
}
return languages
@@ -1600,12 +1643,12 @@ func (cn ContractNotice) GetOfficialLanguages() []string {
// GetDocumentURI returns the document URI from call for tenders document reference
func (cn ContractNotice) GetDocumentURI() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil &&
len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil {
return cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI
if cn.firstLot() != nil &&
cn.firstLot().TenderingTerms != nil &&
len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil {
return cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI
}
return ""
}
@@ -1613,10 +1656,12 @@ func (cn ContractNotice) GetDocumentURI() string {
// GetAllDocumentURIs returns all document URIs from call for tenders document references
func (cn ContractNotice) GetAllDocumentURIs() []string {
var uris []string
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil {
for _, docRef := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference {
for i := range cn.ProcurementProjectLots {
lot := &cn.ProcurementProjectLots[i]
if lot.TenderingTerms == nil {
continue
}
for _, docRef := range lot.TenderingTerms.CallForTendersDocumentReference {
if docRef.Attachment != nil &&
docRef.Attachment.ExternalReference != nil &&
docRef.Attachment.ExternalReference.URI != "" {
@@ -1629,21 +1674,21 @@ func (cn ContractNotice) GetAllDocumentURIs() []string {
// GetTenderDeadline returns the tender submission deadline
func (cn ContractNotice) GetTenderDeadline() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingProcess != nil &&
cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
return cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
if cn.firstLot() != nil &&
cn.firstLot().TenderingProcess != nil &&
cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
return cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
}
return ""
}
// GetBudget returns the estimated contract amount and currency
func (cn ContractNotice) GetBudget() (amount string, currency string) {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
contractAmount := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
contractAmount := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
return contractAmount.Text, contractAmount.CurrencyID
}
return "", ""
@@ -1651,12 +1696,12 @@ func (cn ContractNotice) GetBudget() (amount string, currency string) {
// GetDuration returns the contract duration and unit
func (cn ContractNotice) GetDuration() (duration string, unit string) {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil {
durationMeasure := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure
durationMeasure := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure
return durationMeasure.Text, durationMeasure.UnitCode
}
return "", ""
@@ -1716,9 +1761,14 @@ func (can ContractAwardNotice) GetID() string {
return can.ID
}
// GetNoticeType returns the notice type
// GetNoticeType returns the notice type code (BT-02).
func (can ContractAwardNotice) GetNoticeType() string {
return can.NoticeTypeCode
return strings.TrimSpace(can.NoticeTypeCode.Text)
}
// GetNoticeFormType returns the form type from listName (BT-03).
func (can ContractAwardNotice) GetNoticeFormType() string {
return strings.TrimSpace(can.NoticeTypeCode.ListName)
}
// GetLanguage returns the notice language
@@ -1818,25 +1868,34 @@ func (can ContractAwardNotice) GetProcedureCode() string {
// GetProcurementTitle returns the procurement project title
func (can ContractAwardNotice) GetProcurementTitle() string {
if can.ProcurementProject != nil {
if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Name) != "" {
return can.ProcurementProject.Name
}
if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
return lot.ProcurementProject.Name
}
return ""
}
// GetProcurementDescription returns the procurement project description
func (can ContractAwardNotice) GetProcurementDescription() string {
if can.ProcurementProject != nil {
if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Description) != "" {
return can.ProcurementProject.Description
}
if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
return lot.ProcurementProject.Description
}
return ""
}
// GetContractNature returns the main nature of the contract
func (can ContractAwardNotice) GetContractNature() string {
if can.ProcurementProject != nil {
if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.ProcurementTypeCode) != "" {
return can.ProcurementProject.ProcurementTypeCode
}
if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
return lot.ProcurementProject.ProcurementTypeCode
}
return ""
}
@@ -1846,6 +1905,11 @@ func (can ContractAwardNotice) GetMainClassification() string {
can.ProcurementProject.MainCommodityClassification != nil {
return can.ProcurementProject.MainCommodityClassification.ItemClassificationCode
}
if lot := can.firstAwardLot(); lot != nil &&
lot.ProcurementProject != nil &&
lot.ProcurementProject.MainCommodityClassification != nil {
return lot.ProcurementProject.MainCommodityClassification.ItemClassificationCode
}
return ""
}
@@ -1882,3 +1946,27 @@ func (can ContractAwardNotice) GetOrganizationByID(orgID string) *Organization {
}
return nil
}
// GetWinningTenderPartyID resolves the tendering party reference for the primary winning bid from NoticeResult.
func (can ContractAwardNotice) GetWinningTenderPartyID() string {
nr := can.GetNoticeResult()
if nr == nil {
return ""
}
for _, lr := range nr.LotResult {
winner := pickWinningLotTender(lr.LotTender)
if winner != nil && winner.TenderingParty != nil && winner.TenderingParty.ID != "" {
return winner.TenderingParty.ID
}
}
return ""
}
// GetWinningTenderOrganization resolves the winning tenderer organization from NoticeResult via Organizations.
func (can ContractAwardNotice) GetWinningTenderOrganization() *Organization {
id := can.GetWinningTenderPartyID()
if id == "" {
return nil
}
return can.GetOrganizationByID(id)
}
ted/notice_upsert.go
+80
View File
@@ -0,0 +1,80 @@
package ted
import (
"strconv"
"strings"
"time"
"tm/internal/notice"
)
// noticeNeedsRefresh reports whether incoming scraped data should replace an existing notice row.
func noticeNeedsRefresh(existing, incoming *notice.Notice) bool {
if incoming == nil || existing == nil {
return true
}
if strings.TrimSpace(incoming.ContentXML) != "" &&
strings.TrimSpace(existing.ContentXML) != strings.TrimSpace(incoming.ContentXML) {
return true
}
if normalizePublicationID(existing.NoticePublicationID) != normalizePublicationID(incoming.NoticePublicationID) {
return true
}
if compareNoticeVersions(incoming.NoticeVersion, existing.NoticeVersion) > 0 {
return true
}
return false
}
func normalizePublicationID(s string) string {
return strings.TrimSpace(s)
}
// compareNoticeVersions compares BT-757 notice versions (e.g. "01", "02"). Empty parses as 0.
func compareNoticeVersions(incoming, existing string) int {
vi := parseNoticeVersion(incoming)
ve := parseNoticeVersion(existing)
switch {
case vi > ve:
return 1
case vi < ve:
return -1
default:
return 0
}
}
func parseNoticeVersion(s string) int {
s = strings.TrimSpace(s)
if s == "" {
return 0
}
n, err := strconv.Atoi(s)
if err != nil {
return 0
}
return n
}
// MergeNoticeUpdate replaces scrape-derived fields from src into dst while preserving identity,
// creation time, and optional enrichment metadata already stored on dst.
func MergeNoticeUpdate(dst *notice.Notice, src *notice.Notice) {
preservedID := dst.ID
preservedCreated := dst.CreatedAt
oldEnrichment := dst.ProcessingMetadata.EnrichmentData
oldTranslated := dst.ProcessingMetadata.TranslatedData
oldTranslatedAt := dst.ProcessingMetadata.TranslatedAt
*dst = *src
dst.ID = preservedID
dst.CreatedAt = preservedCreated
dst.UpdatedAt = time.Now().Unix()
dst.ProcessingMetadata.Processed = false
if len(dst.ProcessingMetadata.EnrichmentData) == 0 && len(oldEnrichment) > 0 {
dst.ProcessingMetadata.EnrichmentData = oldEnrichment
}
if len(dst.ProcessingMetadata.TranslatedData) == 0 && len(oldTranslated) > 0 {
dst.ProcessingMetadata.TranslatedData = oldTranslated
dst.ProcessingMetadata.TranslatedAt = oldTranslatedAt
}
}
ted/scraper.go
+64 -41
View File
@@ -4,6 +4,7 @@ import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"net/http"
@@ -12,7 +13,7 @@ import (
"time"
"tm/internal/notice"
"tm/pkg/logger"
"tm/pkg/mongo"
orm "tm/pkg/mongo"
"tm/pkg/notification"
)
@@ -38,16 +39,16 @@ type TEDScraper struct {
logger logger.Logger
httpClient *http.Client
xmlParser *TEDParser
mongoManager *mongo.ConnectionManager
tenderRepo notice.Repository
mongoManager *orm.ConnectionManager
noticeRepo notice.Repository
}
// NewTEDScraper creates a new TED scraper instance
func NewTEDScraper(
config *Config,
logger logger.Logger,
mongoManager *mongo.ConnectionManager,
tenderRepo notice.Repository,
mongoManager *orm.ConnectionManager,
noticeRepo notice.Repository,
notify notification.SDK,
) *TEDScraper {
httpClient := &http.Client{
@@ -61,7 +62,7 @@ func NewTEDScraper(
httpClient: httpClient,
xmlParser: NewTEDParser(),
mongoManager: mongoManager,
tenderRepo: tenderRepo,
noticeRepo: noticeRepo,
}
}
@@ -289,16 +290,18 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
return result, nil
}
func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID)
func (s *TEDScraper) findNoticeByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
if strings.TrimSpace(contractNoticeID) == "" {
return nil, nil
}
existing, err := s.noticeRepo.GetByContractNoticeID(ctx, contractNoticeID)
if err != nil {
if err.Error() == "document not found" {
if errors.Is(err, orm.ErrDocumentNotFound) {
return nil, nil
}
return nil, err
}
return existingTender, nil
return existing, nil
}
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, result *FileProcessingResult) error {
@@ -331,45 +334,66 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
}
t.ContentXML = string(content)
// Check if tender already exists to prevent duplicates
existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID)
existingNotice, err := s.findNoticeByContractNoticeID(ctx, t.ContractNoticeID)
if err != nil {
s.logger.Error("Failed to check existing tender", map[string]interface{}{
s.logger.Error("Failed to check existing notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"error": err.Error(),
})
return fmt.Errorf("failed to check existing tender: %w", err)
return fmt.Errorf("failed to check existing notice: %w", err)
}
if existingTender != nil {
s.logger.Info("Tender already exists, skipping duplicate", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"existing_id": existingTender.ID,
if existingNotice != nil {
if !noticeNeedsRefresh(existingNotice, t) {
s.logger.Debug("Notice unchanged, skipping", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"notice_doc_id": existingNotice.ID.Hex(),
})
return nil
}
MergeNoticeUpdate(existingNotice, t)
err = s.noticeRepo.Update(ctx, existingNotice)
if err != nil {
s.logger.Error("Failed to update existing notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"notice_doc_id": existingNotice.ID.Hex(),
"error": err.Error(),
})
return fmt.Errorf("failed to update notice: %w", err)
}
s.logger.Info("Notice updated (refresh)", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"notice_publication_id": t.NoticePublicationID,
"notice_version": t.NoticeVersion,
"notice_doc_id": existingNotice.ID.Hex(),
"processing_reset_worker": true,
})
return nil
}
// Create new tender
s.logger.Info("Creating new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
})
t.CreatedAt = time.Now().Unix()
t.UpdatedAt = t.CreatedAt
err = s.tenderRepo.Import(ctx, t)
if err != nil {
s.logger.Error("Failed to create new tender", map[string]interface{}{
result.SuccessCount++
} else {
s.logger.Info("Creating new notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"error": err.Error(),
})
return fmt.Errorf("failed to create tender: %w", err)
}
s.logger.Info("Successfully created new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
})
// }
now := time.Now().Unix()
t.CreatedAt = now
t.UpdatedAt = now
err = s.noticeRepo.Import(ctx, t)
if err != nil {
s.logger.Error("Failed to import notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"error": err.Error(),
})
return fmt.Errorf("failed to import notice: %w", err)
}
s.logger.Info("Successfully imported new notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
})
result.SuccessCount++
}
// Log detailed parsed information based on notice type
s.logger.Info("Successfully parsed notice", map[string]interface{}{
@@ -379,7 +403,6 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
"tender_url": t.TenderURL,
})
result.SuccessCount++
return nil
}
ted/service.go
+51
View File
@@ -7,6 +7,57 @@ import (
"time"
)
// ParseIssueDateTimeToUnix combines cbc:IssueDate and cbc:IssueTime (with timezone offsets) into one Unix timestamp.
func ParseIssueDateTimeToUnix(issueDate, issueTime string) int64 {
issueDate = strings.TrimSpace(issueDate)
issueTime = strings.TrimSpace(issueTime)
if issueDate == "" {
return 0
}
if issueTime == "" {
return ParseDateToUnix(issueDate)
}
dateCore := issueDate
if len(issueDate) >= 10 && issueDate[4] == '-' && issueDate[7] == '-' {
dateCore = issueDate[:10]
}
combined := dateCore + "T" + issueTime
if t, err := ParseDate(combined); err == nil {
return t.Unix()
}
return ParseDateToUnix(issueDate)
}
// TransmissionDispatchUnix reads BT-803 (eSender) transmission date/time from extension content (root or eForms extension).
func TransmissionDispatchUnix(ext *UBLExtensions) int64 {
if ext == nil || ext.UBLExtension == nil || ext.UBLExtension.ExtensionContent == nil {
return 0
}
ec := ext.UBLExtension.ExtensionContent
if u := ParseIssueDateTimeToUnix(ec.TransmissionDate, ec.TransmissionTime); u != 0 {
return u
}
if ec.EformsExtension != nil {
if u := ParseIssueDateTimeToUnix(ec.EformsExtension.TransmissionDate, ec.EformsExtension.TransmissionTime); u != 0 {
return u
}
}
return 0
}
// ParseEuropeanAmount normalizes TED amount strings (spaces, comma decimal) for strconv.ParseFloat.
func ParseEuropeanAmount(raw string) (float64, bool) {
s := strings.TrimSpace(raw)
s = strings.ReplaceAll(s, "\u00a0", "")
s = strings.ReplaceAll(s, " ", "")
s = strings.ReplaceAll(s, ",", ".")
if s == "" {
return 0, false
}
v, err := strconv.ParseFloat(s, 64)
return v, err == nil
}
// parseDate parses various date formats used in TED XML
func ParseDate(dateStr string) (time.Time, error) {
if dateStr == "" {