Refactor TED Calendar and EForm Structure for Enhanced Functionality

- Updated the GetOJS function to accept a base URL parameter, improving flexibility in constructing the calendar URL.
- Removed the deprecated eform.go file, streamlining the codebase by eliminating unused components.
- Introduced new eform structures for BusinessRegistrationInformationNotice, ContractNotice, ContractAwardNotice, and PriorInformationNotice, enhancing the handling of TED XML data.
- Implemented mapping functions to convert eform notices to the Tender entity, improving integration with the tender management system.
- Added a service layer to encapsulate business logic related to eform processing, adhering to Clean Architecture principles.
This commit is contained in:
n.nakhostin
2025-09-30 16:51:40 +03:30
parent 05c7eae8a2
commit 44f266189c
9 changed files with 574 additions and 336 deletions
+3 -3
View File
@@ -10,15 +10,15 @@ import (
)
const (
calendarUrl = "https://ted.europa.eu/en/release-calendar/-/download/file/CSV/%d"
calendarUrl = "%s/en/release-calendar/-/download/file/CSV/%d"
)
// Get OJS Number
func GetOJS(year int, date string) (string, error) {
func GetOJS(baseURL string, year int, date string) (string, error) {
var (
ojs string
)
url := fmt.Sprintf(calendarUrl, year)
url := fmt.Sprintf(calendarUrl, baseURL, year)
resp, err := http.Get(url)
if err != nil {
-126
View File
@@ -1,126 +0,0 @@
package ted
import (
"encoding/xml"
"tm/ted/eform"
)
// =====================================================
// ROOT NOTICE STRUCTURE
// =====================================================
// Notice represents the UBL ContractNotice structure used by TED
type ContractNotice struct {
XMLName xml.Name `xml:"ContractNotice"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
UBLExtensions eform.UBLExtensions `xml:"UBLExtensions,omitempty"`
ContractingParty eform.ContractingParty `xml:"ContractingParty"`
TenderingTerms eform.TenderingTerms `xml:"TenderingTerms,omitempty"`
TenderingProcess eform.TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject eform.ProcurementProject `xml:"ProcurementProject"`
ProcurementProjectLot []eform.ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
}
// ContractAwardNotice represents the UBL ContractAwardNotice structure
type ContractAwardNotice struct {
XMLName xml.Name `xml:"ContractAwardNotice"`
UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"`
CustomizationID string `json:"customization_id" xml:"CustomizationID"`
ID string `json:"id" xml:"ID"`
ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"`
IssueDate string `json:"issue_date" xml:"IssueDate"`
IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"`
VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"`
RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"`
NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"`
NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"`
ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"`
TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"`
TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"`
ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"`
ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"`
UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"`
}
// PriorInformationNotice represents the UBL PriorInformationNotice structure
type PriorInformationNotice struct {
XMLName xml.Name `xml:"PriorInformationNotice"`
UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"`
CustomizationID string `json:"customization_id" xml:"CustomizationID"`
ID string `json:"id" xml:"ID"`
ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"`
IssueDate string `json:"issue_date" xml:"IssueDate"`
IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"`
VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"`
RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"`
NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"`
NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"`
ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"`
TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"`
TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"`
ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"`
ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"`
UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"`
}
// BusinessRegistrationInformationNotice represents the UBL BusinessRegistrationInformationNotice structure
type BusinessRegistrationInformationNotice struct {
XMLName xml.Name `xml:"BusinessRegistrationInformationNotice"`
UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"`
CustomizationID string `json:"customization_id" xml:"CustomizationID"`
ID string `json:"id" xml:"ID"`
IssueDate string `json:"issue_date" xml:"IssueDate"`
IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"`
VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"`
RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"`
NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"`
NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"`
UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"`
AdditionalNoticeLanguage []eform.AdditionalNoticeLanguage `json:"additional_notice_language,omitempty" xml:"AdditionalNoticeLanguage,omitempty"`
SenderParty eform.SenderParty `json:"sender_party,omitempty" xml:"SenderParty,omitempty"`
BusinessParty eform.BusinessParty `json:"business_party" xml:"BusinessParty"`
AdditionalDocumentReference []eform.AdditionalDocumentReference `json:"additional_document_reference,omitempty" xml:"AdditionalDocumentReference,omitempty"`
BusinessCapability []eform.BusinessCapability `json:"business_capability,omitempty" xml:"BusinessCapability,omitempty"`
NoticePurpose eform.NoticePurpose `json:"notice_purpose,omitempty" xml:"NoticePurpose,omitempty"`
}
// =====================================================
// BUSINESS LOGIC HELPER METHODS
// =====================================================
// GetMainBuyer returns the lead/main buyer organization
func (n *ContractNotice) GetMainBuyer() *eform.Organization {
if len(n.UBLExtensions.UBLExtension) > 0 {
orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization
for i := range orgs {
for _, role := range orgs[i].Roles {
if role == eform.RoleBuyer || role == eform.RoleLeadBuyer {
return &orgs[i]
}
}
}
}
return nil
}
// GetOrganizationByID retrieves organization by ID
func (n *ContractNotice) GetOrganizationByID(id string) *eform.Organization {
if len(n.UBLExtensions.UBLExtension) > 0 {
orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization
for i := range orgs {
if orgs[i].ID == id {
return &orgs[i]
}
}
}
return nil
}
@@ -0,0 +1,44 @@
package business_registration_information
import (
"encoding/xml"
"tm/internal/tender"
"tm/ted/eform"
)
// BusinessRegistrationInformationNotice represents the UBL BusinessRegistrationInformationNotice structure
type BusinessRegistrationInformationNotice struct {
XMLName xml.Name `xml:"BusinessRegistrationInformationNotice"`
UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"`
CustomizationID string `json:"customization_id" xml:"CustomizationID"`
ID string `json:"id" xml:"ID"`
IssueDate string `json:"issue_date" xml:"IssueDate"`
IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"`
VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"`
RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"`
NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"`
NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"`
UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"`
AdditionalNoticeLanguage []eform.AdditionalNoticeLanguage `json:"additional_notice_language,omitempty" xml:"AdditionalNoticeLanguage,omitempty"`
SenderParty eform.SenderParty `json:"sender_party,omitempty" xml:"SenderParty,omitempty"`
BusinessParty eform.BusinessParty `json:"business_party" xml:"BusinessParty"`
AdditionalDocumentReference []eform.AdditionalDocumentReference `json:"additional_document_reference,omitempty" xml:"AdditionalDocumentReference,omitempty"`
BusinessCapability []eform.BusinessCapability `json:"business_capability,omitempty" xml:"BusinessCapability,omitempty"`
NoticePurpose eform.NoticePurpose `json:"notice_purpose,omitempty" xml:"NoticePurpose,omitempty"`
}
// GetID returns the business registration information notice ID
func (brin BusinessRegistrationInformationNotice) GetID() string {
return brin.ID
}
// Validate validates the parsed model
func (brin BusinessRegistrationInformationNotice) Validate() error {
return nil
}
func (notice BusinessRegistrationInformationNotice) MapToTender() *tender.Tender {
t := new(tender.Tender)
t.ContractNoticeID = notice.GetID()
return t
}
+213
View File
@@ -0,0 +1,213 @@
package contract
import (
"encoding/xml"
"time"
"tm/internal/tender"
"tm/ted/eform"
)
// Notice represents the UBL ContractNotice structure used by TED
type ContractNotice struct {
XMLName xml.Name `xml:"ContractNotice"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
UBLExtensions eform.UBLExtensions `xml:"UBLExtensions,omitempty"`
ContractingParty eform.ContractingParty `xml:"ContractingParty"`
TenderingTerms eform.TenderingTerms `xml:"TenderingTerms,omitempty"`
TenderingProcess eform.TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject eform.ProcurementProject `xml:"ProcurementProject"`
ProcurementProjectLot []eform.ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
}
func (notice ContractNotice) MapToTender() *tender.Tender {
now := time.Now().Unix()
t := &tender.Tender{
ContractNoticeID: notice.ID,
ContractFolderID: notice.ContractFolderID,
NoticeTypeCode: notice.NoticeTypeCode,
// NoticeSubTypeCode: notice.NoticeSubTypeCode,
NoticeLanguageCode: notice.NoticeLanguageCode,
// IssueDate: parseDateToUnixMilli(notice.IssueDate),
// IssueTime: parseTimeToUnixMilli(notice.IssueTime),
// GazetteID: notice.GetOJSID(),
// Title: notice.GetProcurementTitle(),
// Description: notice.GetProcurementDescription(),
// ProcurementTypeCode: notice.GetContractNature(),
// ProcedureCode: notice.GetProcedureCode(),
// MainClassification: notice.GetMainClassification(),
// PlaceOfPerformance: notice.GetPlaceOfPerformance(),
// DocumentURI: notice.GetDocumentURI(),
Status: tender.TenderStatusActive,
Source: tender.TenderSourceTEDScraper,
ProcessingMetadata: tender.ProcessingMetadata{
ScrapedAt: now,
ProcessedAt: now,
ProcessingVersion: "1.0",
},
}
// Set publication information
// if pubInfo := notice.GetPublicationInfo(); pubInfo != nil {
// t.NoticePublicationID = pubInfo.NoticePublicationID
// t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate)
// t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
// }
// Set additional classifications
// if notice.ProcurementProject != nil && notice.ProcurementProject.AdditionalCommodityClassification != nil {
// for _, additionalClass := range notice.ProcurementProject.AdditionalCommodityClassification {
// if additionalClass.ItemClassificationCode != "" {
// t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode)
// }
// }
// }
// Set estimated value and currency
// if amount, currency := notice.GetBudget(); amount != "" {
// if value, err := strconv.ParseFloat(amount, 64); err == nil {
// t.EstimatedValue = value
// t.Currency = currency
// }
// }
// Set duration information
// if duration, unit := notice.GetDuration(); duration != "" {
// t.Duration = duration
// t.DurationUnit = unit
// }
// Set tender deadline
// if deadline := notice.GetTenderDeadline(); deadline != "" {
// t.TenderDeadline = parseDateToUnixMilli(deadline)
// if deadlineTime, err := parseDate(deadline); err == nil {
// applicationDeadline := calculateApplicationDeadline(deadlineTime)
// t.ApplicationDeadline = applicationDeadline.UnixMilli()
// }
// }
// if pubDate, err := parseDate(notice.GetPublicationDate()); err == nil {
// submissionDeadline := calculateSubmissionDeadline(pubDate)
// t.SubmissionDeadline = submissionDeadline.UnixMilli()
// }
// Set SubmissionURL from TenderingTerms
// if notice.ProcurementProjectLot != nil && notice.ProcurementProjectLot.TenderingTerms != nil {
// if tenderingTerms := notice.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
// if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
// if strings.HasPrefix(endpointID, "http") {
// t.SubmissionURL = endpointID
// } else {
// t.SubmissionURL = "https://" + endpointID
// }
// }
// }
// }
// Set location information
// if notice.ProcurementProject != nil && notice.ProcurementProject.RealizedLocation != nil {
// if addr := notice.ProcurementProject.RealizedLocation.Address; addr != nil {
// t.CityName = addr.CityName
// t.PostalCode = addr.PostalZone
// t.RegionCode = addr.CountrySubentityCode
// if addr.Country != nil {
// t.CountryCode = addr.Country.IdentificationCode
// }
// }
// }
// Set buyer organization
// buyerID := notice.GetBuyerID()
// if buyerID != "" {
// if buyerOrg := notice.GetOrganizationByID(buyerID); buyerOrg != nil {
// t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
// }
// }
// Set review organization
// reviewID := notice.GetReviewOrganizationID()
// if reviewID != "" {
// if reviewOrg := notice.GetOrganizationByID(reviewID); reviewOrg != nil {
// t.ReviewOrganization = s.mapOrganization(reviewOrg, "review")
// }
// }
// Set all organizations
// if orgs := notice.GetOrganizations(); orgs != nil {
// for _, org := range orgs.Organization {
// mappedOrg := s.mapOrganization(&org, "")
// if mappedOrg != nil {
// t.Organizations = append(t.Organizations, *mappedOrg)
// }
// }
// }
// Set selection criteria
// if criteria := notice.GetSelectionCriteria(); criteria != nil {
// for _, criterion := range criteria {
// t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{
// TypeCode: criterion.CriterionTypeCode,
// Description: criterion.Description,
// LanguageID: criterion.LanguageID,
// })
// }
// }
// Set official languages
// if languages := notice.GetOfficialLanguages(); languages != nil {
// t.OfficialLanguages = languages
// }
// Generate unique tender ID
// t.TenderID = generateTenderID(t)
return t
}
// GetID returns the contract notice ID
func (cn ContractNotice) GetID() string {
return cn.ID
}
// Validate validates the parsed model
func (cn ContractNotice) Validate() error {
return nil
}
// GetMainBuyer returns the lead/main buyer organization
func (n *ContractNotice) GetMainBuyer() *eform.Organization {
if len(n.UBLExtensions.UBLExtension) > 0 {
orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization
for i := range orgs {
for _, role := range orgs[i].Roles {
if role == eform.RoleBuyer || role == eform.RoleLeadBuyer {
return &orgs[i]
}
}
}
}
return nil
}
// GetOrganizationByID retrieves organization by ID
func (n *ContractNotice) GetOrganizationByID(id string) *eform.Organization {
if len(n.UBLExtensions.UBLExtension) > 0 {
orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization
for i := range orgs {
if orgs[i].ID == id {
return &orgs[i]
}
}
}
return nil
}
@@ -0,0 +1,44 @@
package contract_award
import (
"encoding/xml"
"tm/internal/tender"
"tm/ted/eform"
)
// ContractAwardNotice represents the UBL ContractAwardNotice structure
type ContractAwardNotice struct {
XMLName xml.Name `xml:"ContractAwardNotice"`
UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"`
CustomizationID string `json:"customization_id" xml:"CustomizationID"`
ID string `json:"id" xml:"ID"`
ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"`
IssueDate string `json:"issue_date" xml:"IssueDate"`
IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"`
VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"`
RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"`
NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"`
NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"`
ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"`
TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"`
TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"`
ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"`
ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"`
UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"`
}
// GetID returns the contract award notice ID
func (can ContractAwardNotice) GetID() string {
return can.ID
}
// Validate validates the parsed model
func (can ContractAwardNotice) Validate() error {
return nil
}
func (notice ContractAwardNotice) MapToTender() *tender.Tender {
t := new(tender.Tender)
t.ContractNoticeID = notice.GetID()
return t
}
@@ -0,0 +1,44 @@
package prior_information
import (
"encoding/xml"
"tm/internal/tender"
"tm/ted/eform"
)
// PriorInformationNotice represents the UBL PriorInformationNotice structure
type PriorInformationNotice struct {
XMLName xml.Name `xml:"PriorInformationNotice"`
UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"`
CustomizationID string `json:"customization_id" xml:"CustomizationID"`
ID string `json:"id" xml:"ID"`
ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"`
IssueDate string `json:"issue_date" xml:"IssueDate"`
IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"`
VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"`
RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"`
NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"`
NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"`
ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"`
TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"`
TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"`
ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"`
ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"`
UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"`
}
// GetID returns the prior information notice ID
func (pin PriorInformationNotice) GetID() string {
return pin.ID
}
// Validate validates the parsed model
func (pin PriorInformationNotice) Validate() error {
return nil
}
func (pin PriorInformationNotice) MapToTender() *tender.Tender {
t := new(tender.Tender)
t.ContractNoticeID = pin.GetID()
return t
}
+21 -105
View File
@@ -4,8 +4,11 @@ import (
"encoding/xml"
"fmt"
"strings"
"tm/pkg/notification"
"tm/pkg/xmlparser"
"tm/ted/eform/business_registration_information"
"tm/ted/eform/contract"
"tm/ted/eform/contract_award"
"tm/ted/eform/prior_information"
)
type ParserType string
@@ -18,8 +21,10 @@ const (
)
type TEDParser struct {
Notification notification.SDK
NoticeParser xmlparser.Parser[ContractNotice]
ContractNotice xmlparser.Parser[contract.ContractNotice]
ContractAwardNotice xmlparser.Parser[contract_award.ContractAwardNotice]
PriorInformationNotice xmlparser.Parser[prior_information.PriorInformationNotice]
BusinessRegistrationInformationNotice xmlparser.Parser[business_registration_information.BusinessRegistrationInformationNotice]
}
// NewTEDParser creates a new TED parser
@@ -28,9 +33,16 @@ func NewTEDParser() *TEDParser {
options.MaxSize = 50 * 1024 * 1024 // 50MB limit for TED XML files
options.IgnoreUnknownElements = true // TED XML has many optional fields
parser := xmlparser.NewParser[ContractNotice](options)
contractNoticeParser := xmlparser.NewParser[contract.ContractNotice](options)
contractAwardNoticeParser := xmlparser.NewParser[contract_award.ContractAwardNotice](options)
priorInformationNoticeParser := xmlparser.NewParser[prior_information.PriorInformationNotice](options)
businessRegistrationInformationNoticeParser := xmlparser.NewParser[business_registration_information.BusinessRegistrationInformationNotice](options)
return &TEDParser{
NoticeParser: parser,
ContractNotice: contractNoticeParser,
ContractAwardNotice: contractAwardNoticeParser,
PriorInformationNotice: priorInformationNoticeParser,
BusinessRegistrationInformationNotice: businessRegistrationInformationNoticeParser,
}
}
@@ -45,16 +57,16 @@ func (tp *TEDParser) Parse(content []byte) (interface{}, string, ParserType, err
// Parse based on the detected type
switch ParserType(rootElement) {
case ParserTypeContractNotice:
notice, err := tp.parseContractNotice(content)
notice, err := tp.ContractNotice.ParseBytes(content)
return notice, notice.GetID(), ParserTypeContractNotice, err
case ParserTypeContractAwardNotice:
notice, err := tp.parseContractAwardNotice(content)
notice, err := tp.ContractAwardNotice.ParseBytes(content)
return notice, notice.GetID(), ParserTypeContractAwardNotice, err
case ParserTypePriorInformationNotice:
notice, err := tp.parsePriorInformationNotice(content)
notice, err := tp.PriorInformationNotice.ParseBytes(content)
return notice, notice.GetID(), ParserTypePriorInformationNotice, err
case ParserTypeBusinessRegistrationInformationNotice:
notice, err := tp.parseBusinessRegistrationInformationNotice(content)
notice, err := tp.BusinessRegistrationInformationNotice.ParseBytes(content)
return notice, notice.GetID(), ParserTypeBusinessRegistrationInformationNotice, err
default:
return nil, "", ParserType(""), fmt.Errorf("unsupported notice type: %s", rootElement)
@@ -74,99 +86,3 @@ func detectRootElement(content []byte) (string, error) {
}
}
}
// parseContractNotice parses a ContractNotice
func (tp *TEDParser) parseContractNotice(content []byte) (*ContractNotice, error) {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024
options.IgnoreUnknownElements = true
parser := xmlparser.NewParser[ContractNotice](options)
notice, err := parser.ParseBytes(content)
if err != nil {
return nil, err
}
return notice, nil
}
// parseContractAwardNotice parses a ContractAwardNotice
func (tp *TEDParser) parseContractAwardNotice(content []byte) (*ContractAwardNotice, error) {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024
options.IgnoreUnknownElements = true
parser := xmlparser.NewParser[ContractAwardNotice](options)
notice, err := parser.ParseBytes(content)
if err != nil {
return nil, err
}
return notice, nil
}
// parsePriorInformationNotice parses a PriorInformationNotice
func (tp *TEDParser) parsePriorInformationNotice(content []byte) (*PriorInformationNotice, error) {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024
options.IgnoreUnknownElements = true
parser := xmlparser.NewParser[PriorInformationNotice](options)
notice, err := parser.ParseBytes(content)
if err != nil {
return nil, err
}
return notice, nil
}
// parseBusinessRegistrationInformationNotice parses a BusinessRegistrationInformationNotice
func (tp *TEDParser) parseBusinessRegistrationInformationNotice(content []byte) (*BusinessRegistrationInformationNotice, error) {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024
options.IgnoreUnknownElements = true
parser := xmlparser.NewParser[BusinessRegistrationInformationNotice](options)
notice, err := parser.ParseBytes(content)
if err != nil {
return nil, err
}
return notice, nil
}
// GetID returns the contract notice ID
func (cn ContractNotice) GetID() string {
return cn.ID
}
// GetID returns the contract award notice ID
func (can ContractAwardNotice) GetID() string {
return can.ID
}
// GetID returns the prior information notice ID
func (pin PriorInformationNotice) GetID() string {
return pin.ID
}
// GetID returns the business registration information notice ID
func (brin BusinessRegistrationInformationNotice) GetID() string {
return brin.ID
}
// Validate validates the parsed model
func (cn ContractNotice) Validate() error {
return nil
}
// Validate validates the parsed model
func (can ContractAwardNotice) Validate() error {
return nil
}
// Validate validates the parsed model
func (pin PriorInformationNotice) Validate() error {
return nil
}
// Validate validates the parsed model
func (brin BusinessRegistrationInformationNotice) Validate() error {
return nil
}
+4 -102
View File
@@ -283,7 +283,7 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
})
// Parse XML using the flexible TED parser that detects notice type
_, id, noticeType, err := s.xmlParser.Parse(content)
notice, id, noticeType, err := s.xmlParser.Parse(content)
if err != nil {
s.logger.Error("XML parsing failed", map[string]interface{}{
"file_name": fileName,
@@ -293,6 +293,8 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
return fmt.Errorf("failed to parse XML: %w", err)
}
_ = notice
// Log detailed parsed information based on notice type
s.logger.Info("Successfully parsed notice", map[string]interface{}{
"notice_id": id,
@@ -311,7 +313,7 @@ func (s *TEDScraper) Run(ctx context.Context) error {
})
// get ojs
ojs, err := GetOJS(now.Year(), now.Format("02/01/2006"))
ojs, err := GetOJS(s.config.BaseURL, now.Year(), now.Format("02/01/2006"))
if err != nil {
s.logger.Error("Failed to get OJS", map[string]interface{}{
"error": err.Error(),
@@ -337,103 +339,3 @@ func (s *TEDScraper) Run(ctx context.Context) error {
return nil
}
// parseDate parses various date formats used in TED XML
func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) {
if dateStr == "" {
return time.Time{}, fmt.Errorf("empty date string")
}
// Common TED date formats
formats := []string{
"2006-01-02+07:00",
"2006-01-02-07:00",
"2006-01-02T15:04:05+07:00",
"2006-01-02T15:04:05-07:00",
"2006-01-02T15:04:05Z",
"2006-01-02Z",
"2006-01-02",
time.RFC3339,
time.RFC3339Nano,
}
for _, format := range formats {
if parsedTime, err := time.Parse(format, dateStr); err == nil {
return parsedTime, nil
}
}
return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr)
}
// parseDateToUnixMilli converts string date to Unix milliseconds (int64)
func (s *TEDScraper) parseDateToUnixMilli(dateStr string) int64 {
if dateStr == "" {
return 0
}
if parsedTime, err := s.parseDate(dateStr); err == nil {
return parsedTime.UnixMilli()
}
return 0
}
// parseTimeToUnixMilli converts string time to Unix milliseconds (int64)
func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 {
if timeStr == "" {
return 0
}
// If time string is just time without date, use today's date
if !strings.Contains(timeStr, "T") && !strings.Contains(timeStr, " ") {
// Assume it's just a time like "15:04:05"
today := time.Now().Format("2006-01-02")
timeStr = fmt.Sprintf("%sT%s", today, timeStr)
}
if parsedTime, err := s.parseDate(timeStr); err == nil {
return parsedTime.UnixMilli()
}
return 0
}
// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation
func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string {
if unixMilli == 0 {
return "00000000"
}
return time.UnixMilli(unixMilli).Format("20060102")
}
func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time {
deadline := publicationDate
workingHoursAdded := 0
for workingHoursAdded < 48 {
deadline = deadline.Add(1 * time.Hour)
if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday {
workingHoursAdded++
}
}
return deadline
}
func (s *TEDScraper) calculateApplicationDeadline(tenderDeadline time.Time) time.Time {
applicationDeadline := tenderDeadline
workingDaysSubtracted := 0
for workingDaysSubtracted < 7 {
applicationDeadline = applicationDeadline.AddDate(0, 0, -1)
if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday {
workingDaysSubtracted++
}
}
return applicationDeadline
}
+201
View File
@@ -0,0 +1,201 @@
package ted
import (
"fmt"
"strconv"
"strings"
"time"
"tm/internal/tender"
"tm/ted/eform/business_registration_information"
"tm/ted/eform/contract"
"tm/ted/eform/contract_award"
"tm/ted/eform/prior_information"
)
func EFormMapper(notice interface{}) *tender.Tender {
t := new(tender.Tender)
switch notice := notice.(type) {
case contract.ContractNotice:
return notice.MapToTender()
case contract_award.ContractAwardNotice:
return notice.MapToTender()
case prior_information.PriorInformationNotice:
return notice.MapToTender()
case business_registration_information.BusinessRegistrationInformationNotice:
return notice.MapToTender()
default:
return t
}
}
// parseDate parses various date formats used in TED XML
func parseDate(dateStr string) (time.Time, error) {
if dateStr == "" {
return time.Time{}, fmt.Errorf("empty date string")
}
// Common TED date formats
formats := []string{
"2006-01-02+07:00",
"2006-01-02-07:00",
"2006-01-02T15:04:05+07:00",
"2006-01-02T15:04:05-07:00",
"2006-01-02T15:04:05Z",
"2006-01-02Z",
"2006-01-02",
time.RFC3339,
time.RFC3339Nano,
}
for _, format := range formats {
if parsedTime, err := time.Parse(format, dateStr); err == nil {
return parsedTime, nil
}
}
return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr)
}
// parseDateToUnixMilli converts string date to Unix milliseconds (int64)
func parseDateToUnixMilli(dateStr string) int64 {
if dateStr == "" {
return 0
}
if parsedTime, err := parseDate(dateStr); err == nil {
return parsedTime.UnixMilli()
}
return 0
}
// parseTimeToUnixMilli converts string time to Unix milliseconds (int64)
func parseTimeToUnixMilli(timeStr string) int64 {
if timeStr == "" {
return 0
}
// If time string is just time without date, use today's date
if !strings.Contains(timeStr, "T") && !strings.Contains(timeStr, " ") {
// Assume it's just a time like "15:04:05"
today := time.Now().Format("2006-01-02")
timeStr = fmt.Sprintf("%sT%s", today, timeStr)
}
if parsedTime, err := parseDate(timeStr); err == nil {
return parsedTime.UnixMilli()
}
return 0
}
// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation
func unixMilliToDateString(unixMilli int64) string {
if unixMilli == 0 {
return "00000000"
}
return time.UnixMilli(unixMilli).Format("20060102")
}
func calculateSubmissionDeadline(publicationDate time.Time) time.Time {
deadline := publicationDate
workingHoursAdded := 0
for workingHoursAdded < 48 {
deadline = deadline.Add(1 * time.Hour)
if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday {
workingHoursAdded++
}
}
return deadline
}
func calculateApplicationDeadline(tenderDeadline time.Time) time.Time {
applicationDeadline := tenderDeadline
workingDaysSubtracted := 0
for workingDaysSubtracted < 7 {
applicationDeadline = applicationDeadline.AddDate(0, 0, -1)
if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday {
workingDaysSubtracted++
}
}
return applicationDeadline
}
// padOrTruncate pads a string with zeros or truncates it to the specified length
func padOrTruncate(str string, length int) string {
// Remove any non-alphanumeric characters and convert to uppercase
clean := strings.ToUpper(strings.ReplaceAll(str, "-", ""))
clean = strings.ReplaceAll(clean, " ", "")
if len(clean) >= length {
return clean[:length]
}
// Pad with zeros
return clean + strings.Repeat("0", length-len(clean))
}
// generateTenderURL generates the TED tender URL from NoticePublicationID
func generateTenderURL(noticePublicationID string) string {
if noticePublicationID == "" {
return ""
}
// Remove leading zeros and format for URL
// Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025
parts := strings.Split(noticePublicationID, "-")
if len(parts) != 2 {
return ""
}
// Convert to int and back to string to remove leading zeros
if num, err := strconv.Atoi(parts[0]); err == nil {
cleanID := fmt.Sprintf("%d-%s", num, parts[1])
return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID)
}
return ""
}
// generateTenderID generates a unique tender ID using the format:
// {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
func generateTenderID(t *tender.Tender) string {
var parts []string
// Source (TED)
parts = append(parts, "1")
// TED code (Notice Publication ID, first 8 chars)
tedCode := "00000000"
if t.NoticePublicationID != "" {
tedCode = padOrTruncate(t.NoticePublicationID, 8)
}
parts = append(parts, tedCode)
// Buyer code (first 8 chars of buyer organization ID, or 8 zeros if not available)
// buyerCode := "00000000"
// if t.BuyerOrganization != nil && t.BuyerOrganization.ID != "" {
// buyerCode = padOrTruncate(t.BuyerOrganization.ID, 8)
// }
// parts = append(parts, buyerCode)
// Date (YYYYMMDD format from issue date)
parts = append(parts, padOrTruncate(unixMilliToDateString(t.IssueDate), 4))
// Location code (country code + region code, padded to 6 chars)
locationCode := "EU"
if t.CountryCode != "" {
locationCode = t.CountryCode
}
parts = append(parts, locationCode)
return strings.Join(parts, "")
}