diff --git a/ted/calendar.go b/ted/calendar.go index 3cfd3e8..dc413b9 100644 --- a/ted/calendar.go +++ b/ted/calendar.go @@ -10,15 +10,15 @@ import ( ) const ( - calendarUrl = "https://ted.europa.eu/en/release-calendar/-/download/file/CSV/%d" + calendarUrl = "%s/en/release-calendar/-/download/file/CSV/%d" ) // Get OJS Number -func GetOJS(year int, date string) (string, error) { +func GetOJS(baseURL string, year int, date string) (string, error) { var ( ojs string ) - url := fmt.Sprintf(calendarUrl, year) + url := fmt.Sprintf(calendarUrl, baseURL, year) resp, err := http.Get(url) if err != nil { diff --git a/ted/eform.go b/ted/eform.go deleted file mode 100644 index c478bb9..0000000 --- a/ted/eform.go +++ /dev/null @@ -1,126 +0,0 @@ -package ted - -import ( - "encoding/xml" - "tm/ted/eform" -) - -// ===================================================== -// ROOT NOTICE STRUCTURE -// ===================================================== - -// Notice represents the UBL ContractNotice structure used by TED -type ContractNotice struct { - XMLName xml.Name `xml:"ContractNotice"` - UBLVersionID string `xml:"UBLVersionID"` - CustomizationID string `xml:"CustomizationID"` - ID string `xml:"ID"` - ContractFolderID string `xml:"ContractFolderID"` - IssueDate string `xml:"IssueDate"` - IssueTime string `xml:"IssueTime"` - VersionID string `xml:"VersionID"` - RegulatoryDomain string `xml:"RegulatoryDomain"` - NoticeTypeCode string `xml:"NoticeTypeCode"` - NoticeLanguageCode string `xml:"NoticeLanguageCode"` - UBLExtensions eform.UBLExtensions `xml:"UBLExtensions,omitempty"` - ContractingParty eform.ContractingParty `xml:"ContractingParty"` - TenderingTerms eform.TenderingTerms `xml:"TenderingTerms,omitempty"` - TenderingProcess eform.TenderingProcess `xml:"TenderingProcess,omitempty"` - ProcurementProject eform.ProcurementProject `xml:"ProcurementProject"` - ProcurementProjectLot []eform.ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"` -} - -// ContractAwardNotice represents the UBL ContractAwardNotice structure -type ContractAwardNotice struct { - XMLName xml.Name `xml:"ContractAwardNotice"` - UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"` - CustomizationID string `json:"customization_id" xml:"CustomizationID"` - ID string `json:"id" xml:"ID"` - ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"` - IssueDate string `json:"issue_date" xml:"IssueDate"` - IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"` - VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"` - RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"` - NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"` - NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"` - ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"` - TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"` - TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"` - ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"` - ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"` - UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` -} - -// PriorInformationNotice represents the UBL PriorInformationNotice structure -type PriorInformationNotice struct { - XMLName xml.Name `xml:"PriorInformationNotice"` - UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"` - CustomizationID string `json:"customization_id" xml:"CustomizationID"` - ID string `json:"id" xml:"ID"` - ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"` - IssueDate string `json:"issue_date" xml:"IssueDate"` - IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"` - VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"` - RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"` - NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"` - NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"` - ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"` - TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"` - TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"` - ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"` - ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"` - UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` -} - -// BusinessRegistrationInformationNotice represents the UBL BusinessRegistrationInformationNotice structure -type BusinessRegistrationInformationNotice struct { - XMLName xml.Name `xml:"BusinessRegistrationInformationNotice"` - UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"` - CustomizationID string `json:"customization_id" xml:"CustomizationID"` - ID string `json:"id" xml:"ID"` - IssueDate string `json:"issue_date" xml:"IssueDate"` - IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"` - VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"` - RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"` - NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"` - NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"` - UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` - AdditionalNoticeLanguage []eform.AdditionalNoticeLanguage `json:"additional_notice_language,omitempty" xml:"AdditionalNoticeLanguage,omitempty"` - SenderParty eform.SenderParty `json:"sender_party,omitempty" xml:"SenderParty,omitempty"` - BusinessParty eform.BusinessParty `json:"business_party" xml:"BusinessParty"` - AdditionalDocumentReference []eform.AdditionalDocumentReference `json:"additional_document_reference,omitempty" xml:"AdditionalDocumentReference,omitempty"` - BusinessCapability []eform.BusinessCapability `json:"business_capability,omitempty" xml:"BusinessCapability,omitempty"` - NoticePurpose eform.NoticePurpose `json:"notice_purpose,omitempty" xml:"NoticePurpose,omitempty"` -} - -// ===================================================== -// BUSINESS LOGIC HELPER METHODS -// ===================================================== - -// GetMainBuyer returns the lead/main buyer organization -func (n *ContractNotice) GetMainBuyer() *eform.Organization { - if len(n.UBLExtensions.UBLExtension) > 0 { - orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization - for i := range orgs { - for _, role := range orgs[i].Roles { - if role == eform.RoleBuyer || role == eform.RoleLeadBuyer { - return &orgs[i] - } - } - } - } - return nil -} - -// GetOrganizationByID retrieves organization by ID -func (n *ContractNotice) GetOrganizationByID(id string) *eform.Organization { - if len(n.UBLExtensions.UBLExtension) > 0 { - orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization - for i := range orgs { - if orgs[i].ID == id { - return &orgs[i] - } - } - } - return nil -} diff --git a/ted/eform/business_registration_information/business_registration_information.go b/ted/eform/business_registration_information/business_registration_information.go new file mode 100644 index 0000000..65ffc3e --- /dev/null +++ b/ted/eform/business_registration_information/business_registration_information.go @@ -0,0 +1,44 @@ +package business_registration_information + +import ( + "encoding/xml" + "tm/internal/tender" + "tm/ted/eform" +) + +// BusinessRegistrationInformationNotice represents the UBL BusinessRegistrationInformationNotice structure +type BusinessRegistrationInformationNotice struct { + XMLName xml.Name `xml:"BusinessRegistrationInformationNotice"` + UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"` + CustomizationID string `json:"customization_id" xml:"CustomizationID"` + ID string `json:"id" xml:"ID"` + IssueDate string `json:"issue_date" xml:"IssueDate"` + IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"` + VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"` + RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"` + NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"` + NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"` + UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` + AdditionalNoticeLanguage []eform.AdditionalNoticeLanguage `json:"additional_notice_language,omitempty" xml:"AdditionalNoticeLanguage,omitempty"` + SenderParty eform.SenderParty `json:"sender_party,omitempty" xml:"SenderParty,omitempty"` + BusinessParty eform.BusinessParty `json:"business_party" xml:"BusinessParty"` + AdditionalDocumentReference []eform.AdditionalDocumentReference `json:"additional_document_reference,omitempty" xml:"AdditionalDocumentReference,omitempty"` + BusinessCapability []eform.BusinessCapability `json:"business_capability,omitempty" xml:"BusinessCapability,omitempty"` + NoticePurpose eform.NoticePurpose `json:"notice_purpose,omitempty" xml:"NoticePurpose,omitempty"` +} + +// GetID returns the business registration information notice ID +func (brin BusinessRegistrationInformationNotice) GetID() string { + return brin.ID +} + +// Validate validates the parsed model +func (brin BusinessRegistrationInformationNotice) Validate() error { + return nil +} + +func (notice BusinessRegistrationInformationNotice) MapToTender() *tender.Tender { + t := new(tender.Tender) + t.ContractNoticeID = notice.GetID() + return t +} diff --git a/ted/eform/contract/contract.go b/ted/eform/contract/contract.go new file mode 100644 index 0000000..47d830f --- /dev/null +++ b/ted/eform/contract/contract.go @@ -0,0 +1,213 @@ +package contract + +import ( + "encoding/xml" + "time" + "tm/internal/tender" + "tm/ted/eform" +) + +// Notice represents the UBL ContractNotice structure used by TED +type ContractNotice struct { + XMLName xml.Name `xml:"ContractNotice"` + UBLVersionID string `xml:"UBLVersionID"` + CustomizationID string `xml:"CustomizationID"` + ID string `xml:"ID"` + ContractFolderID string `xml:"ContractFolderID"` + IssueDate string `xml:"IssueDate"` + IssueTime string `xml:"IssueTime"` + VersionID string `xml:"VersionID"` + RegulatoryDomain string `xml:"RegulatoryDomain"` + NoticeTypeCode string `xml:"NoticeTypeCode"` + NoticeLanguageCode string `xml:"NoticeLanguageCode"` + UBLExtensions eform.UBLExtensions `xml:"UBLExtensions,omitempty"` + ContractingParty eform.ContractingParty `xml:"ContractingParty"` + TenderingTerms eform.TenderingTerms `xml:"TenderingTerms,omitempty"` + TenderingProcess eform.TenderingProcess `xml:"TenderingProcess,omitempty"` + ProcurementProject eform.ProcurementProject `xml:"ProcurementProject"` + ProcurementProjectLot []eform.ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"` +} + +func (notice ContractNotice) MapToTender() *tender.Tender { + now := time.Now().Unix() + + t := &tender.Tender{ + ContractNoticeID: notice.ID, + ContractFolderID: notice.ContractFolderID, + NoticeTypeCode: notice.NoticeTypeCode, + // NoticeSubTypeCode: notice.NoticeSubTypeCode, + NoticeLanguageCode: notice.NoticeLanguageCode, + // IssueDate: parseDateToUnixMilli(notice.IssueDate), + // IssueTime: parseTimeToUnixMilli(notice.IssueTime), + // GazetteID: notice.GetOJSID(), + // Title: notice.GetProcurementTitle(), + // Description: notice.GetProcurementDescription(), + // ProcurementTypeCode: notice.GetContractNature(), + // ProcedureCode: notice.GetProcedureCode(), + // MainClassification: notice.GetMainClassification(), + // PlaceOfPerformance: notice.GetPlaceOfPerformance(), + // DocumentURI: notice.GetDocumentURI(), + Status: tender.TenderStatusActive, + Source: tender.TenderSourceTEDScraper, + ProcessingMetadata: tender.ProcessingMetadata{ + ScrapedAt: now, + ProcessedAt: now, + ProcessingVersion: "1.0", + }, + } + + // Set publication information + // if pubInfo := notice.GetPublicationInfo(); pubInfo != nil { + // t.NoticePublicationID = pubInfo.NoticePublicationID + // t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate) + // t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID) + // } + + // Set additional classifications + // if notice.ProcurementProject != nil && notice.ProcurementProject.AdditionalCommodityClassification != nil { + // for _, additionalClass := range notice.ProcurementProject.AdditionalCommodityClassification { + // if additionalClass.ItemClassificationCode != "" { + // t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode) + // } + // } + // } + + // Set estimated value and currency + // if amount, currency := notice.GetBudget(); amount != "" { + // if value, err := strconv.ParseFloat(amount, 64); err == nil { + // t.EstimatedValue = value + // t.Currency = currency + // } + // } + + // Set duration information + // if duration, unit := notice.GetDuration(); duration != "" { + // t.Duration = duration + // t.DurationUnit = unit + // } + + // Set tender deadline + // if deadline := notice.GetTenderDeadline(); deadline != "" { + // t.TenderDeadline = parseDateToUnixMilli(deadline) + + // if deadlineTime, err := parseDate(deadline); err == nil { + // applicationDeadline := calculateApplicationDeadline(deadlineTime) + // t.ApplicationDeadline = applicationDeadline.UnixMilli() + // } + // } + + // if pubDate, err := parseDate(notice.GetPublicationDate()); err == nil { + // submissionDeadline := calculateSubmissionDeadline(pubDate) + // t.SubmissionDeadline = submissionDeadline.UnixMilli() + // } + + // Set SubmissionURL from TenderingTerms + // if notice.ProcurementProjectLot != nil && notice.ProcurementProjectLot.TenderingTerms != nil { + // if tenderingTerms := notice.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil { + // if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" { + // if strings.HasPrefix(endpointID, "http") { + // t.SubmissionURL = endpointID + // } else { + // t.SubmissionURL = "https://" + endpointID + // } + // } + // } + // } + + // Set location information + // if notice.ProcurementProject != nil && notice.ProcurementProject.RealizedLocation != nil { + // if addr := notice.ProcurementProject.RealizedLocation.Address; addr != nil { + // t.CityName = addr.CityName + // t.PostalCode = addr.PostalZone + // t.RegionCode = addr.CountrySubentityCode + // if addr.Country != nil { + // t.CountryCode = addr.Country.IdentificationCode + // } + // } + // } + + // Set buyer organization + // buyerID := notice.GetBuyerID() + // if buyerID != "" { + // if buyerOrg := notice.GetOrganizationByID(buyerID); buyerOrg != nil { + // t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer") + // } + // } + + // Set review organization + // reviewID := notice.GetReviewOrganizationID() + // if reviewID != "" { + // if reviewOrg := notice.GetOrganizationByID(reviewID); reviewOrg != nil { + // t.ReviewOrganization = s.mapOrganization(reviewOrg, "review") + // } + // } + + // Set all organizations + // if orgs := notice.GetOrganizations(); orgs != nil { + // for _, org := range orgs.Organization { + // mappedOrg := s.mapOrganization(&org, "") + // if mappedOrg != nil { + // t.Organizations = append(t.Organizations, *mappedOrg) + // } + // } + // } + + // Set selection criteria + // if criteria := notice.GetSelectionCriteria(); criteria != nil { + // for _, criterion := range criteria { + // t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{ + // TypeCode: criterion.CriterionTypeCode, + // Description: criterion.Description, + // LanguageID: criterion.LanguageID, + // }) + // } + // } + + // Set official languages + // if languages := notice.GetOfficialLanguages(); languages != nil { + // t.OfficialLanguages = languages + // } + + // Generate unique tender ID + // t.TenderID = generateTenderID(t) + + return t +} + +// GetID returns the contract notice ID +func (cn ContractNotice) GetID() string { + return cn.ID +} + +// Validate validates the parsed model +func (cn ContractNotice) Validate() error { + return nil +} + +// GetMainBuyer returns the lead/main buyer organization +func (n *ContractNotice) GetMainBuyer() *eform.Organization { + if len(n.UBLExtensions.UBLExtension) > 0 { + orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization + for i := range orgs { + for _, role := range orgs[i].Roles { + if role == eform.RoleBuyer || role == eform.RoleLeadBuyer { + return &orgs[i] + } + } + } + } + return nil +} + +// GetOrganizationByID retrieves organization by ID +func (n *ContractNotice) GetOrganizationByID(id string) *eform.Organization { + if len(n.UBLExtensions.UBLExtension) > 0 { + orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization + for i := range orgs { + if orgs[i].ID == id { + return &orgs[i] + } + } + } + return nil +} diff --git a/ted/eform/contract_award/contract_award.go b/ted/eform/contract_award/contract_award.go new file mode 100644 index 0000000..24625d9 --- /dev/null +++ b/ted/eform/contract_award/contract_award.go @@ -0,0 +1,44 @@ +package contract_award + +import ( + "encoding/xml" + "tm/internal/tender" + "tm/ted/eform" +) + +// ContractAwardNotice represents the UBL ContractAwardNotice structure +type ContractAwardNotice struct { + XMLName xml.Name `xml:"ContractAwardNotice"` + UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"` + CustomizationID string `json:"customization_id" xml:"CustomizationID"` + ID string `json:"id" xml:"ID"` + ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"` + IssueDate string `json:"issue_date" xml:"IssueDate"` + IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"` + VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"` + RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"` + NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"` + NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"` + ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"` + TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"` + TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"` + ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"` + ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"` + UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` +} + +// GetID returns the contract award notice ID +func (can ContractAwardNotice) GetID() string { + return can.ID +} + +// Validate validates the parsed model +func (can ContractAwardNotice) Validate() error { + return nil +} + +func (notice ContractAwardNotice) MapToTender() *tender.Tender { + t := new(tender.Tender) + t.ContractNoticeID = notice.GetID() + return t +} diff --git a/ted/eform/prior_information/prior_information.go b/ted/eform/prior_information/prior_information.go new file mode 100644 index 0000000..52511eb --- /dev/null +++ b/ted/eform/prior_information/prior_information.go @@ -0,0 +1,44 @@ +package prior_information + +import ( + "encoding/xml" + "tm/internal/tender" + "tm/ted/eform" +) + +// PriorInformationNotice represents the UBL PriorInformationNotice structure +type PriorInformationNotice struct { + XMLName xml.Name `xml:"PriorInformationNotice"` + UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"` + CustomizationID string `json:"customization_id" xml:"CustomizationID"` + ID string `json:"id" xml:"ID"` + ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"` + IssueDate string `json:"issue_date" xml:"IssueDate"` + IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"` + VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"` + RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"` + NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"` + NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"` + ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"` + TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"` + TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"` + ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"` + ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"` + UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` +} + +// GetID returns the prior information notice ID +func (pin PriorInformationNotice) GetID() string { + return pin.ID +} + +// Validate validates the parsed model +func (pin PriorInformationNotice) Validate() error { + return nil +} + +func (pin PriorInformationNotice) MapToTender() *tender.Tender { + t := new(tender.Tender) + t.ContractNoticeID = pin.GetID() + return t +} diff --git a/ted/parser.go b/ted/parser.go index b609491..ef169fb 100644 --- a/ted/parser.go +++ b/ted/parser.go @@ -4,8 +4,11 @@ import ( "encoding/xml" "fmt" "strings" - "tm/pkg/notification" "tm/pkg/xmlparser" + "tm/ted/eform/business_registration_information" + "tm/ted/eform/contract" + "tm/ted/eform/contract_award" + "tm/ted/eform/prior_information" ) type ParserType string @@ -18,8 +21,10 @@ const ( ) type TEDParser struct { - Notification notification.SDK - NoticeParser xmlparser.Parser[ContractNotice] + ContractNotice xmlparser.Parser[contract.ContractNotice] + ContractAwardNotice xmlparser.Parser[contract_award.ContractAwardNotice] + PriorInformationNotice xmlparser.Parser[prior_information.PriorInformationNotice] + BusinessRegistrationInformationNotice xmlparser.Parser[business_registration_information.BusinessRegistrationInformationNotice] } // NewTEDParser creates a new TED parser @@ -28,9 +33,16 @@ func NewTEDParser() *TEDParser { options.MaxSize = 50 * 1024 * 1024 // 50MB limit for TED XML files options.IgnoreUnknownElements = true // TED XML has many optional fields - parser := xmlparser.NewParser[ContractNotice](options) + contractNoticeParser := xmlparser.NewParser[contract.ContractNotice](options) + contractAwardNoticeParser := xmlparser.NewParser[contract_award.ContractAwardNotice](options) + priorInformationNoticeParser := xmlparser.NewParser[prior_information.PriorInformationNotice](options) + businessRegistrationInformationNoticeParser := xmlparser.NewParser[business_registration_information.BusinessRegistrationInformationNotice](options) + return &TEDParser{ - NoticeParser: parser, + ContractNotice: contractNoticeParser, + ContractAwardNotice: contractAwardNoticeParser, + PriorInformationNotice: priorInformationNoticeParser, + BusinessRegistrationInformationNotice: businessRegistrationInformationNoticeParser, } } @@ -45,16 +57,16 @@ func (tp *TEDParser) Parse(content []byte) (interface{}, string, ParserType, err // Parse based on the detected type switch ParserType(rootElement) { case ParserTypeContractNotice: - notice, err := tp.parseContractNotice(content) + notice, err := tp.ContractNotice.ParseBytes(content) return notice, notice.GetID(), ParserTypeContractNotice, err case ParserTypeContractAwardNotice: - notice, err := tp.parseContractAwardNotice(content) + notice, err := tp.ContractAwardNotice.ParseBytes(content) return notice, notice.GetID(), ParserTypeContractAwardNotice, err case ParserTypePriorInformationNotice: - notice, err := tp.parsePriorInformationNotice(content) + notice, err := tp.PriorInformationNotice.ParseBytes(content) return notice, notice.GetID(), ParserTypePriorInformationNotice, err case ParserTypeBusinessRegistrationInformationNotice: - notice, err := tp.parseBusinessRegistrationInformationNotice(content) + notice, err := tp.BusinessRegistrationInformationNotice.ParseBytes(content) return notice, notice.GetID(), ParserTypeBusinessRegistrationInformationNotice, err default: return nil, "", ParserType(""), fmt.Errorf("unsupported notice type: %s", rootElement) @@ -74,99 +86,3 @@ func detectRootElement(content []byte) (string, error) { } } } - -// parseContractNotice parses a ContractNotice -func (tp *TEDParser) parseContractNotice(content []byte) (*ContractNotice, error) { - options := xmlparser.DefaultParserOptions() - options.MaxSize = 50 * 1024 * 1024 - options.IgnoreUnknownElements = true - - parser := xmlparser.NewParser[ContractNotice](options) - notice, err := parser.ParseBytes(content) - if err != nil { - return nil, err - } - return notice, nil -} - -// parseContractAwardNotice parses a ContractAwardNotice -func (tp *TEDParser) parseContractAwardNotice(content []byte) (*ContractAwardNotice, error) { - options := xmlparser.DefaultParserOptions() - options.MaxSize = 50 * 1024 * 1024 - options.IgnoreUnknownElements = true - - parser := xmlparser.NewParser[ContractAwardNotice](options) - notice, err := parser.ParseBytes(content) - if err != nil { - return nil, err - } - return notice, nil -} - -// parsePriorInformationNotice parses a PriorInformationNotice -func (tp *TEDParser) parsePriorInformationNotice(content []byte) (*PriorInformationNotice, error) { - options := xmlparser.DefaultParserOptions() - options.MaxSize = 50 * 1024 * 1024 - options.IgnoreUnknownElements = true - - parser := xmlparser.NewParser[PriorInformationNotice](options) - notice, err := parser.ParseBytes(content) - if err != nil { - return nil, err - } - return notice, nil -} - -// parseBusinessRegistrationInformationNotice parses a BusinessRegistrationInformationNotice -func (tp *TEDParser) parseBusinessRegistrationInformationNotice(content []byte) (*BusinessRegistrationInformationNotice, error) { - options := xmlparser.DefaultParserOptions() - options.MaxSize = 50 * 1024 * 1024 - options.IgnoreUnknownElements = true - - parser := xmlparser.NewParser[BusinessRegistrationInformationNotice](options) - notice, err := parser.ParseBytes(content) - if err != nil { - return nil, err - } - return notice, nil -} - -// GetID returns the contract notice ID -func (cn ContractNotice) GetID() string { - return cn.ID -} - -// GetID returns the contract award notice ID -func (can ContractAwardNotice) GetID() string { - return can.ID -} - -// GetID returns the prior information notice ID -func (pin PriorInformationNotice) GetID() string { - return pin.ID -} - -// GetID returns the business registration information notice ID -func (brin BusinessRegistrationInformationNotice) GetID() string { - return brin.ID -} - -// Validate validates the parsed model -func (cn ContractNotice) Validate() error { - return nil -} - -// Validate validates the parsed model -func (can ContractAwardNotice) Validate() error { - return nil -} - -// Validate validates the parsed model -func (pin PriorInformationNotice) Validate() error { - return nil -} - -// Validate validates the parsed model -func (brin BusinessRegistrationInformationNotice) Validate() error { - return nil -} diff --git a/ted/scraper.go b/ted/scraper.go index 33cb7c7..a33cc4b 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -283,7 +283,7 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file }) // Parse XML using the flexible TED parser that detects notice type - _, id, noticeType, err := s.xmlParser.Parse(content) + notice, id, noticeType, err := s.xmlParser.Parse(content) if err != nil { s.logger.Error("XML parsing failed", map[string]interface{}{ "file_name": fileName, @@ -293,6 +293,8 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file return fmt.Errorf("failed to parse XML: %w", err) } + _ = notice + // Log detailed parsed information based on notice type s.logger.Info("Successfully parsed notice", map[string]interface{}{ "notice_id": id, @@ -311,7 +313,7 @@ func (s *TEDScraper) Run(ctx context.Context) error { }) // get ojs - ojs, err := GetOJS(now.Year(), now.Format("02/01/2006")) + ojs, err := GetOJS(s.config.BaseURL, now.Year(), now.Format("02/01/2006")) if err != nil { s.logger.Error("Failed to get OJS", map[string]interface{}{ "error": err.Error(), @@ -337,103 +339,3 @@ func (s *TEDScraper) Run(ctx context.Context) error { return nil } - -// parseDate parses various date formats used in TED XML -func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) { - if dateStr == "" { - return time.Time{}, fmt.Errorf("empty date string") - } - - // Common TED date formats - formats := []string{ - "2006-01-02+07:00", - "2006-01-02-07:00", - "2006-01-02T15:04:05+07:00", - "2006-01-02T15:04:05-07:00", - "2006-01-02T15:04:05Z", - "2006-01-02Z", - "2006-01-02", - time.RFC3339, - time.RFC3339Nano, - } - - for _, format := range formats { - if parsedTime, err := time.Parse(format, dateStr); err == nil { - return parsedTime, nil - } - } - - return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr) -} - -// parseDateToUnixMilli converts string date to Unix milliseconds (int64) -func (s *TEDScraper) parseDateToUnixMilli(dateStr string) int64 { - if dateStr == "" { - return 0 - } - - if parsedTime, err := s.parseDate(dateStr); err == nil { - return parsedTime.UnixMilli() - } - - return 0 -} - -// parseTimeToUnixMilli converts string time to Unix milliseconds (int64) -func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 { - if timeStr == "" { - return 0 - } - - // If time string is just time without date, use today's date - if !strings.Contains(timeStr, "T") && !strings.Contains(timeStr, " ") { - // Assume it's just a time like "15:04:05" - today := time.Now().Format("2006-01-02") - timeStr = fmt.Sprintf("%sT%s", today, timeStr) - } - - if parsedTime, err := s.parseDate(timeStr); err == nil { - return parsedTime.UnixMilli() - } - - return 0 -} - -// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation -func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string { - if unixMilli == 0 { - return "00000000" - } - - return time.UnixMilli(unixMilli).Format("20060102") -} - -func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time { - deadline := publicationDate - workingHoursAdded := 0 - - for workingHoursAdded < 48 { - deadline = deadline.Add(1 * time.Hour) - - if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday { - workingHoursAdded++ - } - } - - return deadline -} - -func (s *TEDScraper) calculateApplicationDeadline(tenderDeadline time.Time) time.Time { - applicationDeadline := tenderDeadline - workingDaysSubtracted := 0 - - for workingDaysSubtracted < 7 { - applicationDeadline = applicationDeadline.AddDate(0, 0, -1) - - if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday { - workingDaysSubtracted++ - } - } - - return applicationDeadline -} diff --git a/ted/service.go b/ted/service.go new file mode 100644 index 0000000..e01696c --- /dev/null +++ b/ted/service.go @@ -0,0 +1,201 @@ +package ted + +import ( + "fmt" + "strconv" + "strings" + "time" + "tm/internal/tender" + "tm/ted/eform/business_registration_information" + "tm/ted/eform/contract" + "tm/ted/eform/contract_award" + "tm/ted/eform/prior_information" +) + +func EFormMapper(notice interface{}) *tender.Tender { + t := new(tender.Tender) + + switch notice := notice.(type) { + case contract.ContractNotice: + return notice.MapToTender() + case contract_award.ContractAwardNotice: + return notice.MapToTender() + case prior_information.PriorInformationNotice: + return notice.MapToTender() + case business_registration_information.BusinessRegistrationInformationNotice: + return notice.MapToTender() + default: + return t + } +} + +// parseDate parses various date formats used in TED XML +func parseDate(dateStr string) (time.Time, error) { + if dateStr == "" { + return time.Time{}, fmt.Errorf("empty date string") + } + + // Common TED date formats + formats := []string{ + "2006-01-02+07:00", + "2006-01-02-07:00", + "2006-01-02T15:04:05+07:00", + "2006-01-02T15:04:05-07:00", + "2006-01-02T15:04:05Z", + "2006-01-02Z", + "2006-01-02", + time.RFC3339, + time.RFC3339Nano, + } + + for _, format := range formats { + if parsedTime, err := time.Parse(format, dateStr); err == nil { + return parsedTime, nil + } + } + + return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr) +} + +// parseDateToUnixMilli converts string date to Unix milliseconds (int64) +func parseDateToUnixMilli(dateStr string) int64 { + if dateStr == "" { + return 0 + } + + if parsedTime, err := parseDate(dateStr); err == nil { + return parsedTime.UnixMilli() + } + + return 0 +} + +// parseTimeToUnixMilli converts string time to Unix milliseconds (int64) +func parseTimeToUnixMilli(timeStr string) int64 { + if timeStr == "" { + return 0 + } + + // If time string is just time without date, use today's date + if !strings.Contains(timeStr, "T") && !strings.Contains(timeStr, " ") { + // Assume it's just a time like "15:04:05" + today := time.Now().Format("2006-01-02") + timeStr = fmt.Sprintf("%sT%s", today, timeStr) + } + + if parsedTime, err := parseDate(timeStr); err == nil { + return parsedTime.UnixMilli() + } + + return 0 +} + +// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation +func unixMilliToDateString(unixMilli int64) string { + if unixMilli == 0 { + return "00000000" + } + + return time.UnixMilli(unixMilli).Format("20060102") +} + +func calculateSubmissionDeadline(publicationDate time.Time) time.Time { + deadline := publicationDate + workingHoursAdded := 0 + + for workingHoursAdded < 48 { + deadline = deadline.Add(1 * time.Hour) + + if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday { + workingHoursAdded++ + } + } + + return deadline +} + +func calculateApplicationDeadline(tenderDeadline time.Time) time.Time { + applicationDeadline := tenderDeadline + workingDaysSubtracted := 0 + + for workingDaysSubtracted < 7 { + applicationDeadline = applicationDeadline.AddDate(0, 0, -1) + + if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday { + workingDaysSubtracted++ + } + } + + return applicationDeadline +} + +// padOrTruncate pads a string with zeros or truncates it to the specified length +func padOrTruncate(str string, length int) string { + // Remove any non-alphanumeric characters and convert to uppercase + clean := strings.ToUpper(strings.ReplaceAll(str, "-", "")) + clean = strings.ReplaceAll(clean, " ", "") + + if len(clean) >= length { + return clean[:length] + } + + // Pad with zeros + return clean + strings.Repeat("0", length-len(clean)) +} + +// generateTenderURL generates the TED tender URL from NoticePublicationID +func generateTenderURL(noticePublicationID string) string { + if noticePublicationID == "" { + return "" + } + + // Remove leading zeros and format for URL + // Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025 + parts := strings.Split(noticePublicationID, "-") + if len(parts) != 2 { + return "" + } + + // Convert to int and back to string to remove leading zeros + if num, err := strconv.Atoi(parts[0]); err == nil { + cleanID := fmt.Sprintf("%d-%s", num, parts[1]) + return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID) + } + + return "" +} + +// generateTenderID generates a unique tender ID using the format: +// {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} +func generateTenderID(t *tender.Tender) string { + var parts []string + + // Source (TED) + parts = append(parts, "1") + + // TED code (Notice Publication ID, first 8 chars) + tedCode := "00000000" + if t.NoticePublicationID != "" { + tedCode = padOrTruncate(t.NoticePublicationID, 8) + } + parts = append(parts, tedCode) + + // Buyer code (first 8 chars of buyer organization ID, or 8 zeros if not available) + // buyerCode := "00000000" + // if t.BuyerOrganization != nil && t.BuyerOrganization.ID != "" { + // buyerCode = padOrTruncate(t.BuyerOrganization.ID, 8) + // } + // parts = append(parts, buyerCode) + + // Date (YYYYMMDD format from issue date) + parts = append(parts, padOrTruncate(unixMilliToDateString(t.IssueDate), 4)) + + // Location code (country code + region code, padded to 6 chars) + locationCode := "EU" + if t.CountryCode != "" { + locationCode = t.CountryCode + } + parts = append(parts, locationCode) + + return strings.Join(parts, "") +}