Add TED Scraper Configuration and Cron Scheduler Implementation
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
This commit is contained in:
n.nakhostin
+126
@@ -0,0 +1,126 @@
|
||||
package ted
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"tm/ted/eform"
|
||||
)
|
||||
|
||||
// =====================================================
|
||||
// ROOT NOTICE STRUCTURE
|
||||
// =====================================================
|
||||
|
||||
// Notice represents the UBL ContractNotice structure used by TED
|
||||
type ContractNotice struct {
|
||||
XMLName xml.Name `xml:"ContractNotice"`
|
||||
UBLVersionID string `xml:"UBLVersionID"`
|
||||
CustomizationID string `xml:"CustomizationID"`
|
||||
ID string `xml:"ID"`
|
||||
ContractFolderID string `xml:"ContractFolderID"`
|
||||
IssueDate string `xml:"IssueDate"`
|
||||
IssueTime string `xml:"IssueTime"`
|
||||
VersionID string `xml:"VersionID"`
|
||||
RegulatoryDomain string `xml:"RegulatoryDomain"`
|
||||
NoticeTypeCode string `xml:"NoticeTypeCode"`
|
||||
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
|
||||
UBLExtensions eform.UBLExtensions `xml:"UBLExtensions,omitempty"`
|
||||
ContractingParty eform.ContractingParty `xml:"ContractingParty"`
|
||||
TenderingTerms eform.TenderingTerms `xml:"TenderingTerms,omitempty"`
|
||||
TenderingProcess eform.TenderingProcess `xml:"TenderingProcess,omitempty"`
|
||||
ProcurementProject eform.ProcurementProject `xml:"ProcurementProject"`
|
||||
ProcurementProjectLot []eform.ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
|
||||
}
|
||||
|
||||
// ContractAwardNotice represents the UBL ContractAwardNotice structure
|
||||
type ContractAwardNotice struct {
|
||||
XMLName xml.Name `xml:"ContractAwardNotice"`
|
||||
UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"`
|
||||
CustomizationID string `json:"customization_id" xml:"CustomizationID"`
|
||||
ID string `json:"id" xml:"ID"`
|
||||
ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"`
|
||||
IssueDate string `json:"issue_date" xml:"IssueDate"`
|
||||
IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"`
|
||||
VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"`
|
||||
RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"`
|
||||
NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"`
|
||||
NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"`
|
||||
ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"`
|
||||
TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"`
|
||||
TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"`
|
||||
ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"`
|
||||
ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"`
|
||||
UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"`
|
||||
}
|
||||
|
||||
// PriorInformationNotice represents the UBL PriorInformationNotice structure
|
||||
type PriorInformationNotice struct {
|
||||
XMLName xml.Name `xml:"PriorInformationNotice"`
|
||||
UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"`
|
||||
CustomizationID string `json:"customization_id" xml:"CustomizationID"`
|
||||
ID string `json:"id" xml:"ID"`
|
||||
ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"`
|
||||
IssueDate string `json:"issue_date" xml:"IssueDate"`
|
||||
IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"`
|
||||
VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"`
|
||||
RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"`
|
||||
NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"`
|
||||
NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"`
|
||||
ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"`
|
||||
TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"`
|
||||
TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"`
|
||||
ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"`
|
||||
ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"`
|
||||
UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"`
|
||||
}
|
||||
|
||||
// BusinessRegistrationInformationNotice represents the UBL BusinessRegistrationInformationNotice structure
|
||||
type BusinessRegistrationInformationNotice struct {
|
||||
XMLName xml.Name `xml:"BusinessRegistrationInformationNotice"`
|
||||
UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"`
|
||||
CustomizationID string `json:"customization_id" xml:"CustomizationID"`
|
||||
ID string `json:"id" xml:"ID"`
|
||||
IssueDate string `json:"issue_date" xml:"IssueDate"`
|
||||
IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"`
|
||||
VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"`
|
||||
RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"`
|
||||
NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"`
|
||||
NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"`
|
||||
UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"`
|
||||
AdditionalNoticeLanguage []eform.AdditionalNoticeLanguage `json:"additional_notice_language,omitempty" xml:"AdditionalNoticeLanguage,omitempty"`
|
||||
SenderParty eform.SenderParty `json:"sender_party,omitempty" xml:"SenderParty,omitempty"`
|
||||
BusinessParty eform.BusinessParty `json:"business_party" xml:"BusinessParty"`
|
||||
AdditionalDocumentReference []eform.AdditionalDocumentReference `json:"additional_document_reference,omitempty" xml:"AdditionalDocumentReference,omitempty"`
|
||||
BusinessCapability []eform.BusinessCapability `json:"business_capability,omitempty" xml:"BusinessCapability,omitempty"`
|
||||
NoticePurpose eform.NoticePurpose `json:"notice_purpose,omitempty" xml:"NoticePurpose,omitempty"`
|
||||
}
|
||||
|
||||
// =====================================================
|
||||
// BUSINESS LOGIC HELPER METHODS
|
||||
// =====================================================
|
||||
|
||||
// GetMainBuyer returns the lead/main buyer organization
|
||||
func (n *ContractNotice) GetMainBuyer() *eform.Organization {
|
||||
if len(n.UBLExtensions.UBLExtension) > 0 {
|
||||
orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization
|
||||
for i := range orgs {
|
||||
for _, role := range orgs[i].Roles {
|
||||
if role == eform.RoleBuyer || role == eform.RoleLeadBuyer {
|
||||
return &orgs[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetOrganizationByID retrieves organization by ID
|
||||
func (n *ContractNotice) GetOrganizationByID(id string) *eform.Organization {
|
||||
if len(n.UBLExtensions.UBLExtension) > 0 {
|
||||
orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization
|
||||
for i := range orgs {
|
||||
if orgs[i].ID == id {
|
||||
return &orgs[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user