Add TED Scraper Configuration and Cron Scheduler Implementation

- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
2025-09-30 16:03:53 +03:30
parent 0c50935c42
commit 05c7eae8a2
36 changed files with 3427 additions and 311 deletions
@@ -0,0 +1,97 @@
+package eform
+
+// =====================================================
+// ORGANIZATION STRUCTURE
+// =====================================================
+
+// Organization represents any party involved in the procurement
+type Organization struct {
+	ID                  string             `json:"id" xml:"id" validate:"required,orgIDFormat"`
+	Roles               []OrganizationRole `json:"roles" xml:"roles" validate:"required,min=1,dive"`
+	LegalName           string             `json:"legal_name" xml:"legal_name" validate:"required,max=500"`
+	LegalType           *OrganizationType  `json:"legal_type,omitempty" xml:"legal_type,omitempty"`
+	RegistrationID      string             `json:"registration_id,omitempty" xml:"registration_id,omitempty"`
+	VATNumber           string             `json:"vat_number,omitempty" xml:"vat_number,omitempty" validate:"omitempty,vatFormat"`
+	NationalID          string             `json:"national_id,omitempty" xml:"national_id,omitempty"`
+	Address             Address            `json:"address" xml:"address" validate:"required"`
+	ContactPoint        ContactPoint       `json:"contact_point" xml:"contact_point" validate:"required"`
+	BuyerProfile        string             `json:"buyer_profile,omitempty" xml:"buyer_profile,omitempty" validate:"omitempty,url"`
+	ProcurementActivity *ActivityType      `json:"procurement_activity,omitempty" xml:"procurement_activity,omitempty"`
+	SME                 bool               `json:"sme" xml:"sme"`
+}
+
+// OrganizationRole defines the role of an organization in the procurement
+type OrganizationRole string
+
+const (
+	RoleBuyer           OrganizationRole = "buyer"
+	RoleServiceProvider OrganizationRole = "service-provider"
+	RoleTEDeSender      OrganizationRole = "ted-esen"
+	RoleWinner          OrganizationRole = "winner"
+	RoleSubcontractor   OrganizationRole = "subcontractor"
+	RoleReviewBody      OrganizationRole = "review-body"
+	RoleReviewOrg       OrganizationRole = "review-org"
+	RoleMediationBody   OrganizationRole = "mediation-body"
+	RoleLeadBuyer       OrganizationRole = "lead-buyer"
+	RoleCentralBuyer    OrganizationRole = "central-buyer"
+	RoleAcquirer        OrganizationRole = "acquirer"
+)
+
+// OrganizationType represents the legal type of organization
+type OrganizationType string
+
+const (
+	OrgTypeBodyPublicLaw     OrganizationType = "body-pl"
+	OrgTypeCentralGov        OrganizationType = "cga"
+	OrgTypeLocalAuthority    OrganizationType = "la"
+	OrgTypeSubsidized        OrganizationType = "org-sub"
+	OrgTypePublicUndertaking OrganizationType = "pub-undert"
+	OrgTypeEUInstitution     OrganizationType = "eu-ins-bod-ag"
+	OrgTypeOther             OrganizationType = "org-other"
+)
+
+// ActivityType represents main procurement activity
+type ActivityType string
+
+const (
+	ActivityAuthority          ActivityType = "auth-oth"
+	ActivityDefence            ActivityType = "def"
+	ActivityEconomicAffairs    ActivityType = "econ-aff"
+	ActivityEducation          ActivityType = "edu"
+	ActivityEnvironment        ActivityType = "env"
+	ActivityGeneralPublic      ActivityType = "gen-pub"
+	ActivityHealth             ActivityType = "health"
+	ActivityHousingCommunity   ActivityType = "hous-com"
+	ActivityPublicOrderSafety  ActivityType = "pub-os"
+	ActivityRecreationCulture  ActivityType = "rec-com"
+	ActivitySocialProtection   ActivityType = "soc-pro"
+	ActivityWater              ActivityType = "water"
+	ActivityAirport            ActivityType = "airport"
+	ActivityElectricity        ActivityType = "electricity"
+	ActivityExplorationExtract ActivityType = "exploration-extraction"
+	ActivityGasHeat            ActivityType = "gas-heat"
+	ActivityPort               ActivityType = "port"
+	ActivityPost               ActivityType = "post"
+	ActivityRailway            ActivityType = "rail"
+	ActivityUrbanRailway       ActivityType = "urban-railway"
+	ActivityWaterUtility       ActivityType = "water"
+)
+
+// Address represents physical address information
+type Address struct {
+	StreetAddress      string   `json:"street_address" xml:"street_address" validate:"required,max=200"`
+	Locality           string   `json:"locality" xml:"locality" validate:"required,max=100"`
+	PostalCode         string   `json:"postal_code" xml:"postal_code" validate:"required,max=20"`
+	CountrySubdivision string   `json:"country_subdivision,omitempty" xml:"country_subdivision,omitempty" validate:"max=100"`
+	CountryCode        string   `json:"country_code" xml:"country_code" validate:"required,iso3166_1_alpha2"`
+	NUTSCodes          []string `json:"nuts_codes,omitempty" xml:"nuts_codes,omitempty" validate:"dive,nutsCode"`
+}
+
+// ContactPoint represents contact information
+type ContactPoint struct {
+	Name      string `json:"name,omitempty" xml:"name,omitempty" validate:"max=200"`
+	Email     string `json:"email" xml:"email" validate:"required,email"`
+	Telephone string `json:"telephone,omitempty" xml:"telephone,omitempty" validate:"max=30"`
+	Fax       string `json:"fax,omitempty" xml:"fax,omitempty" validate:"max=30"`
+	URL       string `json:"url,omitempty" xml:"url,omitempty" validate:"omitempty,url"`
+}