Add TED Scraper Configuration and Cron Scheduler Implementation

- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
2025-09-30 16:03:53 +03:30
parent 0c50935c42
commit 05c7eae8a2
36 changed files with 3427 additions and 311 deletions
@@ -0,0 +1,244 @@
+package eform
+
+// =====================================================
+// PROCEDURE STRUCTURE
+// =====================================================
+
+// Procedure represents the procurement procedure
+type Procedure struct {
+	ID                        string           `json:"id" xml:"id" validate:"required,procIDFormat"`
+	Title                     string           `json:"title" xml:"title" validate:"required,max=500"`
+	Description               string           `json:"description" xml:"description" validate:"required,max=10000"`
+	ProcedureType             ProcedureType    `json:"procedure_type" xml:"procedure_type" validate:"required"`
+	AcceleratedProcedure      bool             `json:"accelerated_procedure" xml:"accelerated_procedure"`
+	AcceleratedJustification  string           `json:"accelerated_justification,omitempty" xml:"accelerated_justification,omitempty" validate:"required_if=AcceleratedProcedure true"`
+	FrameworkAgreement        bool             `json:"framework_agreement" xml:"framework_agreement"`
+	FAMaxParticipants         *int             `json:"fa_max_participants,omitempty" xml:"fa_max_participants,omitempty" validate:"omitempty,min=1"`
+	FADuration                *Duration        `json:"fa_duration,omitempty" xml:"fa_duration,omitempty"`
+	DynamicPurchasingSystem   bool             `json:"dynamic_purchasing_system" xml:"dynamic_purchasing_system"`
+	ElectronicAuction         bool             `json:"electronic_auction" xml:"electronic_auction"`
+	JointProcurement          bool             `json:"joint_procurement" xml:"joint_procurement"`
+	CentralPurchasing         bool             `json:"central_purchasing" xml:"central_purchasing"`
+	GPA                       bool             `json:"gpa" xml:"gpa"`
+	InternationalAgreements   []string         `json:"international_agreements,omitempty" xml:"international_agreements,omitempty"`
+	MainCPVCode               string           `json:"main_cpv_code" xml:"main_cpv_code" validate:"required,cpvCode"`
+	AdditionalCPVCodes        []string         `json:"additional_cpv_codes,omitempty" xml:"additional_cpv_codes,omitempty" validate:"dive,cpvCode"`
+	ContractNature            ContractNature   `json:"contract_nature" xml:"contract_nature" validate:"required"`
+	EstimatedValue            *Money           `json:"estimated_value,omitempty" xml:"estimated_value,omitempty"`
+	EstimatedValueRange       *ValueRange      `json:"estimated_value_range,omitempty" xml:"estimated_value_range,omitempty"`
+	ProcurementDocumentsURL   string           `json:"procurement_documents_url,omitempty" xml:"procurement_documents_url,omitempty" validate:"omitempty,url"`
+	ProcurementDocsRestricted bool             `json:"procurement_docs_restricted" xml:"procurement_docs_restricted"`
+	SubmissionElectronic      SubmissionMethod `json:"submission_electronic" xml:"submission_electronic" validate:"required"`
+	SubmissionLanguages       []LanguageCode   `json:"submission_languages" xml:"submission_languages" validate:"required,min=1,dive,iso639_1"`
+	TenderValidityDuration    *Duration        `json:"tender_validity_duration,omitempty" xml:"tender_validity_duration,omitempty"`
+	Lots                      []Lot            `json:"lots,omitempty" xml:"lots,omitempty" validate:"dive"`
+	SecondStage               *SecondStage     `json:"second_stage,omitempty" xml:"second_stage,omitempty"`
+	Recurrence                bool             `json:"recurrence" xml:"recurrence"`
+	RecurrenceTiming          string           `json:"recurrence_timing,omitempty" xml:"recurrence_timing,omitempty" validate:"max=1000"`
+}
+
+// ProcedureType represents the type of procurement procedure
+type ProcedureType string
+
+const (
+	ProcTypeOpen                ProcedureType = "open"
+	ProcTypeRestricted          ProcedureType = "restricted"
+	ProcTypeCompetitiveDialogue ProcedureType = "comp-dial"
+	ProcTypeInnovation          ProcedureType = "innovation"
+	ProcTypeNegotiatedWithCall  ProcedureType = "neg-w-call"
+	ProcTypeNegotiatedWoCall    ProcedureType = "neg-wo-call"
+	ProcTypeCompWithNeg         ProcedureType = "comp-w-n"
+	ProcTypeAwardWoCall         ProcedureType = "pt-award-wo-call"
+	ProcTypeOtherSingle         ProcedureType = "oth-single"
+	ProcTypeOtherMultiple       ProcedureType = "oth-mult"
+)
+
+// ContractNature represents the type of contract
+type ContractNature string
+
+const (
+	ContractNatureWorks    ContractNature = "works"
+	ContractNatureSupplies ContractNature = "supplies"
+	ContractNatureServices ContractNature = "services"
+	ContractNatureCombined ContractNature = "combined"
+)
+
+// SubmissionMethod represents how tenders can be submitted
+type SubmissionMethod string
+
+const (
+	SubmissionAllowed    SubmissionMethod = "allowed"
+	SubmissionRequired   SubmissionMethod = "required"
+	SubmissionNotAllowed SubmissionMethod = "not-allowed"
+)
+
+// Duration represents a time duration
+type Duration struct {
+	Type  DurationType `json:"type" xml:"type" validate:"required"`
+	Value *int         `json:"value,omitempty" xml:"value,omitempty" validate:"omitempty,min=1"`
+}
+
+// DurationType represents the unit of duration
+type DurationType string
+
+const (
+	DurationMonths    DurationType = "months"
+	DurationDays      DurationType = "days"
+	DurationYears     DurationType = "years"
+	DurationUnlimited DurationType = "unlimited"
+)
+
+// Money represents a monetary value
+type Money struct {
+	Amount   float64 `json:"amount" xml:"amount" validate:"required,min=0"`
+	Currency string  `json:"currency" xml:"currency" validate:"required,iso4217"`
+}
+
+// ValueRange represents a range of monetary values
+type ValueRange struct {
+	MinValue Money `json:"min_value" xml:"min_value" validate:"required"`
+	MaxValue Money `json:"max_value" xml:"max_value" validate:"required,gtfield=MinValue.Amount"`
+}
+
+// SecondStage represents second stage of two-stage procedures
+type SecondStage struct {
+	MinCandidates     *int   `json:"min_candidates,omitempty" xml:"min_candidates,omitempty" validate:"omitempty,min=1"`
+	MaxCandidates     *int   `json:"max_candidates,omitempty" xml:"max_candidates,omitempty" validate:"omitempty,min=1"`
+	ObjectiveCriteria string `json:"objective_criteria,omitempty" xml:"objective_criteria,omitempty" validate:"max=10000"`
+}
+
+// IsAboveThreshold checks if procedure value is above EU thresholds
+func (p *Procedure) IsAboveThreshold() bool {
+	if p.EstimatedValue == nil {
+		return false
+	}
+
+	// EU thresholds 2024-2025 (in EUR)
+	thresholds := map[ContractNature]float64{
+		ContractNatureWorks:    5382000,
+		ContractNatureSupplies: 143000, // central government
+		ContractNatureServices: 143000, // central government
+	}
+
+	threshold, exists := thresholds[p.ContractNature]
+	if !exists {
+		return false
+	}
+
+	// Convert to EUR if needed
+	valueEUR := p.EstimatedValue.Amount
+	if p.EstimatedValue.Currency != CurrencyEUR {
+		// In real implementation, would use exchange rates
+		// This is simplified
+		return true // Assume above threshold for non-EUR
+	}
+
+	return valueEUR >= threshold
+}
+
+// GetTotalEstimatedValue calculates total value from lots
+func (p *Procedure) GetTotalEstimatedValue() *Money {
+	if len(p.Lots) == 0 {
+		return p.EstimatedValue
+	}
+
+	var total float64
+	var currency string
+
+	for _, lot := range p.Lots {
+		if lot.EstimatedValue != nil {
+			if currency == "" {
+				currency = lot.EstimatedValue.Currency
+			}
+			if lot.EstimatedValue.Currency == currency {
+				total += lot.EstimatedValue.Amount
+			}
+		}
+	}
+
+	if currency != "" {
+		return &Money{
+			Amount:   total,
+			Currency: currency,
+		}
+	}
+
+	return p.EstimatedValue
+}
+
+// IsDividedIntoLots checks if procedure has lots
+func (p *Procedure) IsDividedIntoLots() bool {
+	return len(p.Lots) > 0
+}
+
+// GetLotByID retrieves lot by ID
+func (p *Procedure) GetLotByID(id string) *Lot {
+	for i := range p.Lots {
+		if p.Lots[i].ID == id {
+			return &p.Lots[i]
+		}
+	}
+	return nil
+}
+
+// IsFrameworkAgreement checks if this is a framework agreement
+func (p *Procedure) IsFrameworkAgreement() bool {
+	return p.FrameworkAgreement
+}
+
+// IsDynamicPurchasingSystem checks if this is a DPS
+func (p *Procedure) IsDynamicPurchasingSystem() bool {
+	return p.DynamicPurchasingSystem
+}
+
+// RequiresTwoStage checks if procedure is two-stage
+func (p *Procedure) RequiresTwoStage() bool {
+	twoStageProcs := []ProcedureType{
+		ProcTypeRestricted,
+		ProcTypeCompetitiveDialogue,
+		ProcTypeInnovation,
+		ProcTypeNegotiatedWithCall,
+	}
+
+	for _, t := range twoStageProcs {
+		if p.ProcedureType == t {
+			return true
+		}
+	}
+	return false
+}
+
+// GetAwardedContracts returns all awarded contracts from procedure
+func (p *Procedure) GetAwardedContracts() []SettledContract {
+	var contracts []SettledContract
+
+	for _, lot := range p.Lots {
+		if lot.Result != nil && lot.Result.SettledContract != nil {
+			contracts = append(contracts, *lot.Result.SettledContract)
+		}
+	}
+
+	return contracts
+}
+
+// GetTotalAwardedValue calculates total value of all awarded contracts
+func (p *Procedure) GetTotalAwardedValue() *Money {
+	contracts := p.GetAwardedContracts()
+	if len(contracts) == 0 {
+		return nil
+	}
+
+	var total float64
+	currency := contracts[0].ContractValue.Currency
+
+	for _, contract := range contracts {
+		if contract.ContractValue.Currency == currency {
+			total += contract.ContractValue.Amount
+		}
+	}
+
+	return &Money{
+		Amount:   total,
+		Currency: currency,
+	}
+}