Implement TED Scraper and Configuration Management
- Introduced a new TED scraper in `cmd/scraper` to handle downloading and parsing TED XML files. - Added configuration management for the scraper, including a new `Config` struct and YAML configuration file. - Created necessary handler, service, and repository layers for tender management, adhering to Clean Architecture principles. - Implemented comprehensive logging and error handling throughout the scraper functionality. - Updated API routes to include tender management operations, enhancing the overall system capabilities.
This commit is contained in:
n.nakhostin
@@ -0,0 +1,418 @@
|
||||
package tender
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"tm/pkg/mongo"
|
||||
)
|
||||
|
||||
// Tender represents a tender/contract notice entity
|
||||
type Tender struct {
|
||||
mongo.Model `bson:",inline"`
|
||||
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
|
||||
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
|
||||
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
|
||||
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
|
||||
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
|
||||
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
|
||||
IssueDate int64 `bson:"issue_date" json:"issue_date"` // Unix milliseconds
|
||||
IssueTime int64 `bson:"issue_time" json:"issue_time"` // Unix milliseconds
|
||||
PublicationDate int64 `bson:"publication_date" json:"publication_date"` // Unix milliseconds
|
||||
GazetteID string `bson:"gazette_id" json:"gazette_id"`
|
||||
Title string `bson:"title" json:"title"`
|
||||
Description string `bson:"description" json:"description"`
|
||||
ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"`
|
||||
ProcedureCode string `bson:"procedure_code" json:"procedure_code"`
|
||||
MainClassification string `bson:"main_classification" json:"main_classification"`
|
||||
AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"`
|
||||
EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"`
|
||||
Currency string `bson:"currency" json:"currency"`
|
||||
Duration string `bson:"duration" json:"duration"`
|
||||
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
|
||||
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds
|
||||
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
|
||||
CountryCode string `bson:"country_code" json:"country_code"`
|
||||
RegionCode string `bson:"region_code" json:"region_code"`
|
||||
CityName string `bson:"city_name" json:"city_name"`
|
||||
PostalCode string `bson:"postal_code" json:"postal_code"`
|
||||
DocumentURI string `bson:"document_uri" json:"document_uri"`
|
||||
TenderURL string `bson:"tender_url" json:"tender_url"`
|
||||
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
|
||||
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
|
||||
Organizations []Organization `bson:"organizations" json:"organizations"`
|
||||
SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"`
|
||||
OfficialLanguages []string `bson:"official_languages" json:"official_languages"`
|
||||
Status TenderStatus `bson:"status" json:"status"`
|
||||
Source TenderSource `bson:"source" json:"source"`
|
||||
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
|
||||
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
|
||||
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
|
||||
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds
|
||||
TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
|
||||
|
||||
}
|
||||
|
||||
// Organization represents organization information
|
||||
type Organization struct {
|
||||
ID string `bson:"id" json:"id"`
|
||||
Name string `bson:"name" json:"name"`
|
||||
CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"`
|
||||
WebsiteURI string `bson:"website_uri,omitempty" json:"website_uri,omitempty"`
|
||||
ContactName string `bson:"contact_name,omitempty" json:"contact_name,omitempty"`
|
||||
ContactTelephone string `bson:"contact_telephone,omitempty" json:"contact_telephone,omitempty"`
|
||||
ContactEmail string `bson:"contact_email,omitempty" json:"contact_email,omitempty"`
|
||||
ContactFax string `bson:"contact_fax,omitempty" json:"contact_fax,omitempty"`
|
||||
Address Address `bson:"address" json:"address"`
|
||||
Role string `bson:"role,omitempty" json:"role,omitempty"`
|
||||
}
|
||||
|
||||
// Address represents address information
|
||||
type Address struct {
|
||||
StreetName string `bson:"street_name,omitempty" json:"street_name,omitempty"`
|
||||
CityName string `bson:"city_name,omitempty" json:"city_name,omitempty"`
|
||||
PostalZone string `bson:"postal_zone,omitempty" json:"postal_zone,omitempty"`
|
||||
CountrySubentityCode string `bson:"country_subentity_code,omitempty" json:"country_subentity_code,omitempty"`
|
||||
Department string `bson:"department,omitempty" json:"department,omitempty"`
|
||||
Region string `bson:"region,omitempty" json:"region,omitempty"`
|
||||
CountryCode string `bson:"country_code" json:"country_code"`
|
||||
}
|
||||
|
||||
// SelectionCriterion represents selection criteria
|
||||
type SelectionCriterion struct {
|
||||
TypeCode string `bson:"type_code" json:"type_code"`
|
||||
Description string `bson:"description,omitempty" json:"description,omitempty"`
|
||||
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
|
||||
}
|
||||
|
||||
// ProcessingMetadata contains metadata about how the tender was processed
|
||||
type ProcessingMetadata struct {
|
||||
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
|
||||
ProcessedAt int64 `bson:"processed_at" json:"processed_at"` // Unix milliseconds
|
||||
ProcessingVersion string `bson:"processing_version" json:"processing_version"`
|
||||
ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
|
||||
ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
|
||||
EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
|
||||
}
|
||||
|
||||
// TenderStatus represents the status of a tender
|
||||
type TenderStatus string
|
||||
|
||||
const (
|
||||
TenderStatusActive TenderStatus = "active"
|
||||
TenderStatusExpired TenderStatus = "expired"
|
||||
TenderStatusCancelled TenderStatus = "cancelled"
|
||||
TenderStatusAwarded TenderStatus = "awarded"
|
||||
TenderStatusDraft TenderStatus = "draft"
|
||||
)
|
||||
|
||||
// TenderSource represents the source of tender data
|
||||
type TenderSource string
|
||||
|
||||
const (
|
||||
TenderSourceTEDScraper TenderSource = "ted_scraper"
|
||||
TenderSourceManual TenderSource = "manual"
|
||||
TenderSourceAPI TenderSource = "api"
|
||||
TenderSourceBulkImport TenderSource = "bulk_import"
|
||||
)
|
||||
|
||||
// TenderSearchCriteria represents search criteria for tenders
|
||||
type TenderSearchCriteria struct {
|
||||
Query string `json:"query,omitempty"`
|
||||
NoticeType string `json:"notice_type,omitempty"`
|
||||
ProcurementType string `json:"procurement_type,omitempty"`
|
||||
CountryCodes []string `json:"country_codes,omitempty"`
|
||||
RegionCodes []string `json:"region_codes,omitempty"`
|
||||
Classifications []string `json:"classifications,omitempty"`
|
||||
MinEstimatedValue *float64 `json:"min_estimated_value,omitempty"`
|
||||
MaxEstimatedValue *float64 `json:"max_estimated_value,omitempty"`
|
||||
Currency string `json:"currency,omitempty"`
|
||||
DeadlineFrom *int64 `json:"deadline_from,omitempty"` // Unix milliseconds
|
||||
DeadlineTo *int64 `json:"deadline_to,omitempty"` // Unix milliseconds
|
||||
PublicationDateFrom *int64 `json:"publication_date_from,omitempty"` // Unix milliseconds
|
||||
PublicationDateTo *int64 `json:"publication_date_to,omitempty"` // Unix milliseconds
|
||||
Status []TenderStatus `json:"status,omitempty"`
|
||||
Source []TenderSource `json:"source,omitempty"`
|
||||
BuyerOrganizationID string `json:"buyer_organization_id,omitempty"`
|
||||
Languages []string `json:"languages,omitempty"`
|
||||
}
|
||||
|
||||
// TenderStatistics represents tender statistics
|
||||
type TenderStatistics struct {
|
||||
TotalTenders int64 `json:"total_tenders"`
|
||||
ActiveTenders int64 `json:"active_tenders"`
|
||||
ExpiredTenders int64 `json:"expired_tenders"`
|
||||
TendersByCountry map[string]int64 `json:"tenders_by_country"`
|
||||
TendersByType map[string]int64 `json:"tenders_by_type"`
|
||||
TendersByClassification map[string]int64 `json:"tenders_by_classification"`
|
||||
AverageEstimatedValue float64 `json:"average_estimated_value"`
|
||||
TotalEstimatedValue float64 `json:"total_estimated_value"`
|
||||
LastUpdated int64 `json:"last_updated"` // Unix milliseconds
|
||||
}
|
||||
|
||||
// GetID returns the tender ID
|
||||
func (t *Tender) GetID() string {
|
||||
return t.ID.Hex()
|
||||
}
|
||||
|
||||
// IsActive returns true if the tender is active
|
||||
func (t *Tender) IsActive() bool {
|
||||
return t.Status == TenderStatusActive
|
||||
}
|
||||
|
||||
// IsExpired returns true if the tender deadline has passed
|
||||
func (t *Tender) IsExpired() bool {
|
||||
if t.Status == TenderStatusExpired {
|
||||
return true
|
||||
}
|
||||
|
||||
if t.TenderDeadline == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check if deadline (in milliseconds) is in the past
|
||||
now := time.Now().UnixMilli()
|
||||
return t.TenderDeadline < now
|
||||
}
|
||||
|
||||
// GetTenderURL returns the TED tender URL
|
||||
func (t *Tender) GetTenderURL() string {
|
||||
if t.TenderURL != "" {
|
||||
return t.TenderURL
|
||||
}
|
||||
|
||||
if t.NoticePublicationID != "" {
|
||||
return generateTenderURL(t.NoticePublicationID)
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// generateTenderURL generates the TED tender URL from NoticePublicationID
|
||||
func generateTenderURL(noticePublicationID string) string {
|
||||
// Remove leading zeros and format for URL
|
||||
// Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025
|
||||
parts := strings.Split(noticePublicationID, "-")
|
||||
if len(parts) != 2 {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Convert to int and back to string to remove leading zeros
|
||||
if num, err := strconv.Atoi(parts[0]); err == nil {
|
||||
cleanID := fmt.Sprintf("%d-%s", num, parts[1])
|
||||
return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID)
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// GetMainOrganization returns the buyer organization
|
||||
func (t *Tender) GetMainOrganization() *Organization {
|
||||
return t.BuyerOrganization
|
||||
}
|
||||
|
||||
// HasEstimatedValue returns true if the tender has an estimated value
|
||||
func (t *Tender) HasEstimatedValue() bool {
|
||||
return t.EstimatedValue > 0
|
||||
}
|
||||
|
||||
// GetFormattedEstimatedValue returns formatted estimated value with currency
|
||||
func (t *Tender) GetFormattedEstimatedValue() string {
|
||||
if !t.HasEstimatedValue() {
|
||||
return ""
|
||||
}
|
||||
|
||||
if t.Currency != "" {
|
||||
return fmt.Sprintf("%.2f %s", t.EstimatedValue, t.Currency)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%.2f", t.EstimatedValue)
|
||||
}
|
||||
|
||||
// UpdateStatus updates the tender status and updated timestamp
|
||||
func (t *Tender) UpdateStatus(status TenderStatus) {
|
||||
t.Status = status
|
||||
t.UpdatedAt = time.Now().UnixMilli()
|
||||
}
|
||||
|
||||
// AddProcessingError adds a processing error to metadata
|
||||
func (t *Tender) AddProcessingError(error string) {
|
||||
if t.ProcessingMetadata.ParsingErrors == nil {
|
||||
t.ProcessingMetadata.ParsingErrors = make([]string, 0)
|
||||
}
|
||||
t.ProcessingMetadata.ParsingErrors = append(t.ProcessingMetadata.ParsingErrors, error)
|
||||
t.UpdatedAt = time.Now().UnixMilli()
|
||||
}
|
||||
|
||||
// AddValidationError adds a validation error to metadata
|
||||
func (t *Tender) AddValidationError(error string) {
|
||||
if t.ProcessingMetadata.ValidationErrors == nil {
|
||||
t.ProcessingMetadata.ValidationErrors = make([]string, 0)
|
||||
}
|
||||
t.ProcessingMetadata.ValidationErrors = append(t.ProcessingMetadata.ValidationErrors, error)
|
||||
t.UpdatedAt = time.Now().UnixMilli()
|
||||
}
|
||||
|
||||
// HasErrors returns true if the tender has processing or validation errors
|
||||
func (t *Tender) HasErrors() bool {
|
||||
return len(t.ProcessingMetadata.ParsingErrors) > 0 || len(t.ProcessingMetadata.ValidationErrors) > 0
|
||||
}
|
||||
|
||||
// GetAllErrors returns all processing and validation errors
|
||||
func (t *Tender) GetAllErrors() []string {
|
||||
var allErrors []string
|
||||
allErrors = append(allErrors, t.ProcessingMetadata.ParsingErrors...)
|
||||
allErrors = append(allErrors, t.ProcessingMetadata.ValidationErrors...)
|
||||
return allErrors
|
||||
}
|
||||
|
||||
// ScrapingJobType represents the type of scraping job
|
||||
type ScrapingJobType string
|
||||
|
||||
const (
|
||||
ScrapingJobTypeTEDScraper ScrapingJobType = "ted_scraper"
|
||||
ScrapingJobTypeManual ScrapingJobType = "manual"
|
||||
ScrapingJobTypeAPI ScrapingJobType = "api"
|
||||
ScrapingJobTypeBulkImport ScrapingJobType = "bulk_import"
|
||||
)
|
||||
|
||||
// ScrapingJobStatus represents the status of a scraping job
|
||||
type ScrapingJobStatus string
|
||||
|
||||
const (
|
||||
ScrapingJobStatusPending ScrapingJobStatus = "pending"
|
||||
ScrapingJobStatusRunning ScrapingJobStatus = "running"
|
||||
ScrapingJobStatusCompleted ScrapingJobStatus = "completed"
|
||||
ScrapingJobStatusFailed ScrapingJobStatus = "failed"
|
||||
ScrapingJobStatusCancelled ScrapingJobStatus = "cancelled"
|
||||
)
|
||||
|
||||
// JobProgress represents the progress of a scraping job
|
||||
type JobProgress struct {
|
||||
ProcessedCount int `bson:"processed_count" json:"processed_count"`
|
||||
TotalCount int `bson:"total_count" json:"total_count"`
|
||||
SuccessCount int `bson:"success_count" json:"success_count"`
|
||||
ErrorCount int `bson:"error_count" json:"error_count"`
|
||||
Percentage float64 `bson:"percentage" json:"percentage"`
|
||||
CurrentStep string `bson:"current_step" json:"current_step"`
|
||||
}
|
||||
|
||||
// JobResults represents the results of a scraping job
|
||||
type JobResults struct {
|
||||
TotalProcessed int `bson:"total_processed" json:"total_processed"`
|
||||
TotalCreated int `bson:"total_created" json:"total_created"`
|
||||
TotalUpdated int `bson:"total_updated" json:"total_updated"`
|
||||
TotalSkipped int `bson:"total_skipped" json:"total_skipped"`
|
||||
TotalErrors int `bson:"total_errors" json:"total_errors"`
|
||||
ProcessingErrors []string `bson:"processing_errors,omitempty" json:"processing_errors,omitempty"`
|
||||
Summary string `bson:"summary,omitempty" json:"summary,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapingJob represents a scraping job entity
|
||||
type ScrapingJob struct {
|
||||
ID string `bson:"_id,omitempty" json:"id"`
|
||||
Type ScrapingJobType `bson:"type" json:"type"`
|
||||
Status ScrapingJobStatus `bson:"status" json:"status"`
|
||||
StartedAt int64 `bson:"started_at" json:"started_at"` // Unix seconds
|
||||
CompletedAt *int64 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` // Unix seconds
|
||||
Duration *int64 `bson:"duration,omitempty" json:"duration,omitempty"` // Duration in seconds
|
||||
Progress JobProgress `bson:"progress" json:"progress"`
|
||||
Results JobResults `bson:"results" json:"results"`
|
||||
Errors []string `bson:"errors,omitempty" json:"errors,omitempty"`
|
||||
Config map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"`
|
||||
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
|
||||
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
|
||||
}
|
||||
|
||||
// ScrapingState represents the state of the scraping process
|
||||
type ScrapingState struct {
|
||||
ID string `bson:"_id,omitempty" json:"id"`
|
||||
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
|
||||
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
|
||||
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"` // Unix seconds
|
||||
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"` // Unix seconds
|
||||
IsRunning bool `bson:"is_running" json:"is_running"`
|
||||
CurrentJobID string `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
|
||||
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
|
||||
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
|
||||
}
|
||||
|
||||
// GetJobID returns the scraping job ID
|
||||
func (j *ScrapingJob) GetJobID() string {
|
||||
return j.ID
|
||||
}
|
||||
|
||||
// IsActive returns true if the job is actively running
|
||||
func (j *ScrapingJob) IsActive() bool {
|
||||
return j.Status == ScrapingJobStatusRunning || j.Status == ScrapingJobStatusPending
|
||||
}
|
||||
|
||||
// IsCompleted returns true if the job has completed (successfully or with failure)
|
||||
func (j *ScrapingJob) IsCompleted() bool {
|
||||
return j.Status == ScrapingJobStatusCompleted || j.Status == ScrapingJobStatusFailed || j.Status == ScrapingJobStatusCancelled
|
||||
}
|
||||
|
||||
// GetProgressPercentage calculates and returns the progress percentage
|
||||
func (j *ScrapingJob) GetProgressPercentage() float64 {
|
||||
if j.Progress.TotalCount == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(j.Progress.ProcessedCount) / float64(j.Progress.TotalCount) * 100
|
||||
}
|
||||
|
||||
// UpdateProgress updates the job progress
|
||||
func (j *ScrapingJob) UpdateProgress(processed, total, success, errors int, step string) {
|
||||
j.Progress.ProcessedCount = processed
|
||||
j.Progress.TotalCount = total
|
||||
j.Progress.SuccessCount = success
|
||||
j.Progress.ErrorCount = errors
|
||||
j.Progress.CurrentStep = step
|
||||
j.Progress.Percentage = j.GetProgressPercentage()
|
||||
j.UpdatedAt = time.Now().Unix()
|
||||
}
|
||||
|
||||
// AddError adds an error to the job
|
||||
func (j *ScrapingJob) AddError(error string) {
|
||||
if j.Errors == nil {
|
||||
j.Errors = make([]string, 0)
|
||||
}
|
||||
j.Errors = append(j.Errors, error)
|
||||
j.UpdatedAt = time.Now().Unix()
|
||||
}
|
||||
|
||||
// MarkCompleted marks the job as completed
|
||||
func (j *ScrapingJob) MarkCompleted(status ScrapingJobStatus, results JobResults) {
|
||||
j.Status = status
|
||||
j.Results = results
|
||||
completedAt := time.Now().Unix()
|
||||
j.CompletedAt = &completedAt
|
||||
if j.StartedAt > 0 {
|
||||
duration := completedAt - j.StartedAt
|
||||
j.Duration = &duration
|
||||
}
|
||||
j.UpdatedAt = completedAt
|
||||
}
|
||||
|
||||
// IsStateRunning returns true if the scraping state indicates a job is running
|
||||
func (s *ScrapingState) IsStateRunning() bool {
|
||||
return s.IsRunning
|
||||
}
|
||||
|
||||
// MarkAsRunning marks the state as running with a job ID
|
||||
func (s *ScrapingState) MarkAsRunning(jobID string) {
|
||||
s.IsRunning = true
|
||||
s.CurrentJobID = jobID
|
||||
s.LastRunAt = time.Now().Unix()
|
||||
s.UpdatedAt = time.Now().Unix()
|
||||
}
|
||||
|
||||
// MarkAsCompleted marks the state as completed
|
||||
func (s *ScrapingState) MarkAsCompleted(year, number int) {
|
||||
s.IsRunning = false
|
||||
s.CurrentJobID = ""
|
||||
s.LastProcessedYear = year
|
||||
s.LastProcessedNumber = number
|
||||
s.NextRunAt = time.Now().Add(4 * time.Hour).Unix() // Schedule next run in 4 hours
|
||||
s.UpdatedAt = time.Now().Unix()
|
||||
}
|
||||
Reference in New Issue
Block a user