Implement TED Scraper and Configuration Management
- Introduced a new TED scraper in `cmd/scraper` to handle downloading and parsing TED XML files. - Added configuration management for the scraper, including a new `Config` struct and YAML configuration file. - Created necessary handler, service, and repository layers for tender management, adhering to Clean Architecture principles. - Implemented comprehensive logging and error handling throughout the scraper functionality. - Updated API routes to include tender management operations, enhancing the overall system capabilities.
This commit is contained in:
n.nakhostin
@@ -0,0 +1,293 @@
|
||||
package tender
|
||||
|
||||
// CreateTenderRequest represents a request to create a new tender
|
||||
type CreateTenderRequest struct {
|
||||
ContractNoticeID string `json:"contract_notice_id" valid:"required~Contract notice ID is required"`
|
||||
NoticePublicationID string `json:"notice_publication_id" valid:"required~Notice publication ID is required"`
|
||||
ContractFolderID string `json:"contract_folder_id" valid:"required~Contract folder ID is required"`
|
||||
NoticeTypeCode string `json:"notice_type_code" valid:"required~Notice type code is required"`
|
||||
NoticeSubTypeCode string `json:"notice_sub_type_code,omitempty"`
|
||||
NoticeLanguageCode string `json:"notice_language_code,omitempty" valid:"stringlength(3|3)~Language code must be 3 characters"`
|
||||
IssueDate int64 `json:"issue_date" valid:"required~Issue date is required"` // Unix milliseconds
|
||||
IssueTime int64 `json:"issue_time,omitempty"` // Unix milliseconds
|
||||
PublicationDate int64 `json:"publication_date,omitempty"` // Unix milliseconds
|
||||
GazetteID string `json:"gazette_id,omitempty"`
|
||||
Title string `json:"title" valid:"required~Title is required,stringlength(1|500)~Title must be between 1 and 500 characters"`
|
||||
Description string `json:"description,omitempty" valid:"stringlength(0|5000)~Description must not exceed 5000 characters"`
|
||||
ProcurementTypeCode string `json:"procurement_type_code,omitempty"`
|
||||
ProcedureCode string `json:"procedure_code,omitempty"`
|
||||
MainClassification string `json:"main_classification,omitempty"`
|
||||
AdditionalClassifications []string `json:"additional_classifications,omitempty"`
|
||||
EstimatedValue float64 `json:"estimated_value,omitempty" valid:"range(0|999999999999)~Estimated value must be between 0 and 999999999999"`
|
||||
Currency string `json:"currency,omitempty" valid:"stringlength(3|3)~Currency must be 3 characters"`
|
||||
Duration string `json:"duration,omitempty"`
|
||||
DurationUnit string `json:"duration_unit,omitempty"`
|
||||
TenderDeadline int64 `json:"tender_deadline,omitempty"` // Unix milliseconds
|
||||
PlaceOfPerformance string `json:"place_of_performance,omitempty"`
|
||||
CountryCode string `json:"country_code,omitempty" valid:"stringlength(2|2)~Country code must be 2 characters"`
|
||||
RegionCode string `json:"region_code,omitempty"`
|
||||
CityName string `json:"city_name,omitempty"`
|
||||
PostalCode string `json:"postal_code,omitempty"`
|
||||
DocumentURI string `json:"document_uri,omitempty" valid:"url~Document URI must be a valid URL"`
|
||||
TenderURL string `json:"tender_url,omitempty" valid:"url~Tender URL must be a valid URL"`
|
||||
BuyerOrganization *Organization `json:"buyer_organization,omitempty"`
|
||||
ReviewOrganization *Organization `json:"review_organization,omitempty"`
|
||||
Organizations []Organization `json:"organizations,omitempty"`
|
||||
SelectionCriteria []SelectionCriterion `json:"selection_criteria,omitempty"`
|
||||
OfficialLanguages []string `json:"official_languages,omitempty"`
|
||||
Source TenderSource `json:"source" valid:"required~Source is required"`
|
||||
SourceFileURL string `json:"source_file_url,omitempty" valid:"url~Source file URL must be a valid URL"`
|
||||
SourceFileName string `json:"source_file_name,omitempty"`
|
||||
}
|
||||
|
||||
// UpdateTenderRequest represents a request to update an existing tender
|
||||
type UpdateTenderRequest struct {
|
||||
ID string `json:"id" valid:"required~Tender ID is required"`
|
||||
Title *string `json:"title,omitempty" valid:"stringlength(1|500)~Title must be between 1 and 500 characters"`
|
||||
Description *string `json:"description,omitempty" valid:"stringlength(0|5000)~Description must not exceed 5000 characters"`
|
||||
Status *TenderStatus `json:"status,omitempty"`
|
||||
EstimatedValue *float64 `json:"estimated_value,omitempty" valid:"range(0|999999999999)~Estimated value must be between 0 and 999999999999"`
|
||||
Currency *string `json:"currency,omitempty" valid:"stringlength(3|3)~Currency must be 3 characters"`
|
||||
TenderDeadline *int64 `json:"tender_deadline,omitempty"` // Unix milliseconds
|
||||
}
|
||||
|
||||
// ListTendersRequest represents a request to list tenders with pagination and filtering
|
||||
type ListTendersRequest struct {
|
||||
Criteria TenderSearchCriteria `json:"criteria"`
|
||||
Limit int `json:"limit" valid:"range(1|100)~Limit must be between 1 and 100"`
|
||||
Offset int `json:"offset" valid:"range(0|999999)~Offset must be non-negative"`
|
||||
}
|
||||
|
||||
// ListTendersResponse represents the response for listing tenders
|
||||
type ListTendersResponse struct {
|
||||
Tenders []Tender `json:"tenders"`
|
||||
Total int64 `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
// SearchTendersRequest represents a request to search tenders
|
||||
type SearchTendersRequest struct {
|
||||
Query string `json:"query" valid:"required~Search query is required,stringlength(1|200)~Query must be between 1 and 200 characters"`
|
||||
Criteria TenderSearchCriteria `json:"criteria"`
|
||||
}
|
||||
|
||||
// SearchTendersResponse represents the response for searching tenders
|
||||
type SearchTendersResponse struct {
|
||||
Tenders []Tender `json:"tenders"`
|
||||
Query string `json:"query"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// StartScrapingJobRequest represents a request to start a scraping job
|
||||
type StartScrapingJobRequest struct {
|
||||
Type ScrapingJobType `json:"type" valid:"required~Job type is required"`
|
||||
Config map[string]interface{} `json:"config,omitempty"`
|
||||
}
|
||||
|
||||
// ListScrapingJobsRequest represents a request to list scraping jobs
|
||||
type ListScrapingJobsRequest struct {
|
||||
Limit int `json:"limit" valid:"range(1|100)~Limit must be between 1 and 100"`
|
||||
Offset int `json:"offset" valid:"range(0|999999)~Offset must be non-negative"`
|
||||
}
|
||||
|
||||
// ListScrapingJobsResponse represents the response for listing scraping jobs
|
||||
type ListScrapingJobsResponse struct {
|
||||
Jobs []ScrapingJob `json:"jobs"`
|
||||
Total int64 `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
}
|
||||
|
||||
// BatchProcessingResult represents the result of batch processing
|
||||
type BatchProcessingResult struct {
|
||||
Total int `json:"total"`
|
||||
Processed int `json:"processed"`
|
||||
Errors []string `json:"errors"`
|
||||
Tenders []*Tender `json:"tenders"`
|
||||
SuccessRate float64 `json:"success_rate"`
|
||||
}
|
||||
|
||||
// DashboardData represents dashboard data
|
||||
type DashboardData struct {
|
||||
Statistics *TenderStatistics `json:"statistics"`
|
||||
RecentTenders []Tender `json:"recent_tenders"`
|
||||
ActiveJobs []ScrapingJob `json:"active_jobs"`
|
||||
}
|
||||
|
||||
// TenderResponse represents a tender in API responses
|
||||
type TenderResponse struct {
|
||||
ID string `json:"id"`
|
||||
ContractNoticeID string `json:"contract_notice_id"`
|
||||
NoticePublicationID string `json:"notice_publication_id"`
|
||||
ContractFolderID string `json:"contract_folder_id"`
|
||||
NoticeTypeCode string `json:"notice_type_code"`
|
||||
NoticeSubTypeCode string `json:"notice_sub_type_code"`
|
||||
NoticeLanguageCode string `json:"notice_language_code"`
|
||||
IssueDate int64 `json:"issue_date"`
|
||||
IssueTime int64 `json:"issue_time"`
|
||||
PublicationDate int64 `json:"publication_date"`
|
||||
GazetteID string `json:"gazette_id"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
ProcurementTypeCode string `json:"procurement_type_code"`
|
||||
ProcedureCode string `json:"procedure_code"`
|
||||
MainClassification string `json:"main_classification"`
|
||||
AdditionalClassifications []string `json:"additional_classifications"`
|
||||
EstimatedValue float64 `json:"estimated_value"`
|
||||
Currency string `json:"currency"`
|
||||
FormattedEstimatedValue string `json:"formatted_estimated_value"`
|
||||
Duration string `json:"duration"`
|
||||
DurationUnit string `json:"duration_unit"`
|
||||
TenderDeadline int64 `json:"tender_deadline"`
|
||||
PlaceOfPerformance string `json:"place_of_performance"`
|
||||
CountryCode string `json:"country_code"`
|
||||
RegionCode string `json:"region_code"`
|
||||
CityName string `json:"city_name"`
|
||||
PostalCode string `json:"postal_code"`
|
||||
DocumentURI string `json:"document_uri"`
|
||||
TenderURL string `json:"tender_url"`
|
||||
BuyerOrganization *Organization `json:"buyer_organization"`
|
||||
ReviewOrganization *Organization `json:"review_organization,omitempty"`
|
||||
Organizations []Organization `json:"organizations"`
|
||||
SelectionCriteria []SelectionCriterion `json:"selection_criteria"`
|
||||
OfficialLanguages []string `json:"official_languages"`
|
||||
Status TenderStatus `json:"status"`
|
||||
Source TenderSource `json:"source"`
|
||||
SourceFileURL string `json:"source_file_url"`
|
||||
SourceFileName string `json:"source_file_name"`
|
||||
ProcessingMetadata ProcessingMetadata `json:"processing_metadata"`
|
||||
IsActive bool `json:"is_active"`
|
||||
IsExpired bool `json:"is_expired"`
|
||||
HasErrors bool `json:"has_errors"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ScrapingJobResponse represents a scraping job in API responses
|
||||
type ScrapingJobResponse struct {
|
||||
ID string `json:"id"`
|
||||
Type ScrapingJobType `json:"type"`
|
||||
Status ScrapingJobStatus `json:"status"`
|
||||
StartedAt int64 `json:"started_at"`
|
||||
CompletedAt *int64 `json:"completed_at,omitempty"`
|
||||
Duration *int64 `json:"duration,omitempty"`
|
||||
Progress JobProgress `json:"progress"`
|
||||
Results JobResults `json:"results"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
Config map[string]interface{} `json:"config,omitempty"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ScrapingStateResponse represents scraping state in API responses
|
||||
type ScrapingStateResponse struct {
|
||||
ID string `json:"id"`
|
||||
LastProcessedYear int `json:"last_processed_year"`
|
||||
LastProcessedNumber int `json:"last_processed_number"`
|
||||
LastRunAt int64 `json:"last_run_at"`
|
||||
NextRunAt int64 `json:"next_run_at"`
|
||||
IsRunning bool `json:"is_running"`
|
||||
CurrentJobID string `json:"current_job_id,omitempty"`
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
UpdatedAt int64 `json:"updated_at"`
|
||||
}
|
||||
|
||||
// ValidationErrorResponse represents validation errors in API responses
|
||||
type ValidationErrorResponse struct {
|
||||
Field string `json:"field"`
|
||||
Message string `json:"message"`
|
||||
Value string `json:"value,omitempty"`
|
||||
}
|
||||
|
||||
// ErrorResponse represents error responses
|
||||
type ErrorResponse struct {
|
||||
Error string `json:"error"`
|
||||
Message string `json:"message"`
|
||||
ValidationErrors []ValidationErrorResponse `json:"validation_errors,omitempty"`
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
// ToTenderResponse converts a Tender entity to TenderResponse
|
||||
func (t *Tender) ToTenderResponse() *TenderResponse {
|
||||
return &TenderResponse{
|
||||
ID: t.ID.Hex(),
|
||||
ContractNoticeID: t.ContractNoticeID,
|
||||
NoticePublicationID: t.NoticePublicationID,
|
||||
ContractFolderID: t.ContractFolderID,
|
||||
NoticeTypeCode: t.NoticeTypeCode,
|
||||
NoticeSubTypeCode: t.NoticeSubTypeCode,
|
||||
NoticeLanguageCode: t.NoticeLanguageCode,
|
||||
IssueDate: t.IssueDate,
|
||||
IssueTime: t.IssueTime,
|
||||
PublicationDate: t.PublicationDate,
|
||||
GazetteID: t.GazetteID,
|
||||
Title: t.Title,
|
||||
Description: t.Description,
|
||||
ProcurementTypeCode: t.ProcurementTypeCode,
|
||||
ProcedureCode: t.ProcedureCode,
|
||||
MainClassification: t.MainClassification,
|
||||
AdditionalClassifications: t.AdditionalClassifications,
|
||||
EstimatedValue: t.EstimatedValue,
|
||||
Currency: t.Currency,
|
||||
FormattedEstimatedValue: t.GetFormattedEstimatedValue(),
|
||||
Duration: t.Duration,
|
||||
DurationUnit: t.DurationUnit,
|
||||
TenderDeadline: t.TenderDeadline,
|
||||
PlaceOfPerformance: t.PlaceOfPerformance,
|
||||
CountryCode: t.CountryCode,
|
||||
RegionCode: t.RegionCode,
|
||||
CityName: t.CityName,
|
||||
PostalCode: t.PostalCode,
|
||||
DocumentURI: t.DocumentURI,
|
||||
TenderURL: t.TenderURL,
|
||||
BuyerOrganization: t.BuyerOrganization,
|
||||
ReviewOrganization: t.ReviewOrganization,
|
||||
Organizations: t.Organizations,
|
||||
SelectionCriteria: t.SelectionCriteria,
|
||||
OfficialLanguages: t.OfficialLanguages,
|
||||
Status: t.Status,
|
||||
Source: t.Source,
|
||||
SourceFileURL: t.SourceFileURL,
|
||||
SourceFileName: t.SourceFileName,
|
||||
ProcessingMetadata: t.ProcessingMetadata,
|
||||
IsActive: t.IsActive(),
|
||||
IsExpired: t.IsExpired(),
|
||||
HasErrors: t.HasErrors(),
|
||||
CreatedAt: t.CreatedAt,
|
||||
UpdatedAt: t.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// ToScrapingJobResponse converts a ScrapingJob entity to ScrapingJobResponse
|
||||
func (j *ScrapingJob) ToScrapingJobResponse() *ScrapingJobResponse {
|
||||
return &ScrapingJobResponse{
|
||||
ID: j.ID,
|
||||
Type: j.Type,
|
||||
Status: j.Status,
|
||||
StartedAt: j.StartedAt,
|
||||
CompletedAt: j.CompletedAt,
|
||||
Duration: j.Duration,
|
||||
Progress: j.Progress,
|
||||
Results: j.Results,
|
||||
Errors: j.Errors,
|
||||
Config: j.Config,
|
||||
CreatedAt: j.CreatedAt,
|
||||
UpdatedAt: j.UpdatedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// ToScrapingStateResponse converts a ScrapingState entity to ScrapingStateResponse
|
||||
func (s *ScrapingState) ToScrapingStateResponse() *ScrapingStateResponse {
|
||||
return &ScrapingStateResponse{
|
||||
ID: s.ID,
|
||||
LastProcessedYear: s.LastProcessedYear,
|
||||
LastProcessedNumber: s.LastProcessedNumber,
|
||||
LastRunAt: s.LastRunAt,
|
||||
NextRunAt: s.NextRunAt,
|
||||
IsRunning: s.IsRunning,
|
||||
CurrentJobID: s.CurrentJobID,
|
||||
CreatedAt: s.CreatedAt,
|
||||
UpdatedAt: s.UpdatedAt,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user