n.nakhostin
96c21b8b78
- Introduced a new TED scraper in `cmd/scraper` to handle downloading and parsing TED XML files. - Added configuration management for the scraper, including a new `Config` struct and YAML configuration file. - Created necessary handler, service, and repository layers for tender management, adhering to Clean Architecture principles. - Implemented comprehensive logging and error handling throughout the scraper functionality. - Updated API routes to include tender management operations, enhancing the overall system capabilities.
294 lines
15 KiB
Go
294 lines
15 KiB
Go
package tender
|
|
|
|
// CreateTenderRequest represents a request to create a new tender
|
|
type CreateTenderRequest struct {
|
|
ContractNoticeID string `json:"contract_notice_id" valid:"required~Contract notice ID is required"`
|
|
NoticePublicationID string `json:"notice_publication_id" valid:"required~Notice publication ID is required"`
|
|
ContractFolderID string `json:"contract_folder_id" valid:"required~Contract folder ID is required"`
|
|
NoticeTypeCode string `json:"notice_type_code" valid:"required~Notice type code is required"`
|
|
NoticeSubTypeCode string `json:"notice_sub_type_code,omitempty"`
|
|
NoticeLanguageCode string `json:"notice_language_code,omitempty" valid:"stringlength(3|3)~Language code must be 3 characters"`
|
|
IssueDate int64 `json:"issue_date" valid:"required~Issue date is required"` // Unix milliseconds
|
|
IssueTime int64 `json:"issue_time,omitempty"` // Unix milliseconds
|
|
PublicationDate int64 `json:"publication_date,omitempty"` // Unix milliseconds
|
|
GazetteID string `json:"gazette_id,omitempty"`
|
|
Title string `json:"title" valid:"required~Title is required,stringlength(1|500)~Title must be between 1 and 500 characters"`
|
|
Description string `json:"description,omitempty" valid:"stringlength(0|5000)~Description must not exceed 5000 characters"`
|
|
ProcurementTypeCode string `json:"procurement_type_code,omitempty"`
|
|
ProcedureCode string `json:"procedure_code,omitempty"`
|
|
MainClassification string `json:"main_classification,omitempty"`
|
|
AdditionalClassifications []string `json:"additional_classifications,omitempty"`
|
|
EstimatedValue float64 `json:"estimated_value,omitempty" valid:"range(0|999999999999)~Estimated value must be between 0 and 999999999999"`
|
|
Currency string `json:"currency,omitempty" valid:"stringlength(3|3)~Currency must be 3 characters"`
|
|
Duration string `json:"duration,omitempty"`
|
|
DurationUnit string `json:"duration_unit,omitempty"`
|
|
TenderDeadline int64 `json:"tender_deadline,omitempty"` // Unix milliseconds
|
|
PlaceOfPerformance string `json:"place_of_performance,omitempty"`
|
|
CountryCode string `json:"country_code,omitempty" valid:"stringlength(2|2)~Country code must be 2 characters"`
|
|
RegionCode string `json:"region_code,omitempty"`
|
|
CityName string `json:"city_name,omitempty"`
|
|
PostalCode string `json:"postal_code,omitempty"`
|
|
DocumentURI string `json:"document_uri,omitempty" valid:"url~Document URI must be a valid URL"`
|
|
TenderURL string `json:"tender_url,omitempty" valid:"url~Tender URL must be a valid URL"`
|
|
BuyerOrganization *Organization `json:"buyer_organization,omitempty"`
|
|
ReviewOrganization *Organization `json:"review_organization,omitempty"`
|
|
Organizations []Organization `json:"organizations,omitempty"`
|
|
SelectionCriteria []SelectionCriterion `json:"selection_criteria,omitempty"`
|
|
OfficialLanguages []string `json:"official_languages,omitempty"`
|
|
Source TenderSource `json:"source" valid:"required~Source is required"`
|
|
SourceFileURL string `json:"source_file_url,omitempty" valid:"url~Source file URL must be a valid URL"`
|
|
SourceFileName string `json:"source_file_name,omitempty"`
|
|
}
|
|
|
|
// UpdateTenderRequest represents a request to update an existing tender
|
|
type UpdateTenderRequest struct {
|
|
ID string `json:"id" valid:"required~Tender ID is required"`
|
|
Title *string `json:"title,omitempty" valid:"stringlength(1|500)~Title must be between 1 and 500 characters"`
|
|
Description *string `json:"description,omitempty" valid:"stringlength(0|5000)~Description must not exceed 5000 characters"`
|
|
Status *TenderStatus `json:"status,omitempty"`
|
|
EstimatedValue *float64 `json:"estimated_value,omitempty" valid:"range(0|999999999999)~Estimated value must be between 0 and 999999999999"`
|
|
Currency *string `json:"currency,omitempty" valid:"stringlength(3|3)~Currency must be 3 characters"`
|
|
TenderDeadline *int64 `json:"tender_deadline,omitempty"` // Unix milliseconds
|
|
}
|
|
|
|
// ListTendersRequest represents a request to list tenders with pagination and filtering
|
|
type ListTendersRequest struct {
|
|
Criteria TenderSearchCriteria `json:"criteria"`
|
|
Limit int `json:"limit" valid:"range(1|100)~Limit must be between 1 and 100"`
|
|
Offset int `json:"offset" valid:"range(0|999999)~Offset must be non-negative"`
|
|
}
|
|
|
|
// ListTendersResponse represents the response for listing tenders
|
|
type ListTendersResponse struct {
|
|
Tenders []Tender `json:"tenders"`
|
|
Total int64 `json:"total"`
|
|
Limit int `json:"limit"`
|
|
Offset int `json:"offset"`
|
|
}
|
|
|
|
// SearchTendersRequest represents a request to search tenders
|
|
type SearchTendersRequest struct {
|
|
Query string `json:"query" valid:"required~Search query is required,stringlength(1|200)~Query must be between 1 and 200 characters"`
|
|
Criteria TenderSearchCriteria `json:"criteria"`
|
|
}
|
|
|
|
// SearchTendersResponse represents the response for searching tenders
|
|
type SearchTendersResponse struct {
|
|
Tenders []Tender `json:"tenders"`
|
|
Query string `json:"query"`
|
|
Total int `json:"total"`
|
|
}
|
|
|
|
// StartScrapingJobRequest represents a request to start a scraping job
|
|
type StartScrapingJobRequest struct {
|
|
Type ScrapingJobType `json:"type" valid:"required~Job type is required"`
|
|
Config map[string]interface{} `json:"config,omitempty"`
|
|
}
|
|
|
|
// ListScrapingJobsRequest represents a request to list scraping jobs
|
|
type ListScrapingJobsRequest struct {
|
|
Limit int `json:"limit" valid:"range(1|100)~Limit must be between 1 and 100"`
|
|
Offset int `json:"offset" valid:"range(0|999999)~Offset must be non-negative"`
|
|
}
|
|
|
|
// ListScrapingJobsResponse represents the response for listing scraping jobs
|
|
type ListScrapingJobsResponse struct {
|
|
Jobs []ScrapingJob `json:"jobs"`
|
|
Total int64 `json:"total"`
|
|
Limit int `json:"limit"`
|
|
Offset int `json:"offset"`
|
|
}
|
|
|
|
// BatchProcessingResult represents the result of batch processing
|
|
type BatchProcessingResult struct {
|
|
Total int `json:"total"`
|
|
Processed int `json:"processed"`
|
|
Errors []string `json:"errors"`
|
|
Tenders []*Tender `json:"tenders"`
|
|
SuccessRate float64 `json:"success_rate"`
|
|
}
|
|
|
|
// DashboardData represents dashboard data
|
|
type DashboardData struct {
|
|
Statistics *TenderStatistics `json:"statistics"`
|
|
RecentTenders []Tender `json:"recent_tenders"`
|
|
ActiveJobs []ScrapingJob `json:"active_jobs"`
|
|
}
|
|
|
|
// TenderResponse represents a tender in API responses
|
|
type TenderResponse struct {
|
|
ID string `json:"id"`
|
|
ContractNoticeID string `json:"contract_notice_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticeTypeCode string `json:"notice_type_code"`
|
|
NoticeSubTypeCode string `json:"notice_sub_type_code"`
|
|
NoticeLanguageCode string `json:"notice_language_code"`
|
|
IssueDate int64 `json:"issue_date"`
|
|
IssueTime int64 `json:"issue_time"`
|
|
PublicationDate int64 `json:"publication_date"`
|
|
GazetteID string `json:"gazette_id"`
|
|
Title string `json:"title"`
|
|
Description string `json:"description"`
|
|
ProcurementTypeCode string `json:"procurement_type_code"`
|
|
ProcedureCode string `json:"procedure_code"`
|
|
MainClassification string `json:"main_classification"`
|
|
AdditionalClassifications []string `json:"additional_classifications"`
|
|
EstimatedValue float64 `json:"estimated_value"`
|
|
Currency string `json:"currency"`
|
|
FormattedEstimatedValue string `json:"formatted_estimated_value"`
|
|
Duration string `json:"duration"`
|
|
DurationUnit string `json:"duration_unit"`
|
|
TenderDeadline int64 `json:"tender_deadline"`
|
|
PlaceOfPerformance string `json:"place_of_performance"`
|
|
CountryCode string `json:"country_code"`
|
|
RegionCode string `json:"region_code"`
|
|
CityName string `json:"city_name"`
|
|
PostalCode string `json:"postal_code"`
|
|
DocumentURI string `json:"document_uri"`
|
|
TenderURL string `json:"tender_url"`
|
|
BuyerOrganization *Organization `json:"buyer_organization"`
|
|
ReviewOrganization *Organization `json:"review_organization,omitempty"`
|
|
Organizations []Organization `json:"organizations"`
|
|
SelectionCriteria []SelectionCriterion `json:"selection_criteria"`
|
|
OfficialLanguages []string `json:"official_languages"`
|
|
Status TenderStatus `json:"status"`
|
|
Source TenderSource `json:"source"`
|
|
SourceFileURL string `json:"source_file_url"`
|
|
SourceFileName string `json:"source_file_name"`
|
|
ProcessingMetadata ProcessingMetadata `json:"processing_metadata"`
|
|
IsActive bool `json:"is_active"`
|
|
IsExpired bool `json:"is_expired"`
|
|
HasErrors bool `json:"has_errors"`
|
|
CreatedAt int64 `json:"created_at"`
|
|
UpdatedAt int64 `json:"updated_at"`
|
|
}
|
|
|
|
// ScrapingJobResponse represents a scraping job in API responses
|
|
type ScrapingJobResponse struct {
|
|
ID string `json:"id"`
|
|
Type ScrapingJobType `json:"type"`
|
|
Status ScrapingJobStatus `json:"status"`
|
|
StartedAt int64 `json:"started_at"`
|
|
CompletedAt *int64 `json:"completed_at,omitempty"`
|
|
Duration *int64 `json:"duration,omitempty"`
|
|
Progress JobProgress `json:"progress"`
|
|
Results JobResults `json:"results"`
|
|
Errors []string `json:"errors,omitempty"`
|
|
Config map[string]interface{} `json:"config,omitempty"`
|
|
CreatedAt int64 `json:"created_at"`
|
|
UpdatedAt int64 `json:"updated_at"`
|
|
}
|
|
|
|
// ScrapingStateResponse represents scraping state in API responses
|
|
type ScrapingStateResponse struct {
|
|
ID string `json:"id"`
|
|
LastProcessedYear int `json:"last_processed_year"`
|
|
LastProcessedNumber int `json:"last_processed_number"`
|
|
LastRunAt int64 `json:"last_run_at"`
|
|
NextRunAt int64 `json:"next_run_at"`
|
|
IsRunning bool `json:"is_running"`
|
|
CurrentJobID string `json:"current_job_id,omitempty"`
|
|
CreatedAt int64 `json:"created_at"`
|
|
UpdatedAt int64 `json:"updated_at"`
|
|
}
|
|
|
|
// ValidationErrorResponse represents validation errors in API responses
|
|
type ValidationErrorResponse struct {
|
|
Field string `json:"field"`
|
|
Message string `json:"message"`
|
|
Value string `json:"value,omitempty"`
|
|
}
|
|
|
|
// ErrorResponse represents error responses
|
|
type ErrorResponse struct {
|
|
Error string `json:"error"`
|
|
Message string `json:"message"`
|
|
ValidationErrors []ValidationErrorResponse `json:"validation_errors,omitempty"`
|
|
Timestamp int64 `json:"timestamp"`
|
|
}
|
|
|
|
// ToTenderResponse converts a Tender entity to TenderResponse
|
|
func (t *Tender) ToTenderResponse() *TenderResponse {
|
|
return &TenderResponse{
|
|
ID: t.ID.Hex(),
|
|
ContractNoticeID: t.ContractNoticeID,
|
|
NoticePublicationID: t.NoticePublicationID,
|
|
ContractFolderID: t.ContractFolderID,
|
|
NoticeTypeCode: t.NoticeTypeCode,
|
|
NoticeSubTypeCode: t.NoticeSubTypeCode,
|
|
NoticeLanguageCode: t.NoticeLanguageCode,
|
|
IssueDate: t.IssueDate,
|
|
IssueTime: t.IssueTime,
|
|
PublicationDate: t.PublicationDate,
|
|
GazetteID: t.GazetteID,
|
|
Title: t.Title,
|
|
Description: t.Description,
|
|
ProcurementTypeCode: t.ProcurementTypeCode,
|
|
ProcedureCode: t.ProcedureCode,
|
|
MainClassification: t.MainClassification,
|
|
AdditionalClassifications: t.AdditionalClassifications,
|
|
EstimatedValue: t.EstimatedValue,
|
|
Currency: t.Currency,
|
|
FormattedEstimatedValue: t.GetFormattedEstimatedValue(),
|
|
Duration: t.Duration,
|
|
DurationUnit: t.DurationUnit,
|
|
TenderDeadline: t.TenderDeadline,
|
|
PlaceOfPerformance: t.PlaceOfPerformance,
|
|
CountryCode: t.CountryCode,
|
|
RegionCode: t.RegionCode,
|
|
CityName: t.CityName,
|
|
PostalCode: t.PostalCode,
|
|
DocumentURI: t.DocumentURI,
|
|
TenderURL: t.TenderURL,
|
|
BuyerOrganization: t.BuyerOrganization,
|
|
ReviewOrganization: t.ReviewOrganization,
|
|
Organizations: t.Organizations,
|
|
SelectionCriteria: t.SelectionCriteria,
|
|
OfficialLanguages: t.OfficialLanguages,
|
|
Status: t.Status,
|
|
Source: t.Source,
|
|
SourceFileURL: t.SourceFileURL,
|
|
SourceFileName: t.SourceFileName,
|
|
ProcessingMetadata: t.ProcessingMetadata,
|
|
IsActive: t.IsActive(),
|
|
IsExpired: t.IsExpired(),
|
|
HasErrors: t.HasErrors(),
|
|
CreatedAt: t.CreatedAt,
|
|
UpdatedAt: t.UpdatedAt,
|
|
}
|
|
}
|
|
|
|
// ToScrapingJobResponse converts a ScrapingJob entity to ScrapingJobResponse
|
|
func (j *ScrapingJob) ToScrapingJobResponse() *ScrapingJobResponse {
|
|
return &ScrapingJobResponse{
|
|
ID: j.ID,
|
|
Type: j.Type,
|
|
Status: j.Status,
|
|
StartedAt: j.StartedAt,
|
|
CompletedAt: j.CompletedAt,
|
|
Duration: j.Duration,
|
|
Progress: j.Progress,
|
|
Results: j.Results,
|
|
Errors: j.Errors,
|
|
Config: j.Config,
|
|
CreatedAt: j.CreatedAt,
|
|
UpdatedAt: j.UpdatedAt,
|
|
}
|
|
}
|
|
|
|
// ToScrapingStateResponse converts a ScrapingState entity to ScrapingStateResponse
|
|
func (s *ScrapingState) ToScrapingStateResponse() *ScrapingStateResponse {
|
|
return &ScrapingStateResponse{
|
|
ID: s.ID,
|
|
LastProcessedYear: s.LastProcessedYear,
|
|
LastProcessedNumber: s.LastProcessedNumber,
|
|
LastRunAt: s.LastRunAt,
|
|
NextRunAt: s.NextRunAt,
|
|
IsRunning: s.IsRunning,
|
|
CurrentJobID: s.CurrentJobID,
|
|
CreatedAt: s.CreatedAt,
|
|
UpdatedAt: s.UpdatedAt,
|
|
}
|
|
}
|