mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Tender Management System by Removing Scraping Functionality and Enhancing Tender API

Browse Source 
- Removed all scraping-related functionality, including routes, handlers, services, and repository methods, to streamline the tender management system.
- Updated the tender API to focus solely on tender management, enhancing endpoints for listing, retrieving, and updating tenders.
- Improved Swagger documentation to reflect the removal of scraping endpoints and the addition of new tender-related features.
- Ensured adherence to Clean Architecture principles throughout the refactoring process, maintaining a clear separation of concerns.
This commit is contained in:
n.nakhostin
2025-08-16 16:52:17 +03:30
parent 21b736b55b
commit e986ee4135
11 changed files with 239 additions and 4266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tender/entity.go
+4 -228
View File
@@ -128,10 +128,10 @@ type TenderSearchCriteria struct {
MinEstimatedValue *float64 `json:"min_estimated_value,omitempty"`
MaxEstimatedValue *float64 `json:"max_estimated_value,omitempty"`
Currency string `json:"currency,omitempty"`
DeadlineFrom *int64 `json:"deadline_from,omitempty"` // Unix milliseconds
DeadlineTo *int64 `json:"deadline_to,omitempty"` // Unix milliseconds
PublicationDateFrom *int64 `json:"publication_date_from,omitempty"` // Unix milliseconds
PublicationDateTo *int64 `json:"publication_date_to,omitempty"` // Unix milliseconds
DeadlineFrom *int64 `json:"deadline_from,omitempty"`
DeadlineTo *int64 `json:"deadline_to,omitempty"`
PublicationDateFrom *int64 `json:"publication_date_from,omitempty"`
PublicationDateTo *int64 `json:"publication_date_to,omitempty"`
Status []TenderStatus `json:"status,omitempty"`
Source []TenderSource `json:"source,omitempty"`
BuyerOrganizationID string `json:"buyer_organization_id,omitempty"`
@@ -236,230 +236,6 @@ func (t *Tender) UpdateStatus(status TenderStatus) {
t.UpdatedAt = time.Now().UnixMilli()
}
// AddProcessingError adds a processing error to metadata
func (t *Tender) AddProcessingError(error string) {
if t.ProcessingMetadata.ParsingErrors == nil {
t.ProcessingMetadata.ParsingErrors = make([]string, 0)
}
t.ProcessingMetadata.ParsingErrors = append(t.ProcessingMetadata.ParsingErrors, error)
t.UpdatedAt = time.Now().UnixMilli()
}
// AddValidationError adds a validation error to metadata
func (t *Tender) AddValidationError(error string) {
if t.ProcessingMetadata.ValidationErrors == nil {
t.ProcessingMetadata.ValidationErrors = make([]string, 0)
}
t.ProcessingMetadata.ValidationErrors = append(t.ProcessingMetadata.ValidationErrors, error)
t.UpdatedAt = time.Now().UnixMilli()
}
// HasErrors returns true if the tender has processing or validation errors
func (t *Tender) HasErrors() bool {
return len(t.ProcessingMetadata.ParsingErrors) > 0 || len(t.ProcessingMetadata.ValidationErrors) > 0
}
// GetAllErrors returns all processing and validation errors
func (t *Tender) GetAllErrors() []string {
var allErrors []string
allErrors = append(allErrors, t.ProcessingMetadata.ParsingErrors...)
allErrors = append(allErrors, t.ProcessingMetadata.ValidationErrors...)
return allErrors
}
// ScrapingJobType represents the type of scraping job
type ScrapingJobType string
const (
ScrapingJobTypeTEDScraper ScrapingJobType = "ted_scraper"
ScrapingJobTypeManual ScrapingJobType = "manual"
ScrapingJobTypeAPI ScrapingJobType = "api"
ScrapingJobTypeBulkImport ScrapingJobType = "bulk_import"
)
// ScrapingJobStatus represents the status of a scraping job
type ScrapingJobStatus string
const (
ScrapingJobStatusPending ScrapingJobStatus = "pending"
ScrapingJobStatusRunning ScrapingJobStatus = "running"
ScrapingJobStatusCompleted ScrapingJobStatus = "completed"
ScrapingJobStatusFailed ScrapingJobStatus = "failed"
ScrapingJobStatusCancelled ScrapingJobStatus = "cancelled"
)
// JobProgress represents the progress of a scraping job
type JobProgress struct {
ProcessedCount int `bson:"processed_count" json:"processed_count"`
TotalCount int `bson:"total_count" json:"total_count"`
SuccessCount int `bson:"success_count" json:"success_count"`
ErrorCount int `bson:"error_count" json:"error_count"`
Percentage float64 `bson:"percentage" json:"percentage"`
CurrentStep string `bson:"current_step" json:"current_step"`
}
// JobResults represents the results of a scraping job
type JobResults struct {
TotalProcessed int `bson:"total_processed" json:"total_processed"`
TotalCreated int `bson:"total_created" json:"total_created"`
TotalUpdated int `bson:"total_updated" json:"total_updated"`
TotalSkipped int `bson:"total_skipped" json:"total_skipped"`
TotalErrors int `bson:"total_errors" json:"total_errors"`
ProcessingErrors []string `bson:"processing_errors,omitempty" json:"processing_errors,omitempty"`
Summary string `bson:"summary,omitempty" json:"summary,omitempty"`
}
// ScrapingJob represents a scraping job entity
type ScrapingJob struct {
ID string `bson:"_id,omitempty" json:"id"`
Type ScrapingJobType `bson:"type" json:"type"`
Status ScrapingJobStatus `bson:"status" json:"status"`
StartedAt int64 `bson:"started_at" json:"started_at"` // Unix seconds
CompletedAt *int64 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` // Unix seconds
Duration *int64 `bson:"duration,omitempty" json:"duration,omitempty"` // Duration in seconds
Progress JobProgress `bson:"progress" json:"progress"`
Results JobResults `bson:"results" json:"results"`
Errors []string `bson:"errors,omitempty" json:"errors,omitempty"`
Config map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"`
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
}
// ScrapingState represents the state of the scraping process
type ScrapingState struct {
ID string `bson:"_id,omitempty" json:"id"`
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"` // Unix seconds
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"` // Unix seconds
IsRunning bool `bson:"is_running" json:"is_running"`
CurrentJobID string `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
}
// GetJobID returns the scraping job ID
func (j *ScrapingJob) GetJobID() string {
return j.ID
}
// IsActive returns true if the job is actively running
func (j *ScrapingJob) IsActive() bool {
return j.Status == ScrapingJobStatusRunning || j.Status == ScrapingJobStatusPending
}
// IsCompleted returns true if the job has completed (successfully or with failure)
func (j *ScrapingJob) IsCompleted() bool {
return j.Status == ScrapingJobStatusCompleted || j.Status == ScrapingJobStatusFailed || j.Status == ScrapingJobStatusCancelled
}
// GetProgressPercentage calculates and returns the progress percentage
func (j *ScrapingJob) GetProgressPercentage() float64 {
if j.Progress.TotalCount == 0 {
return 0
}
return float64(j.Progress.ProcessedCount) / float64(j.Progress.TotalCount) * 100
}
// UpdateProgress updates the job progress
func (j *ScrapingJob) UpdateProgress(processed, total, success, errors int, step string) {
j.Progress.ProcessedCount = processed
j.Progress.TotalCount = total
j.Progress.SuccessCount = success
j.Progress.ErrorCount = errors
j.Progress.CurrentStep = step
j.Progress.Percentage = j.GetProgressPercentage()
j.UpdatedAt = time.Now().Unix()
}
// AddError adds an error to the job
func (j *ScrapingJob) AddError(error string) {
if j.Errors == nil {
j.Errors = make([]string, 0)
}
j.Errors = append(j.Errors, error)
j.UpdatedAt = time.Now().Unix()
}
// MarkCompleted marks the job as completed
func (j *ScrapingJob) MarkCompleted(status ScrapingJobStatus, results JobResults) {
j.Status = status
j.Results = results
completedAt := time.Now().Unix()
j.CompletedAt = &completedAt
if j.StartedAt > 0 {
duration := completedAt - j.StartedAt
j.Duration = &duration
}
j.UpdatedAt = completedAt
}
// IsStateRunning returns true if the scraping state indicates a job is running
func (s *ScrapingState) IsStateRunning() bool {
return s.IsRunning
}
// MarkAsRunning marks the state as running with a job ID
func (s *ScrapingState) MarkAsRunning(jobID string) {
s.IsRunning = true
s.CurrentJobID = jobID
s.LastRunAt = time.Now().Unix()
s.UpdatedAt = time.Now().Unix()
}
// MarkAsCompleted marks the state as completed
func (s *ScrapingState) MarkAsCompleted(year, number int) {
s.IsRunning = false
s.CurrentJobID = ""
s.LastProcessedYear = year
s.LastProcessedNumber = number
s.NextRunAt = time.Now().Add(4 * time.Hour).Unix() // Schedule next run in 4 hours
s.UpdatedAt = time.Now().Unix()
}
// CalculateMatchPercentage calculates the match percentage between tender classifications and company CPV codes
func (t *Tender) CalculateMatchPercentage(companyCPVCodes []string) int {
if len(companyCPVCodes) == 0 {
return 0
}
mainMatch := false
additionalMatch := false
// Check if main classification matches any company CPV codes
if t.MainClassification != "" {
for _, companyCode := range companyCPVCodes {
if companyCode == t.MainClassification {
mainMatch = true
break
}
}
}
// Check if any additional classification matches any company CPV codes
if len(t.AdditionalClassifications) > 0 {
for _, additionalCode := range t.AdditionalClassifications {
for _, companyCode := range companyCPVCodes {
if companyCode == additionalCode {
additionalMatch = true
break
}
}
if additionalMatch {
break
}
}
}
// Calculate percentage based on matches
if mainMatch && additionalMatch {
return 100
} else if mainMatch || additionalMatch {
return 50
}
return 0
}
// GetDaysUntilDeadline returns the number of days until the tender deadline
func (t *Tender) GetDaysUntilDeadline() int {
if t.TenderDeadline == 0 {