Add TED Scraper Configuration and Cron Scheduler Implementation
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
This commit is contained in:
n.nakhostin
@@ -1,153 +0,0 @@
|
||||
package scraping
|
||||
|
||||
import "time"
|
||||
|
||||
// ScrapingJobType represents the type of scraping job
|
||||
type ScrapingJobType string
|
||||
|
||||
const (
|
||||
ScrapingJobTypeTEDScraper ScrapingJobType = "ted_scraper"
|
||||
ScrapingJobTypeManual ScrapingJobType = "manual"
|
||||
ScrapingJobTypeAPI ScrapingJobType = "api"
|
||||
ScrapingJobTypeBulkImport ScrapingJobType = "bulk_import"
|
||||
)
|
||||
|
||||
// ScrapingJobStatus represents the status of a scraping job
|
||||
type ScrapingJobStatus string
|
||||
|
||||
const (
|
||||
ScrapingJobStatusPending ScrapingJobStatus = "pending"
|
||||
ScrapingJobStatusRunning ScrapingJobStatus = "running"
|
||||
ScrapingJobStatusCompleted ScrapingJobStatus = "completed"
|
||||
ScrapingJobStatusFailed ScrapingJobStatus = "failed"
|
||||
ScrapingJobStatusCancelled ScrapingJobStatus = "cancelled"
|
||||
)
|
||||
|
||||
// JobProgress represents the progress of a scraping job
|
||||
type JobProgress struct {
|
||||
ProcessedCount int `bson:"processed_count" json:"processed_count"`
|
||||
TotalCount int `bson:"total_count" json:"total_count"`
|
||||
SuccessCount int `bson:"success_count" json:"success_count"`
|
||||
ErrorCount int `bson:"error_count" json:"error_count"`
|
||||
Percentage float64 `bson:"percentage" json:"percentage"`
|
||||
CurrentStep string `bson:"current_step" json:"current_step"`
|
||||
}
|
||||
|
||||
// JobResults represents the results of a scraping job
|
||||
type JobResults struct {
|
||||
TotalProcessed int `bson:"total_processed" json:"total_processed"`
|
||||
TotalCreated int `bson:"total_created" json:"total_created"`
|
||||
TotalUpdated int `bson:"total_updated" json:"total_updated"`
|
||||
TotalSkipped int `bson:"total_skipped" json:"total_skipped"`
|
||||
TotalErrors int `bson:"total_errors" json:"total_errors"`
|
||||
ProcessingErrors []string `bson:"processing_errors,omitempty" json:"processing_errors,omitempty"`
|
||||
Summary string `bson:"summary,omitempty" json:"summary,omitempty"`
|
||||
}
|
||||
|
||||
// ScrapingJob represents a scraping job entity
|
||||
type ScrapingJob struct {
|
||||
ID string `bson:"_id,omitempty" json:"id"`
|
||||
Type ScrapingJobType `bson:"type" json:"type"`
|
||||
Status ScrapingJobStatus `bson:"status" json:"status"`
|
||||
StartedAt int64 `bson:"started_at" json:"started_at"` // Unix seconds
|
||||
CompletedAt *int64 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` // Unix seconds
|
||||
Duration *int64 `bson:"duration,omitempty" json:"duration,omitempty"` // Duration in seconds
|
||||
Progress JobProgress `bson:"progress" json:"progress"`
|
||||
Results JobResults `bson:"results" json:"results"`
|
||||
Errors []string `bson:"errors,omitempty" json:"errors,omitempty"`
|
||||
Config map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"`
|
||||
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
|
||||
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
|
||||
}
|
||||
|
||||
// ScrapingState represents the state of the scraping process
|
||||
type ScrapingState struct {
|
||||
ID string `bson:"_id,omitempty" json:"id"`
|
||||
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
|
||||
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
|
||||
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"` // Unix seconds
|
||||
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"` // Unix seconds
|
||||
IsRunning bool `bson:"is_running" json:"is_running"`
|
||||
CurrentJobID string `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
|
||||
CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds
|
||||
UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds
|
||||
}
|
||||
|
||||
// GetJobID returns the scraping job ID
|
||||
func (j *ScrapingJob) GetJobID() string {
|
||||
return j.ID
|
||||
}
|
||||
|
||||
// IsActive returns true if the job is actively running
|
||||
func (j *ScrapingJob) IsActive() bool {
|
||||
return j.Status == ScrapingJobStatusRunning || j.Status == ScrapingJobStatusPending
|
||||
}
|
||||
|
||||
// IsCompleted returns true if the job has completed (successfully or with failure)
|
||||
func (j *ScrapingJob) IsCompleted() bool {
|
||||
return j.Status == ScrapingJobStatusCompleted || j.Status == ScrapingJobStatusFailed || j.Status == ScrapingJobStatusCancelled
|
||||
}
|
||||
|
||||
// GetProgressPercentage calculates and returns the progress percentage
|
||||
func (j *ScrapingJob) GetProgressPercentage() float64 {
|
||||
if j.Progress.TotalCount == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(j.Progress.ProcessedCount) / float64(j.Progress.TotalCount) * 100
|
||||
}
|
||||
|
||||
// UpdateProgress updates the job progress
|
||||
func (j *ScrapingJob) UpdateProgress(processed, total, success, errors int, step string) {
|
||||
j.Progress.ProcessedCount = processed
|
||||
j.Progress.TotalCount = total
|
||||
j.Progress.SuccessCount = success
|
||||
j.Progress.ErrorCount = errors
|
||||
j.Progress.CurrentStep = step
|
||||
j.Progress.Percentage = j.GetProgressPercentage()
|
||||
j.UpdatedAt = time.Now().Unix()
|
||||
}
|
||||
|
||||
// AddError adds an error to the job
|
||||
func (j *ScrapingJob) AddError(error string) {
|
||||
if j.Errors == nil {
|
||||
j.Errors = make([]string, 0)
|
||||
}
|
||||
j.Errors = append(j.Errors, error)
|
||||
j.UpdatedAt = time.Now().Unix()
|
||||
}
|
||||
|
||||
// MarkCompleted marks the job as completed
|
||||
func (j *ScrapingJob) MarkCompleted(status ScrapingJobStatus, results JobResults) {
|
||||
j.Status = status
|
||||
j.Results = results
|
||||
completedAt := time.Now().Unix()
|
||||
j.CompletedAt = &completedAt
|
||||
if j.StartedAt > 0 {
|
||||
duration := completedAt - j.StartedAt
|
||||
j.Duration = &duration
|
||||
}
|
||||
j.UpdatedAt = completedAt
|
||||
}
|
||||
|
||||
// IsStateRunning returns true if the scraping state indicates a job is running
|
||||
func (s *ScrapingState) IsStateRunning() bool {
|
||||
return s.IsRunning
|
||||
}
|
||||
|
||||
// MarkAsRunning marks the state as running with a job ID
|
||||
func (s *ScrapingState) MarkAsRunning(jobID string) {
|
||||
s.IsRunning = true
|
||||
s.CurrentJobID = jobID
|
||||
s.LastRunAt = time.Now().Unix()
|
||||
s.UpdatedAt = time.Now().Unix()
|
||||
}
|
||||
|
||||
// MarkAsCompleted marks the state as completed
|
||||
func (s *ScrapingState) MarkAsCompleted(year, number int) {
|
||||
s.IsRunning = false
|
||||
s.CurrentJobID = ""
|
||||
s.LastProcessedYear = year
|
||||
s.LastProcessedNumber = number
|
||||
s.NextRunAt = time.Now().Add(4 * time.Hour).Unix() // Schedule next run in 4 hours
|
||||
s.UpdatedAt = time.Now().Unix()
|
||||
}
|
||||
Reference in New Issue
Block a user