package scraping import "time" // ScrapingJobType represents the type of scraping job type ScrapingJobType string const ( ScrapingJobTypeTEDScraper ScrapingJobType = "ted_scraper" ScrapingJobTypeManual ScrapingJobType = "manual" ScrapingJobTypeAPI ScrapingJobType = "api" ScrapingJobTypeBulkImport ScrapingJobType = "bulk_import" ) // ScrapingJobStatus represents the status of a scraping job type ScrapingJobStatus string const ( ScrapingJobStatusPending ScrapingJobStatus = "pending" ScrapingJobStatusRunning ScrapingJobStatus = "running" ScrapingJobStatusCompleted ScrapingJobStatus = "completed" ScrapingJobStatusFailed ScrapingJobStatus = "failed" ScrapingJobStatusCancelled ScrapingJobStatus = "cancelled" ) // JobProgress represents the progress of a scraping job type JobProgress struct { ProcessedCount int `bson:"processed_count" json:"processed_count"` TotalCount int `bson:"total_count" json:"total_count"` SuccessCount int `bson:"success_count" json:"success_count"` ErrorCount int `bson:"error_count" json:"error_count"` Percentage float64 `bson:"percentage" json:"percentage"` CurrentStep string `bson:"current_step" json:"current_step"` } // JobResults represents the results of a scraping job type JobResults struct { TotalProcessed int `bson:"total_processed" json:"total_processed"` TotalCreated int `bson:"total_created" json:"total_created"` TotalUpdated int `bson:"total_updated" json:"total_updated"` TotalSkipped int `bson:"total_skipped" json:"total_skipped"` TotalErrors int `bson:"total_errors" json:"total_errors"` ProcessingErrors []string `bson:"processing_errors,omitempty" json:"processing_errors,omitempty"` Summary string `bson:"summary,omitempty" json:"summary,omitempty"` } // ScrapingJob represents a scraping job entity type ScrapingJob struct { ID string `bson:"_id,omitempty" json:"id"` Type ScrapingJobType `bson:"type" json:"type"` Status ScrapingJobStatus `bson:"status" json:"status"` StartedAt int64 `bson:"started_at" json:"started_at"` // Unix seconds CompletedAt *int64 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` // Unix seconds Duration *int64 `bson:"duration,omitempty" json:"duration,omitempty"` // Duration in seconds Progress JobProgress `bson:"progress" json:"progress"` Results JobResults `bson:"results" json:"results"` Errors []string `bson:"errors,omitempty" json:"errors,omitempty"` Config map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"` CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds } // ScrapingState represents the state of the scraping process type ScrapingState struct { ID string `bson:"_id,omitempty" json:"id"` LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"` LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"` LastRunAt int64 `bson:"last_run_at" json:"last_run_at"` // Unix seconds NextRunAt int64 `bson:"next_run_at" json:"next_run_at"` // Unix seconds IsRunning bool `bson:"is_running" json:"is_running"` CurrentJobID string `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"` CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds } // GetJobID returns the scraping job ID func (j *ScrapingJob) GetJobID() string { return j.ID } // IsActive returns true if the job is actively running func (j *ScrapingJob) IsActive() bool { return j.Status == ScrapingJobStatusRunning || j.Status == ScrapingJobStatusPending } // IsCompleted returns true if the job has completed (successfully or with failure) func (j *ScrapingJob) IsCompleted() bool { return j.Status == ScrapingJobStatusCompleted || j.Status == ScrapingJobStatusFailed || j.Status == ScrapingJobStatusCancelled } // GetProgressPercentage calculates and returns the progress percentage func (j *ScrapingJob) GetProgressPercentage() float64 { if j.Progress.TotalCount == 0 { return 0 } return float64(j.Progress.ProcessedCount) / float64(j.Progress.TotalCount) * 100 } // UpdateProgress updates the job progress func (j *ScrapingJob) UpdateProgress(processed, total, success, errors int, step string) { j.Progress.ProcessedCount = processed j.Progress.TotalCount = total j.Progress.SuccessCount = success j.Progress.ErrorCount = errors j.Progress.CurrentStep = step j.Progress.Percentage = j.GetProgressPercentage() j.UpdatedAt = time.Now().Unix() } // AddError adds an error to the job func (j *ScrapingJob) AddError(error string) { if j.Errors == nil { j.Errors = make([]string, 0) } j.Errors = append(j.Errors, error) j.UpdatedAt = time.Now().Unix() } // MarkCompleted marks the job as completed func (j *ScrapingJob) MarkCompleted(status ScrapingJobStatus, results JobResults) { j.Status = status j.Results = results completedAt := time.Now().Unix() j.CompletedAt = &completedAt if j.StartedAt > 0 { duration := completedAt - j.StartedAt j.Duration = &duration } j.UpdatedAt = completedAt } // IsStateRunning returns true if the scraping state indicates a job is running func (s *ScrapingState) IsStateRunning() bool { return s.IsRunning } // MarkAsRunning marks the state as running with a job ID func (s *ScrapingState) MarkAsRunning(jobID string) { s.IsRunning = true s.CurrentJobID = jobID s.LastRunAt = time.Now().Unix() s.UpdatedAt = time.Now().Unix() } // MarkAsCompleted marks the state as completed func (s *ScrapingState) MarkAsCompleted(year, number int) { s.IsRunning = false s.CurrentJobID = "" s.LastProcessedYear = year s.LastProcessedNumber = number s.NextRunAt = time.Now().Add(4 * time.Hour).Unix() // Schedule next run in 4 hours s.UpdatedAt = time.Now().Unix() }