tm_back/internal/scraping/scraping.go

package scraping

import "time"

// ScrapingJobType represents the type of scraping job
type ScrapingJobType string

const (
	ScrapingJobTypeTEDScraper ScrapingJobType = "ted_scraper"
	ScrapingJobTypeManual     ScrapingJobType = "manual"
	ScrapingJobTypeAPI        ScrapingJobType = "api"
	ScrapingJobTypeBulkImport ScrapingJobType = "bulk_import"
)

// ScrapingJobStatus represents the status of a scraping job
type ScrapingJobStatus string

const (
	ScrapingJobStatusPending   ScrapingJobStatus = "pending"
	ScrapingJobStatusRunning   ScrapingJobStatus = "running"
	ScrapingJobStatusCompleted ScrapingJobStatus = "completed"
	ScrapingJobStatusFailed    ScrapingJobStatus = "failed"
	ScrapingJobStatusCancelled ScrapingJobStatus = "cancelled"
)

// JobProgress represents the progress of a scraping job
type JobProgress struct {
	ProcessedCount int     `bson:"processed_count" json:"processed_count"`
	TotalCount     int     `bson:"total_count" json:"total_count"`
	SuccessCount   int     `bson:"success_count" json:"success_count"`
	ErrorCount     int     `bson:"error_count" json:"error_count"`
	Percentage     float64 `bson:"percentage" json:"percentage"`
	CurrentStep    string  `bson:"current_step" json:"current_step"`
}

// JobResults represents the results of a scraping job
type JobResults struct {
	TotalProcessed   int      `bson:"total_processed" json:"total_processed"`
	TotalCreated     int      `bson:"total_created" json:"total_created"`
	TotalUpdated     int      `bson:"total_updated" json:"total_updated"`
	TotalSkipped     int      `bson:"total_skipped" json:"total_skipped"`
	TotalErrors      int      `bson:"total_errors" json:"total_errors"`
	ProcessingErrors []string `bson:"processing_errors,omitempty" json:"processing_errors,omitempty"`
	Summary          string   `bson:"summary,omitempty" json:"summary,omitempty"`
}

// ScrapingJob represents a scraping job entity
type ScrapingJob struct {
	ID          string                 `bson:"_id,omitempty" json:"id"`
	Type        ScrapingJobType        `bson:"type" json:"type"`
	Status      ScrapingJobStatus      `bson:"status" json:"status"`
	StartedAt   int64                  `bson:"started_at" json:"started_at"`                         // Unix seconds
	CompletedAt *int64                 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` // Unix seconds
	Duration    *int64                 `bson:"duration,omitempty" json:"duration,omitempty"`         // Duration in seconds
	Progress    JobProgress            `bson:"progress" json:"progress"`
	Results     JobResults             `bson:"results" json:"results"`
	Errors      []string               `bson:"errors,omitempty" json:"errors,omitempty"`
	Config      map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"`
	CreatedAt   int64                  `bson:"created_at" json:"created_at"` // Unix seconds
	UpdatedAt   int64                  `bson:"updated_at" json:"updated_at"` // Unix seconds
}

// ScrapingState represents the state of the scraping process
type ScrapingState struct {
	ID                  string `bson:"_id,omitempty" json:"id"`
	LastProcessedYear   int    `bson:"last_processed_year" json:"last_processed_year"`
	LastProcessedNumber int    `bson:"last_processed_number" json:"last_processed_number"`
	LastRunAt           int64  `bson:"last_run_at" json:"last_run_at"` // Unix seconds
	NextRunAt           int64  `bson:"next_run_at" json:"next_run_at"` // Unix seconds
	IsRunning           bool   `bson:"is_running" json:"is_running"`
	CurrentJobID        string `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
	CreatedAt           int64  `bson:"created_at" json:"created_at"` // Unix seconds
	UpdatedAt           int64  `bson:"updated_at" json:"updated_at"` // Unix seconds
}

// GetJobID returns the scraping job ID
func (j *ScrapingJob) GetJobID() string {
	return j.ID
}

// IsActive returns true if the job is actively running
func (j *ScrapingJob) IsActive() bool {
	return j.Status == ScrapingJobStatusRunning || j.Status == ScrapingJobStatusPending
}

// IsCompleted returns true if the job has completed (successfully or with failure)
func (j *ScrapingJob) IsCompleted() bool {
	return j.Status == ScrapingJobStatusCompleted || j.Status == ScrapingJobStatusFailed || j.Status == ScrapingJobStatusCancelled
}

// GetProgressPercentage calculates and returns the progress percentage
func (j *ScrapingJob) GetProgressPercentage() float64 {
	if j.Progress.TotalCount == 0 {
		return 0
	}
	return float64(j.Progress.ProcessedCount) / float64(j.Progress.TotalCount) * 100
}

// UpdateProgress updates the job progress
func (j *ScrapingJob) UpdateProgress(processed, total, success, errors int, step string) {
	j.Progress.ProcessedCount = processed
	j.Progress.TotalCount = total
	j.Progress.SuccessCount = success
	j.Progress.ErrorCount = errors
	j.Progress.CurrentStep = step
	j.Progress.Percentage = j.GetProgressPercentage()
	j.UpdatedAt = time.Now().Unix()
}

// AddError adds an error to the job
func (j *ScrapingJob) AddError(error string) {
	if j.Errors == nil {
		j.Errors = make([]string, 0)
	}
	j.Errors = append(j.Errors, error)
	j.UpdatedAt = time.Now().Unix()
}

// MarkCompleted marks the job as completed
func (j *ScrapingJob) MarkCompleted(status ScrapingJobStatus, results JobResults) {
	j.Status = status
	j.Results = results
	completedAt := time.Now().Unix()
	j.CompletedAt = &completedAt
	if j.StartedAt > 0 {
		duration := completedAt - j.StartedAt
		j.Duration = &duration
	}
	j.UpdatedAt = completedAt
}

// IsStateRunning returns true if the scraping state indicates a job is running
func (s *ScrapingState) IsStateRunning() bool {
	return s.IsRunning
}

// MarkAsRunning marks the state as running with a job ID
func (s *ScrapingState) MarkAsRunning(jobID string) {
	s.IsRunning = true
	s.CurrentJobID = jobID
	s.LastRunAt = time.Now().Unix()
	s.UpdatedAt = time.Now().Unix()
}

// MarkAsCompleted marks the state as completed
func (s *ScrapingState) MarkAsCompleted(year, number int) {
	s.IsRunning = false
	s.CurrentJobID = ""
	s.LastProcessedYear = year
	s.LastProcessedNumber = number
	s.NextRunAt = time.Now().Add(4 * time.Hour).Unix() // Schedule next run in 4 hours
	s.UpdatedAt = time.Now().Unix()
}