m.nazemi
54 lines
1.3 KiB
Go
54 lines
1.3 KiB
Go
package queue
|
|
|
|
import (
|
|
"time"
|
|
)
|
|
|
|
/*
|
|
Queue Configuration and Constants
|
|
|
|
This package provides a Redis-based job queue system for managing
|
|
document scraping tasks in a distributed manner.
|
|
*/
|
|
|
|
const (
|
|
// Queue names
|
|
DocumentScraperQueue = "scraper:document:queue"
|
|
DocumentScraperDLQ = "scraper:document:dlq" // Dead Letter Queue
|
|
|
|
// Job status keys
|
|
JobStatusPrefix = "job:"
|
|
QueueMetricsKey = "queue:metrics"
|
|
|
|
// Default timeouts
|
|
DefaultJobTimeout = 5 * time.Minute
|
|
DefaultMaxRetries = 3
|
|
DefaultRetryBackoff = 10 * time.Second
|
|
MaxRetryBackoff = 5 * time.Minute // Cap retry backoff at 5 minutes
|
|
InProgressExpiration = 10 * time.Minute // Job must complete within this time
|
|
)
|
|
|
|
// QueueConfig represents configuration for the queue service
|
|
type QueueConfig struct {
|
|
RedisURL string
|
|
MaxRetries int
|
|
RetryBackoff time.Duration
|
|
JobTimeout time.Duration
|
|
PoolSize int
|
|
IsDLQEnabled bool
|
|
MaxQueueLength int // 0 means unlimited
|
|
}
|
|
|
|
// DefaultConfig returns default queue configuration
|
|
func DefaultConfig() QueueConfig {
|
|
return QueueConfig{
|
|
RedisURL: "redis://localhost:6379",
|
|
MaxRetries: DefaultMaxRetries,
|
|
RetryBackoff: DefaultRetryBackoff,
|
|
JobTimeout: DefaultJobTimeout,
|
|
PoolSize: 10,
|
|
IsDLQEnabled: true,
|
|
MaxQueueLength: 0, // Unlimited by default
|
|
}
|
|
}
|