package queue import ( "time" ) /* Queue Configuration and Constants This package provides a Redis-based job queue system for managing document scraping tasks in a distributed manner. */ const ( // Queue names DocumentScraperQueue = "scraper:document:queue" DocumentScraperDLQ = "scraper:document:dlq" // Dead Letter Queue // Job status keys JobStatusPrefix = "job:" QueueMetricsKey = "queue:metrics" // Default timeouts DefaultJobTimeout = 5 * time.Minute DefaultMaxRetries = 3 DefaultRetryBackoff = 10 * time.Second MaxRetryBackoff = 5 * time.Minute // Cap retry backoff at 5 minutes InProgressExpiration = 10 * time.Minute // Job must complete within this time ) // QueueConfig represents configuration for the queue service type QueueConfig struct { RedisURL string MaxRetries int RetryBackoff time.Duration JobTimeout time.Duration PoolSize int IsDLQEnabled bool MaxQueueLength int // 0 means unlimited } // DefaultConfig returns default queue configuration func DefaultConfig() QueueConfig { return QueueConfig{ RedisURL: "redis://localhost:6379", MaxRetries: DefaultMaxRetries, RetryBackoff: DefaultRetryBackoff, JobTimeout: DefaultJobTimeout, PoolSize: 10, IsDLQEnabled: true, MaxQueueLength: 0, // Unlimited by default } }