agent SDK fix

cmd/worker/bootstrap/bootstrap.go

@@ -13,6 +13,7 @@ import (
 	"tm/pkg/mongo"
 	"tm/pkg/notification"
 	"tm/pkg/ollama"
+	"tm/pkg/queue"
 	"tm/pkg/schedule"
 	"tm/pkg/scraper"
 )
@@ -81,6 +82,8 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 	// Debug: Log worker config
 	appLogger.Info("Worker configuration", map[string]interface{}{
 		"worker_interval": config.Worker.Interval,
+		"queue_enabled":   config.Queue.Enabled,
+		"queue_mode":      config.Queue.Mode,
 	})
 	// Initialize repositories
 	noticeRepo := notice.NewRepository(mongoManager, appLogger)
@@ -91,11 +94,87 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 	noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, scraperService)
 	noticeWorker.Run()
 
-	// Initialize Document Scraper Worker
-	// appLogger.Info("Initializing document scraper worker", map[string]interface{}{})
-	// scraperWorker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
-	// appLogger.Info("Starting document scraper worker", map[string]interface{}{})
-	// scraperWorker.Run()
+	// Initialize Queue-based scraping if enabled
+	var queueService queue.Queue
+	if config.Queue.Enabled {
+		var err error
+		queueConfig := queue.QueueConfig{
+			RedisURL:       config.Queue.RedisURL,
+			MaxRetries:     config.Queue.MaxRetries,
+			RetryBackoff:   config.Queue.RetryBackoff,
+			JobTimeout:     config.Queue.JobTimeout,
+			PoolSize:       config.Queue.PoolSize,
+			IsDLQEnabled:   config.Queue.IsDLQEnabled,
+			MaxQueueLength: 0, // Unlimited
+		}
+
+		queueService, err = queue.NewRedisQueue(queueConfig, appLogger)
+		if err != nil {
+			appLogger.Error("Failed to initialize queue service", map[string]interface{}{
+				"error": err.Error(),
+			})
+			appLogger.Warn("Queue-based scraping is disabled, falling back to scheduled scraping", map[string]interface{}{})
+			config.Queue.Enabled = false
+		} else {
+			appLogger.Info("Queue service initialized successfully", map[string]interface{}{
+				"mode": config.Queue.Mode,
+			})
+		}
+	}
+
+	// Start Queue Consumer Worker (24/7 continuous processing)
+	if config.Queue.Enabled && (config.Queue.Mode == "consumer" || config.Queue.Mode == "both") {
+		appLogger.Info("Starting queue consumer worker", map[string]interface{}{
+			"concurrency": config.Queue.Concurrency,
+		})
+		consumerWorker := workers.NewDocumentScraperConsumerWorker(
+			mongoManager,
+			appLogger,
+			tenderRepo,
+			queueService,
+			scraperService,
+			config.Queue.Concurrency,
+		)
+		go consumerWorker.Run()
+	} else if !config.Queue.Enabled {
+		appLogger.Info("Queue consumer worker disabled, using scheduled scraping instead", map[string]interface{}{})
+	}
+
+	// Start Queue Producer Worker (enqueues unscraped tenders periodically)
+	if config.Queue.Enabled && (config.Queue.Mode == "producer" || config.Queue.Mode == "both") {
+		appLogger.Info("Scheduling queue producer worker", map[string]interface{}{
+			"batch_size": config.Queue.ProducerBatch,
+			"interval":   config.Queue.ProducerInterval,
+		})
+
+		schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
+			Name: "Queue Producer Worker Job",
+			Func: func() {
+				producer := workers.NewQueueProducerWorker(
+					mongoManager,
+					appLogger,
+					tenderRepo,
+					queueService,
+					config.Queue.ProducerBatch,
+				)
+				producer.Run()
+			},
+			Expr: config.Queue.ProducerInterval,
+		})
+	}
+
+	// Fallback: Keep the old scheduled scraper for backward compatibility
+	if !config.Queue.Enabled {
+		appLogger.Info("Scheduling fallback document scraper worker", map[string]interface{}{})
+		schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
+			Name: "Document Scraper Worker Job (Fallback)",
+			Func: func() {
+				worker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
+				worker.Run()
+			},
+			Expr: "0 10 * * * *", // Run at 10 AM every day
+		})
+	}
 
 	// Initialize Document Summarization Worker (only if MinIO is available)
 	if minioService != nil {
@@ -133,17 +212,6 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 		},
 		Expr: workerInterval,
 	})
-
-	// Schedule Document Scraper Worker job (runs after notice worker)
-	schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
-		Name: "Document Scraper Worker Job",
-		Func: func() {
-			worker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
-			worker.Run()
-		},
-		Expr: "0 10 * * * *", // Run at 10 AM every day (1 hour after notice worker)
-	})
-
 }
 
 // Init Notification Service

cmd/worker/bootstrap/config.go

@@ -15,7 +15,23 @@ type Config struct {
 	GLM          GLMConfig
 	Scraper      config.ScraperConfig
 	MinIO        config.MinIOConfig
-	AlertMail    string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
+	Queue        QueueConfig `env:"QUEUE_" envPrefix:"QUEUE_"`
+	AlertMail    string      `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
+}
+
+// QueueConfig defines the queue configuration
+type QueueConfig struct {
+	Enabled          bool          `env:"QUEUE_ENABLED" envDefault:"true"`
+	RedisURL         string        `env:"QUEUE_REDIS_URL" envDefault:"redis://localhost:6379"`
+	Mode             string        `env:"QUEUE_MODE" envDefault:"consumer"` // "producer", "consumer", or "both"
+	MaxRetries       int           `env:"QUEUE_MAX_RETRIES" envDefault:"3"`
+	RetryBackoff     time.Duration `env:"QUEUE_RETRY_BACKOFF" envDefault:"10s"`
+	JobTimeout       time.Duration `env:"QUEUE_JOB_TIMEOUT" envDefault:"5m"`
+	PoolSize         int           `env:"QUEUE_POOL_SIZE" envDefault:"10"`
+	IsDLQEnabled     bool          `env:"QUEUE_DLQ_ENABLED" envDefault:"true"`
+	Concurrency      int           `env:"QUEUE_CONCURRENCY" envDefault:"5"`
+	ProducerBatch    int           `env:"QUEUE_PRODUCER_BATCH" envDefault:"100"`
+	ProducerInterval string        `env:"QUEUE_PRODUCER_INTERVAL" envDefault:"0 */6 * * * *"` // Every 6 hours
 }
 
 type WorkerConfig struct {