agent SDK fix

2026-04-12 12:29:35 +03:30
parent fd89371bdf
commit bf3f21fb51
8 changed files with 1186 additions and 20 deletions
@@ -13,6 +13,7 @@ import (
 	"tm/pkg/mongo"
 	"tm/pkg/notification"
 	"tm/pkg/ollama"
+	"tm/pkg/queue"
 	"tm/pkg/schedule"
 	"tm/pkg/scraper"
 )
@@ -81,6 +82,8 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 	// Debug: Log worker config
 	appLogger.Info("Worker configuration", map[string]interface{}{
 		"worker_interval": config.Worker.Interval,
+		"queue_enabled":   config.Queue.Enabled,
+		"queue_mode":      config.Queue.Mode,
 	})
 	// Initialize repositories
 	noticeRepo := notice.NewRepository(mongoManager, appLogger)
@@ -91,11 +94,87 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 	noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, scraperService)
 	noticeWorker.Run()

-	// Initialize Document Scraper Worker
-	// appLogger.Info("Initializing document scraper worker", map[string]interface{}{})
-	// scraperWorker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
-	// appLogger.Info("Starting document scraper worker", map[string]interface{}{})
-	// scraperWorker.Run()
+	// Initialize Queue-based scraping if enabled
+	var queueService queue.Queue
+	if config.Queue.Enabled {
+		var err error
+		queueConfig := queue.QueueConfig{
+			RedisURL:       config.Queue.RedisURL,
+			MaxRetries:     config.Queue.MaxRetries,
+			RetryBackoff:   config.Queue.RetryBackoff,
+			JobTimeout:     config.Queue.JobTimeout,
+			PoolSize:       config.Queue.PoolSize,
+			IsDLQEnabled:   config.Queue.IsDLQEnabled,
+			MaxQueueLength: 0, // Unlimited
+		}
+
+		queueService, err = queue.NewRedisQueue(queueConfig, appLogger)
+		if err != nil {
+			appLogger.Error("Failed to initialize queue service", map[string]interface{}{
+				"error": err.Error(),
+			})
+			appLogger.Warn("Queue-based scraping is disabled, falling back to scheduled scraping", map[string]interface{}{})
+			config.Queue.Enabled = false
+		} else {
+			appLogger.Info("Queue service initialized successfully", map[string]interface{}{
+				"mode": config.Queue.Mode,
+			})
+		}
+	}
+
+	// Start Queue Consumer Worker (24/7 continuous processing)
+	if config.Queue.Enabled && (config.Queue.Mode == "consumer" || config.Queue.Mode == "both") {
+		appLogger.Info("Starting queue consumer worker", map[string]interface{}{
+			"concurrency": config.Queue.Concurrency,
+		})
+		consumerWorker := workers.NewDocumentScraperConsumerWorker(
+			mongoManager,
+			appLogger,
+			tenderRepo,
+			queueService,
+			scraperService,
+			config.Queue.Concurrency,
+		)
+		go consumerWorker.Run()
+	} else if !config.Queue.Enabled {
+		appLogger.Info("Queue consumer worker disabled, using scheduled scraping instead", map[string]interface{}{})
+	}
+
+	// Start Queue Producer Worker (enqueues unscraped tenders periodically)
+	if config.Queue.Enabled && (config.Queue.Mode == "producer" || config.Queue.Mode == "both") {
+		appLogger.Info("Scheduling queue producer worker", map[string]interface{}{
+			"batch_size": config.Queue.ProducerBatch,
+			"interval":   config.Queue.ProducerInterval,
+		})
+
+		schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
+			Name: "Queue Producer Worker Job",
+			Func: func() {
+				producer := workers.NewQueueProducerWorker(
+					mongoManager,
+					appLogger,
+					tenderRepo,
+					queueService,
+					config.Queue.ProducerBatch,
+				)
+				producer.Run()
+			},
+			Expr: config.Queue.ProducerInterval,
+		})
+	}
+
+	// Fallback: Keep the old scheduled scraper for backward compatibility
+	if !config.Queue.Enabled {
+		appLogger.Info("Scheduling fallback document scraper worker", map[string]interface{}{})
+		schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
+			Name: "Document Scraper Worker Job (Fallback)",
+			Func: func() {
+				worker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
+				worker.Run()
+			},
+			Expr: "0 10 * * * *", // Run at 10 AM every day
+		})
+	}

 	// Initialize Document Summarization Worker (only if MinIO is available)
 	if minioService != nil {
@@ -133,17 +212,6 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 		},
 		Expr: workerInterval,
 	})
-
-	// Schedule Document Scraper Worker job (runs after notice worker)
-	schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
-		Name: "Document Scraper Worker Job",
-		Func: func() {
-			worker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
-			worker.Run()
-		},
-		Expr: "0 10 * * * *", // Run at 10 AM every day (1 hour after notice worker)
-	})
-
 }

 // Init Notification Service