Enhance company recommendation caching and pipeline integration

- Updated the company service to include a new method for scheduling the refresh of cached AI recommendations after the AI pipeline execution.
- Introduced a new interface for managing cached recommendation refreshes, improving the separation of concerns within the service layer.
- Enhanced the worker initialization to include Redis client support, allowing for better management of recommendation caching.
- Added functionality to list company IDs with existing recommendation caches, ensuring efficient updates post-pipeline runs.
- Implemented unit tests to validate the new recommendation refresh logic and ensure proper handling of various scenarios.

This update significantly improves the handling of AI recommendations by integrating caching mechanisms with the AI pipeline, enhancing overall system performance and responsiveness.

cmd/web/main.go

@@ -240,7 +240,7 @@ func main() {
 
 	userService := user.NewService(userRepository, logger, userAuthService, userValidator, notificationSDK, redisClient, auditLogger)
 	categoryService := company_category.NewService(categoryRepository, logger)
-	companyService := company.NewService(companyRepository, categoryService, fileStoreService, aiRecommendationClient, redisClient, conf.AISummarizer.RecommendationCacheTTL, logger)
+	companyService := company.NewService(companyRepository, categoryService, fileStoreService, aiRecommendationClient, aiPipelineClient, redisClient, conf.AISummarizer.RecommendationCacheTTL, logger)
 	customerService := customer.NewService(customerRepository, logger, customerAuthService, companyService, redisClient, notificationSDK, customerValidator, auditLogger)
 	tenderService := tender.NewService(tenderRepository, companyService, customerService, tenderApprovalRepository, logger, aiSummarizerClient, aiSummarizerStorage, conf.AISummarizer.DefaultLanguage)
 	feedbackService := feedback.NewService(feedbackRepo, tenderService, companyService, logger)
@@ -254,7 +254,7 @@ func main() {
 	documentScraperService := document_scraper.NewService(tenderRepository, aiPipelineClient, logger)
 	dashboardRepository := dashboard.NewRepository(mongoManager, logger, procedureDocumentsLister)
 	dashboardService := dashboard.NewService(dashboardRepository, logger, redisClient)
-	aiPipelineService := ai_pipeline.NewService(aiPipelineClient, logger, tenderService)
+	aiPipelineService := ai_pipeline.NewService(aiPipelineClient, logger, tenderService, companyService)
 	auditLogRepository := auditlog.NewRepository(auditStore)
 	auditLogService := auditlog.NewService(auditLogRepository, logger)
 	logger.Info("Services initialized successfully", map[string]interface{}{

cmd/worker/bootstrap/bootstrap.go

@@ -8,6 +8,8 @@ import (
 	"sync"
 	"time"
 	"tm/cmd/worker/workers"
+	"tm/internal/company"
+	"tm/internal/company_category"
 	"tm/internal/notice"
 	notificationDomain "tm/internal/notification"
 	"tm/internal/tender"
@@ -17,6 +19,7 @@ import (
 	"tm/pkg/mongo"
 	"tm/pkg/notification"
 	"tm/pkg/ollama"
+	"tm/pkg/redis"
 	"tm/pkg/schedule"
 )
 
@@ -82,8 +85,8 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
 	return connectionManager
 }
 
-// Init Worker
-func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient) *schedule.CronScheduler {
+// InitWorker
+func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient, redisClient redis.Client) *schedule.CronScheduler {
 	// Debug: Log worker config
 	appLogger.Info("Worker configuration", map[string]interface{}{
 		"worker_interval":            config.Worker.Interval,
@@ -103,6 +106,26 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 	tenderRepo := tender.NewRepository(mongoManager, appLogger)
 	notificationRepo := notificationDomain.NewRepository(mongoManager, appLogger)
 
+	var recommendationRefresher company.Service
+	if config.Worker.RecommendationRefreshAfterPipelineEnabled && aiClient != nil && redisClient != nil {
+		companyRepo := company.NewRepository(mongoManager, appLogger)
+		categoryRepo := company_category.NewRepository(mongoManager, appLogger)
+		categoryService := company_category.NewService(categoryRepo, appLogger)
+		recommendationRefresher = company.NewService(
+			companyRepo,
+			categoryService,
+			nil,
+			aiClient,
+			aiClient,
+			redisClient,
+			config.AISummarizer.RecommendationCacheTTL,
+			appLogger,
+		)
+		appLogger.Info("Company recommendation refresh after pipeline enabled", map[string]interface{}{})
+	} else if config.Worker.RecommendationRefreshAfterPipelineEnabled {
+		appLogger.Warn("Company recommendation refresh after pipeline disabled: AI client or Redis unavailable", map[string]interface{}{})
+	}
+
 	// Create a single shared cron scheduler for all recurring jobs
 	scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
 
@@ -275,6 +298,10 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 				"trigger": trigger,
 				"date":    today.Format("02/01/2006"),
 			})
+
+			if recommendationRefresher != nil {
+				recommendationRefresher.ScheduleRefreshCachedAIRecommendationsAfterPipeline()
+			}
 		}
 
 		scheduler.AddJob(schedule.Job{
@@ -387,6 +414,35 @@ func InitAISummarizerStorage(conf AISummarizerConfig, log logger.Logger) *ai_sum
 	return storage
 }
 
+// InitRedis initializes the Redis client used by worker jobs.
+func InitRedis(conf config.RedisConfig, log logger.Logger) redis.Client {
+	connectionConfig := &redis.Config{
+		Host:     conf.Host,
+		Port:     conf.Port,
+		Password: conf.Password,
+		DB:       conf.DB,
+		PoolSize: conf.PoolSize,
+	}
+
+	redisClient, err := redis.NewClient(connectionConfig, log)
+	if err != nil {
+		log.Error("Failed to initialize Redis connection for worker", map[string]interface{}{
+			"error": err.Error(),
+			"host":  conf.Host,
+			"port":  conf.Port,
+		})
+		panic(fmt.Sprintf("Failed to initialize Redis connection: %v", err))
+	}
+
+	log.Info("Redis connection initialized for worker", map[string]interface{}{
+		"host":      conf.Host,
+		"port":      conf.Port,
+		"pool_size": conf.PoolSize,
+	})
+
+	return redisClient
+}
+
 func parseTranslationLanguages(conf AISummarizerConfig) []string {
 	raw := strings.TrimSpace(conf.TranslationLanguages)
 	if raw == "" {

cmd/worker/bootstrap/config.go

@@ -8,6 +8,7 @@ import (
 // Config defines the worker application configuration
 type Config struct {
 	Database     config.DatabaseConfig
+	Redis        config.RedisConfig
 	Logging      config.LoggingConfig
 	Notification config.NotificationConfig
 	Ollama       config.OllamaConfig
@@ -15,7 +16,7 @@ type Config struct {
 	Scraper      config.ScraperConfig
 	MinIO        config.MinIOConfig
 	AISummarizer AISummarizerConfig
-	AlertMail    string      `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
+	AlertMail    string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
 }
 
 type WorkerConfig struct {
@@ -40,20 +41,23 @@ type WorkerConfig struct {
 	AIPipelineAutoEnabled bool `env:"WORKER_AI_PIPELINE_AUTO_ENABLED" envDefault:"true"`
 	// AIPipelineAutoInterval runs daily-run once per day by default at 11:00 UTC (after TED scrape at 10:00).
 	AIPipelineAutoInterval string `env:"WORKER_AI_PIPELINE_AUTO_INTERVAL" envDefault:"0 0 11 * * *"`
+	// RecommendationRefreshAfterPipelineEnabled refreshes cached company recommendations after the daily AI pipeline run.
+	RecommendationRefreshAfterPipelineEnabled bool `env:"WORKER_RECOMMENDATION_REFRESH" envDefault:"true"`
 }
 
 // AISummarizerConfig holds configuration for the external AI summarizer service.
 type AISummarizerConfig struct {
-	APIBaseURL            string        `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
-	APITimeout            time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
-	APIRetryCount         int           `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
-	APIRetryDelay         time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
-	DefaultLanguage       string        `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
-	TranslationLanguages  string        `env:"AI_SUMMARIZER_TRANSLATION_LANGUAGES" envDefault:"en"`
-	MinioEndpoint         string        `env:"AI_SUMMARIZER_MINIO_ENDPOINT" envDefault:""`
-	MinioAccessKey        string        `env:"AI_SUMMARIZER_MINIO_ACCESS_KEY" envDefault:""`
-	MinioSecretKey        string        `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
-	MinioUseSSL           bool          `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
-	MinioRegion           string        `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
-	MinioBucket           string        `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens"`
+	APIBaseURL             string        `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
+	APITimeout             time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
+	APIRetryCount          int           `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
+	APIRetryDelay          time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
+	DefaultLanguage        string        `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
+	TranslationLanguages   string        `env:"AI_SUMMARIZER_TRANSLATION_LANGUAGES" envDefault:"en"`
+	MinioEndpoint          string        `env:"AI_SUMMARIZER_MINIO_ENDPOINT" envDefault:""`
+	MinioAccessKey         string        `env:"AI_SUMMARIZER_MINIO_ACCESS_KEY" envDefault:""`
+	MinioSecretKey         string        `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
+	MinioUseSSL            bool          `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
+	MinioRegion            string        `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
+	MinioBucket            string        `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens"`
+	RecommendationCacheTTL time.Duration `env:"AI_RECOMMENDATION_CACHE_TTL" envDefault:"0"`
 }

cmd/worker/main.go

@@ -40,9 +40,19 @@ func main() {
 	// Initialize AI summarizer client (translation worker)
 	aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, mongoManager, appLogger)
 	aiSummarizerStorage := bootstrap.InitAISummarizerStorage(config.AISummarizer, appLogger)
+	redisClient := bootstrap.InitRedis(config.Redis, appLogger)
+	defer func() {
+		if redisClient != nil {
+			if err := redisClient.Close(); err != nil {
+				appLogger.Error("Failed to close Redis connection", map[string]interface{}{
+					"error": err.Error(),
+				})
+			}
+		}
+	}()
 
 	// Initialize Worker
-	scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, aiSummarizerClient, aiSummarizerStorage)
+	scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, aiSummarizerClient, aiSummarizerStorage, redisClient)
 
 	// Set up signal handling for graceful shutdown
 	signalChan := make(chan os.Signal, 1)