Enhance company recommendation caching and pipeline integration
continuous-integration/drone/push Build is passing
continuous-integration/drone/push Build is passing
- Updated the company service to include a new method for scheduling the refresh of cached AI recommendations after the AI pipeline execution. - Introduced a new interface for managing cached recommendation refreshes, improving the separation of concerns within the service layer. - Enhanced the worker initialization to include Redis client support, allowing for better management of recommendation caching. - Added functionality to list company IDs with existing recommendation caches, ensuring efficient updates post-pipeline runs. - Implemented unit tests to validate the new recommendation refresh logic and ensure proper handling of various scenarios. This update significantly improves the handling of AI recommendations by integrating caching mechanisms with the AI pipeline, enhancing overall system performance and responsiveness.
This commit is contained in:
@@ -8,6 +8,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
"tm/cmd/worker/workers"
|
||||
"tm/internal/company"
|
||||
"tm/internal/company_category"
|
||||
"tm/internal/notice"
|
||||
notificationDomain "tm/internal/notification"
|
||||
"tm/internal/tender"
|
||||
@@ -17,6 +19,7 @@ import (
|
||||
"tm/pkg/mongo"
|
||||
"tm/pkg/notification"
|
||||
"tm/pkg/ollama"
|
||||
"tm/pkg/redis"
|
||||
"tm/pkg/schedule"
|
||||
)
|
||||
|
||||
@@ -82,8 +85,8 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
|
||||
return connectionManager
|
||||
}
|
||||
|
||||
// Init Worker
|
||||
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient) *schedule.CronScheduler {
|
||||
// InitWorker
|
||||
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient, redisClient redis.Client) *schedule.CronScheduler {
|
||||
// Debug: Log worker config
|
||||
appLogger.Info("Worker configuration", map[string]interface{}{
|
||||
"worker_interval": config.Worker.Interval,
|
||||
@@ -103,6 +106,26 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
tenderRepo := tender.NewRepository(mongoManager, appLogger)
|
||||
notificationRepo := notificationDomain.NewRepository(mongoManager, appLogger)
|
||||
|
||||
var recommendationRefresher company.Service
|
||||
if config.Worker.RecommendationRefreshAfterPipelineEnabled && aiClient != nil && redisClient != nil {
|
||||
companyRepo := company.NewRepository(mongoManager, appLogger)
|
||||
categoryRepo := company_category.NewRepository(mongoManager, appLogger)
|
||||
categoryService := company_category.NewService(categoryRepo, appLogger)
|
||||
recommendationRefresher = company.NewService(
|
||||
companyRepo,
|
||||
categoryService,
|
||||
nil,
|
||||
aiClient,
|
||||
aiClient,
|
||||
redisClient,
|
||||
config.AISummarizer.RecommendationCacheTTL,
|
||||
appLogger,
|
||||
)
|
||||
appLogger.Info("Company recommendation refresh after pipeline enabled", map[string]interface{}{})
|
||||
} else if config.Worker.RecommendationRefreshAfterPipelineEnabled {
|
||||
appLogger.Warn("Company recommendation refresh after pipeline disabled: AI client or Redis unavailable", map[string]interface{}{})
|
||||
}
|
||||
|
||||
// Create a single shared cron scheduler for all recurring jobs
|
||||
scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
|
||||
|
||||
@@ -275,6 +298,10 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
"trigger": trigger,
|
||||
"date": today.Format("02/01/2006"),
|
||||
})
|
||||
|
||||
if recommendationRefresher != nil {
|
||||
recommendationRefresher.ScheduleRefreshCachedAIRecommendationsAfterPipeline()
|
||||
}
|
||||
}
|
||||
|
||||
scheduler.AddJob(schedule.Job{
|
||||
@@ -387,6 +414,35 @@ func InitAISummarizerStorage(conf AISummarizerConfig, log logger.Logger) *ai_sum
|
||||
return storage
|
||||
}
|
||||
|
||||
// InitRedis initializes the Redis client used by worker jobs.
|
||||
func InitRedis(conf config.RedisConfig, log logger.Logger) redis.Client {
|
||||
connectionConfig := &redis.Config{
|
||||
Host: conf.Host,
|
||||
Port: conf.Port,
|
||||
Password: conf.Password,
|
||||
DB: conf.DB,
|
||||
PoolSize: conf.PoolSize,
|
||||
}
|
||||
|
||||
redisClient, err := redis.NewClient(connectionConfig, log)
|
||||
if err != nil {
|
||||
log.Error("Failed to initialize Redis connection for worker", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"host": conf.Host,
|
||||
"port": conf.Port,
|
||||
})
|
||||
panic(fmt.Sprintf("Failed to initialize Redis connection: %v", err))
|
||||
}
|
||||
|
||||
log.Info("Redis connection initialized for worker", map[string]interface{}{
|
||||
"host": conf.Host,
|
||||
"port": conf.Port,
|
||||
"pool_size": conf.PoolSize,
|
||||
})
|
||||
|
||||
return redisClient
|
||||
}
|
||||
|
||||
func parseTranslationLanguages(conf AISummarizerConfig) []string {
|
||||
raw := strings.TrimSpace(conf.TranslationLanguages)
|
||||
if raw == "" {
|
||||
|
||||
Reference in New Issue
Block a user