Implement AI analysis trigger endpoint and refactor related components
- Added a new endpoint to trigger on-demand agentic analysis for tenders via the AI service. - Introduced `TriggerAIAnalyze` method in the `TenderHandler` to handle requests and responses for AI analysis. - Updated the `tenderService` to include `TriggerAIAnalyze` method, which validates input and interacts with the AI summarizer client. - Enhanced the `AIAnalyzeResponse` and `AIAnalyzeDocument` structures to support the new analysis feature. - Refactored the `DocumentSummarizationWorker` and `TranslationWorker` to remove deprecated MinIO dependencies, streamlining the AI service interactions. This update improves the functionality of the tender management system by allowing users to trigger AI analysis on-demand, enhancing the overall user experience and system capabilities.
This commit is contained in:
Mazyar
@@ -10,11 +10,9 @@ import (
|
||||
ai_summarizer "tm/pkg/ai_summarizer"
|
||||
"tm/pkg/config"
|
||||
"tm/pkg/logger"
|
||||
"tm/pkg/minio"
|
||||
"tm/pkg/mongo"
|
||||
"tm/pkg/notification"
|
||||
"tm/pkg/ollama"
|
||||
"tm/pkg/queue"
|
||||
"tm/pkg/schedule"
|
||||
)
|
||||
|
||||
@@ -78,7 +76,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
|
||||
}
|
||||
|
||||
// Init Worker
|
||||
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient) *schedule.CronScheduler {
|
||||
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient) *schedule.CronScheduler {
|
||||
// Debug: Log worker config
|
||||
appLogger.Info("Worker configuration", map[string]interface{}{
|
||||
"worker_interval": config.Worker.Interval,
|
||||
@@ -97,70 +95,24 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
// Create a single shared cron scheduler for all recurring jobs
|
||||
scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
|
||||
|
||||
var queueService queue.Queue
|
||||
if config.Queue.Enabled {
|
||||
var err error
|
||||
queueConfig := queue.QueueConfig{
|
||||
RedisURL: config.Queue.RedisURL,
|
||||
MaxRetries: config.Queue.MaxRetries,
|
||||
RetryBackoff: config.Queue.RetryBackoff,
|
||||
JobTimeout: config.Queue.JobTimeout,
|
||||
PoolSize: config.Queue.PoolSize,
|
||||
IsDLQEnabled: config.Queue.IsDLQEnabled,
|
||||
MaxQueueLength: 0, // Unlimited
|
||||
}
|
||||
|
||||
queueService, err = queue.NewRedisQueue(queueConfig, appLogger)
|
||||
if err != nil {
|
||||
appLogger.Error("Failed to initialize queue service", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
appLogger.Warn("Queue-based scraping is disabled, falling back to scheduled scraping", map[string]interface{}{})
|
||||
config.Queue.Enabled = false
|
||||
} else {
|
||||
appLogger.Info("Queue service initialized successfully", map[string]interface{}{
|
||||
"mode": config.Queue.Mode,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Start Queue Producer Worker (enqueues unscraped tenders periodically)
|
||||
if config.Queue.Enabled && (config.Queue.Mode == "producer" || config.Queue.Mode == "both") {
|
||||
appLogger.Info("Scheduling queue producer worker", map[string]interface{}{
|
||||
"batch_size": config.Queue.ProducerBatch,
|
||||
"interval": config.Queue.ProducerInterval,
|
||||
})
|
||||
|
||||
scheduler.AddJob(schedule.Job{
|
||||
Name: "Queue Producer Worker Job",
|
||||
Func: func() {
|
||||
producer := workers.NewQueueProducerWorker(
|
||||
mongoManager,
|
||||
appLogger,
|
||||
tenderRepo,
|
||||
queueService,
|
||||
config.Queue.ProducerBatch,
|
||||
)
|
||||
producer.Run()
|
||||
},
|
||||
Expr: config.Queue.ProducerInterval,
|
||||
appLogger.Warn("Redis document-scraper queue is deprecated; document scraping is owned by the AI service pipeline", map[string]interface{}{
|
||||
"queue_mode": config.Queue.Mode,
|
||||
})
|
||||
}
|
||||
|
||||
// Document summarization via external AI service (requires MinIO + AI_SUMMARIZER_API_BASE_URL)
|
||||
if minioService != nil && aiClient != nil {
|
||||
// Document summarization via external AI service (requires AI_SUMMARIZER_API_BASE_URL)
|
||||
if aiClient != nil {
|
||||
scheduler.AddJob(schedule.Job{
|
||||
Name: "Document Summarization Worker Job",
|
||||
Func: func() {
|
||||
worker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, minioService, aiClient)
|
||||
worker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, aiClient, aiStorage)
|
||||
worker.Run()
|
||||
},
|
||||
Expr: "0 11 * * * *", // Run at 11 AM every day (1 hour after document scraping)
|
||||
Expr: "0 11 * * * *", // Run at 11 AM every day (after AI pipeline scrape)
|
||||
})
|
||||
} else if minioService != nil {
|
||||
appLogger.Warn("Document summarization worker not scheduled: AI summarizer API is not configured", map[string]interface{}{})
|
||||
} else {
|
||||
appLogger.Warn("MinIO service not available, document summarization will be skipped", map[string]interface{}{})
|
||||
appLogger.Warn("Document summarization worker not scheduled: AI summarizer API is not configured", map[string]interface{}{})
|
||||
}
|
||||
|
||||
// Schedule Notice Worker job (no blocking run at startup; first tick runs on schedule)
|
||||
@@ -395,54 +347,3 @@ func InitOllamaService(conf config.OllamaConfig, log logger.Logger) *ollama.SDK
|
||||
|
||||
return ollamaSDK
|
||||
}
|
||||
|
||||
// Init MinIO Service
|
||||
func InitMinIOService(conf config.MinIOConfig, log logger.Logger) *minio.Service {
|
||||
// Convert bootstrap config to minio config
|
||||
minioConfig := minio.Config{
|
||||
Endpoint: conf.Endpoint,
|
||||
AccessKeyID: conf.AccessKeyID,
|
||||
SecretAccessKey: conf.SecretAccessKey,
|
||||
UseSSL: conf.UseSSL,
|
||||
Region: conf.Region,
|
||||
DefaultBucket: conf.DefaultBucket,
|
||||
HierarchicalBucket: conf.HierarchicalBucket,
|
||||
}
|
||||
|
||||
// Validate config
|
||||
if err := minioConfig.Validate(); err != nil {
|
||||
log.Warn("Invalid MinIO configuration, MinIO features will be disabled", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil // Return nil instead of panicking
|
||||
}
|
||||
|
||||
// Create connection manager
|
||||
connManager, err := minio.NewConnectionManager(minioConfig, log)
|
||||
if err != nil {
|
||||
log.Error("Failed to create MinIO connection manager", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
panic(fmt.Sprintf("Failed to create MinIO connection manager: %v", err))
|
||||
}
|
||||
|
||||
// Connect to MinIO
|
||||
if err := connManager.Connect(); err != nil {
|
||||
log.Warn("Failed to connect to MinIO, MinIO features will be disabled", map[string]interface{}{
|
||||
"endpoint": minioConfig.Endpoint,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil // Return nil instead of panicking
|
||||
}
|
||||
|
||||
// Create service
|
||||
service := minio.NewService(connManager, minioConfig, log)
|
||||
|
||||
log.Info("MinIO service initialized successfully", map[string]interface{}{
|
||||
"endpoint": conf.Endpoint,
|
||||
"default_bucket": conf.DefaultBucket,
|
||||
"hierarchical_bucket": conf.HierarchicalBucket,
|
||||
})
|
||||
|
||||
return service
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user