merge tenders which have the same procedure id - worker performance fix
This commit is contained in:
Mazyar
@@ -8,7 +8,6 @@ import (
|
||||
"tm/internal/tender"
|
||||
ai_summarizer "tm/pkg/ai_summarizer"
|
||||
"tm/pkg/config"
|
||||
"tm/pkg/glm"
|
||||
"tm/pkg/logger"
|
||||
"tm/pkg/minio"
|
||||
"tm/pkg/mongo"
|
||||
@@ -78,12 +77,15 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
|
||||
}
|
||||
|
||||
// Init Worker
|
||||
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client) *schedule.CronScheduler {
|
||||
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client) *schedule.CronScheduler {
|
||||
// Debug: Log worker config
|
||||
appLogger.Info("Worker configuration", map[string]interface{}{
|
||||
"worker_interval": config.Worker.Interval,
|
||||
"queue_enabled": config.Queue.Enabled,
|
||||
"queue_mode": config.Queue.Mode,
|
||||
"worker_interval": config.Worker.Interval,
|
||||
"notice_concurrency": config.Worker.NoticeWorkerConcurrency,
|
||||
"notice_fetch_batch": config.Worker.NoticeFetchBatchSize,
|
||||
"notice_processing_limit": config.Worker.NoticeProcessingLimit,
|
||||
"queue_enabled": config.Queue.Enabled,
|
||||
"queue_mode": config.Queue.Mode,
|
||||
})
|
||||
// Initialize repositories
|
||||
noticeRepo := notice.NewRepository(mongoManager, appLogger)
|
||||
@@ -92,14 +94,6 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
// Create a single shared cron scheduler for all recurring jobs
|
||||
scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
|
||||
|
||||
// Initialize Notice Worker
|
||||
appLogger.Info("Starting notice worker", map[string]interface{}{
|
||||
"processing_limit": config.Worker.NoticeProcessingLimit,
|
||||
})
|
||||
noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, ¬ify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit, config.Worker.DeleteProcessedNotices)
|
||||
noticeWorker.Run()
|
||||
|
||||
// Initialize Queue-based scraping if enabled
|
||||
var queueService queue.Queue
|
||||
if config.Queue.Enabled {
|
||||
var err error
|
||||
@@ -150,26 +144,23 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
})
|
||||
}
|
||||
|
||||
// Initialize Document Summarization Worker (only if MinIO is available)
|
||||
if minioService != nil {
|
||||
appLogger.Info("Starting document summarization worker", map[string]interface{}{})
|
||||
summarizerWorker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, minioService, glmService)
|
||||
summarizerWorker.Run()
|
||||
|
||||
// Schedule Document Summarization Worker job
|
||||
// Document summarization via external AI service (requires MinIO + AI_SUMMARIZER_API_BASE_URL)
|
||||
if minioService != nil && aiClient != nil {
|
||||
scheduler.AddJob(schedule.Job{
|
||||
Name: "Document Summarization Worker Job",
|
||||
Func: func() {
|
||||
worker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, minioService, glmService)
|
||||
worker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, minioService, aiClient)
|
||||
worker.Run()
|
||||
},
|
||||
Expr: "0 11 * * * *", // Run at 11 AM every day (1 hour after document scraping)
|
||||
})
|
||||
} else if minioService != nil {
|
||||
appLogger.Warn("Document summarization worker not scheduled: AI summarizer API is not configured", map[string]interface{}{})
|
||||
} else {
|
||||
appLogger.Warn("MinIO service not available, document summarization will be skipped", map[string]interface{}{})
|
||||
}
|
||||
|
||||
// Schedule Notice Worker job
|
||||
// Schedule Notice Worker job (no blocking run at startup; first tick runs on schedule)
|
||||
workerInterval := config.Worker.Interval
|
||||
if workerInterval == "" {
|
||||
workerInterval = "* 10 * * * *" // Default: every hour at minute 10
|
||||
@@ -181,7 +172,17 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
scheduler.AddJob(schedule.Job{
|
||||
Name: "Notice Worker Job",
|
||||
Func: func() {
|
||||
worker := workers.NewNoticeWorker(mongoManager, appLogger, ¬ify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit, config.Worker.DeleteProcessedNotices)
|
||||
worker := workers.NewNoticeWorker(
|
||||
mongoManager,
|
||||
appLogger,
|
||||
¬ify,
|
||||
noticeRepo,
|
||||
tenderRepo,
|
||||
config.Worker.NoticeProcessingLimit,
|
||||
config.Worker.NoticeWorkerConcurrency,
|
||||
config.Worker.NoticeFetchBatchSize,
|
||||
config.Worker.DeleteProcessedNotices,
|
||||
)
|
||||
worker.Run()
|
||||
},
|
||||
Expr: workerInterval,
|
||||
@@ -214,6 +215,22 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
|
||||
}
|
||||
|
||||
// Kick off one notice-processing pass without blocking startup (cron continues on schedule)
|
||||
go func() {
|
||||
w := workers.NewNoticeWorker(
|
||||
mongoManager,
|
||||
appLogger,
|
||||
¬ify,
|
||||
noticeRepo,
|
||||
tenderRepo,
|
||||
config.Worker.NoticeProcessingLimit,
|
||||
config.Worker.NoticeWorkerConcurrency,
|
||||
config.Worker.NoticeFetchBatchSize,
|
||||
config.Worker.DeleteProcessedNotices,
|
||||
)
|
||||
w.Run()
|
||||
}()
|
||||
|
||||
// Start the cron scheduler so all recurring jobs actually run
|
||||
scheduler.Start()
|
||||
appLogger.Info("Cron scheduler started with all worker jobs", map[string]interface{}{})
|
||||
@@ -306,52 +323,6 @@ func InitOllamaService(conf config.OllamaConfig, log logger.Logger) *ollama.SDK
|
||||
return ollamaSDK
|
||||
}
|
||||
|
||||
// Init GLM Service
|
||||
func InitGLMService(conf GLMConfig, log logger.Logger) *glm.SDK {
|
||||
glmSDK, err := glm.New(
|
||||
&glm.Config{
|
||||
BaseURL: conf.BaseURL,
|
||||
APIKey: conf.APIKey,
|
||||
Timeout: conf.Timeout,
|
||||
RetryAttempts: conf.RetryAttempts,
|
||||
RetryDelay: conf.RetryDelay,
|
||||
EnableLogging: conf.EnableLogging,
|
||||
UserAgent: conf.UserAgent,
|
||||
DefaultModel: conf.DefaultModel,
|
||||
DefaultMaxTokens: conf.DefaultMaxTokens,
|
||||
DefaultTopP: conf.DefaultTopP,
|
||||
DefaultTemperature: conf.DefaultTemperature,
|
||||
DefaultFrequencyPenalty: conf.DefaultFrequencyPenalty,
|
||||
DefaultPresencePenalty: conf.DefaultPresencePenalty,
|
||||
EnableStreaming: conf.EnableStreaming,
|
||||
EnableThinking: conf.EnableThinking,
|
||||
TranslationOptions: glm.TranslationOption{
|
||||
Model: conf.TranslateOptions.Model,
|
||||
MaxTokens: conf.TranslateOptions.MaxTokens,
|
||||
Temperature: conf.TranslateOptions.Temperature,
|
||||
FrequencyPenalty: conf.TranslateOptions.FrequencyPenalty,
|
||||
PresencePenalty: conf.TranslateOptions.PresencePenalty,
|
||||
TopP: conf.TranslateOptions.TopP,
|
||||
EnableThinking: conf.TranslateOptions.EnableThinking,
|
||||
},
|
||||
}, log)
|
||||
|
||||
if err != nil {
|
||||
log.Error("Failed to initialize GLM SDK", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
log.Warn("GLM service will be disabled, translation features will not be available", map[string]interface{}{})
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Info("GLM SDK initialized successfully", map[string]interface{}{
|
||||
"base_url": conf.BaseURL,
|
||||
"timeout": conf.Timeout,
|
||||
})
|
||||
|
||||
return glmSDK
|
||||
}
|
||||
|
||||
// Init MinIO Service
|
||||
func InitMinIOService(conf config.MinIOConfig, log logger.Logger) *minio.Service {
|
||||
// Convert bootstrap config to minio config
|
||||
|
||||
@@ -12,7 +12,6 @@ type Config struct {
|
||||
Notification config.NotificationConfig
|
||||
Ollama config.OllamaConfig
|
||||
Worker WorkerConfig
|
||||
GLM GLMConfig
|
||||
Scraper config.ScraperConfig
|
||||
MinIO config.MinIOConfig
|
||||
AISummarizer AISummarizerConfig
|
||||
@@ -39,8 +38,10 @@ type WorkerConfig struct {
|
||||
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
|
||||
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
|
||||
DeleteProcessedNotices bool `env:"WORKER_DELETE_PROCESSED_NOTICES" envDefault:"true"`
|
||||
NoticeWorkerConcurrency int `env:"WORKER_NOTICE_CONCURRENCY" envDefault:"8"`
|
||||
NoticeFetchBatchSize int `env:"WORKER_NOTICE_FETCH_BATCH" envDefault:"50"`
|
||||
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
|
||||
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
|
||||
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
|
||||
}
|
||||
|
||||
// AISummarizerConfig holds configuration for the external AI summarizer service.
|
||||
@@ -51,32 +52,3 @@ type AISummarizerConfig struct {
|
||||
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
|
||||
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
|
||||
}
|
||||
|
||||
type GLMConfig struct {
|
||||
BaseURL string `env:"GLM_BASE_URL" envDefault:"https://api.z.ai"`
|
||||
APIKey string `env:"GLM_API_KEY" envDefault:""`
|
||||
Timeout time.Duration `env:"GLM_TIMEOUT" envDefault:"300s"`
|
||||
RetryAttempts int `env:"GLM_RETRY_ATTEMPTS" envDefault:"3"`
|
||||
RetryDelay time.Duration `env:"GLM_RETRY_DELAY" envDefault:"2s"`
|
||||
EnableLogging bool `env:"GLM_ENABLE_LOGGING" envDefault:"true"`
|
||||
UserAgent string `env:"GLM_USER_AGENT" envDefault:"Opplens-GLMSDK/1.0"`
|
||||
DefaultModel string `env:"GLM_DEFAULT_MODEL" envDefault:"glm-4.5"`
|
||||
DefaultMaxTokens int `env:"GLM_DEFAULT_MAX_TOKENS" envDefault:"4096"`
|
||||
DefaultTopP float64 `env:"GLM_DEFAULT_TOP_P" envDefault:"1.0"`
|
||||
DefaultTemperature float64 `env:"GLM_DEFAULT_TEMPERATURE" envDefault:"0.7"`
|
||||
DefaultFrequencyPenalty float64 `env:"GLM_DEFAULT_FREQUENCY_PENALTY" envDefault:"0"`
|
||||
DefaultPresencePenalty float64 `env:"GLM_DEFAULT_PRESENCE_PENALTY" envDefault:"0"`
|
||||
EnableStreaming bool `env:"GLM_ENABLE_STREAMING" envDefault:"false"`
|
||||
EnableThinking bool `env:"GLM_ENABLE_THINKING" envDefault:"false"`
|
||||
TranslateOptions TranslateOptions
|
||||
}
|
||||
|
||||
type TranslateOptions struct {
|
||||
Model string `env:"GLM_TRANSLATION_MODEL" envDefault:"glm-4.5"`
|
||||
MaxTokens int `env:"GLM_TRANSLATION_MAX_TOKENS" envDefault:"2048"`
|
||||
Temperature float64 `env:"GLM_TRANSLATION_TEMPERATURE" envDefault:"0.3"`
|
||||
FrequencyPenalty float64 `env:"GLM_TRANSLATION_FREQUENCY_PENALTY" envDefault:"0.1"`
|
||||
PresencePenalty float64 `env:"GLM_TRANSLATION_PRESENCE_PENALTY" envDefault:"0.1"`
|
||||
TopP float64 `env:"GLM_TRANSLATION_TOP_P" envDefault:"1.0"`
|
||||
EnableThinking bool `env:"GLM_TRANSLATION_ENABLE_THINKING" envDefault:"false"`
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user