mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

worker concurrency config

Browse Source
This commit is contained in:
Mazyar
2026-05-12 23:56:10 +03:30
parent 5e8d4f67b2
commit be53fa8054
3 changed files with 51 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/bootstrap/bootstrap.go
+12 -6
View File
@@ -80,12 +80,14 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client) *schedule.CronScheduler {
// Debug: Log worker config
appLogger.Info("Worker configuration", map[string]interface{}{
"worker_interval": config.Worker.Interval,
"notice_concurrency": config.Worker.NoticeWorkerConcurrency,
"notice_fetch_batch": config.Worker.NoticeFetchBatchSize,
"notice_processing_limit": config.Worker.NoticeProcessingLimit,
"queue_enabled": config.Queue.Enabled,
"queue_mode": config.Queue.Mode,
"worker_interval": config.Worker.Interval,
"notice_concurrency": config.Worker.NoticeWorkerConcurrency,
"notice_fetch_batch": config.Worker.NoticeFetchBatchSize,
"notice_processing_limit": config.Worker.NoticeProcessingLimit,
"notice_batch_pause": config.Worker.NoticeBatchPause.String(),
"notice_fetch_error_backoff": config.Worker.NoticeFetchErrorBackoff.String(),
"queue_enabled": config.Queue.Enabled,
"queue_mode": config.Queue.Mode,
})
// Initialize repositories
noticeRepo := notice.NewRepository(mongoManager, appLogger)
@@ -182,6 +184,8 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
config.Worker.NoticeWorkerConcurrency,
config.Worker.NoticeFetchBatchSize,
config.Worker.DeleteProcessedNotices,
config.Worker.NoticeBatchPause,
config.Worker.NoticeFetchErrorBackoff,
)
worker.Run()
},
@@ -227,6 +231,8 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
config.Worker.NoticeWorkerConcurrency,
config.Worker.NoticeFetchBatchSize,
config.Worker.DeleteProcessedNotices,
config.Worker.NoticeBatchPause,
config.Worker.NoticeFetchErrorBackoff,
)
w.Run()
}()
cmd/worker/bootstrap/config.go
+11 -7
View File
@@ -35,13 +35,17 @@ type QueueConfig struct {
}
type WorkerConfig struct {
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
DeleteProcessedNotices bool `env:"WORKER_DELETE_PROCESSED_NOTICES" envDefault:"true"`
NoticeWorkerConcurrency int `env:"WORKER_NOTICE_CONCURRENCY" envDefault:"8"`
NoticeFetchBatchSize int `env:"WORKER_NOTICE_FETCH_BATCH" envDefault:"50"`
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
DeleteProcessedNotices bool `env:"WORKER_DELETE_PROCESSED_NOTICES" envDefault:"true"`
NoticeWorkerConcurrency int `env:"WORKER_NOTICE_CONCURRENCY" envDefault:"8"`
NoticeFetchBatchSize int `env:"WORKER_NOTICE_FETCH_BATCH" envDefault:"50"`
// NoticeBatchPause is slept after each notice batch finishes, before loading the next batch (0 = no pause).
NoticeBatchPause time.Duration `env:"WORKER_NOTICE_BATCH_PAUSE" envDefault:"0s"`
// NoticeFetchErrorBackoff is slept after a failed notice fetch before retrying the loop (0 = retry immediately).
NoticeFetchErrorBackoff time.Duration `env:"WORKER_NOTICE_FETCH_ERROR_BACKOFF" envDefault:"2s"`
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
}
// AISummarizerConfig holds configuration for the external AI summarizer service.