mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AI translate refactor

Browse Source
This commit is contained in:
Mazyar
2026-05-16 12:56:10 +03:30
parent ca490f3acf
commit 6701428b09
10 changed files with 510 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/bootstrap/bootstrap.go
+70 -7
View File
@@ -2,6 +2,7 @@ package bootstrap
import (
"fmt"
"strings"
"time"
"tm/cmd/worker/workers"
"tm/internal/notice"
@@ -77,7 +78,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
}
// Init Worker
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client) *schedule.CronScheduler {
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient) *schedule.CronScheduler {
// Debug: Log worker config
appLogger.Info("Worker configuration", map[string]interface{}{
"worker_interval": config.Worker.Interval,
@@ -192,9 +193,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
Expr: workerInterval,
})
// Initialize translation backfill worker for default language
// Initialize translation backfill worker for configured languages
if aiClient != nil {
targetLanguage := config.AISummarizer.DefaultLanguage
targetLanguages := parseTranslationLanguages(config.AISummarizer)
scheduler.AddJob(schedule.Job{
Name: "Tender Translation Worker Job",
Func: func() {
@@ -203,7 +204,8 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
appLogger,
tenderRepo,
aiClient,
targetLanguage,
aiStorage,
targetLanguages,
config.Worker.TranslationBatchSize,
)
worker.Run()
@@ -211,9 +213,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
Expr: config.Worker.TranslationInterval,
})
appLogger.Info("Scheduled tender translation worker", map[string]interface{}{
"target_language": targetLanguage,
"interval": config.Worker.TranslationInterval,
"batch_size": config.Worker.TranslationBatchSize,
"target_languages": targetLanguages,
"interval": config.Worker.TranslationInterval,
"batch_size": config.Worker.TranslationBatchSize,
})
} else {
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
@@ -274,6 +276,67 @@ func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summ
return client
}
// InitAISummarizerStorage initializes the AI summarizer MinIO storage client for workers.
func InitAISummarizerStorage(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.StorageClient {
if conf.MinioEndpoint == "" {
log.Warn("AI Summarizer MinIO endpoint not configured, translation sync from storage will be unavailable", map[string]interface{}{})
return nil
}
cfg := &ai_summarizer.Config{
APIBaseURL: conf.APIBaseURL,
APITimeout: conf.APITimeout,
APIRetryCount: conf.APIRetryCount,
APIRetryDelay: conf.APIRetryDelay,
MinioEndpoint: conf.MinioEndpoint,
MinioAccessKey: conf.MinioAccessKey,
MinioSecretKey: conf.MinioSecretKey,
MinioUseSSL: conf.MinioUseSSL,
MinioRegion: conf.MinioRegion,
MinioBucket: conf.MinioBucket,
}
storage, err := ai_summarizer.NewStorageClient(cfg, log)
if err != nil {
log.Error("Failed to initialize AI Summarizer storage client for worker", map[string]interface{}{
"error": err.Error(),
})
return nil
}
log.Info("AI Summarizer storage client initialized for worker", map[string]interface{}{
"endpoint": conf.MinioEndpoint,
"bucket": conf.MinioBucket,
})
return storage
}
func parseTranslationLanguages(conf AISummarizerConfig) []string {
raw := strings.TrimSpace(conf.TranslationLanguages)
if raw == "" {
raw = conf.DefaultLanguage
}
parts := strings.Split(raw, ",")
langs := make([]string, 0, len(parts))
seen := make(map[string]struct{}, len(parts))
for _, part := range parts {
lang := strings.ToLower(strings.TrimSpace(part))
if lang == "" {
continue
}
if _, dup := seen[lang]; dup {
continue
}
seen[lang] = struct{}{}
langs = append(langs, lang)
}
if len(langs) == 0 {
return []string{"en"}
}
return langs
}
// Init Notification Service
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
notificationConfig := &notification.Config{
cmd/worker/bootstrap/config.go
+12 -5
View File
@@ -50,9 +50,16 @@ type WorkerConfig struct {
// AISummarizerConfig holds configuration for the external AI summarizer service.
type AISummarizerConfig struct {
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
TranslationLanguages string `env:"AI_SUMMARIZER_TRANSLATION_LANGUAGES" envDefault:"en"`
MinioEndpoint string `env:"AI_SUMMARIZER_MINIO_ENDPOINT" envDefault:""`
MinioAccessKey string `env:"AI_SUMMARIZER_MINIO_ACCESS_KEY" envDefault:""`
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens-documents"`
}