mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AI translate refactor

Browse Source
This commit is contained in:
Mazyar
2026-05-16 12:56:10 +03:30
parent ca490f3acf
commit 6701428b09
10 changed files with 510 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/bootstrap/bootstrap.go
+70 -7
View File
@@ -2,6 +2,7 @@ package bootstrap
import (
"fmt"
"strings"
"time"
"tm/cmd/worker/workers"
"tm/internal/notice"
@@ -77,7 +78,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
}
// Init Worker
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client) *schedule.CronScheduler {
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient) *schedule.CronScheduler {
// Debug: Log worker config
appLogger.Info("Worker configuration", map[string]interface{}{
"worker_interval": config.Worker.Interval,
@@ -192,9 +193,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
Expr: workerInterval,
})
// Initialize translation backfill worker for default language
// Initialize translation backfill worker for configured languages
if aiClient != nil {
targetLanguage := config.AISummarizer.DefaultLanguage
targetLanguages := parseTranslationLanguages(config.AISummarizer)
scheduler.AddJob(schedule.Job{
Name: "Tender Translation Worker Job",
Func: func() {
@@ -203,7 +204,8 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
appLogger,
tenderRepo,
aiClient,
targetLanguage,
aiStorage,
targetLanguages,
config.Worker.TranslationBatchSize,
)
worker.Run()
@@ -211,9 +213,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
Expr: config.Worker.TranslationInterval,
})
appLogger.Info("Scheduled tender translation worker", map[string]interface{}{
"target_language": targetLanguage,
"interval": config.Worker.TranslationInterval,
"batch_size": config.Worker.TranslationBatchSize,
"target_languages": targetLanguages,
"interval": config.Worker.TranslationInterval,
"batch_size": config.Worker.TranslationBatchSize,
})
} else {
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
@@ -274,6 +276,67 @@ func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summ
return client
}
// InitAISummarizerStorage initializes the AI summarizer MinIO storage client for workers.
func InitAISummarizerStorage(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.StorageClient {
if conf.MinioEndpoint == "" {
log.Warn("AI Summarizer MinIO endpoint not configured, translation sync from storage will be unavailable", map[string]interface{}{})
return nil
}
cfg := &ai_summarizer.Config{
APIBaseURL: conf.APIBaseURL,
APITimeout: conf.APITimeout,
APIRetryCount: conf.APIRetryCount,
APIRetryDelay: conf.APIRetryDelay,
MinioEndpoint: conf.MinioEndpoint,
MinioAccessKey: conf.MinioAccessKey,
MinioSecretKey: conf.MinioSecretKey,
MinioUseSSL: conf.MinioUseSSL,
MinioRegion: conf.MinioRegion,
MinioBucket: conf.MinioBucket,
}
storage, err := ai_summarizer.NewStorageClient(cfg, log)
if err != nil {
log.Error("Failed to initialize AI Summarizer storage client for worker", map[string]interface{}{
"error": err.Error(),
})
return nil
}
log.Info("AI Summarizer storage client initialized for worker", map[string]interface{}{
"endpoint": conf.MinioEndpoint,
"bucket": conf.MinioBucket,
})
return storage
}
func parseTranslationLanguages(conf AISummarizerConfig) []string {
raw := strings.TrimSpace(conf.TranslationLanguages)
if raw == "" {
raw = conf.DefaultLanguage
}
parts := strings.Split(raw, ",")
langs := make([]string, 0, len(parts))
seen := make(map[string]struct{}, len(parts))
for _, part := range parts {
lang := strings.ToLower(strings.TrimSpace(part))
if lang == "" {
continue
}
if _, dup := seen[lang]; dup {
continue
}
seen[lang] = struct{}{}
langs = append(langs, lang)
}
if len(langs) == 0 {
return []string{"en"}
}
return langs
}
// Init Notification Service
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
notificationConfig := &notification.Config{
cmd/worker/bootstrap/config.go
+12 -5
View File
@@ -50,9 +50,16 @@ type WorkerConfig struct {
// AISummarizerConfig holds configuration for the external AI summarizer service.
type AISummarizerConfig struct {
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
TranslationLanguages string `env:"AI_SUMMARIZER_TRANSLATION_LANGUAGES" envDefault:"en"`
MinioEndpoint string `env:"AI_SUMMARIZER_MINIO_ENDPOINT" envDefault:""`
MinioAccessKey string `env:"AI_SUMMARIZER_MINIO_ACCESS_KEY" envDefault:""`
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens-documents"`
}
cmd/worker/main.go
+2 -1
View File
@@ -45,9 +45,10 @@ func main() {
// Initialize AI summarizer client (translation + document summarization workers)
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, appLogger)
aiSummarizerStorage := bootstrap.InitAISummarizerStorage(config.AISummarizer, appLogger)
// Initialize Worker
scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, minioService, aiSummarizerClient)
scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, minioService, aiSummarizerClient, aiSummarizerStorage)
// Set up signal handling for graceful shutdown
signalChan := make(chan os.Signal, 1)
cmd/worker/workers/translation.go
+135 -65
View File
@@ -2,6 +2,7 @@ package workers
import (
"context"
"errors"
"strings"
"time"
"tm/internal/tender"
@@ -10,32 +11,56 @@ import (
"tm/pkg/mongo"
)
// TranslationWorker backfills missing tender translations using AI summarizer API.
// TranslationWorker backfills missing tender translations using the AI pipeline
// and on-demand translate API, syncing results from MinIO when available.
type TranslationWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tender.TenderRepository
AIClient *ai_summarizer.Client
TargetLanguage string
BatchSize int
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tender.TenderRepository
AIClient *ai_summarizer.Client
AIStorage *ai_summarizer.StorageClient
TargetLanguages []string
BatchSize int
}
// NewTranslationWorker creates a translation worker for a target language.
func NewTranslationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, aiClient *ai_summarizer.Client, targetLanguage string, batchSize int) *TranslationWorker {
if strings.TrimSpace(targetLanguage) == "" {
targetLanguage = "en"
// NewTranslationWorker creates a translation worker for the given target languages.
func NewTranslationWorker(
mongo *mongo.ConnectionManager,
logger logger.Logger,
tenderRepo tender.TenderRepository,
aiClient *ai_summarizer.Client,
aiStorage *ai_summarizer.StorageClient,
targetLanguages []string,
batchSize int,
) *TranslationWorker {
langs := make([]string, 0, len(targetLanguages))
seen := make(map[string]struct{}, len(targetLanguages))
for _, lang := range targetLanguages {
clean := strings.ToLower(strings.TrimSpace(lang))
if clean == "" {
continue
}
if _, dup := seen[clean]; dup {
continue
}
seen[clean] = struct{}{}
langs = append(langs, clean)
}
if len(langs) == 0 {
langs = []string{"en"}
}
if batchSize <= 0 {
batchSize = 20
}
return &TranslationWorker{
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
AIClient: aiClient,
TargetLanguage: strings.ToLower(strings.TrimSpace(targetLanguage)),
BatchSize: batchSize,
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
AIClient: aiClient,
AIStorage: aiStorage,
TargetLanguages: langs,
BatchSize: batchSize,
}
}
@@ -43,48 +68,60 @@ func NewTranslationWorker(mongo *mongo.ConnectionManager, logger logger.Logger,
func (w *TranslationWorker) Run() {
if w.AIClient == nil {
w.Logger.Warn("Translation worker skipped: AI client is not configured", map[string]interface{}{
"target_language": w.TargetLanguage,
"target_languages": w.TargetLanguages,
})
return
}
w.Logger.Info("Translation worker started", map[string]interface{}{
"target_language": w.TargetLanguage,
"batch_size": w.BatchSize,
"target_languages": w.TargetLanguages,
"batch_size": w.BatchSize,
})
if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
w.Logger.Warn("Failed to trigger AI pipeline translate (continuing with per-tender sync)", map[string]interface{}{
"target_languages": w.TargetLanguages,
"error": err.Error(),
})
}
for _, language := range w.TargetLanguages {
w.runForLanguage(language)
}
w.Logger.Info("Translation worker completed", map[string]interface{}{
"target_languages": w.TargetLanguages,
})
}
func (w *TranslationWorker) runForLanguage(language string) {
for {
// Always use skip=0 because successfully translated tenders are excluded by filter.
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), w.TargetLanguage, w.BatchSize, 0)
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, 0)
if err != nil {
w.Logger.Error("Failed to fetch untranslated tenders", map[string]interface{}{
"target_language": w.TargetLanguage,
"target_language": language,
"error": err.Error(),
})
break
return
}
w.Logger.Info("Fetched untranslated tenders batch", map[string]interface{}{
"target_language": w.TargetLanguage,
"target_language": language,
"batch_count": len(tenders),
"total_count": totalCount,
})
if len(tenders) == 0 {
break
return
}
for _, t := range tenders {
w.translateTender(&t)
w.translateTender(&t, language)
}
}
w.Logger.Info("Translation worker completed", map[string]interface{}{
"target_language": w.TargetLanguage,
})
}
func (w *TranslationWorker) translateTender(t *tender.Tender) {
func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
if t.NoticePublicationID == "" {
return
}
@@ -94,64 +131,97 @@ func (w *TranslationWorker) translateTender(t *tender.Tender) {
})
return
}
if existing, ok := t.Translations[w.TargetLanguage]; ok && strings.TrimSpace(existing.Title) != "" {
if existing, ok := t.Translations[language]; ok && strings.TrimSpace(existing.Title) != "" {
w.Logger.Debug("Skipping tender translation because target language already exists", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"target_language": language,
})
return
}
resp, err := w.AIClient.FetchTranslationOnDemand(context.Background(), ai_summarizer.TranslateRequest{
NoticeID: t.NoticePublicationID,
ContractFolderID: t.ContractFolderID,
NoticePublicationID: t.NoticePublicationID,
Title: t.Title,
Description: t.Description,
Language: w.TargetLanguage,
})
title, description, source, err := w.resolveTranslation(t, language)
if err != nil {
w.Logger.Error("Failed to translate tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"target_language": language,
"translation_source": source,
"translation_error": err.Error(),
})
return
}
if t.Translations == nil {
t.Translations = make(map[string]tender.TranslationEntry)
}
now := time.Now().Unix()
// Only update the target-language slot. Existing translations in other
// languages (e.g., "de", "fr") are preserved and never modified.
t.Translations[w.TargetLanguage] = tender.TranslationEntry{
Title: resp.TranslatedTitle,
Description: resp.TranslatedDescription,
Language: w.TargetLanguage,
TranslatedAt: now,
}
if t.ProcessingMetadata.TranslatedData == nil {
t.ProcessingMetadata.TranslatedData = make(map[string]string)
}
t.ProcessingMetadata.TranslatedData[w.TargetLanguage] = "done"
t.ProcessingMetadata.TranslatedAt = now
persistTranslation(t, language, title, description)
if err := w.TenderRepo.Update(context.Background(), t); err != nil {
w.Logger.Error("Failed to persist translated tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"target_language": language,
"translation_source": source,
"persistence_error": err.Error(),
})
return
}
w.Logger.Info("Tender translated successfully", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"translation_source": source,
})
}
func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string) (title, description, source string, err error) {
if w.AIStorage != nil {
stored, storageErr := w.AIStorage.GetTranslationFromStorage(
context.Background(),
t.ContractFolderID,
t.NoticePublicationID,
language,
)
if storageErr == nil {
return stored.Title, stored.Description, "storage", nil
}
if !errors.Is(storageErr, ai_summarizer.ErrTranslationNotReady) &&
!errors.Is(storageErr, ai_summarizer.ErrObjectNotFound) &&
!errors.Is(storageErr, ai_summarizer.ErrNoTranslation) {
w.Logger.Debug("Translation not available from storage yet", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"error": storageErr.Error(),
})
}
}
resp, apiErr := w.AIClient.FetchTranslationOnDemand(context.Background(), ai_summarizer.TranslateRequest{
ContractFolderID: t.ContractFolderID,
NoticePublicationID: t.NoticePublicationID,
Title: t.Title,
Description: t.Description,
Language: language,
})
if apiErr != nil {
return "", "", "api", apiErr
}
return resp.TranslatedTitle, resp.TranslatedDescription, "api", nil
}
func persistTranslation(t *tender.Tender, language, title, description string) {
if t.Translations == nil {
t.Translations = make(map[string]tender.TranslationEntry)
}
now := time.Now().Unix()
t.Translations[language] = tender.TranslationEntry{
Title: title,
Description: description,
Language: language,
TranslatedAt: now,
}
if t.ProcessingMetadata.TranslatedData == nil {
t.ProcessingMetadata.TranslatedData = make(map[string]string)
}
t.ProcessingMetadata.TranslatedData[language] = "done"
t.ProcessingMetadata.TranslatedAt = now
}