mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AI translate refactor

Browse Source
This commit is contained in:
Mazyar
2026-05-16 12:56:10 +03:30
parent ca490f3acf
commit 6701428b09
10 changed files with 510 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/workers/translation.go
+135 -65
View File
@@ -2,6 +2,7 @@ package workers
import (
"context"
"errors"
"strings"
"time"
"tm/internal/tender"
@@ -10,32 +11,56 @@ import (
"tm/pkg/mongo"
)
// TranslationWorker backfills missing tender translations using AI summarizer API.
// TranslationWorker backfills missing tender translations using the AI pipeline
// and on-demand translate API, syncing results from MinIO when available.
type TranslationWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tender.TenderRepository
AIClient *ai_summarizer.Client
TargetLanguage string
BatchSize int
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tender.TenderRepository
AIClient *ai_summarizer.Client
AIStorage *ai_summarizer.StorageClient
TargetLanguages []string
BatchSize int
}
// NewTranslationWorker creates a translation worker for a target language.
func NewTranslationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, aiClient *ai_summarizer.Client, targetLanguage string, batchSize int) *TranslationWorker {
if strings.TrimSpace(targetLanguage) == "" {
targetLanguage = "en"
// NewTranslationWorker creates a translation worker for the given target languages.
func NewTranslationWorker(
mongo *mongo.ConnectionManager,
logger logger.Logger,
tenderRepo tender.TenderRepository,
aiClient *ai_summarizer.Client,
aiStorage *ai_summarizer.StorageClient,
targetLanguages []string,
batchSize int,
) *TranslationWorker {
langs := make([]string, 0, len(targetLanguages))
seen := make(map[string]struct{}, len(targetLanguages))
for _, lang := range targetLanguages {
clean := strings.ToLower(strings.TrimSpace(lang))
if clean == "" {
continue
}
if _, dup := seen[clean]; dup {
continue
}
seen[clean] = struct{}{}
langs = append(langs, clean)
}
if len(langs) == 0 {
langs = []string{"en"}
}
if batchSize <= 0 {
batchSize = 20
}
return &TranslationWorker{
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
AIClient: aiClient,
TargetLanguage: strings.ToLower(strings.TrimSpace(targetLanguage)),
BatchSize: batchSize,
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
AIClient: aiClient,
AIStorage: aiStorage,
TargetLanguages: langs,
BatchSize: batchSize,
}
}
@@ -43,48 +68,60 @@ func NewTranslationWorker(mongo *mongo.ConnectionManager, logger logger.Logger,
func (w *TranslationWorker) Run() {
if w.AIClient == nil {
w.Logger.Warn("Translation worker skipped: AI client is not configured", map[string]interface{}{
"target_language": w.TargetLanguage,
"target_languages": w.TargetLanguages,
})
return
}
w.Logger.Info("Translation worker started", map[string]interface{}{
"target_language": w.TargetLanguage,
"batch_size": w.BatchSize,
"target_languages": w.TargetLanguages,
"batch_size": w.BatchSize,
})
if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
w.Logger.Warn("Failed to trigger AI pipeline translate (continuing with per-tender sync)", map[string]interface{}{
"target_languages": w.TargetLanguages,
"error": err.Error(),
})
}
for _, language := range w.TargetLanguages {
w.runForLanguage(language)
}
w.Logger.Info("Translation worker completed", map[string]interface{}{
"target_languages": w.TargetLanguages,
})
}
func (w *TranslationWorker) runForLanguage(language string) {
for {
// Always use skip=0 because successfully translated tenders are excluded by filter.
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), w.TargetLanguage, w.BatchSize, 0)
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, 0)
if err != nil {
w.Logger.Error("Failed to fetch untranslated tenders", map[string]interface{}{
"target_language": w.TargetLanguage,
"target_language": language,
"error": err.Error(),
})
break
return
}
w.Logger.Info("Fetched untranslated tenders batch", map[string]interface{}{
"target_language": w.TargetLanguage,
"target_language": language,
"batch_count": len(tenders),
"total_count": totalCount,
})
if len(tenders) == 0 {
break
return
}
for _, t := range tenders {
w.translateTender(&t)
w.translateTender(&t, language)
}
}
w.Logger.Info("Translation worker completed", map[string]interface{}{
"target_language": w.TargetLanguage,
})
}
func (w *TranslationWorker) translateTender(t *tender.Tender) {
func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
if t.NoticePublicationID == "" {
return
}
@@ -94,64 +131,97 @@ func (w *TranslationWorker) translateTender(t *tender.Tender) {
})
return
}
if existing, ok := t.Translations[w.TargetLanguage]; ok && strings.TrimSpace(existing.Title) != "" {
if existing, ok := t.Translations[language]; ok && strings.TrimSpace(existing.Title) != "" {
w.Logger.Debug("Skipping tender translation because target language already exists", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"target_language": language,
})
return
}
resp, err := w.AIClient.FetchTranslationOnDemand(context.Background(), ai_summarizer.TranslateRequest{
NoticeID: t.NoticePublicationID,
ContractFolderID: t.ContractFolderID,
NoticePublicationID: t.NoticePublicationID,
Title: t.Title,
Description: t.Description,
Language: w.TargetLanguage,
})
title, description, source, err := w.resolveTranslation(t, language)
if err != nil {
w.Logger.Error("Failed to translate tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"target_language": language,
"translation_source": source,
"translation_error": err.Error(),
})
return
}
if t.Translations == nil {
t.Translations = make(map[string]tender.TranslationEntry)
}
now := time.Now().Unix()
// Only update the target-language slot. Existing translations in other
// languages (e.g., "de", "fr") are preserved and never modified.
t.Translations[w.TargetLanguage] = tender.TranslationEntry{
Title: resp.TranslatedTitle,
Description: resp.TranslatedDescription,
Language: w.TargetLanguage,
TranslatedAt: now,
}
if t.ProcessingMetadata.TranslatedData == nil {
t.ProcessingMetadata.TranslatedData = make(map[string]string)
}
t.ProcessingMetadata.TranslatedData[w.TargetLanguage] = "done"
t.ProcessingMetadata.TranslatedAt = now
persistTranslation(t, language, title, description)
if err := w.TenderRepo.Update(context.Background(), t); err != nil {
w.Logger.Error("Failed to persist translated tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"target_language": language,
"translation_source": source,
"persistence_error": err.Error(),
})
return
}
w.Logger.Info("Tender translated successfully", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"translation_source": source,
})
}
func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string) (title, description, source string, err error) {
if w.AIStorage != nil {
stored, storageErr := w.AIStorage.GetTranslationFromStorage(
context.Background(),
t.ContractFolderID,
t.NoticePublicationID,
language,
)
if storageErr == nil {
return stored.Title, stored.Description, "storage", nil
}
if !errors.Is(storageErr, ai_summarizer.ErrTranslationNotReady) &&
!errors.Is(storageErr, ai_summarizer.ErrObjectNotFound) &&
!errors.Is(storageErr, ai_summarizer.ErrNoTranslation) {
w.Logger.Debug("Translation not available from storage yet", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"error": storageErr.Error(),
})
}
}
resp, apiErr := w.AIClient.FetchTranslationOnDemand(context.Background(), ai_summarizer.TranslateRequest{
ContractFolderID: t.ContractFolderID,
NoticePublicationID: t.NoticePublicationID,
Title: t.Title,
Description: t.Description,
Language: language,
})
if apiErr != nil {
return "", "", "api", apiErr
}
return resp.TranslatedTitle, resp.TranslatedDescription, "api", nil
}
func persistTranslation(t *tender.Tender, language, title, description string) {
if t.Translations == nil {
t.Translations = make(map[string]tender.TranslationEntry)
}
now := time.Now().Unix()
t.Translations[language] = tender.TranslationEntry{
Title: title,
Description: description,
Language: language,
TranslatedAt: now,
}
if t.ProcessingMetadata.TranslatedData == nil {
t.ProcessingMetadata.TranslatedData = make(map[string]string)
}
t.ProcessingMetadata.TranslatedData[language] = "done"
t.ProcessingMetadata.TranslatedAt = now
}