mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cache translation in minio instead of mongo

Browse Source
This commit is contained in:
Mazyar
2026-05-19 19:45:07 +03:30
parent f7763cf997
commit e9ec33f0d4
5 changed files with 41 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/workers/translation.go
+32 -41
View File
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"strings"
"time"
"tm/internal/tender"
"tm/pkg/ai_summarizer"
"tm/pkg/logger"
@@ -12,7 +11,7 @@ import (
)
// TranslationWorker backfills missing tender translations using the AI pipeline
// and on-demand translate API, syncing results from MinIO when available.
// and on-demand translate API. Results are persisted by the AI service in MinIO only.
type TranslationWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
@@ -95,20 +94,22 @@ func (w *TranslationWorker) Run() {
}
func (w *TranslationWorker) runForLanguage(language string) {
skip := 0
for {
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, 0)
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, skip)
if err != nil {
w.Logger.Error("Failed to fetch untranslated tenders", map[string]interface{}{
w.Logger.Error("Failed to fetch tenders for translation backfill", map[string]interface{}{
"target_language": language,
"error": err.Error(),
})
return
}
w.Logger.Info("Fetched untranslated tenders batch", map[string]interface{}{
w.Logger.Info("Fetched tenders for translation backfill", map[string]interface{}{
"target_language": language,
"batch_count": len(tenders),
"total_count": totalCount,
"skip": skip,
})
if len(tenders) == 0 {
@@ -118,6 +119,11 @@ func (w *TranslationWorker) runForLanguage(language string) {
for _, t := range tenders {
w.translateTender(&t, language)
}
skip += len(tenders)
if int64(skip) >= totalCount {
return
}
}
}
@@ -131,8 +137,8 @@ func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
})
return
}
if existing, ok := t.Translations[language]; ok && strings.TrimSpace(existing.Title) != "" {
w.Logger.Debug("Skipping tender translation because target language already exists", map[string]interface{}{
if w.hasTranslationInStorage(t, language) {
w.Logger.Debug("Skipping tender translation: already available in MinIO", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
@@ -143,23 +149,11 @@ func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
title, description, source, err := w.resolveTranslation(t, language)
if err != nil {
w.Logger.Error("Failed to translate tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"translation_source": source,
"translation_error": err.Error(),
})
return
}
persistTranslation(t, language, title, description)
if err := w.TenderRepo.Update(context.Background(), t); err != nil {
w.Logger.Error("Failed to persist translated tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"translation_source": source,
"persistence_error": err.Error(),
"translation_error": err.Error(),
})
return
}
@@ -169,9 +163,24 @@ func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
"notice_id": t.NoticePublicationID,
"target_language": language,
"translation_source": source,
"title_len": len(title),
"description_len": len(description),
})
}
func (w *TranslationWorker) hasTranslationInStorage(t *tender.Tender, language string) bool {
if w.AIStorage == nil {
return false
}
stored, err := w.AIStorage.GetTranslationFromStorage(
context.Background(),
t.ContractFolderID,
t.NoticePublicationID,
language,
)
return err == nil && strings.TrimSpace(stored.Title) != ""
}
func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string) (title, description, source string, err error) {
if w.AIStorage != nil {
stored, storageErr := w.AIStorage.GetTranslationFromStorage(
@@ -207,21 +216,3 @@ func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string
}
return resp.TranslatedTitle, resp.TranslatedDescription, "api", nil
}
func persistTranslation(t *tender.Tender, language, title, description string) {
if t.Translations == nil {
t.Translations = make(map[string]tender.TranslationEntry)
}
now := time.Now().Unix()
t.Translations[language] = tender.TranslationEntry{
Title: title,
Description: description,
Language: language,
TranslatedAt: now,
}
if t.ProcessingMetadata.TranslatedData == nil {
t.ProcessingMetadata.TranslatedData = make(map[string]string)
}
t.ProcessingMetadata.TranslatedData[language] = "done"
t.ProcessingMetadata.TranslatedAt = now
}