mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cache translation in minio instead of mongo

Browse Source
This commit is contained in:
Mazyar
2026-05-19 19:45:07 +03:30
parent f7763cf997
commit e9ec33f0d4
5 changed files with 41 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/workers/translation.go
+32 -41
View File
@@ -4,7 +4,6 @@ import (
"context"
"errors"
"strings"
"time"
"tm/internal/tender"
"tm/pkg/ai_summarizer"
"tm/pkg/logger"
@@ -12,7 +11,7 @@ import (
)
// TranslationWorker backfills missing tender translations using the AI pipeline
// and on-demand translate API, syncing results from MinIO when available.
// and on-demand translate API. Results are persisted by the AI service in MinIO only.
type TranslationWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
@@ -95,20 +94,22 @@ func (w *TranslationWorker) Run() {
}
func (w *TranslationWorker) runForLanguage(language string) {
skip := 0
for {
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, 0)
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, skip)
if err != nil {
w.Logger.Error("Failed to fetch untranslated tenders", map[string]interface{}{
w.Logger.Error("Failed to fetch tenders for translation backfill", map[string]interface{}{
"target_language": language,
"error": err.Error(),
})
return
}
w.Logger.Info("Fetched untranslated tenders batch", map[string]interface{}{
w.Logger.Info("Fetched tenders for translation backfill", map[string]interface{}{
"target_language": language,
"batch_count": len(tenders),
"total_count": totalCount,
"skip": skip,
})
if len(tenders) == 0 {
@@ -118,6 +119,11 @@ func (w *TranslationWorker) runForLanguage(language string) {
for _, t := range tenders {
w.translateTender(&t, language)
}
skip += len(tenders)
if int64(skip) >= totalCount {
return
}
}
}
@@ -131,8 +137,8 @@ func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
})
return
}
if existing, ok := t.Translations[language]; ok && strings.TrimSpace(existing.Title) != "" {
w.Logger.Debug("Skipping tender translation because target language already exists", map[string]interface{}{
if w.hasTranslationInStorage(t, language) {
w.Logger.Debug("Skipping tender translation: already available in MinIO", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
@@ -143,23 +149,11 @@ func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
title, description, source, err := w.resolveTranslation(t, language)
if err != nil {
w.Logger.Error("Failed to translate tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"translation_source": source,
"translation_error": err.Error(),
})
return
}
persistTranslation(t, language, title, description)
if err := w.TenderRepo.Update(context.Background(), t); err != nil {
w.Logger.Error("Failed to persist translated tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": language,
"translation_source": source,
"persistence_error": err.Error(),
"translation_error": err.Error(),
})
return
}
@@ -169,9 +163,24 @@ func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
"notice_id": t.NoticePublicationID,
"target_language": language,
"translation_source": source,
"title_len": len(title),
"description_len": len(description),
})
}
func (w *TranslationWorker) hasTranslationInStorage(t *tender.Tender, language string) bool {
if w.AIStorage == nil {
return false
}
stored, err := w.AIStorage.GetTranslationFromStorage(
context.Background(),
t.ContractFolderID,
t.NoticePublicationID,
language,
)
return err == nil && strings.TrimSpace(stored.Title) != ""
}
func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string) (title, description, source string, err error) {
if w.AIStorage != nil {
stored, storageErr := w.AIStorage.GetTranslationFromStorage(
@@ -207,21 +216,3 @@ func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string
}
return resp.TranslatedTitle, resp.TranslatedDescription, "api", nil
}
func persistTranslation(t *tender.Tender, language, title, description string) {
if t.Translations == nil {
t.Translations = make(map[string]tender.TranslationEntry)
}
now := time.Now().Unix()
t.Translations[language] = tender.TranslationEntry{
Title: title,
Description: description,
Language: language,
TranslatedAt: now,
}
if t.ProcessingMetadata.TranslatedData == nil {
t.ProcessingMetadata.TranslatedData = make(map[string]string)
}
t.ProcessingMetadata.TranslatedData[language] = "done"
t.ProcessingMetadata.TranslatedAt = now
}
internal/tender/form.go
+2 -11
View File
@@ -255,19 +255,11 @@ func (t *Tender) ToResponse() *TenderResponse {
return t.ToResponseWithLanguage("")
}
// ToResponseWithLanguage converts a Tender entity to TenderResponse using a preferred language.
// When language is provided and translation exists, translated title/description are returned.
// ToResponseWithLanguage converts a Tender entity to TenderResponse.
func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse {
title := t.Title
description := t.Description
usedLanguage := ""
if language != "" {
if entry, ok := t.Translations[language]; ok {
title = entry.Title
description = entry.Description
usedLanguage = language
}
}
_ = language
mainDesc := firstNonEmpty(
strings.TrimSpace(t.ProcessingMetadata.EnrichmentData[enrichmentMainCPVDescriptionKey]),
@@ -349,7 +341,6 @@ func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse {
TenderID: t.TenderID,
ProjectName: strings.TrimSpace(t.ProjectName),
CreatedAt: t.CreatedAt,
Language: usedLanguage,
OverallSummary: strings.TrimSpace(t.AIOverallSummary),
}
internal/tender/handler.go
+2 -2
View File
@@ -552,7 +552,7 @@ func (h *TenderHandler) TriggerAISummarize(c echo.Context) error {
// TriggerAITranslate triggers on-demand AI translation for a tender
// @Summary Trigger AI translation
// @Description Trigger on-demand AI translation for tender title and description and store it in MongoDB
// @Description Trigger on-demand AI translation for tender title and description (cached in MinIO by the AI service)
// @Tags Admin-Tenders
// @Accept json
// @Produce json
@@ -622,7 +622,7 @@ func (h *TenderHandler) GetPublicAISummary(c echo.Context) error {
// TriggerPublicAITranslate triggers on-demand AI translation for a tender (public endpoint)
// @Summary Trigger AI translation (public)
// @Description Trigger on-demand AI translation for tender title and description and store it in MongoDB
// @Description Trigger on-demand AI translation for tender title and description (cached in MinIO by the AI service)
// @Tags Tenders
// @Accept json
// @Produce json
internal/tender/repository.go
+3 -18
View File
@@ -41,7 +41,6 @@ func collectionName() string {
return "tenders"
}
// SearchPageResult is the paginated outcome of a tender search/list query.
type SearchPageResult struct {
Items []Tender
@@ -544,29 +543,15 @@ func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, sk
return result.Items, result.TotalCount, nil
}
// GetUnTranslatedTenders retrieves tenders that don't have a translation for the given language yet.
// GetUnTranslatedTenders returns tenders eligible for translation backfill.
func (r *tenderRepository) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) {
translationKey := "translations." + language
metadataKey := "processing_metadata.translated_data." + language
_ = language
filter := bson.M{
"notice_publication_id": bson.M{"$ne": ""},
"contract_folder_id": bson.M{"$ne": ""},
"title": bson.M{"$ne": ""},
"description": bson.M{"$ne": ""},
"$and": []bson.M{
{
"$or": []bson.M{
{translationKey: bson.M{"$exists": false}},
{translationKey: nil},
},
},
{
"$or": []bson.M{
{metadataKey: bson.M{"$exists": false}},
{metadataKey: bson.M{"$ne": "done"}},
},
},
},
}
pagination := orm.Pagination{
internal/tender/service.go
+2 -37
View File
@@ -269,14 +269,11 @@ func (s *tenderService) Update(ctx context.Context, req UpdateTenderRequest) (*T
}
// Update fields if provided
translationSourceChanged := false
if req.Title != nil {
tender.Title = *req.Title
translationSourceChanged = true
}
if req.Description != nil {
tender.Description = *req.Description
translationSourceChanged = true
}
if req.Status != nil {
tender.Status = *req.Status
@@ -290,11 +287,6 @@ func (s *tenderService) Update(ctx context.Context, req UpdateTenderRequest) (*T
if req.TenderDeadline != nil {
tender.TenderDeadline = *req.TenderDeadline
}
if translationSourceChanged {
tender.Translations = nil
tender.ProcessingMetadata.TranslatedData = nil
tender.ProcessingMetadata.TranslatedAt = 0
}
if err := s.repository.Update(ctx, tender); err != nil {
s.logger.Error("Failed to update tender", map[string]interface{}{
@@ -1206,8 +1198,8 @@ func (s *tenderService) persistScrapedDocuments(ctx context.Context, t *Tender,
}
}
// TriggerAITranslate triggers on-demand AI translation for a tender and stores
// translated result per language in MongoDB for future reuse.
// TriggerAITranslate triggers on-demand AI translation for a tender. Results are
// read from MinIO when available, otherwise fetched from the AI service (which persists to MinIO).
func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error) {
if id == "" {
return nil, fmt.Errorf("tender ID cannot be empty")
@@ -1239,10 +1231,6 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
return nil, fmt.Errorf("tender has no contract folder ID")
}
if tr, ok := t.Translations[targetLanguage]; ok && strings.TrimSpace(tr.Title) != "" {
return s.buildAITranslateResponse(t, targetLanguage, tr.Title, tr.Description), nil
}
title, description, err := s.resolveTranslation(ctx, t, targetLanguage)
if err != nil {
s.logger.Error("AI translation failed", map[string]interface{}{
@@ -1254,11 +1242,6 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
return nil, fmt.Errorf("AI translation request failed: %w", err)
}
s.storeTranslation(t, targetLanguage, title, description)
if err := s.repository.Update(ctx, t); err != nil {
return nil, fmt.Errorf("failed to persist translation: %w", err)
}
return s.buildAITranslateResponse(t, targetLanguage, title, description), nil
}
@@ -1302,24 +1285,6 @@ func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, langu
return resp.TranslatedTitle, resp.TranslatedDescription, nil
}
func (s *tenderService) storeTranslation(t *Tender, language, title, description string) {
if t.Translations == nil {
t.Translations = make(map[string]TranslationEntry)
}
now := time.Now().Unix()
t.Translations[language] = TranslationEntry{
Title: title,
Description: description,
Language: language,
TranslatedAt: now,
}
if t.ProcessingMetadata.TranslatedData == nil {
t.ProcessingMetadata.TranslatedData = make(map[string]string)
}
t.ProcessingMetadata.TranslatedData[language] = "done"
t.ProcessingMetadata.TranslatedAt = now
}
func (s *tenderService) pickResponseLanguage(language *string) string {
if language != nil {
clean := strings.ToLower(strings.TrimSpace(*language))