From e9ec33f0d4e8584cf44607b43405ff5030c640a1 Mon Sep 17 00:00:00 2001 From: Mazyar Date: Tue, 19 May 2026 19:45:07 +0330 Subject: [PATCH] cache translation in minio instead of mongo --- cmd/worker/workers/translation.go | 73 ++++++++++++++----------------- internal/tender/form.go | 13 +----- internal/tender/handler.go | 4 +- internal/tender/repository.go | 21 ++------- internal/tender/service.go | 39 +---------------- 5 files changed, 41 insertions(+), 109 deletions(-) diff --git a/cmd/worker/workers/translation.go b/cmd/worker/workers/translation.go index 815b268..dc4fdcf 100644 --- a/cmd/worker/workers/translation.go +++ b/cmd/worker/workers/translation.go @@ -4,7 +4,6 @@ import ( "context" "errors" "strings" - "time" "tm/internal/tender" "tm/pkg/ai_summarizer" "tm/pkg/logger" @@ -12,7 +11,7 @@ import ( ) // TranslationWorker backfills missing tender translations using the AI pipeline -// and on-demand translate API, syncing results from MinIO when available. +// and on-demand translate API. Results are persisted by the AI service in MinIO only. type TranslationWorker struct { Mongo *mongo.ConnectionManager Logger logger.Logger @@ -95,20 +94,22 @@ func (w *TranslationWorker) Run() { } func (w *TranslationWorker) runForLanguage(language string) { + skip := 0 for { - tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, 0) + tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, skip) if err != nil { - w.Logger.Error("Failed to fetch untranslated tenders", map[string]interface{}{ + w.Logger.Error("Failed to fetch tenders for translation backfill", map[string]interface{}{ "target_language": language, "error": err.Error(), }) return } - w.Logger.Info("Fetched untranslated tenders batch", map[string]interface{}{ + w.Logger.Info("Fetched tenders for translation backfill", map[string]interface{}{ "target_language": language, "batch_count": len(tenders), "total_count": totalCount, + "skip": skip, }) if len(tenders) == 0 { @@ -118,6 +119,11 @@ func (w *TranslationWorker) runForLanguage(language string) { for _, t := range tenders { w.translateTender(&t, language) } + + skip += len(tenders) + if int64(skip) >= totalCount { + return + } } } @@ -131,8 +137,8 @@ func (w *TranslationWorker) translateTender(t *tender.Tender, language string) { }) return } - if existing, ok := t.Translations[language]; ok && strings.TrimSpace(existing.Title) != "" { - w.Logger.Debug("Skipping tender translation because target language already exists", map[string]interface{}{ + if w.hasTranslationInStorage(t, language) { + w.Logger.Debug("Skipping tender translation: already available in MinIO", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_id": t.NoticePublicationID, "target_language": language, @@ -143,23 +149,11 @@ func (w *TranslationWorker) translateTender(t *tender.Tender, language string) { title, description, source, err := w.resolveTranslation(t, language) if err != nil { w.Logger.Error("Failed to translate tender", map[string]interface{}{ - "tender_id": t.ID.Hex(), - "notice_id": t.NoticePublicationID, - "target_language": language, + "tender_id": t.ID.Hex(), + "notice_id": t.NoticePublicationID, + "target_language": language, "translation_source": source, - "translation_error": err.Error(), - }) - return - } - - persistTranslation(t, language, title, description) - if err := w.TenderRepo.Update(context.Background(), t); err != nil { - w.Logger.Error("Failed to persist translated tender", map[string]interface{}{ - "tender_id": t.ID.Hex(), - "notice_id": t.NoticePublicationID, - "target_language": language, - "translation_source": source, - "persistence_error": err.Error(), + "translation_error": err.Error(), }) return } @@ -169,9 +163,24 @@ func (w *TranslationWorker) translateTender(t *tender.Tender, language string) { "notice_id": t.NoticePublicationID, "target_language": language, "translation_source": source, + "title_len": len(title), + "description_len": len(description), }) } +func (w *TranslationWorker) hasTranslationInStorage(t *tender.Tender, language string) bool { + if w.AIStorage == nil { + return false + } + stored, err := w.AIStorage.GetTranslationFromStorage( + context.Background(), + t.ContractFolderID, + t.NoticePublicationID, + language, + ) + return err == nil && strings.TrimSpace(stored.Title) != "" +} + func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string) (title, description, source string, err error) { if w.AIStorage != nil { stored, storageErr := w.AIStorage.GetTranslationFromStorage( @@ -207,21 +216,3 @@ func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string } return resp.TranslatedTitle, resp.TranslatedDescription, "api", nil } - -func persistTranslation(t *tender.Tender, language, title, description string) { - if t.Translations == nil { - t.Translations = make(map[string]tender.TranslationEntry) - } - now := time.Now().Unix() - t.Translations[language] = tender.TranslationEntry{ - Title: title, - Description: description, - Language: language, - TranslatedAt: now, - } - if t.ProcessingMetadata.TranslatedData == nil { - t.ProcessingMetadata.TranslatedData = make(map[string]string) - } - t.ProcessingMetadata.TranslatedData[language] = "done" - t.ProcessingMetadata.TranslatedAt = now -} diff --git a/internal/tender/form.go b/internal/tender/form.go index c9fe943..4b66b36 100644 --- a/internal/tender/form.go +++ b/internal/tender/form.go @@ -255,19 +255,11 @@ func (t *Tender) ToResponse() *TenderResponse { return t.ToResponseWithLanguage("") } -// ToResponseWithLanguage converts a Tender entity to TenderResponse using a preferred language. -// When language is provided and translation exists, translated title/description are returned. +// ToResponseWithLanguage converts a Tender entity to TenderResponse. func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse { title := t.Title description := t.Description - usedLanguage := "" - if language != "" { - if entry, ok := t.Translations[language]; ok { - title = entry.Title - description = entry.Description - usedLanguage = language - } - } + _ = language mainDesc := firstNonEmpty( strings.TrimSpace(t.ProcessingMetadata.EnrichmentData[enrichmentMainCPVDescriptionKey]), @@ -349,7 +341,6 @@ func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse { TenderID: t.TenderID, ProjectName: strings.TrimSpace(t.ProjectName), CreatedAt: t.CreatedAt, - Language: usedLanguage, OverallSummary: strings.TrimSpace(t.AIOverallSummary), } diff --git a/internal/tender/handler.go b/internal/tender/handler.go index 90262e7..654c617 100644 --- a/internal/tender/handler.go +++ b/internal/tender/handler.go @@ -552,7 +552,7 @@ func (h *TenderHandler) TriggerAISummarize(c echo.Context) error { // TriggerAITranslate triggers on-demand AI translation for a tender // @Summary Trigger AI translation -// @Description Trigger on-demand AI translation for tender title and description and store it in MongoDB +// @Description Trigger on-demand AI translation for tender title and description (cached in MinIO by the AI service) // @Tags Admin-Tenders // @Accept json // @Produce json @@ -622,7 +622,7 @@ func (h *TenderHandler) GetPublicAISummary(c echo.Context) error { // TriggerPublicAITranslate triggers on-demand AI translation for a tender (public endpoint) // @Summary Trigger AI translation (public) -// @Description Trigger on-demand AI translation for tender title and description and store it in MongoDB +// @Description Trigger on-demand AI translation for tender title and description (cached in MinIO by the AI service) // @Tags Tenders // @Accept json // @Produce json diff --git a/internal/tender/repository.go b/internal/tender/repository.go index a76dd83..8dcf1b5 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -41,7 +41,6 @@ func collectionName() string { return "tenders" } - // SearchPageResult is the paginated outcome of a tender search/list query. type SearchPageResult struct { Items []Tender @@ -544,29 +543,15 @@ func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, sk return result.Items, result.TotalCount, nil } -// GetUnTranslatedTenders retrieves tenders that don't have a translation for the given language yet. +// GetUnTranslatedTenders returns tenders eligible for translation backfill. func (r *tenderRepository) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) { - translationKey := "translations." + language - metadataKey := "processing_metadata.translated_data." + language + _ = language filter := bson.M{ "notice_publication_id": bson.M{"$ne": ""}, + "contract_folder_id": bson.M{"$ne": ""}, "title": bson.M{"$ne": ""}, "description": bson.M{"$ne": ""}, - "$and": []bson.M{ - { - "$or": []bson.M{ - {translationKey: bson.M{"$exists": false}}, - {translationKey: nil}, - }, - }, - { - "$or": []bson.M{ - {metadataKey: bson.M{"$exists": false}}, - {metadataKey: bson.M{"$ne": "done"}}, - }, - }, - }, } pagination := orm.Pagination{ diff --git a/internal/tender/service.go b/internal/tender/service.go index a70d5dd..9bf1517 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -269,14 +269,11 @@ func (s *tenderService) Update(ctx context.Context, req UpdateTenderRequest) (*T } // Update fields if provided - translationSourceChanged := false if req.Title != nil { tender.Title = *req.Title - translationSourceChanged = true } if req.Description != nil { tender.Description = *req.Description - translationSourceChanged = true } if req.Status != nil { tender.Status = *req.Status @@ -290,11 +287,6 @@ func (s *tenderService) Update(ctx context.Context, req UpdateTenderRequest) (*T if req.TenderDeadline != nil { tender.TenderDeadline = *req.TenderDeadline } - if translationSourceChanged { - tender.Translations = nil - tender.ProcessingMetadata.TranslatedData = nil - tender.ProcessingMetadata.TranslatedAt = 0 - } if err := s.repository.Update(ctx, tender); err != nil { s.logger.Error("Failed to update tender", map[string]interface{}{ @@ -1206,8 +1198,8 @@ func (s *tenderService) persistScrapedDocuments(ctx context.Context, t *Tender, } } -// TriggerAITranslate triggers on-demand AI translation for a tender and stores -// translated result per language in MongoDB for future reuse. +// TriggerAITranslate triggers on-demand AI translation for a tender. Results are +// read from MinIO when available, otherwise fetched from the AI service (which persists to MinIO). func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error) { if id == "" { return nil, fmt.Errorf("tender ID cannot be empty") @@ -1239,10 +1231,6 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str return nil, fmt.Errorf("tender has no contract folder ID") } - if tr, ok := t.Translations[targetLanguage]; ok && strings.TrimSpace(tr.Title) != "" { - return s.buildAITranslateResponse(t, targetLanguage, tr.Title, tr.Description), nil - } - title, description, err := s.resolveTranslation(ctx, t, targetLanguage) if err != nil { s.logger.Error("AI translation failed", map[string]interface{}{ @@ -1254,11 +1242,6 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str return nil, fmt.Errorf("AI translation request failed: %w", err) } - s.storeTranslation(t, targetLanguage, title, description) - if err := s.repository.Update(ctx, t); err != nil { - return nil, fmt.Errorf("failed to persist translation: %w", err) - } - return s.buildAITranslateResponse(t, targetLanguage, title, description), nil } @@ -1302,24 +1285,6 @@ func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, langu return resp.TranslatedTitle, resp.TranslatedDescription, nil } -func (s *tenderService) storeTranslation(t *Tender, language, title, description string) { - if t.Translations == nil { - t.Translations = make(map[string]TranslationEntry) - } - now := time.Now().Unix() - t.Translations[language] = TranslationEntry{ - Title: title, - Description: description, - Language: language, - TranslatedAt: now, - } - if t.ProcessingMetadata.TranslatedData == nil { - t.ProcessingMetadata.TranslatedData = make(map[string]string) - } - t.ProcessingMetadata.TranslatedData[language] = "done" - t.ProcessingMetadata.TranslatedAt = now -} - func (s *tenderService) pickResponseLanguage(language *string) string { if language != nil { clean := strings.ToLower(strings.TrimSpace(*language))