mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

AI translate refactor

Browse Source
This commit is contained in:
Mazyar
2026-05-16 12:56:10 +03:30
parent ca490f3acf
commit 6701428b09
10 changed files with 510 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/ai_summarizer/client.go
+80 -11
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"net/http"
"strings"
"time"
"tm/pkg/logger"
@@ -93,8 +94,9 @@ func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody Translate
for attempt := 0; attempt <= c.config.APIRetryCount; attempt++ {
if attempt > 0 {
c.logger.Warn("Retrying AI translate request", map[string]interface{}{
"notice_id": reqBody.NoticeID,
"attempt": attempt,
"notice_publication_id": reqBody.NoticePublicationID,
"language": reqBody.Language,
"attempt": attempt,
})
select {
case <-ctx.Done():
@@ -107,9 +109,10 @@ func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody Translate
if err != nil {
lastErr = err
c.logger.Error("AI translate request failed", map[string]interface{}{
"notice_id": reqBody.NoticeID,
"attempt": attempt,
"error": err.Error(),
"notice_publication_id": reqBody.NoticePublicationID,
"language": reqBody.Language,
"attempt": attempt,
"error": err.Error(),
})
continue
}
@@ -120,6 +123,50 @@ func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody Translate
return nil, fmt.Errorf("ai_summarizer: all %d translation attempts failed, last error: %w", c.config.APIRetryCount+1, lastErr)
}
// TriggerPipelineTranslate calls POST /pipeline/translate to enqueue batch translation
// for the given target languages.
func (c *Client) TriggerPipelineTranslate(ctx context.Context, languages []string) (*PipelineTranslateResponse, error) {
reqBody := PipelineTranslateRequest{Languages: languages}
jsonBody, err := json.Marshal(reqBody)
if err != nil {
return nil, fmt.Errorf("ai_summarizer: failed to marshal pipeline translate request: %w", err)
}
url := c.config.APIBaseURL + "/pipeline/translate"
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
if err != nil {
return nil, fmt.Errorf("ai_summarizer: failed to create pipeline translate request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
httpResp, err := c.httpClient.Do(req)
if err != nil {
return nil, fmt.Errorf("ai_summarizer: pipeline translate request error: %w", err)
}
defer httpResp.Body.Close()
bodyBytes, err := io.ReadAll(httpResp.Body)
if err != nil {
return nil, fmt.Errorf("ai_summarizer: failed to read pipeline translate response: %w", err)
}
if httpResp.StatusCode >= 400 {
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
}
var result PipelineTranslateResponse
if err := json.Unmarshal(bodyBytes, &result); err != nil {
return nil, fmt.Errorf("ai_summarizer: failed to decode pipeline translate response: %w", err)
}
c.logger.Info("Pipeline translate triggered", map[string]interface{}{
"status": result.Status,
"languages": result.Languages,
})
return &result, nil
}
// doPost performs a single POST request and parses the response.
func (c *Client) doPost(ctx context.Context, url string, jsonBody []byte) (*SummarizeResponse, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
@@ -193,15 +240,37 @@ func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byt
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
}
var result TranslateResponse
if err := json.Unmarshal(bodyBytes, &result); err != nil {
return nil, fmt.Errorf("ai_summarizer: failed to decode translation response JSON: %w", err)
result, err := decodeTranslateResponse(bodyBytes)
if err != nil {
return nil, err
}
c.logger.Info("AI translate request succeeded", map[string]interface{}{
"notice_id": result.NoticeID,
"language": result.Language,
"notice_publication_id": result.NoticePublicationID,
"language": result.Language,
})
return &result, nil
return result, nil
}
type translateAPIEnvelope struct {
Success bool `json:"success"`
Message string `json:"message"`
Data TranslateResponse `json:"data"`
}
func decodeTranslateResponse(bodyBytes []byte) (*TranslateResponse, error) {
var envelope translateAPIEnvelope
if err := json.Unmarshal(bodyBytes, &envelope); err == nil && envelope.Success && envelope.Data.Language != "" {
return &envelope.Data, nil
}
var direct TranslateResponse
if err := json.Unmarshal(bodyBytes, &direct); err != nil {
return nil, fmt.Errorf("ai_summarizer: failed to decode translation response JSON: %w", err)
}
if strings.TrimSpace(direct.Language) == "" {
return nil, fmt.Errorf("ai_summarizer: translation response missing language")
}
return &direct, nil
}
pkg/ai_summarizer/entities.go
+86 -13
View File
@@ -44,29 +44,72 @@ type StoredDocument struct {
// TranslateRequest represents the request payload sent to the AI service
// POST /ai/translate endpoint.
type TranslateRequest struct {
NoticeID string `json:"notice_id"` // required (TED-style notice key)
ContractFolderID string `json:"contract_folder_id"` // required by AI service (procedure / MinIO prefix)
NoticePublicationID string `json:"notice_publication_id"` // required by AI service
Title string `json:"title"` // required
Description string `json:"description"` // required
Language string `json:"language,omitempty"` // optional target language override
ContractFolderID string `json:"contract_folder_id"`
NoticePublicationID string `json:"notice_publication_id"`
Title string `json:"title"`
Description string `json:"description"`
Language string `json:"language"`
}
// TranslateResponse represents the response from the AI translation API.
// TranslateResponse represents the translation payload from POST /ai/translate.
type TranslateResponse struct {
NoticeID string `json:"notice_id"`
ContractFolderID string `json:"contract_folder_id"`
NoticePublicationID string `json:"notice_publication_id"`
Language string `json:"language"`
TranslatedTitle string `json:"translated_title"`
TranslatedDescription string `json:"translated_description"`
Language string `json:"language"`
}
// PipelineTranslateRequest represents the request payload for POST /pipeline/translate.
type PipelineTranslateRequest struct {
Languages []string `json:"languages"`
}
// PipelineTranslateResponse represents the response from POST /pipeline/translate.
type PipelineTranslateResponse struct {
Status string `json:"status"`
Languages []string `json:"languages"`
}
// StoredTranslation is one language entry inside tender.json translations map.
type StoredTranslation struct {
Title string `json:"title"`
Description string `json:"description"`
}
// TranslationStatus tracks which languages the async pipeline has finished.
type TranslationStatus struct {
Done []string `json:"done"`
DoneUpper []string `json:"Done"`
}
func (s TranslationStatus) languagesDone() []string {
if len(s.Done) > 0 {
return s.Done
}
return s.DoneUpper
}
func languageInDone(done []string, language string) bool {
lang := strings.ToLower(strings.TrimSpace(language))
for _, item := range done {
if strings.ToLower(strings.TrimSpace(item)) == lang {
return true
}
}
return false
}
// TenderJSON represents the structure of the <notice_id>/tender.json file
// stored in the AI service's MinIO bucket.
type TenderJSON struct {
OverallSummary *string `json:"overall_summary"`
OverallSummaryCamel *string `json:"overallSummary"`
SummarizeStatus SummarizeStatus `json:"summarize_status"`
SummarizeStatusCamel SummarizeStatus `json:"summarizeStatus"`
OverallSummary *string `json:"overall_summary"`
OverallSummaryCamel *string `json:"overallSummary"`
SummarizeStatus SummarizeStatus `json:"summarize_status"`
SummarizeStatusCamel SummarizeStatus `json:"summarizeStatus"`
Translations map[string]StoredTranslation `json:"translations"`
TranslationStatus TranslationStatus `json:"translation_status"`
TranslationStatusCamel TranslationStatus `json:"translationStatus"`
}
// resolved returns whether summarization is marked done and the best-effort
@@ -90,3 +133,33 @@ type SummarizeStatus struct {
func (s SummarizeStatus) isDone() bool {
return s.Done || s.DoneUpper
}
// translationsDone returns languages marked complete by the translation pipeline.
func (t *TenderJSON) translationsDone() []string {
done := t.TranslationStatus.languagesDone()
if len(done) > 0 {
return done
}
return t.TranslationStatusCamel.languagesDone()
}
// translationForLanguage returns stored translation text when the pipeline marked
// the language done and non-empty title exists.
func (t *TenderJSON) translationForLanguage(language string) (StoredTranslation, bool) {
if t == nil || !languageInDone(t.translationsDone(), language) {
return StoredTranslation{}, false
}
lang := strings.ToLower(strings.TrimSpace(language))
if t.Translations != nil {
for key, entry := range t.Translations {
if strings.ToLower(strings.TrimSpace(key)) != lang {
continue
}
if strings.TrimSpace(entry.Title) == "" {
continue
}
return entry, true
}
}
return StoredTranslation{}, false
}
pkg/ai_summarizer/errors.go
+8
View File
@@ -12,6 +12,14 @@ var (
// overall_summary is empty or null.
ErrNoOverallSummary = errors.New("ai_summarizer: overall_summary is empty")
// ErrTranslationNotReady is returned when tender.json exists but the
// requested language is not listed in translation_status.done yet.
ErrTranslationNotReady = errors.New("ai_summarizer: translation not ready yet")
// ErrNoTranslation is returned when translation_status marks a language done
// but translations[language] is missing or empty.
ErrNoTranslation = errors.New("ai_summarizer: translation is empty")
// ErrObjectNotFound is returned when the tender.json object does not
// exist in the MinIO bucket (HTTP 404 / NoSuchKey).
ErrObjectNotFound = errors.New("ai_summarizer: tender.json not found in storage")
pkg/ai_summarizer/storage.go
+55 -30
View File
@@ -295,9 +295,46 @@ func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, contractFolde
return s.getSummaryFromObjectKey(ctx, prefix+"tender.json", contractFolderID, noticePublicationID)
}
// GetTranslationFromStorage reads tender.json and returns the translation for a language
// when translation_status.done includes that language.
func (s *StorageClient) GetTranslationFromStorage(ctx context.Context, contractFolderID, noticePublicationID, language string) (StoredTranslation, error) {
if strings.TrimSpace(contractFolderID) == "" {
return StoredTranslation{}, fmt.Errorf("ai_summarizer: contractFolderID is required")
}
if strings.TrimSpace(noticePublicationID) == "" {
return StoredTranslation{}, fmt.Errorf("ai_summarizer: noticePublicationID is required")
}
if strings.TrimSpace(language) == "" {
return StoredTranslation{}, fmt.Errorf("ai_summarizer: language is required")
}
prefix, err := s.resolveNoticeStoragePrefix(ctx, contractFolderID, noticePublicationID)
if err != nil {
return StoredTranslation{}, err
}
return s.getTranslationFromObjectKey(ctx, prefix+"tender.json", language)
}
// getTranslationFromObjectKey reads tender.json at objectKey and returns translation for language.
func (s *StorageClient) getTranslationFromObjectKey(ctx context.Context, objectKey, language string) (StoredTranslation, error) {
tenderJSON, err := s.readTenderJSONAtKey(ctx, objectKey)
if err != nil {
return StoredTranslation{}, err
}
if !languageInDone(tenderJSON.translationsDone(), language) {
return StoredTranslation{}, ErrTranslationNotReady
}
entry, ok := tenderJSON.translationForLanguage(language)
if !ok {
return StoredTranslation{}, ErrNoTranslation
}
return entry, nil
}
// getSummaryFromObjectKey reads tender.json at objectKey and returns overall_summary when ready.
// contractFolderID and noticePublicationID are used for logging only (may be empty when unknown).
func (s *StorageClient) getSummaryFromObjectKey(ctx context.Context, objectKey, contractFolderID, noticePublicationID string) (string, error) {
func (s *StorageClient) readTenderJSONAtKey(ctx context.Context, objectKey string) (*TenderJSON, error) {
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectKey, minio.GetObjectOptions{})
if err != nil {
if isMinioNotFound(err) {
@@ -305,13 +342,13 @@ func (s *StorageClient) getSummaryFromObjectKey(ctx context.Context, objectKey,
"bucket": s.config.MinioBucket,
"object_key": objectKey,
})
return "", ErrObjectNotFound
return nil, ErrObjectNotFound
}
var minioErr minio.ErrorResponse
if errors.As(err, &minioErr) && minioErr.StatusCode == http.StatusForbidden {
return "", fmt.Errorf("ai_summarizer: access denied to bucket %s: %w", s.config.MinioBucket, err)
return nil, fmt.Errorf("ai_summarizer: access denied to bucket %s: %w", s.config.MinioBucket, err)
}
return "", fmt.Errorf("ai_summarizer: failed to get object: %w", err)
return nil, fmt.Errorf("ai_summarizer: failed to get object: %w", err)
}
defer object.Close()
@@ -322,14 +359,22 @@ func (s *StorageClient) getSummaryFromObjectKey(ctx context.Context, objectKey,
"bucket": s.config.MinioBucket,
"object_key": objectKey,
})
return "", ErrObjectNotFound
return nil, ErrObjectNotFound
}
return "", fmt.Errorf("ai_summarizer: failed to read tender.json: %w", err)
return nil, fmt.Errorf("ai_summarizer: failed to read tender.json: %w", err)
}
var tenderJSON TenderJSON
if err := json.Unmarshal(data, &tenderJSON); err != nil {
return "", fmt.Errorf("ai_summarizer: failed to parse tender.json: %w", err)
return nil, fmt.Errorf("ai_summarizer: failed to parse tender.json: %w", err)
}
return &tenderJSON, nil
}
func (s *StorageClient) getSummaryFromObjectKey(ctx context.Context, objectKey, contractFolderID, noticePublicationID string) (string, error) {
tenderJSON, err := s.readTenderJSONAtKey(ctx, objectKey)
if err != nil {
return "", err
}
done, overall := tenderJSON.resolved()
@@ -368,32 +413,12 @@ func (s *StorageClient) IsSummaryReady(ctx context.Context, contractFolderID, no
}
objectPath := prefix + "tender.json"
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectPath, minio.GetObjectOptions{})
tenderJSON, err := s.readTenderJSONAtKey(ctx, objectPath)
if err != nil {
if isMinioNotFound(err) {
if errors.Is(err, ErrObjectNotFound) {
return false, nil
}
return false, fmt.Errorf("ai_summarizer: failed to check summary readiness: %w", err)
}
data, readErr := io.ReadAll(object)
closeErr := object.Close()
if readErr != nil {
if isMinioNotFound(readErr) {
return false, nil
}
return false, fmt.Errorf("ai_summarizer: failed to read tender.json: %w", readErr)
}
if closeErr != nil {
s.logger.Warn("Failed to close tender.json object after readiness check", map[string]interface{}{
"object_key": objectPath,
"error": closeErr.Error(),
})
}
var tenderJSON TenderJSON
if err := json.Unmarshal(data, &tenderJSON); err != nil {
return false, fmt.Errorf("ai_summarizer: failed to parse tender.json: %w", err)
return false, err
}
done, _ := tenderJSON.resolved()