AI translate refactor
This commit is contained in:
@@ -2,6 +2,7 @@ package bootstrap
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"tm/cmd/worker/workers"
|
"tm/cmd/worker/workers"
|
||||||
"tm/internal/notice"
|
"tm/internal/notice"
|
||||||
@@ -77,7 +78,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Init Worker
|
// Init Worker
|
||||||
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client) *schedule.CronScheduler {
|
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient) *schedule.CronScheduler {
|
||||||
// Debug: Log worker config
|
// Debug: Log worker config
|
||||||
appLogger.Info("Worker configuration", map[string]interface{}{
|
appLogger.Info("Worker configuration", map[string]interface{}{
|
||||||
"worker_interval": config.Worker.Interval,
|
"worker_interval": config.Worker.Interval,
|
||||||
@@ -192,9 +193,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
|||||||
Expr: workerInterval,
|
Expr: workerInterval,
|
||||||
})
|
})
|
||||||
|
|
||||||
// Initialize translation backfill worker for default language
|
// Initialize translation backfill worker for configured languages
|
||||||
if aiClient != nil {
|
if aiClient != nil {
|
||||||
targetLanguage := config.AISummarizer.DefaultLanguage
|
targetLanguages := parseTranslationLanguages(config.AISummarizer)
|
||||||
scheduler.AddJob(schedule.Job{
|
scheduler.AddJob(schedule.Job{
|
||||||
Name: "Tender Translation Worker Job",
|
Name: "Tender Translation Worker Job",
|
||||||
Func: func() {
|
Func: func() {
|
||||||
@@ -203,7 +204,8 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
|||||||
appLogger,
|
appLogger,
|
||||||
tenderRepo,
|
tenderRepo,
|
||||||
aiClient,
|
aiClient,
|
||||||
targetLanguage,
|
aiStorage,
|
||||||
|
targetLanguages,
|
||||||
config.Worker.TranslationBatchSize,
|
config.Worker.TranslationBatchSize,
|
||||||
)
|
)
|
||||||
worker.Run()
|
worker.Run()
|
||||||
@@ -211,9 +213,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
|||||||
Expr: config.Worker.TranslationInterval,
|
Expr: config.Worker.TranslationInterval,
|
||||||
})
|
})
|
||||||
appLogger.Info("Scheduled tender translation worker", map[string]interface{}{
|
appLogger.Info("Scheduled tender translation worker", map[string]interface{}{
|
||||||
"target_language": targetLanguage,
|
"target_languages": targetLanguages,
|
||||||
"interval": config.Worker.TranslationInterval,
|
"interval": config.Worker.TranslationInterval,
|
||||||
"batch_size": config.Worker.TranslationBatchSize,
|
"batch_size": config.Worker.TranslationBatchSize,
|
||||||
})
|
})
|
||||||
} else {
|
} else {
|
||||||
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
|
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
|
||||||
@@ -274,6 +276,67 @@ func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summ
|
|||||||
return client
|
return client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InitAISummarizerStorage initializes the AI summarizer MinIO storage client for workers.
|
||||||
|
func InitAISummarizerStorage(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.StorageClient {
|
||||||
|
if conf.MinioEndpoint == "" {
|
||||||
|
log.Warn("AI Summarizer MinIO endpoint not configured, translation sync from storage will be unavailable", map[string]interface{}{})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := &ai_summarizer.Config{
|
||||||
|
APIBaseURL: conf.APIBaseURL,
|
||||||
|
APITimeout: conf.APITimeout,
|
||||||
|
APIRetryCount: conf.APIRetryCount,
|
||||||
|
APIRetryDelay: conf.APIRetryDelay,
|
||||||
|
MinioEndpoint: conf.MinioEndpoint,
|
||||||
|
MinioAccessKey: conf.MinioAccessKey,
|
||||||
|
MinioSecretKey: conf.MinioSecretKey,
|
||||||
|
MinioUseSSL: conf.MinioUseSSL,
|
||||||
|
MinioRegion: conf.MinioRegion,
|
||||||
|
MinioBucket: conf.MinioBucket,
|
||||||
|
}
|
||||||
|
|
||||||
|
storage, err := ai_summarizer.NewStorageClient(cfg, log)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Failed to initialize AI Summarizer storage client for worker", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("AI Summarizer storage client initialized for worker", map[string]interface{}{
|
||||||
|
"endpoint": conf.MinioEndpoint,
|
||||||
|
"bucket": conf.MinioBucket,
|
||||||
|
})
|
||||||
|
|
||||||
|
return storage
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseTranslationLanguages(conf AISummarizerConfig) []string {
|
||||||
|
raw := strings.TrimSpace(conf.TranslationLanguages)
|
||||||
|
if raw == "" {
|
||||||
|
raw = conf.DefaultLanguage
|
||||||
|
}
|
||||||
|
parts := strings.Split(raw, ",")
|
||||||
|
langs := make([]string, 0, len(parts))
|
||||||
|
seen := make(map[string]struct{}, len(parts))
|
||||||
|
for _, part := range parts {
|
||||||
|
lang := strings.ToLower(strings.TrimSpace(part))
|
||||||
|
if lang == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, dup := seen[lang]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[lang] = struct{}{}
|
||||||
|
langs = append(langs, lang)
|
||||||
|
}
|
||||||
|
if len(langs) == 0 {
|
||||||
|
return []string{"en"}
|
||||||
|
}
|
||||||
|
return langs
|
||||||
|
}
|
||||||
|
|
||||||
// Init Notification Service
|
// Init Notification Service
|
||||||
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
|
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
|
||||||
notificationConfig := ¬ification.Config{
|
notificationConfig := ¬ification.Config{
|
||||||
|
|||||||
@@ -50,9 +50,16 @@ type WorkerConfig struct {
|
|||||||
|
|
||||||
// AISummarizerConfig holds configuration for the external AI summarizer service.
|
// AISummarizerConfig holds configuration for the external AI summarizer service.
|
||||||
type AISummarizerConfig struct {
|
type AISummarizerConfig struct {
|
||||||
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
|
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
|
||||||
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
|
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
|
||||||
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
|
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
|
||||||
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
|
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
|
||||||
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
|
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
|
||||||
|
TranslationLanguages string `env:"AI_SUMMARIZER_TRANSLATION_LANGUAGES" envDefault:"en"`
|
||||||
|
MinioEndpoint string `env:"AI_SUMMARIZER_MINIO_ENDPOINT" envDefault:""`
|
||||||
|
MinioAccessKey string `env:"AI_SUMMARIZER_MINIO_ACCESS_KEY" envDefault:""`
|
||||||
|
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
|
||||||
|
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
|
||||||
|
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
|
||||||
|
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens-documents"`
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-1
@@ -45,9 +45,10 @@ func main() {
|
|||||||
|
|
||||||
// Initialize AI summarizer client (translation + document summarization workers)
|
// Initialize AI summarizer client (translation + document summarization workers)
|
||||||
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, appLogger)
|
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, appLogger)
|
||||||
|
aiSummarizerStorage := bootstrap.InitAISummarizerStorage(config.AISummarizer, appLogger)
|
||||||
|
|
||||||
// Initialize Worker
|
// Initialize Worker
|
||||||
scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, minioService, aiSummarizerClient)
|
scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, minioService, aiSummarizerClient, aiSummarizerStorage)
|
||||||
|
|
||||||
// Set up signal handling for graceful shutdown
|
// Set up signal handling for graceful shutdown
|
||||||
signalChan := make(chan os.Signal, 1)
|
signalChan := make(chan os.Signal, 1)
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package workers
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"tm/internal/tender"
|
"tm/internal/tender"
|
||||||
@@ -10,32 +11,56 @@ import (
|
|||||||
"tm/pkg/mongo"
|
"tm/pkg/mongo"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TranslationWorker backfills missing tender translations using AI summarizer API.
|
// TranslationWorker backfills missing tender translations using the AI pipeline
|
||||||
|
// and on-demand translate API, syncing results from MinIO when available.
|
||||||
type TranslationWorker struct {
|
type TranslationWorker struct {
|
||||||
Mongo *mongo.ConnectionManager
|
Mongo *mongo.ConnectionManager
|
||||||
Logger logger.Logger
|
Logger logger.Logger
|
||||||
TenderRepo tender.TenderRepository
|
TenderRepo tender.TenderRepository
|
||||||
AIClient *ai_summarizer.Client
|
AIClient *ai_summarizer.Client
|
||||||
TargetLanguage string
|
AIStorage *ai_summarizer.StorageClient
|
||||||
BatchSize int
|
TargetLanguages []string
|
||||||
|
BatchSize int
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewTranslationWorker creates a translation worker for a target language.
|
// NewTranslationWorker creates a translation worker for the given target languages.
|
||||||
func NewTranslationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, aiClient *ai_summarizer.Client, targetLanguage string, batchSize int) *TranslationWorker {
|
func NewTranslationWorker(
|
||||||
if strings.TrimSpace(targetLanguage) == "" {
|
mongo *mongo.ConnectionManager,
|
||||||
targetLanguage = "en"
|
logger logger.Logger,
|
||||||
|
tenderRepo tender.TenderRepository,
|
||||||
|
aiClient *ai_summarizer.Client,
|
||||||
|
aiStorage *ai_summarizer.StorageClient,
|
||||||
|
targetLanguages []string,
|
||||||
|
batchSize int,
|
||||||
|
) *TranslationWorker {
|
||||||
|
langs := make([]string, 0, len(targetLanguages))
|
||||||
|
seen := make(map[string]struct{}, len(targetLanguages))
|
||||||
|
for _, lang := range targetLanguages {
|
||||||
|
clean := strings.ToLower(strings.TrimSpace(lang))
|
||||||
|
if clean == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, dup := seen[clean]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[clean] = struct{}{}
|
||||||
|
langs = append(langs, clean)
|
||||||
|
}
|
||||||
|
if len(langs) == 0 {
|
||||||
|
langs = []string{"en"}
|
||||||
}
|
}
|
||||||
if batchSize <= 0 {
|
if batchSize <= 0 {
|
||||||
batchSize = 20
|
batchSize = 20
|
||||||
}
|
}
|
||||||
|
|
||||||
return &TranslationWorker{
|
return &TranslationWorker{
|
||||||
Mongo: mongo,
|
Mongo: mongo,
|
||||||
Logger: logger,
|
Logger: logger,
|
||||||
TenderRepo: tenderRepo,
|
TenderRepo: tenderRepo,
|
||||||
AIClient: aiClient,
|
AIClient: aiClient,
|
||||||
TargetLanguage: strings.ToLower(strings.TrimSpace(targetLanguage)),
|
AIStorage: aiStorage,
|
||||||
BatchSize: batchSize,
|
TargetLanguages: langs,
|
||||||
|
BatchSize: batchSize,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -43,48 +68,60 @@ func NewTranslationWorker(mongo *mongo.ConnectionManager, logger logger.Logger,
|
|||||||
func (w *TranslationWorker) Run() {
|
func (w *TranslationWorker) Run() {
|
||||||
if w.AIClient == nil {
|
if w.AIClient == nil {
|
||||||
w.Logger.Warn("Translation worker skipped: AI client is not configured", map[string]interface{}{
|
w.Logger.Warn("Translation worker skipped: AI client is not configured", map[string]interface{}{
|
||||||
"target_language": w.TargetLanguage,
|
"target_languages": w.TargetLanguages,
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Logger.Info("Translation worker started", map[string]interface{}{
|
w.Logger.Info("Translation worker started", map[string]interface{}{
|
||||||
"target_language": w.TargetLanguage,
|
"target_languages": w.TargetLanguages,
|
||||||
"batch_size": w.BatchSize,
|
"batch_size": w.BatchSize,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
|
||||||
|
w.Logger.Warn("Failed to trigger AI pipeline translate (continuing with per-tender sync)", map[string]interface{}{
|
||||||
|
"target_languages": w.TargetLanguages,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, language := range w.TargetLanguages {
|
||||||
|
w.runForLanguage(language)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Logger.Info("Translation worker completed", map[string]interface{}{
|
||||||
|
"target_languages": w.TargetLanguages,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *TranslationWorker) runForLanguage(language string) {
|
||||||
for {
|
for {
|
||||||
// Always use skip=0 because successfully translated tenders are excluded by filter.
|
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, 0)
|
||||||
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), w.TargetLanguage, w.BatchSize, 0)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.Logger.Error("Failed to fetch untranslated tenders", map[string]interface{}{
|
w.Logger.Error("Failed to fetch untranslated tenders", map[string]interface{}{
|
||||||
"target_language": w.TargetLanguage,
|
"target_language": language,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
break
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Logger.Info("Fetched untranslated tenders batch", map[string]interface{}{
|
w.Logger.Info("Fetched untranslated tenders batch", map[string]interface{}{
|
||||||
"target_language": w.TargetLanguage,
|
"target_language": language,
|
||||||
"batch_count": len(tenders),
|
"batch_count": len(tenders),
|
||||||
"total_count": totalCount,
|
"total_count": totalCount,
|
||||||
})
|
})
|
||||||
|
|
||||||
if len(tenders) == 0 {
|
if len(tenders) == 0 {
|
||||||
break
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, t := range tenders {
|
for _, t := range tenders {
|
||||||
w.translateTender(&t)
|
w.translateTender(&t, language)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Logger.Info("Translation worker completed", map[string]interface{}{
|
|
||||||
"target_language": w.TargetLanguage,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *TranslationWorker) translateTender(t *tender.Tender) {
|
func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
|
||||||
if t.NoticePublicationID == "" {
|
if t.NoticePublicationID == "" {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -94,64 +131,97 @@ func (w *TranslationWorker) translateTender(t *tender.Tender) {
|
|||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if existing, ok := t.Translations[w.TargetLanguage]; ok && strings.TrimSpace(existing.Title) != "" {
|
if existing, ok := t.Translations[language]; ok && strings.TrimSpace(existing.Title) != "" {
|
||||||
w.Logger.Debug("Skipping tender translation because target language already exists", map[string]interface{}{
|
w.Logger.Debug("Skipping tender translation because target language already exists", map[string]interface{}{
|
||||||
"tender_id": t.ID.Hex(),
|
"tender_id": t.ID.Hex(),
|
||||||
"notice_id": t.NoticePublicationID,
|
"notice_id": t.NoticePublicationID,
|
||||||
"target_language": w.TargetLanguage,
|
"target_language": language,
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := w.AIClient.FetchTranslationOnDemand(context.Background(), ai_summarizer.TranslateRequest{
|
title, description, source, err := w.resolveTranslation(t, language)
|
||||||
NoticeID: t.NoticePublicationID,
|
|
||||||
ContractFolderID: t.ContractFolderID,
|
|
||||||
NoticePublicationID: t.NoticePublicationID,
|
|
||||||
Title: t.Title,
|
|
||||||
Description: t.Description,
|
|
||||||
Language: w.TargetLanguage,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
w.Logger.Error("Failed to translate tender", map[string]interface{}{
|
w.Logger.Error("Failed to translate tender", map[string]interface{}{
|
||||||
"tender_id": t.ID.Hex(),
|
"tender_id": t.ID.Hex(),
|
||||||
"notice_id": t.NoticePublicationID,
|
"notice_id": t.NoticePublicationID,
|
||||||
"target_language": w.TargetLanguage,
|
"target_language": language,
|
||||||
|
"translation_source": source,
|
||||||
"translation_error": err.Error(),
|
"translation_error": err.Error(),
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if t.Translations == nil {
|
persistTranslation(t, language, title, description)
|
||||||
t.Translations = make(map[string]tender.TranslationEntry)
|
|
||||||
}
|
|
||||||
now := time.Now().Unix()
|
|
||||||
// Only update the target-language slot. Existing translations in other
|
|
||||||
// languages (e.g., "de", "fr") are preserved and never modified.
|
|
||||||
t.Translations[w.TargetLanguage] = tender.TranslationEntry{
|
|
||||||
Title: resp.TranslatedTitle,
|
|
||||||
Description: resp.TranslatedDescription,
|
|
||||||
Language: w.TargetLanguage,
|
|
||||||
TranslatedAt: now,
|
|
||||||
}
|
|
||||||
if t.ProcessingMetadata.TranslatedData == nil {
|
|
||||||
t.ProcessingMetadata.TranslatedData = make(map[string]string)
|
|
||||||
}
|
|
||||||
t.ProcessingMetadata.TranslatedData[w.TargetLanguage] = "done"
|
|
||||||
t.ProcessingMetadata.TranslatedAt = now
|
|
||||||
|
|
||||||
if err := w.TenderRepo.Update(context.Background(), t); err != nil {
|
if err := w.TenderRepo.Update(context.Background(), t); err != nil {
|
||||||
w.Logger.Error("Failed to persist translated tender", map[string]interface{}{
|
w.Logger.Error("Failed to persist translated tender", map[string]interface{}{
|
||||||
"tender_id": t.ID.Hex(),
|
"tender_id": t.ID.Hex(),
|
||||||
"notice_id": t.NoticePublicationID,
|
"notice_id": t.NoticePublicationID,
|
||||||
"target_language": w.TargetLanguage,
|
"target_language": language,
|
||||||
|
"translation_source": source,
|
||||||
"persistence_error": err.Error(),
|
"persistence_error": err.Error(),
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
w.Logger.Info("Tender translated successfully", map[string]interface{}{
|
w.Logger.Info("Tender translated successfully", map[string]interface{}{
|
||||||
"tender_id": t.ID.Hex(),
|
"tender_id": t.ID.Hex(),
|
||||||
"notice_id": t.NoticePublicationID,
|
"notice_id": t.NoticePublicationID,
|
||||||
"target_language": w.TargetLanguage,
|
"target_language": language,
|
||||||
|
"translation_source": source,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string) (title, description, source string, err error) {
|
||||||
|
if w.AIStorage != nil {
|
||||||
|
stored, storageErr := w.AIStorage.GetTranslationFromStorage(
|
||||||
|
context.Background(),
|
||||||
|
t.ContractFolderID,
|
||||||
|
t.NoticePublicationID,
|
||||||
|
language,
|
||||||
|
)
|
||||||
|
if storageErr == nil {
|
||||||
|
return stored.Title, stored.Description, "storage", nil
|
||||||
|
}
|
||||||
|
if !errors.Is(storageErr, ai_summarizer.ErrTranslationNotReady) &&
|
||||||
|
!errors.Is(storageErr, ai_summarizer.ErrObjectNotFound) &&
|
||||||
|
!errors.Is(storageErr, ai_summarizer.ErrNoTranslation) {
|
||||||
|
w.Logger.Debug("Translation not available from storage yet", map[string]interface{}{
|
||||||
|
"tender_id": t.ID.Hex(),
|
||||||
|
"notice_id": t.NoticePublicationID,
|
||||||
|
"target_language": language,
|
||||||
|
"error": storageErr.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, apiErr := w.AIClient.FetchTranslationOnDemand(context.Background(), ai_summarizer.TranslateRequest{
|
||||||
|
ContractFolderID: t.ContractFolderID,
|
||||||
|
NoticePublicationID: t.NoticePublicationID,
|
||||||
|
Title: t.Title,
|
||||||
|
Description: t.Description,
|
||||||
|
Language: language,
|
||||||
|
})
|
||||||
|
if apiErr != nil {
|
||||||
|
return "", "", "api", apiErr
|
||||||
|
}
|
||||||
|
return resp.TranslatedTitle, resp.TranslatedDescription, "api", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func persistTranslation(t *tender.Tender, language, title, description string) {
|
||||||
|
if t.Translations == nil {
|
||||||
|
t.Translations = make(map[string]tender.TranslationEntry)
|
||||||
|
}
|
||||||
|
now := time.Now().Unix()
|
||||||
|
t.Translations[language] = tender.TranslationEntry{
|
||||||
|
Title: title,
|
||||||
|
Description: description,
|
||||||
|
Language: language,
|
||||||
|
TranslatedAt: now,
|
||||||
|
}
|
||||||
|
if t.ProcessingMetadata.TranslatedData == nil {
|
||||||
|
t.ProcessingMetadata.TranslatedData = make(map[string]string)
|
||||||
|
}
|
||||||
|
t.ProcessingMetadata.TranslatedData[language] = "done"
|
||||||
|
t.ProcessingMetadata.TranslatedAt = now
|
||||||
|
}
|
||||||
|
|||||||
@@ -517,7 +517,8 @@ type AITranslateRequest struct {
|
|||||||
|
|
||||||
// AITranslateResponse represents the response for AI translation trigger.
|
// AITranslateResponse represents the response for AI translation trigger.
|
||||||
type AITranslateResponse struct {
|
type AITranslateResponse struct {
|
||||||
NoticeID string `json:"notice_id"`
|
ContractFolderID string `json:"contract_folder_id"`
|
||||||
|
NoticePublicationID string `json:"notice_publication_id"`
|
||||||
Language string `json:"language"`
|
Language string `json:"language"`
|
||||||
TranslatedTitle string `json:"translated_title"`
|
TranslatedTitle string `json:"translated_title"`
|
||||||
TranslatedDescription string `json:"translated_description"`
|
TranslatedDescription string `json:"translated_description"`
|
||||||
|
|||||||
+60
-33
@@ -21,6 +21,7 @@ import (
|
|||||||
type AISummarizerClient interface {
|
type AISummarizerClient interface {
|
||||||
FetchSummaryOnDemand(ctx context.Context, reqBody ai_summarizer.SummarizeRequest) (*ai_summarizer.SummarizeResponse, error)
|
FetchSummaryOnDemand(ctx context.Context, reqBody ai_summarizer.SummarizeRequest) (*ai_summarizer.SummarizeResponse, error)
|
||||||
FetchTranslationOnDemand(ctx context.Context, reqBody ai_summarizer.TranslateRequest) (*ai_summarizer.TranslateResponse, error)
|
FetchTranslationOnDemand(ctx context.Context, reqBody ai_summarizer.TranslateRequest) (*ai_summarizer.TranslateResponse, error)
|
||||||
|
TriggerPipelineTranslate(ctx context.Context, languages []string) (*ai_summarizer.PipelineTranslateResponse, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage.
|
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage.
|
||||||
@@ -28,6 +29,7 @@ type AISummarizerClient interface {
|
|||||||
// PROC_<contractFolderID>/<noticePublicationID>/ in the shared bucket.
|
// PROC_<contractFolderID>/<noticePublicationID>/ in the shared bucket.
|
||||||
type AISummarizerStorage interface {
|
type AISummarizerStorage interface {
|
||||||
GetSummaryFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) (string, error)
|
GetSummaryFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) (string, error)
|
||||||
|
GetTranslationFromStorage(ctx context.Context, contractFolderID, noticePublicationID, language string) (ai_summarizer.StoredTranslation, error)
|
||||||
ListDocuments(ctx context.Context, contractFolderID, noticePublicationID string) ([]ai_summarizer.StoredDocument, error)
|
ListDocuments(ctx context.Context, contractFolderID, noticePublicationID string) ([]ai_summarizer.StoredDocument, error)
|
||||||
DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error)
|
DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error)
|
||||||
}
|
}
|
||||||
@@ -1051,25 +1053,13 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
|
|||||||
return nil, fmt.Errorf("tender has no contract folder ID")
|
return nil, fmt.Errorf("tender has no contract folder ID")
|
||||||
}
|
}
|
||||||
|
|
||||||
if tr, ok := t.Translations[targetLanguage]; ok {
|
if tr, ok := t.Translations[targetLanguage]; ok && strings.TrimSpace(tr.Title) != "" {
|
||||||
return &AITranslateResponse{
|
return s.buildAITranslateResponse(t, targetLanguage, tr.Title, tr.Description), nil
|
||||||
NoticeID: t.NoticePublicationID,
|
|
||||||
Language: targetLanguage,
|
|
||||||
TranslatedTitle: tr.Title,
|
|
||||||
TranslatedDescription: tr.Description,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
resp, err := s.aiSummarizerClient.FetchTranslationOnDemand(ctx, ai_summarizer.TranslateRequest{
|
title, description, err := s.resolveTranslation(ctx, t, targetLanguage)
|
||||||
NoticeID: t.NoticePublicationID,
|
|
||||||
ContractFolderID: t.ContractFolderID,
|
|
||||||
NoticePublicationID: t.NoticePublicationID,
|
|
||||||
Title: t.Title,
|
|
||||||
Description: t.Description,
|
|
||||||
Language: targetLanguage,
|
|
||||||
})
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("On-demand AI translation failed", map[string]interface{}{
|
s.logger.Error("AI translation failed", map[string]interface{}{
|
||||||
"tender_id": id,
|
"tender_id": id,
|
||||||
"notice_id": t.NoticePublicationID,
|
"notice_id": t.NoticePublicationID,
|
||||||
"language": targetLanguage,
|
"language": targetLanguage,
|
||||||
@@ -1078,33 +1068,70 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
|
|||||||
return nil, fmt.Errorf("AI translation request failed: %w", err)
|
return nil, fmt.Errorf("AI translation request failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s.storeTranslation(t, targetLanguage, title, description)
|
||||||
|
if err := s.repository.Update(ctx, t); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to persist translation: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return s.buildAITranslateResponse(t, targetLanguage, title, description), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *tenderService) buildAITranslateResponse(t *Tender, language, title, description string) *AITranslateResponse {
|
||||||
|
return &AITranslateResponse{
|
||||||
|
ContractFolderID: t.ContractFolderID,
|
||||||
|
NoticePublicationID: t.NoticePublicationID,
|
||||||
|
Language: language,
|
||||||
|
TranslatedTitle: title,
|
||||||
|
TranslatedDescription: description,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, language string) (string, string, error) {
|
||||||
|
if s.aiSummarizerStorage != nil {
|
||||||
|
stored, err := s.aiSummarizerStorage.GetTranslationFromStorage(ctx, t.ContractFolderID, t.NoticePublicationID, language)
|
||||||
|
if err == nil {
|
||||||
|
return stored.Title, stored.Description, nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ai_summarizer.ErrTranslationNotReady) &&
|
||||||
|
!errors.Is(err, ai_summarizer.ErrObjectNotFound) &&
|
||||||
|
!errors.Is(err, ai_summarizer.ErrNoTranslation) {
|
||||||
|
s.logger.Debug("Translation not available from storage", map[string]interface{}{
|
||||||
|
"tender_id": t.ID.Hex(),
|
||||||
|
"language": language,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := s.aiSummarizerClient.FetchTranslationOnDemand(ctx, ai_summarizer.TranslateRequest{
|
||||||
|
ContractFolderID: t.ContractFolderID,
|
||||||
|
NoticePublicationID: t.NoticePublicationID,
|
||||||
|
Title: t.Title,
|
||||||
|
Description: t.Description,
|
||||||
|
Language: language,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return "", "", err
|
||||||
|
}
|
||||||
|
return resp.TranslatedTitle, resp.TranslatedDescription, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *tenderService) storeTranslation(t *Tender, language, title, description string) {
|
||||||
if t.Translations == nil {
|
if t.Translations == nil {
|
||||||
t.Translations = make(map[string]TranslationEntry)
|
t.Translations = make(map[string]TranslationEntry)
|
||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now().Unix()
|
||||||
t.Translations[targetLanguage] = TranslationEntry{
|
t.Translations[language] = TranslationEntry{
|
||||||
Title: resp.TranslatedTitle,
|
Title: title,
|
||||||
Description: resp.TranslatedDescription,
|
Description: description,
|
||||||
Language: targetLanguage,
|
Language: language,
|
||||||
TranslatedAt: now,
|
TranslatedAt: now,
|
||||||
}
|
}
|
||||||
if t.ProcessingMetadata.TranslatedData == nil {
|
if t.ProcessingMetadata.TranslatedData == nil {
|
||||||
t.ProcessingMetadata.TranslatedData = make(map[string]string)
|
t.ProcessingMetadata.TranslatedData = make(map[string]string)
|
||||||
}
|
}
|
||||||
t.ProcessingMetadata.TranslatedData[targetLanguage] = "done"
|
t.ProcessingMetadata.TranslatedData[language] = "done"
|
||||||
t.ProcessingMetadata.TranslatedAt = now
|
t.ProcessingMetadata.TranslatedAt = now
|
||||||
|
|
||||||
if err := s.repository.Update(ctx, t); err != nil {
|
|
||||||
return nil, fmt.Errorf("failed to persist translation: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &AITranslateResponse{
|
|
||||||
NoticeID: resp.NoticeID,
|
|
||||||
Language: targetLanguage,
|
|
||||||
TranslatedTitle: resp.TranslatedTitle,
|
|
||||||
TranslatedDescription: resp.TranslatedDescription,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *tenderService) pickResponseLanguage(language *string) string {
|
func (s *tenderService) pickResponseLanguage(language *string) string {
|
||||||
|
|||||||
+80
-11
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
@@ -93,8 +94,9 @@ func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody Translate
|
|||||||
for attempt := 0; attempt <= c.config.APIRetryCount; attempt++ {
|
for attempt := 0; attempt <= c.config.APIRetryCount; attempt++ {
|
||||||
if attempt > 0 {
|
if attempt > 0 {
|
||||||
c.logger.Warn("Retrying AI translate request", map[string]interface{}{
|
c.logger.Warn("Retrying AI translate request", map[string]interface{}{
|
||||||
"notice_id": reqBody.NoticeID,
|
"notice_publication_id": reqBody.NoticePublicationID,
|
||||||
"attempt": attempt,
|
"language": reqBody.Language,
|
||||||
|
"attempt": attempt,
|
||||||
})
|
})
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
@@ -107,9 +109,10 @@ func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody Translate
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
lastErr = err
|
lastErr = err
|
||||||
c.logger.Error("AI translate request failed", map[string]interface{}{
|
c.logger.Error("AI translate request failed", map[string]interface{}{
|
||||||
"notice_id": reqBody.NoticeID,
|
"notice_publication_id": reqBody.NoticePublicationID,
|
||||||
"attempt": attempt,
|
"language": reqBody.Language,
|
||||||
"error": err.Error(),
|
"attempt": attempt,
|
||||||
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -120,6 +123,50 @@ func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody Translate
|
|||||||
return nil, fmt.Errorf("ai_summarizer: all %d translation attempts failed, last error: %w", c.config.APIRetryCount+1, lastErr)
|
return nil, fmt.Errorf("ai_summarizer: all %d translation attempts failed, last error: %w", c.config.APIRetryCount+1, lastErr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TriggerPipelineTranslate calls POST /pipeline/translate to enqueue batch translation
|
||||||
|
// for the given target languages.
|
||||||
|
func (c *Client) TriggerPipelineTranslate(ctx context.Context, languages []string) (*PipelineTranslateResponse, error) {
|
||||||
|
reqBody := PipelineTranslateRequest{Languages: languages}
|
||||||
|
jsonBody, err := json.Marshal(reqBody)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: failed to marshal pipeline translate request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
url := c.config.APIBaseURL + "/pipeline/translate"
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: failed to create pipeline translate request: %w", err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
|
||||||
|
httpResp, err := c.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: pipeline translate request error: %w", err)
|
||||||
|
}
|
||||||
|
defer httpResp.Body.Close()
|
||||||
|
|
||||||
|
bodyBytes, err := io.ReadAll(httpResp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: failed to read pipeline translate response: %w", err)
|
||||||
|
}
|
||||||
|
if httpResp.StatusCode >= 400 {
|
||||||
|
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
var result PipelineTranslateResponse
|
||||||
|
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: failed to decode pipeline translate response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.logger.Info("Pipeline translate triggered", map[string]interface{}{
|
||||||
|
"status": result.Status,
|
||||||
|
"languages": result.Languages,
|
||||||
|
})
|
||||||
|
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
// doPost performs a single POST request and parses the response.
|
// doPost performs a single POST request and parses the response.
|
||||||
func (c *Client) doPost(ctx context.Context, url string, jsonBody []byte) (*SummarizeResponse, error) {
|
func (c *Client) doPost(ctx context.Context, url string, jsonBody []byte) (*SummarizeResponse, error) {
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
|
||||||
@@ -193,15 +240,37 @@ func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byt
|
|||||||
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
||||||
}
|
}
|
||||||
|
|
||||||
var result TranslateResponse
|
result, err := decodeTranslateResponse(bodyBytes)
|
||||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("ai_summarizer: failed to decode translation response JSON: %w", err)
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
c.logger.Info("AI translate request succeeded", map[string]interface{}{
|
c.logger.Info("AI translate request succeeded", map[string]interface{}{
|
||||||
"notice_id": result.NoticeID,
|
"notice_publication_id": result.NoticePublicationID,
|
||||||
"language": result.Language,
|
"language": result.Language,
|
||||||
})
|
})
|
||||||
|
|
||||||
return &result, nil
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type translateAPIEnvelope struct {
|
||||||
|
Success bool `json:"success"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
Data TranslateResponse `json:"data"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeTranslateResponse(bodyBytes []byte) (*TranslateResponse, error) {
|
||||||
|
var envelope translateAPIEnvelope
|
||||||
|
if err := json.Unmarshal(bodyBytes, &envelope); err == nil && envelope.Success && envelope.Data.Language != "" {
|
||||||
|
return &envelope.Data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var direct TranslateResponse
|
||||||
|
if err := json.Unmarshal(bodyBytes, &direct); err != nil {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: failed to decode translation response JSON: %w", err)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(direct.Language) == "" {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: translation response missing language")
|
||||||
|
}
|
||||||
|
return &direct, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,29 +44,72 @@ type StoredDocument struct {
|
|||||||
// TranslateRequest represents the request payload sent to the AI service
|
// TranslateRequest represents the request payload sent to the AI service
|
||||||
// POST /ai/translate endpoint.
|
// POST /ai/translate endpoint.
|
||||||
type TranslateRequest struct {
|
type TranslateRequest struct {
|
||||||
NoticeID string `json:"notice_id"` // required (TED-style notice key)
|
ContractFolderID string `json:"contract_folder_id"`
|
||||||
ContractFolderID string `json:"contract_folder_id"` // required by AI service (procedure / MinIO prefix)
|
NoticePublicationID string `json:"notice_publication_id"`
|
||||||
NoticePublicationID string `json:"notice_publication_id"` // required by AI service
|
Title string `json:"title"`
|
||||||
Title string `json:"title"` // required
|
Description string `json:"description"`
|
||||||
Description string `json:"description"` // required
|
Language string `json:"language"`
|
||||||
Language string `json:"language,omitempty"` // optional target language override
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// TranslateResponse represents the response from the AI translation API.
|
// TranslateResponse represents the translation payload from POST /ai/translate.
|
||||||
type TranslateResponse struct {
|
type TranslateResponse struct {
|
||||||
NoticeID string `json:"notice_id"`
|
ContractFolderID string `json:"contract_folder_id"`
|
||||||
|
NoticePublicationID string `json:"notice_publication_id"`
|
||||||
|
Language string `json:"language"`
|
||||||
TranslatedTitle string `json:"translated_title"`
|
TranslatedTitle string `json:"translated_title"`
|
||||||
TranslatedDescription string `json:"translated_description"`
|
TranslatedDescription string `json:"translated_description"`
|
||||||
Language string `json:"language"`
|
}
|
||||||
|
|
||||||
|
// PipelineTranslateRequest represents the request payload for POST /pipeline/translate.
|
||||||
|
type PipelineTranslateRequest struct {
|
||||||
|
Languages []string `json:"languages"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PipelineTranslateResponse represents the response from POST /pipeline/translate.
|
||||||
|
type PipelineTranslateResponse struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
Languages []string `json:"languages"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// StoredTranslation is one language entry inside tender.json translations map.
|
||||||
|
type StoredTranslation struct {
|
||||||
|
Title string `json:"title"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranslationStatus tracks which languages the async pipeline has finished.
|
||||||
|
type TranslationStatus struct {
|
||||||
|
Done []string `json:"done"`
|
||||||
|
DoneUpper []string `json:"Done"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s TranslationStatus) languagesDone() []string {
|
||||||
|
if len(s.Done) > 0 {
|
||||||
|
return s.Done
|
||||||
|
}
|
||||||
|
return s.DoneUpper
|
||||||
|
}
|
||||||
|
|
||||||
|
func languageInDone(done []string, language string) bool {
|
||||||
|
lang := strings.ToLower(strings.TrimSpace(language))
|
||||||
|
for _, item := range done {
|
||||||
|
if strings.ToLower(strings.TrimSpace(item)) == lang {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// TenderJSON represents the structure of the <notice_id>/tender.json file
|
// TenderJSON represents the structure of the <notice_id>/tender.json file
|
||||||
// stored in the AI service's MinIO bucket.
|
// stored in the AI service's MinIO bucket.
|
||||||
type TenderJSON struct {
|
type TenderJSON struct {
|
||||||
OverallSummary *string `json:"overall_summary"`
|
OverallSummary *string `json:"overall_summary"`
|
||||||
OverallSummaryCamel *string `json:"overallSummary"`
|
OverallSummaryCamel *string `json:"overallSummary"`
|
||||||
SummarizeStatus SummarizeStatus `json:"summarize_status"`
|
SummarizeStatus SummarizeStatus `json:"summarize_status"`
|
||||||
SummarizeStatusCamel SummarizeStatus `json:"summarizeStatus"`
|
SummarizeStatusCamel SummarizeStatus `json:"summarizeStatus"`
|
||||||
|
Translations map[string]StoredTranslation `json:"translations"`
|
||||||
|
TranslationStatus TranslationStatus `json:"translation_status"`
|
||||||
|
TranslationStatusCamel TranslationStatus `json:"translationStatus"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// resolved returns whether summarization is marked done and the best-effort
|
// resolved returns whether summarization is marked done and the best-effort
|
||||||
@@ -90,3 +133,33 @@ type SummarizeStatus struct {
|
|||||||
func (s SummarizeStatus) isDone() bool {
|
func (s SummarizeStatus) isDone() bool {
|
||||||
return s.Done || s.DoneUpper
|
return s.Done || s.DoneUpper
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// translationsDone returns languages marked complete by the translation pipeline.
|
||||||
|
func (t *TenderJSON) translationsDone() []string {
|
||||||
|
done := t.TranslationStatus.languagesDone()
|
||||||
|
if len(done) > 0 {
|
||||||
|
return done
|
||||||
|
}
|
||||||
|
return t.TranslationStatusCamel.languagesDone()
|
||||||
|
}
|
||||||
|
|
||||||
|
// translationForLanguage returns stored translation text when the pipeline marked
|
||||||
|
// the language done and non-empty title exists.
|
||||||
|
func (t *TenderJSON) translationForLanguage(language string) (StoredTranslation, bool) {
|
||||||
|
if t == nil || !languageInDone(t.translationsDone(), language) {
|
||||||
|
return StoredTranslation{}, false
|
||||||
|
}
|
||||||
|
lang := strings.ToLower(strings.TrimSpace(language))
|
||||||
|
if t.Translations != nil {
|
||||||
|
for key, entry := range t.Translations {
|
||||||
|
if strings.ToLower(strings.TrimSpace(key)) != lang {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(entry.Title) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return entry, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return StoredTranslation{}, false
|
||||||
|
}
|
||||||
|
|||||||
@@ -12,6 +12,14 @@ var (
|
|||||||
// overall_summary is empty or null.
|
// overall_summary is empty or null.
|
||||||
ErrNoOverallSummary = errors.New("ai_summarizer: overall_summary is empty")
|
ErrNoOverallSummary = errors.New("ai_summarizer: overall_summary is empty")
|
||||||
|
|
||||||
|
// ErrTranslationNotReady is returned when tender.json exists but the
|
||||||
|
// requested language is not listed in translation_status.done yet.
|
||||||
|
ErrTranslationNotReady = errors.New("ai_summarizer: translation not ready yet")
|
||||||
|
|
||||||
|
// ErrNoTranslation is returned when translation_status marks a language done
|
||||||
|
// but translations[language] is missing or empty.
|
||||||
|
ErrNoTranslation = errors.New("ai_summarizer: translation is empty")
|
||||||
|
|
||||||
// ErrObjectNotFound is returned when the tender.json object does not
|
// ErrObjectNotFound is returned when the tender.json object does not
|
||||||
// exist in the MinIO bucket (HTTP 404 / NoSuchKey).
|
// exist in the MinIO bucket (HTTP 404 / NoSuchKey).
|
||||||
ErrObjectNotFound = errors.New("ai_summarizer: tender.json not found in storage")
|
ErrObjectNotFound = errors.New("ai_summarizer: tender.json not found in storage")
|
||||||
|
|||||||
@@ -295,9 +295,46 @@ func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, contractFolde
|
|||||||
return s.getSummaryFromObjectKey(ctx, prefix+"tender.json", contractFolderID, noticePublicationID)
|
return s.getSummaryFromObjectKey(ctx, prefix+"tender.json", contractFolderID, noticePublicationID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetTranslationFromStorage reads tender.json and returns the translation for a language
|
||||||
|
// when translation_status.done includes that language.
|
||||||
|
func (s *StorageClient) GetTranslationFromStorage(ctx context.Context, contractFolderID, noticePublicationID, language string) (StoredTranslation, error) {
|
||||||
|
if strings.TrimSpace(contractFolderID) == "" {
|
||||||
|
return StoredTranslation{}, fmt.Errorf("ai_summarizer: contractFolderID is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(noticePublicationID) == "" {
|
||||||
|
return StoredTranslation{}, fmt.Errorf("ai_summarizer: noticePublicationID is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(language) == "" {
|
||||||
|
return StoredTranslation{}, fmt.Errorf("ai_summarizer: language is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
prefix, err := s.resolveNoticeStoragePrefix(ctx, contractFolderID, noticePublicationID)
|
||||||
|
if err != nil {
|
||||||
|
return StoredTranslation{}, err
|
||||||
|
}
|
||||||
|
return s.getTranslationFromObjectKey(ctx, prefix+"tender.json", language)
|
||||||
|
}
|
||||||
|
|
||||||
|
// getTranslationFromObjectKey reads tender.json at objectKey and returns translation for language.
|
||||||
|
func (s *StorageClient) getTranslationFromObjectKey(ctx context.Context, objectKey, language string) (StoredTranslation, error) {
|
||||||
|
tenderJSON, err := s.readTenderJSONAtKey(ctx, objectKey)
|
||||||
|
if err != nil {
|
||||||
|
return StoredTranslation{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !languageInDone(tenderJSON.translationsDone(), language) {
|
||||||
|
return StoredTranslation{}, ErrTranslationNotReady
|
||||||
|
}
|
||||||
|
entry, ok := tenderJSON.translationForLanguage(language)
|
||||||
|
if !ok {
|
||||||
|
return StoredTranslation{}, ErrNoTranslation
|
||||||
|
}
|
||||||
|
return entry, nil
|
||||||
|
}
|
||||||
|
|
||||||
// getSummaryFromObjectKey reads tender.json at objectKey and returns overall_summary when ready.
|
// getSummaryFromObjectKey reads tender.json at objectKey and returns overall_summary when ready.
|
||||||
// contractFolderID and noticePublicationID are used for logging only (may be empty when unknown).
|
// contractFolderID and noticePublicationID are used for logging only (may be empty when unknown).
|
||||||
func (s *StorageClient) getSummaryFromObjectKey(ctx context.Context, objectKey, contractFolderID, noticePublicationID string) (string, error) {
|
func (s *StorageClient) readTenderJSONAtKey(ctx context.Context, objectKey string) (*TenderJSON, error) {
|
||||||
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectKey, minio.GetObjectOptions{})
|
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectKey, minio.GetObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isMinioNotFound(err) {
|
if isMinioNotFound(err) {
|
||||||
@@ -305,13 +342,13 @@ func (s *StorageClient) getSummaryFromObjectKey(ctx context.Context, objectKey,
|
|||||||
"bucket": s.config.MinioBucket,
|
"bucket": s.config.MinioBucket,
|
||||||
"object_key": objectKey,
|
"object_key": objectKey,
|
||||||
})
|
})
|
||||||
return "", ErrObjectNotFound
|
return nil, ErrObjectNotFound
|
||||||
}
|
}
|
||||||
var minioErr minio.ErrorResponse
|
var minioErr minio.ErrorResponse
|
||||||
if errors.As(err, &minioErr) && minioErr.StatusCode == http.StatusForbidden {
|
if errors.As(err, &minioErr) && minioErr.StatusCode == http.StatusForbidden {
|
||||||
return "", fmt.Errorf("ai_summarizer: access denied to bucket %s: %w", s.config.MinioBucket, err)
|
return nil, fmt.Errorf("ai_summarizer: access denied to bucket %s: %w", s.config.MinioBucket, err)
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("ai_summarizer: failed to get object: %w", err)
|
return nil, fmt.Errorf("ai_summarizer: failed to get object: %w", err)
|
||||||
}
|
}
|
||||||
defer object.Close()
|
defer object.Close()
|
||||||
|
|
||||||
@@ -322,14 +359,22 @@ func (s *StorageClient) getSummaryFromObjectKey(ctx context.Context, objectKey,
|
|||||||
"bucket": s.config.MinioBucket,
|
"bucket": s.config.MinioBucket,
|
||||||
"object_key": objectKey,
|
"object_key": objectKey,
|
||||||
})
|
})
|
||||||
return "", ErrObjectNotFound
|
return nil, ErrObjectNotFound
|
||||||
}
|
}
|
||||||
return "", fmt.Errorf("ai_summarizer: failed to read tender.json: %w", err)
|
return nil, fmt.Errorf("ai_summarizer: failed to read tender.json: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var tenderJSON TenderJSON
|
var tenderJSON TenderJSON
|
||||||
if err := json.Unmarshal(data, &tenderJSON); err != nil {
|
if err := json.Unmarshal(data, &tenderJSON); err != nil {
|
||||||
return "", fmt.Errorf("ai_summarizer: failed to parse tender.json: %w", err)
|
return nil, fmt.Errorf("ai_summarizer: failed to parse tender.json: %w", err)
|
||||||
|
}
|
||||||
|
return &tenderJSON, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *StorageClient) getSummaryFromObjectKey(ctx context.Context, objectKey, contractFolderID, noticePublicationID string) (string, error) {
|
||||||
|
tenderJSON, err := s.readTenderJSONAtKey(ctx, objectKey)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
}
|
}
|
||||||
|
|
||||||
done, overall := tenderJSON.resolved()
|
done, overall := tenderJSON.resolved()
|
||||||
@@ -368,32 +413,12 @@ func (s *StorageClient) IsSummaryReady(ctx context.Context, contractFolderID, no
|
|||||||
}
|
}
|
||||||
|
|
||||||
objectPath := prefix + "tender.json"
|
objectPath := prefix + "tender.json"
|
||||||
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectPath, minio.GetObjectOptions{})
|
tenderJSON, err := s.readTenderJSONAtKey(ctx, objectPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if isMinioNotFound(err) {
|
if errors.Is(err, ErrObjectNotFound) {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
return false, fmt.Errorf("ai_summarizer: failed to check summary readiness: %w", err)
|
return false, err
|
||||||
}
|
|
||||||
|
|
||||||
data, readErr := io.ReadAll(object)
|
|
||||||
closeErr := object.Close()
|
|
||||||
if readErr != nil {
|
|
||||||
if isMinioNotFound(readErr) {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
return false, fmt.Errorf("ai_summarizer: failed to read tender.json: %w", readErr)
|
|
||||||
}
|
|
||||||
if closeErr != nil {
|
|
||||||
s.logger.Warn("Failed to close tender.json object after readiness check", map[string]interface{}{
|
|
||||||
"object_key": objectPath,
|
|
||||||
"error": closeErr.Error(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
var tenderJSON TenderJSON
|
|
||||||
if err := json.Unmarshal(data, &tenderJSON); err != nil {
|
|
||||||
return false, fmt.Errorf("ai_summarizer: failed to parse tender.json: %w", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
done, _ := tenderJSON.resolved()
|
done, _ := tenderJSON.resolved()
|
||||||
|
|||||||
Reference in New Issue
Block a user