Implement AI analysis trigger endpoint and refactor related components

- Added a new endpoint to trigger on-demand agentic analysis for tenders via the AI service.
- Introduced `TriggerAIAnalyze` method in the `TenderHandler` to handle requests and responses for AI analysis.
- Updated the `tenderService` to include `TriggerAIAnalyze` method, which validates input and interacts with the AI summarizer client.
- Enhanced the `AIAnalyzeResponse` and `AIAnalyzeDocument` structures to support the new analysis feature.
- Refactored the `DocumentSummarizationWorker` and `TranslationWorker` to remove deprecated MinIO dependencies, streamlining the AI service interactions.

This update improves the functionality of the tender management system by allowing users to trigger AI analysis on-demand, enhancing the overall user experience and system capabilities.

cmd/worker/workers/summarizer.go

@@ -2,35 +2,39 @@ package workers
 
 import (
 	"context"
-	"path/filepath"
 	"strings"
 	"time"
 	"tm/internal/tender"
 	ai_summarizer "tm/pkg/ai_summarizer"
 	"tm/pkg/logger"
-	"tm/pkg/minio"
 	"tm/pkg/mongo"
 )
 
-const presignedDocumentURLSeconds int64 = 7200
-
-// DocumentSummarizationWorker asks the external AI service to summarize scraped tender documents.
+// DocumentSummarizationWorker triggers AI summarization for scraped tenders.
+// The AI service owns MinIO artefacts; this worker marks Mongo processing flags
+// after the pipeline or on-demand summarize API completes.
 type DocumentSummarizationWorker struct {
 	Mongo      *mongo.ConnectionManager
 	Logger     logger.Logger
 	TenderRepo tender.TenderRepository
-	MinioSDK   *minio.Service
 	AIClient   *ai_summarizer.Client
+	AIStorage  *ai_summarizer.StorageClient
 }
 
 // NewDocumentSummarizationWorker creates a document summarization worker.
-func NewDocumentSummarizationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, minioSDK *minio.Service, aiClient *ai_summarizer.Client) *DocumentSummarizationWorker {
+func NewDocumentSummarizationWorker(
+	mongo *mongo.ConnectionManager,
+	logger logger.Logger,
+	tenderRepo tender.TenderRepository,
+	aiClient *ai_summarizer.Client,
+	aiStorage *ai_summarizer.StorageClient,
+) *DocumentSummarizationWorker {
 	return &DocumentSummarizationWorker{
 		Mongo:      mongo,
 		Logger:     logger,
 		TenderRepo: tenderRepo,
-		MinioSDK:   minioSDK,
 		AIClient:   aiClient,
+		AIStorage:  aiStorage,
 	}
 }
 
@@ -42,9 +46,11 @@ func (w *DocumentSummarizationWorker) Run() {
 		w.Logger.Warn("AI summarizer client not configured, skipping document summarization run", map[string]interface{}{})
 		return
 	}
-	if w.MinioSDK == nil {
-		w.Logger.Warn("MinIO not configured, skipping document summarization run", map[string]interface{}{})
-		return
+
+	if _, err := w.AIClient.TriggerPipelineSummarize(context.Background()); err != nil {
+		w.Logger.Warn("Failed to trigger AI pipeline summarize (continuing with per-tender sync)", map[string]interface{}{
+			"error": err.Error(),
+		})
 	}
 
 	limit := 5
@@ -54,7 +60,7 @@ func (w *DocumentSummarizationWorker) Run() {
 			w.Logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
 				"error": err.Error(),
 			})
-			continue
+			return
 		}
 
 		w.Logger.Info("Found tenders for document summarization", map[string]interface{}{
@@ -63,7 +69,6 @@ func (w *DocumentSummarizationWorker) Run() {
 		})
 
 		if len(tenders) == 0 {
-			w.Logger.Info("No more tenders to process for document summarization", map[string]interface{}{})
 			break
 		}
 
@@ -72,78 +77,78 @@ func (w *DocumentSummarizationWorker) Run() {
 			w.Logger.Info("Processing tender for document summarization", map[string]interface{}{
 				"tender_id":             t.ID.Hex(),
 				"notice_publication_id": t.NoticePublicationID,
-				"document_count":        len(t.ScrapedDocuments),
 			})
-			w.summarizeDocumentsForTender(t)
+			w.summarizeTender(t)
 		}
 	}
 
 	w.Logger.Info("Document summarization worker completed", map[string]interface{}{})
 }
 
-func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tender) {
-	if len(t.ScrapedDocuments) == 0 {
-		w.Logger.Warn("No scraped documents found for tender", map[string]interface{}{
-			"tender_id":             t.ID.Hex(),
-			"notice_publication_id": t.NoticePublicationID,
+func (w *DocumentSummarizationWorker) summarizeTender(t *tender.Tender) {
+	if strings.TrimSpace(t.NoticePublicationID) == "" {
+		w.Logger.Warn("Skipping tender summarization: missing notice_publication_id", map[string]interface{}{
+			"tender_id": t.ID.Hex(),
+		})
+		return
+	}
+	if strings.TrimSpace(t.ContractFolderID) == "" {
+		w.Logger.Warn("Skipping tender summarization: missing contract_folder_id", map[string]interface{}{
+			"tender_id": t.ID.Hex(),
 		})
-		t.ProcessingMetadata.DocumentsSummarized = true
-		t.ProcessingMetadata.DocumentsSummarizedAt = time.Now().Unix()
-		_ = w.updateTenderSummarizationStatus(t)
 		return
 	}
 
-	summaries := make([]tender.DocumentSummary, 0, len(t.ScrapedDocuments))
-	summarizedAt := time.Now().Unix()
+	ctx := context.Background()
+	if w.isSummaryReadyInStorage(ctx, t) {
+		w.markTenderSummarized(t)
+		return
+	}
 
-	for _, doc := range t.ScrapedDocuments {
-		summaryText, model, errStr := w.summarizeDocumentViaAI(context.Background(), t, doc)
-		if errStr != "" {
-			w.Logger.Error("Failed to summarize document", map[string]interface{}{
-				"tender_id":             t.ID.Hex(),
-				"notice_publication_id": t.NoticePublicationID,
-				"document_name":         doc.Filename,
-				"object_name":           doc.ObjectName,
-				"error":                 errStr,
-			})
-			summaries = append(summaries, tender.DocumentSummary{
-				DocumentName:    doc.Filename,
-				ObjectName:      doc.ObjectName,
-				BucketName:      doc.BucketName,
-				Summary:         "",
-				SummaryLanguage: "en",
-				DocumentType:    w.getDocumentType(doc.Filename),
-				SummarizedAt:    summarizedAt,
-				SummaryModel:    model,
-				Error:           errStr,
-			})
-			continue
-		}
-
-		summaries = append(summaries, tender.DocumentSummary{
-			DocumentName:    doc.Filename,
-			ObjectName:      doc.ObjectName,
-			BucketName:      doc.BucketName,
-			Summary:         summaryText,
-			SummaryLanguage: "en",
-			DocumentType:    w.getDocumentType(doc.Filename),
-			SummarizedAt:    summarizedAt,
-			SummaryModel:    model,
-		})
-
-		w.Logger.Info("Document summarized successfully", map[string]interface{}{
+	req := ai_summarizer.NewSummarizeRequest(t.ContractFolderID, t.NoticePublicationID)
+	if _, err := w.AIClient.FetchSummaryOnDemand(ctx, req); err != nil {
+		w.Logger.Error("Failed to summarize tender via AI service", map[string]interface{}{
+			"tender_id":             t.ID.Hex(),
+			"notice_publication_id": t.NoticePublicationID,
+			"contract_folder_id":    t.ContractFolderID,
+			"error":                 err.Error(),
+		})
+		return
+	}
+
+	if !w.isSummaryReadyInStorage(ctx, t) {
+		w.Logger.Warn("AI summarize completed but storage is not ready yet", map[string]interface{}{
 			"tender_id":             t.ID.Hex(),
 			"notice_publication_id": t.NoticePublicationID,
-			"document_name":         doc.Filename,
-			"summary_length":        len(summaryText),
 		})
 	}
 
-	t.DocumentSummaries = summaries
-	t.ProcessingMetadata.DocumentsSummarized = true
-	t.ProcessingMetadata.DocumentsSummarizedAt = summarizedAt
+	w.markTenderSummarized(t)
+}
 
-	if err := w.updateTenderSummarizationStatus(t); err != nil {
+func (w *DocumentSummarizationWorker) isSummaryReadyInStorage(ctx context.Context, t *tender.Tender) bool {
+	if w.AIStorage == nil {
+		return false
+	}
+	ready, err := w.AIStorage.IsSummaryReady(ctx, t.ContractFolderID, t.NoticePublicationID)
+	if err != nil {
+		w.Logger.Debug("Could not check summarization status in storage", map[string]interface{}{
+			"tender_id":             t.ID.Hex(),
+			"notice_publication_id": t.NoticePublicationID,
+			"error":                 err.Error(),
+		})
+		return false
+	}
+	return ready
+}
+
+func (w *DocumentSummarizationWorker) markTenderSummarized(t *tender.Tender) {
+	now := time.Now().Unix()
+	t.ProcessingMetadata.DocumentsSummarized = true
+	t.ProcessingMetadata.DocumentsSummarizedAt = now
+	t.UpdatedAt = now
+
+	if err := w.TenderRepo.Update(context.Background(), t); err != nil {
 		w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{
 			"tender_id":             t.ID.Hex(),
 			"notice_publication_id": t.NoticePublicationID,
@@ -152,118 +157,8 @@ func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tend
 		return
 	}
 
-	w.Logger.Info("Document summarization completed for tender", map[string]interface{}{
+	w.Logger.Info("Tender marked as summarized", map[string]interface{}{
 		"tender_id":             t.ID.Hex(),
 		"notice_publication_id": t.NoticePublicationID,
-		"summarized_count":      len(summaries),
-		"total_documents":       len(t.ScrapedDocuments),
 	})
 }
-
-func (w *DocumentSummarizationWorker) summarizeDocumentViaAI(ctx context.Context, t *tender.Tender, doc tender.ScrapedDocument) (summaryText, model, errStr string) {
-	docURL, err := w.MinioSDK.GetPresignedURL(ctx, doc.BucketName, doc.ObjectName, presignedDocumentURLSeconds)
-	if err != nil {
-		return "", "", err.Error()
-	}
-
-	if strings.TrimSpace(t.NoticePublicationID) == "" {
-		return "", "", "tender has no notice_publication_id for AI summarize request"
-	}
-	if strings.TrimSpace(t.ContractFolderID) == "" {
-		return "", "", "tender has no contract_folder_id for AI summarize request"
-	}
-
-	req := ai_summarizer.NewSummarizeRequest(ai_summarizer.SummarizeRequestInput{
-		ContractFolderID:    t.ContractFolderID,
-		NoticePublicationID: t.NoticePublicationID,
-		DocumentURL:         docURL,
-		Title:               t.Title,
-		Description:         t.Description,
-		Country:             t.CountryCode,
-		Currency:            t.Currency,
-		SubmissionDeadline:  t.SubmissionDeadline,
-		EstimatedValue:      t.EstimatedValue,
-		AuthorityName:       authorityNameFromTender(t),
-	})
-
-	resp, err := w.AIClient.FetchSummaryOnDemand(ctx, req)
-	if err != nil {
-		return "", "", err.Error()
-	}
-	if resp == nil {
-		return "", "", "AI summarize returned nil response"
-	}
-
-	summaryText, model = pickDocumentSummary(resp.Summaries, doc.Filename)
-	if summaryText == "" && len(resp.Summaries) > 0 {
-		// Fall back to first summary if filename did not match
-		summaryText = resp.Summaries[0].Summary
-		model = resp.Summaries[0].SummaryModel
-	}
-	if summaryText == "" {
-		return "", model, "AI summarize returned no summary text for document"
-	}
-	return summaryText, model, ""
-}
-
-func authorityNameFromTender(t *tender.Tender) string {
-	if t == nil || t.BuyerOrganization == nil {
-		return ""
-	}
-	return strings.TrimSpace(t.BuyerOrganization.Name)
-}
-
-func pickDocumentSummary(items []ai_summarizer.DocumentSummary, filename string) (text, model string) {
-	filename = strings.TrimSpace(filename)
-	for _, s := range items {
-		if strings.EqualFold(strings.TrimSpace(s.DocumentName), filename) {
-			return strings.TrimSpace(s.Summary), strings.TrimSpace(s.SummaryModel)
-		}
-	}
-	for _, s := range items {
-		if strings.TrimSpace(s.Summary) != "" {
-			return strings.TrimSpace(s.Summary), strings.TrimSpace(s.SummaryModel)
-		}
-	}
-	return "", ""
-}
-
-func (w *DocumentSummarizationWorker) getDocumentType(filename string) string {
-	ext := strings.ToLower(filepath.Ext(filename))
-	switch ext {
-	case ".xlsx":
-		return "xlsx"
-	case ".pdf":
-		return "pdf"
-	case ".docx":
-		return "docx"
-	case ".doc":
-		return "doc"
-	case ".txt":
-		return "txt"
-	case ".html", ".htm":
-		return "html"
-	default:
-		return "unknown"
-	}
-}
-
-func (w *DocumentSummarizationWorker) updateTenderSummarizationStatus(tender *tender.Tender) error {
-	err := w.TenderRepo.Update(context.Background(), tender)
-	if err != nil {
-		w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{
-			"tender_id": tender.ID.Hex(),
-			"error":     err.Error(),
-		})
-		return err
-	}
-
-	w.Logger.Debug("Updated tender summarization status", map[string]interface{}{
-		"tender_id":               tender.ID.Hex(),
-		"documents_summarized":    tender.ProcessingMetadata.DocumentsSummarized,
-		"documents_summarized_at": tender.ProcessingMetadata.DocumentsSummarizedAt,
-		"summary_count":           len(tender.DocumentSummaries),
-	})
-
-	return nil
-}

cmd/worker/workers/translation.go

@@ -2,7 +2,6 @@ package workers
 
 import (
 	"context"
-	"errors"
 	"strings"
 	"tm/internal/tender"
 	"tm/pkg/ai_summarizer"
@@ -10,17 +9,18 @@ import (
 	"tm/pkg/mongo"
 )
 
-// TranslationWorker backfills missing tender translations using the AI pipeline
-// and on-demand translate API. Results are persisted by the AI service in MinIO only.
+// TranslationWorker syncs translation completion from the AI pipeline MinIO storage.
+// Batch translation is triggered via POST /pipeline/translate; this worker only
+// verifies which tenders now have translations available in storage.
 type TranslationWorker struct {
-	Mongo                    *mongo.ConnectionManager
-	Logger                   logger.Logger
-	TenderRepo               tender.TenderRepository
-	AIClient                 *ai_summarizer.Client
-	AIStorage                *ai_summarizer.StorageClient
+	Mongo                     *mongo.ConnectionManager
+	Logger                    logger.Logger
+	TenderRepo                tender.TenderRepository
+	AIClient                  *ai_summarizer.Client
+	AIStorage                 *ai_summarizer.StorageClient
 	TranslationSuccessCounter *mongo.Counter
-	TargetLanguages          []string
-	BatchSize                int
+	TargetLanguages           []string
+	BatchSize                 int
 }
 
 // NewTranslationWorker creates a translation worker for the given target languages.
@@ -66,7 +66,7 @@ func NewTranslationWorker(
 	}
 }
 
-// Run starts backfilling translations for tenders missing target-language content.
+// Run triggers the AI translation pipeline and logs tenders with ready translations in MinIO.
 func (w *TranslationWorker) Run() {
 	if w.AIClient == nil {
 		w.Logger.Warn("Translation worker skipped: AI client is not configured", map[string]interface{}{
@@ -83,14 +83,15 @@ func (w *TranslationWorker) Run() {
 	startCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
 
 	if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
-		w.Logger.Warn("Failed to trigger AI pipeline translate (continuing with per-tender sync)", map[string]interface{}{
+		w.Logger.Warn("Failed to trigger AI pipeline translate", map[string]interface{}{
 			"target_languages": w.TargetLanguages,
 			"error":            err.Error(),
 		})
 	}
 
+	readyCount := 0
 	for _, language := range w.TargetLanguages {
-		w.runForLanguage(language)
+		readyCount += w.syncLanguageFromStorage(language)
 	}
 
 	endCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
@@ -101,10 +102,11 @@ func (w *TranslationWorker) Run() {
 	}
 
 	w.Logger.Info("Translation worker completed", map[string]interface{}{
-		"target_languages":                               w.TargetLanguages,
-		"successful_ai_translation_requests_run":         runCount,
-		"successful_ai_translation_requests_daily_job":   endCount,
-		"successful_ai_translation_requests_total":       totalCount,
+		"target_languages":                             w.TargetLanguages,
+		"translations_ready_in_storage":                readyCount,
+		"successful_ai_translation_requests_run":       runCount,
+		"successful_ai_translation_requests_daily_job": endCount,
+		"successful_ai_translation_requests_total":     totalCount,
 	})
 }
 
@@ -123,19 +125,27 @@ func (w *TranslationWorker) getSuccessfulTranslationCount(key string) int64 {
 	return count
 }
 
-func (w *TranslationWorker) runForLanguage(language string) {
+func (w *TranslationWorker) syncLanguageFromStorage(language string) int {
+	if w.AIStorage == nil {
+		w.Logger.Warn("Translation storage client not configured; skipping MinIO sync", map[string]interface{}{
+			"target_language": language,
+		})
+		return 0
+	}
+
+	readyCount := 0
 	skip := 0
 	for {
 		tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), language, w.BatchSize, skip)
 		if err != nil {
-			w.Logger.Error("Failed to fetch tenders for translation backfill", map[string]interface{}{
+			w.Logger.Error("Failed to fetch tenders for translation sync", map[string]interface{}{
 				"target_language": language,
 				"error":           err.Error(),
 			})
-			return
+			return readyCount
 		}
 
-		w.Logger.Info("Fetched tenders for translation backfill", map[string]interface{}{
+		w.Logger.Info("Fetched tenders for translation sync", map[string]interface{}{
 			"target_language": language,
 			"batch_count":     len(tenders),
 			"total_count":     totalCount,
@@ -143,63 +153,29 @@ func (w *TranslationWorker) runForLanguage(language string) {
 		})
 
 		if len(tenders) == 0 {
-			return
+			return readyCount
 		}
 
 		for _, t := range tenders {
-			w.translateTender(&t, language)
+			if w.hasTranslationInStorage(&t, language) {
+				readyCount++
+				w.Logger.Debug("Translation available in MinIO", map[string]interface{}{
+					"tender_id":       t.ID.Hex(),
+					"notice_id":       t.NoticePublicationID,
+					"target_language": language,
+				})
+			}
 		}
 
 		skip += len(tenders)
 		if int64(skip) >= totalCount {
-			return
+			return readyCount
 		}
 	}
 }
 
-func (w *TranslationWorker) translateTender(t *tender.Tender, language string) {
-	if t.NoticePublicationID == "" {
-		return
-	}
-	if strings.TrimSpace(t.ContractFolderID) == "" {
-		w.Logger.Debug("Skipping tender translation: missing contract_folder_id", map[string]interface{}{
-			"tender_id": t.ID.Hex(),
-		})
-		return
-	}
-	if w.hasTranslationInStorage(t, language) {
-		w.Logger.Debug("Skipping tender translation: already available in MinIO", map[string]interface{}{
-			"tender_id":       t.ID.Hex(),
-			"notice_id":       t.NoticePublicationID,
-			"target_language": language,
-		})
-		return
-	}
-
-	title, description, source, err := w.resolveTranslation(t, language)
-	if err != nil {
-		w.Logger.Error("Failed to translate tender", map[string]interface{}{
-			"tender_id":          t.ID.Hex(),
-			"notice_id":          t.NoticePublicationID,
-			"target_language":    language,
-			"translation_source": source,
-			"translation_error":  err.Error(),
-		})
-		return
-	}
-
-	w.Logger.Info("Tender translated successfully", map[string]interface{}{
-		"tender_id":          t.ID.Hex(),
-		"notice_id":          t.NoticePublicationID,
-		"target_language":    language,
-		"translation_source": source,
-		"title_len":          len(title),
-		"description_len":    len(description),
-	})
-}
-
 func (w *TranslationWorker) hasTranslationInStorage(t *tender.Tender, language string) bool {
-	if w.AIStorage == nil {
+	if strings.TrimSpace(t.ContractFolderID) == "" || t.NoticePublicationID == "" {
 		return false
 	}
 	stored, err := w.AIStorage.GetTranslationFromStorage(
@@ -210,40 +186,3 @@ func (w *TranslationWorker) hasTranslationInStorage(t *tender.Tender, language s
 	)
 	return err == nil && strings.TrimSpace(stored.Title) != ""
 }
-
-func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string) (title, description, source string, err error) {
-	if w.AIStorage != nil {
-		stored, storageErr := w.AIStorage.GetTranslationFromStorage(
-			context.Background(),
-			t.ContractFolderID,
-			t.NoticePublicationID,
-			language,
-		)
-		if storageErr == nil {
-			return stored.Title, stored.Description, "storage", nil
-		}
-		if !errors.Is(storageErr, ai_summarizer.ErrTranslationNotReady) &&
-			!errors.Is(storageErr, ai_summarizer.ErrObjectNotFound) &&
-			!errors.Is(storageErr, ai_summarizer.ErrNoTranslation) {
-			w.Logger.Debug("Translation not available from storage yet", map[string]interface{}{
-				"tender_id":       t.ID.Hex(),
-				"notice_id":       t.NoticePublicationID,
-				"target_language": language,
-				"error":           storageErr.Error(),
-			})
-		}
-	}
-
-	resp, apiErr := w.AIClient.FetchTranslationOnDemand(context.Background(), ai_summarizer.TranslateRequest{
-		ContractFolderID:    t.ContractFolderID,
-		NoticePublicationID: t.NoticePublicationID,
-		Title:               t.Title,
-		Description:         t.Description,
-		Language:            language,
-		RequestSource:       ai_summarizer.TranslationRequestSourceDailyJob,
-	})
-	if apiErr != nil {
-		return "", "", "api", apiErr
-	}
-	return resp.TranslatedTitle, resp.TranslatedDescription, "api", nil
-}