Implement AI pipeline operations in the admin API

- Added new routes and handlers for AI pipeline operations, including scraping documents, batch summarization, translation, and syncing with the Opplens AI service.
- Introduced request forms for handling tender references and batch operations.
- Enhanced the AI service with methods for triggering batch operations and managing pipeline runs.
- Updated Swagger documentation to reflect the new AI pipeline endpoints and their functionalities.

This update integrates comprehensive AI pipeline capabilities into the tender management system, improving operational efficiency and user experience.

cmd/worker/workers/summarizer.go

@@ -47,10 +47,13 @@ func (w *DocumentSummarizationWorker) Run() {
 		return
 	}
 
-	if _, err := w.AIClient.TriggerPipelineSummarize(context.Background()); err != nil {
-		w.Logger.Warn("Failed to trigger AI pipeline summarize (continuing with per-tender sync)", map[string]interface{}{
-			"error": err.Error(),
-		})
+	if refs := w.collectUnSummarizedTenderRefs(context.Background()); len(refs) > 0 {
+		if _, err := w.AIClient.TriggerSummarizeBatch(context.Background(), refs); err != nil {
+			w.Logger.Warn("Failed to trigger AI summarize batch (continuing with per-tender sync)", map[string]interface{}{
+				"tender_count": len(refs),
+				"error":        err.Error(),
+			})
+		}
 	}
 
 	limit := 5
@@ -92,6 +95,38 @@ func (w *DocumentSummarizationWorker) Run() {
 	w.Logger.Info("Document summarization worker completed", map[string]interface{}{})
 }
 
+func (w *DocumentSummarizationWorker) collectUnSummarizedTenderRefs(ctx context.Context) []ai_summarizer.TenderRef {
+	limit := 100
+	skip := 0
+	refs := make([]ai_summarizer.TenderRef, 0)
+
+	for {
+		tenders, totalCount, err := w.TenderRepo.GetUnSummarizedTenders(ctx, limit, skip)
+		if err != nil {
+			w.Logger.Warn("Failed to collect tenders for summarize batch", map[string]interface{}{
+				"error": err.Error(),
+			})
+			return refs
+		}
+		if len(tenders) == 0 {
+			return refs
+		}
+
+		for i := range tenders {
+			t := &tenders[i]
+			if strings.TrimSpace(t.ContractFolderID) == "" || strings.TrimSpace(t.NoticePublicationID) == "" {
+				continue
+			}
+			refs = append(refs, ai_summarizer.NewTenderRef(t.ContractFolderID, t.NoticePublicationID))
+		}
+
+		skip += len(tenders)
+		if int64(skip) >= totalCount {
+			return refs
+		}
+	}
+}
+
 func (w *DocumentSummarizationWorker) summarizeTender(t *tender.Tender) {
 	if strings.TrimSpace(t.NoticePublicationID) == "" {
 		w.Logger.Warn("Skipping tender summarization: missing notice_publication_id", map[string]interface{}{

cmd/worker/workers/translation.go

@@ -10,7 +10,7 @@ import (
 )
 
 // TranslationWorker syncs translation completion from the AI pipeline MinIO storage.
-// Batch translation is triggered via POST /pipeline/translate; this worker only
+// Batch translation is triggered via POST /ai/translate/batch; this worker only
 // verifies which tenders now have translations available in storage.
 type TranslationWorker struct {
 	Mongo                     *mongo.ConnectionManager
@@ -82,11 +82,14 @@ func (w *TranslationWorker) Run() {
 
 	startCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
 
-	if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
-		w.Logger.Warn("Failed to trigger AI pipeline translate", map[string]interface{}{
-			"target_languages": w.TargetLanguages,
-			"error":            err.Error(),
-		})
+	if refs := w.collectUnTranslatedTenderRefs(context.Background()); len(refs) > 0 {
+		if _, err := w.AIClient.TriggerTranslateBatch(context.Background(), refs, w.TargetLanguages); err != nil {
+			w.Logger.Warn("Failed to trigger AI translate batch", map[string]interface{}{
+				"tender_count":     len(refs),
+				"target_languages": w.TargetLanguages,
+				"error":            err.Error(),
+			})
+		}
 	}
 
 	readyCount := 0
@@ -186,3 +189,44 @@ func (w *TranslationWorker) hasTranslationInStorage(t *tender.Tender, language s
 	)
 	return err == nil && strings.TrimSpace(stored.Title) != ""
 }
+
+func (w *TranslationWorker) collectUnTranslatedTenderRefs(ctx context.Context) []ai_summarizer.TenderRef {
+	seen := make(map[string]struct{})
+	refs := make([]ai_summarizer.TenderRef, 0)
+
+	for _, language := range w.TargetLanguages {
+		skip := 0
+		for {
+			tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(ctx, language, w.BatchSize, skip)
+			if err != nil {
+				w.Logger.Warn("Failed to collect tenders for translate batch", map[string]interface{}{
+					"target_language": language,
+					"error":           err.Error(),
+				})
+				break
+			}
+			if len(tenders) == 0 {
+				break
+			}
+
+			for _, t := range tenders {
+				if strings.TrimSpace(t.ContractFolderID) == "" || strings.TrimSpace(t.NoticePublicationID) == "" {
+					continue
+				}
+				key := t.ContractFolderID + "|" + t.NoticePublicationID
+				if _, dup := seen[key]; dup {
+					continue
+				}
+				seen[key] = struct{}{}
+				refs = append(refs, ai_summarizer.NewTenderRef(t.ContractFolderID, t.NoticePublicationID))
+			}
+
+			skip += len(tenders)
+			if int64(skip) >= totalCount {
+				break
+			}
+		}
+	}
+
+	return refs
+}