Implement AI pipeline operations in the admin API

- Added new routes and handlers for AI pipeline operations, including scraping documents, batch summarization, translation, and syncing with the Opplens AI service.
- Introduced request forms for handling tender references and batch operations.
- Enhanced the AI service with methods for triggering batch operations and managing pipeline runs.
- Updated Swagger documentation to reflect the new AI pipeline endpoints and their functionalities.

This update integrates comprehensive AI pipeline capabilities into the tender management system, improving operational efficiency and user experience.

cmd/web/main.go

@@ -59,6 +59,9 @@ package main
 // @tag.name Admin-CMS
 // @tag.description Administrative CMS management operations for web panel including CRUD operations, search, statistics, and comprehensive filtering with pagination
 
+// @tag.name Admin-AI-Pipeline
+// @tag.description Administrative Opplens AI pipeline operations including scrape, summarize, translate, sync, and background pipeline runs
+
 // @tag.name Authorization
 // @tag.description Customer authentication and authorization operations for mobile application including login, logout, token refresh, and profile access
 
@@ -98,6 +101,7 @@ import (
 	"net/http"
 	"time"
 	"tm/internal/assets"
+	"tm/internal/ai_pipeline"
 	"tm/internal/cms"
 	"tm/internal/company"
 	"tm/internal/company_category"
@@ -168,9 +172,11 @@ func main() {
 	var aiSummarizerClient tender.AISummarizerClient
 	var aiSummarizerStorage tender.AISummarizerStorage
 	var aiRecommendationClient company.AIRecommendationClient
+	var aiPipelineClient ai_pipeline.Client
 
 	if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, mongoManager, logger); c != nil {
 		aiSummarizerClient = c
+		aiPipelineClient = c
 		aiRecommendationClient = c
 	}
 	if s := bootstrap.InitAISummarizerStorage(conf.AISummarizer, logger); s != nil {
@@ -226,8 +232,9 @@ func main() {
 	documentScraperService := document_scraper.NewService(tenderRepository, logger)
 	dashboardRepository := dashboard.NewRepository(mongoManager, logger)
 	dashboardService := dashboard.NewService(dashboardRepository, logger)
+	aiPipelineService := ai_pipeline.NewService(aiPipelineClient, logger)
 	logger.Info("Services initialized successfully", map[string]interface{}{
-		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper", "dashboard"},
+		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper", "dashboard", "ai_pipeline"},
 	})
 
 	// Initialize handlers with services
@@ -247,8 +254,9 @@ func main() {
 	fileStoreHandler := filestore.NewHandler(fileStoreService, logger)
 	documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger)
 	dashboardHandler := dashboard.NewHandler(dashboardService)
+	aiPipelineHandler := ai_pipeline.NewHandler(aiPipelineService)
 	logger.Info("Handlers initialized successfully", map[string]interface{}{
-		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper", "dashboard"},
+		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper", "dashboard", "ai_pipeline"},
 	})
 
 	// Initialize HTTP server
@@ -257,7 +265,7 @@ func main() {
 	router.SetupCSPSecurity(e)
 
 	// Register routes
-	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, dashboardHandler, xssPolicy)
+	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, dashboardHandler, aiPipelineHandler, xssPolicy)
 	router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, xssPolicy)
 	router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey)
 

cmd/web/router/routes.go

@@ -2,6 +2,7 @@ package router
 
 import (
 	"tm/internal/assets"
+	"tm/internal/ai_pipeline"
 	"tm/internal/cms"
 	"tm/internal/company"
 	"tm/internal/company_category"
@@ -34,7 +35,7 @@ func SetupCSPSecurity(e *echo.Echo) {
 	e.POST("/api/v1/csp-report", security.CSPReportHandler)
 }
 
-func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, dashboardHandler *dashboard.Handler, xssPolicy *security.XSSPolicy) {
+func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, dashboardHandler *dashboard.Handler, aiPipelineHandler *ai_pipeline.Handler, xssPolicy *security.XSSPolicy) {
 	adminV1 := e.Group("/admin/v1")
 
 	// Admin Users Routes
@@ -225,6 +226,27 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
 		dashboardGP.GET("/recent", dashboardHandler.Recent)
 		dashboardGP.GET("/statistics", dashboardHandler.Statistics)
 	}
+
+	// AI Pipeline Routes (proxy to Opplens AI service)
+	aiPipelineGP := adminV1.Group("/ai-pipeline")
+	{
+		aiPipelineGP.Use(userHandler.AuthMiddleware())
+		aiPipelineGP.POST("/scrape/documents", aiPipelineHandler.ScrapeDocuments)
+		aiPipelineGP.POST("/scrape/documents/batch", aiPipelineHandler.ScrapeDocumentsBatch)
+		aiPipelineGP.POST("/summarize/batch", aiPipelineHandler.SummarizeBatch)
+		aiPipelineGP.POST("/translate/batch", aiPipelineHandler.TranslateBatch)
+		aiPipelineGP.POST("/sync", aiPipelineHandler.Sync)
+		aiPipelineGP.POST("/run", aiPipelineHandler.Run)
+		aiPipelineGP.POST("/run/batch", aiPipelineHandler.RunBatch)
+		aiPipelineGP.POST("/analyze", aiPipelineHandler.Analyze)
+		aiPipelineGP.POST("/daily-run", aiPipelineHandler.DailyRun)
+		aiPipelineGP.POST("/auto", aiPipelineHandler.Auto)
+		aiPipelineGP.GET("/last-run", aiPipelineHandler.GetLastRun)
+		aiPipelineGP.GET("/last-auto-run", aiPipelineHandler.GetLastAutoRun)
+		aiPipelineGP.GET("/report", aiPipelineHandler.GetReport)
+		aiPipelineGP.GET("/stats", aiPipelineHandler.GetStats)
+		aiPipelineGP.GET("/minio-stats", aiPipelineHandler.GetMinioStats)
+	}
 }
 
 func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) {