Mazyar
e5fa0dfe47
continuous-integration/drone/push Build is passing
- Introduced the GetScrapePortals method in the AI pipeline handler to list document scraping portals supported by the Opplens AI service. - Updated the service layer to include GetScrapePortals, which retrieves the portals from the client and handles errors appropriately. - Enhanced the routes to register the new endpoint for retrieving scrape portals. - Added a new error type for invalid date ranges in the document scraper, improving validation and error handling. This update expands the AI pipeline capabilities, allowing for better management of document scraping portals within the tender management system.
452 lines
17 KiB
Go
452 lines
17 KiB
Go
package ai_pipeline
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
|
|
"tm/pkg/ai_summarizer"
|
|
"tm/pkg/response"
|
|
|
|
"github.com/labstack/echo/v4"
|
|
)
|
|
|
|
// Handler handles admin HTTP requests for Opplens AI pipeline operations.
|
|
type Handler struct {
|
|
service Service
|
|
}
|
|
|
|
// NewHandler creates a new AI pipeline handler.
|
|
func NewHandler(service Service) *Handler {
|
|
return &Handler{service: service}
|
|
}
|
|
|
|
// ScrapeDocuments scrapes documents for one tender via the Opplens AI service.
|
|
// @Summary Scrape tender documents
|
|
// @Description Download all documents for one tender through the Opplens AI service (synchronous, cached)
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Param request body ScrapeDocumentsForm true "Tender identifiers"
|
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.ScrapeDocumentsResponse}
|
|
// @Failure 400 {object} response.APIResponse
|
|
// @Failure 404 {object} response.APIResponse
|
|
// @Failure 422 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/scrape/documents [post]
|
|
func (h *Handler) ScrapeDocuments(c echo.Context) error {
|
|
form, err := response.Parse[ScrapeDocumentsForm](c)
|
|
if err != nil {
|
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
|
}
|
|
|
|
result, err := h.service.ScrapeDocuments(c.Request().Context(), *form)
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Scrape documents")
|
|
}
|
|
|
|
return response.Success(c, result, "Documents scraped successfully")
|
|
}
|
|
|
|
// ScrapeDocumentsBatch enqueues background document scraping for many tenders.
|
|
// @Summary Batch scrape tender documents
|
|
// @Description Enqueue background document scraping for multiple tenders via the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Param request body TenderBatchForm true "Tenders to scrape"
|
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
|
// @Failure 400 {object} response.APIResponse
|
|
// @Failure 422 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/scrape/documents/batch [post]
|
|
func (h *Handler) ScrapeDocumentsBatch(c echo.Context) error {
|
|
form, err := response.Parse[TenderBatchForm](c)
|
|
if err != nil {
|
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
|
}
|
|
|
|
result, err := h.service.ScrapeDocumentsBatch(c.Request().Context(), *form)
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Scrape documents batch")
|
|
}
|
|
|
|
return response.Accepted(c, result, "Scrape batch accepted")
|
|
}
|
|
|
|
// GetScrapePortals lists document scraping portals supported by the Opplens AI service.
|
|
// @Summary List document scraping portals
|
|
// @Description Retrieve the list of document scraping portals supported by the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.ScrapePortalsResponse} "Scrape portals retrieved successfully"
|
|
// @Failure 401 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/scrape/portals [get]
|
|
func (h *Handler) GetScrapePortals(c echo.Context) error {
|
|
result, err := h.service.GetScrapePortals(c.Request().Context())
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Get scrape portals")
|
|
}
|
|
|
|
return response.Success(c, result, "Scrape portals retrieved successfully")
|
|
}
|
|
|
|
// SummarizeBatch enqueues background summarization for many tenders.
|
|
// @Summary Batch summarize tenders
|
|
// @Description Enqueue background AI summarization for multiple tenders via the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Param request body TenderBatchForm true "Tenders to summarize"
|
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
|
// @Failure 400 {object} response.APIResponse
|
|
// @Failure 422 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/summarize/batch [post]
|
|
func (h *Handler) SummarizeBatch(c echo.Context) error {
|
|
form, err := response.Parse[TenderBatchForm](c)
|
|
if err != nil {
|
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
|
}
|
|
|
|
result, err := h.service.SummarizeBatch(c.Request().Context(), *form)
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Summarize batch")
|
|
}
|
|
|
|
return response.Accepted(c, result, "Summarize batch accepted")
|
|
}
|
|
|
|
// TranslateBatch enqueues background translation for many tenders.
|
|
// @Summary Batch translate tenders
|
|
// @Description Enqueue background AI translation for multiple tenders via the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Param request body TranslateBatchForm true "Tenders and target languages"
|
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
|
// @Failure 400 {object} response.APIResponse
|
|
// @Failure 422 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/translate/batch [post]
|
|
func (h *Handler) TranslateBatch(c echo.Context) error {
|
|
form, err := response.Parse[TranslateBatchForm](c)
|
|
if err != nil {
|
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
|
}
|
|
|
|
result, err := h.service.TranslateBatch(c.Request().Context(), *form)
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Translate batch")
|
|
}
|
|
|
|
return response.Accepted(c, result, "Translate batch accepted")
|
|
}
|
|
|
|
// Sync pulls the tender list from this backend into the Opplens AI service.
|
|
// @Summary Sync tenders to AI pipeline
|
|
// @Description Pull the tender list from the backend and store it in the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineSyncResponse}
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/sync [post]
|
|
func (h *Handler) Sync(c echo.Context) error {
|
|
result, err := h.service.Sync(c.Request().Context())
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Pipeline sync")
|
|
}
|
|
|
|
return response.Success(c, result, "Pipeline sync completed successfully")
|
|
}
|
|
|
|
// Run runs the full AI pipeline for one tender (scrape → translate → summarize).
|
|
// @Summary Run full AI pipeline for one tender
|
|
// @Description Run scrape, translate, and summarize for one tender via the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Param request body PipelineRunForm true "Tender and languages"
|
|
// @Success 200 {object} response.APIResponse{data=object}
|
|
// @Failure 400 {object} response.APIResponse
|
|
// @Failure 404 {object} response.APIResponse
|
|
// @Failure 422 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/run [post]
|
|
func (h *Handler) Run(c echo.Context) error {
|
|
form, err := response.Parse[PipelineRunForm](c)
|
|
if err != nil {
|
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
|
}
|
|
|
|
result, err := h.service.Run(c.Request().Context(), *form)
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Pipeline run")
|
|
}
|
|
|
|
return response.Success(c, result, "Pipeline run completed successfully")
|
|
}
|
|
|
|
// RunBatch enqueues the full AI pipeline for many tenders.
|
|
// @Summary Batch run full AI pipeline
|
|
// @Description Enqueue scrape, translate, and summarize for multiple tenders via the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Accept json
|
|
// @Produce json
|
|
// @Param request body PipelineRunBatchForm true "Tenders and languages"
|
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
|
// @Failure 400 {object} response.APIResponse
|
|
// @Failure 422 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/run/batch [post]
|
|
func (h *Handler) RunBatch(c echo.Context) error {
|
|
form, err := response.Parse[PipelineRunBatchForm](c)
|
|
if err != nil {
|
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
|
}
|
|
|
|
result, err := h.service.RunBatch(c.Request().Context(), *form)
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Pipeline run batch")
|
|
}
|
|
|
|
return response.Accepted(c, result, "Pipeline run batch accepted")
|
|
}
|
|
|
|
// Analyze triggers analysis across stored tenders in the Opplens AI service.
|
|
// @Summary Trigger AI analysis pipeline
|
|
// @Description Trigger agentic analysis across stored tenders via the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineActionResponse}
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/analyze [post]
|
|
func (h *Handler) Analyze(c echo.Context) error {
|
|
result, err := h.service.Analyze(c.Request().Context())
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Pipeline analyze")
|
|
}
|
|
|
|
return response.Success(c, result, "Pipeline analyze triggered successfully")
|
|
}
|
|
|
|
// DailyRun syncs tenders and runs the full pipeline on newly synced tenders.
|
|
// @Summary Start daily AI pipeline run
|
|
// @Description Sync tenders, then run the full pipeline on newly synced tenders (background, single-flight)
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.PipelineStartedResponse}
|
|
// @Failure 409 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/daily-run [post]
|
|
func (h *Handler) DailyRun(c echo.Context) error {
|
|
result, err := h.service.DailyRun(c.Request().Context())
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Pipeline daily-run")
|
|
}
|
|
|
|
return response.Accepted(c, result, "Pipeline daily-run started")
|
|
}
|
|
|
|
// Auto syncs tenders and completes all missing AI work across stored tenders.
|
|
// @Summary Start auto AI pipeline run
|
|
// @Description Sync tenders, then complete all missing scrape/translate/summarize work (background, single-flight, idempotent)
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.PipelineStartedResponse}
|
|
// @Failure 409 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/auto [post]
|
|
func (h *Handler) Auto(c echo.Context) error {
|
|
result, err := h.service.Auto(c.Request().Context())
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Pipeline auto")
|
|
}
|
|
|
|
return response.Accepted(c, result, "Pipeline auto started")
|
|
}
|
|
|
|
// GetLastRun returns the result of the last daily-run.
|
|
// @Summary Get last daily-run report
|
|
// @Description Retrieve the report for the last pipeline daily-run from the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/last-run [get]
|
|
func (h *Handler) GetLastRun(c echo.Context) error {
|
|
result, err := h.service.GetLastRun(c.Request().Context())
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Get last pipeline run")
|
|
}
|
|
|
|
return response.Success(c, result, "Last pipeline run retrieved successfully")
|
|
}
|
|
|
|
// GetLastAutoRun returns the result of the last auto run.
|
|
// @Summary Get last auto-run report
|
|
// @Description Retrieve the report for the last pipeline auto run from the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/last-auto-run [get]
|
|
func (h *Handler) GetLastAutoRun(c echo.Context) error {
|
|
result, err := h.service.GetLastAutoRun(c.Request().Context())
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Get last pipeline auto run")
|
|
}
|
|
|
|
return response.Success(c, result, "Last pipeline auto run retrieved successfully")
|
|
}
|
|
|
|
// GetReport returns an event-log summary for a time window.
|
|
// @Summary Get AI pipeline event report
|
|
// @Description Retrieve an event-log summary for a time window from the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Param window query string false "Time window" Enums(last_hour,last_24h,today,yesterday,last_7_days,last_30_days,all,daily) default(last_24h)
|
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
|
// @Failure 400 {object} response.APIResponse
|
|
// @Failure 422 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/report [get]
|
|
func (h *Handler) GetReport(c echo.Context) error {
|
|
form, err := response.Parse[PipelineWindowQueryForm](c)
|
|
if err != nil {
|
|
return response.ValidationError(c, "Invalid query parameters", err.Error())
|
|
}
|
|
|
|
result, err := h.service.GetReport(c.Request().Context(), form.Window)
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Get pipeline report")
|
|
}
|
|
|
|
return response.Success(c, result, "Pipeline report retrieved successfully")
|
|
}
|
|
|
|
// GetStats returns event-log and storage inventory for a time window.
|
|
// @Summary Get AI pipeline stats
|
|
// @Description Retrieve event-log summary and MinIO inventory for a time window from the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Param window query string false "Time window" Enums(last_hour,last_24h,today,yesterday,last_7_days,last_30_days,all,daily) default(last_24h)
|
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineStatsResponse}
|
|
// @Failure 400 {object} response.APIResponse
|
|
// @Failure 422 {object} response.APIResponse
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/stats [get]
|
|
func (h *Handler) GetStats(c echo.Context) error {
|
|
form, err := response.Parse[PipelineWindowQueryForm](c)
|
|
if err != nil {
|
|
return response.ValidationError(c, "Invalid query parameters", err.Error())
|
|
}
|
|
|
|
result, err := h.service.GetStats(c.Request().Context(), form.Window)
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Get pipeline stats")
|
|
}
|
|
|
|
return response.Success(c, result, "Pipeline stats retrieved successfully")
|
|
}
|
|
|
|
// GetMinioStats returns storage inventory from the Opplens AI service.
|
|
// @Summary Get AI pipeline MinIO stats
|
|
// @Description Retrieve storage inventory only from the Opplens AI service
|
|
// @Tags Admin-AI-Pipeline
|
|
// @Produce json
|
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineMinioStatsResponse}
|
|
// @Failure 500 {object} response.APIResponse
|
|
// @Failure 503 {object} response.APIResponse
|
|
// @Security BearerAuth
|
|
// @Router /admin/v1/ai-pipeline/minio-stats [get]
|
|
func (h *Handler) GetMinioStats(c echo.Context) error {
|
|
result, err := h.service.GetMinioStats(c.Request().Context())
|
|
if err != nil {
|
|
return mapPipelineHTTPError(c, err, "Get pipeline minio stats")
|
|
}
|
|
|
|
return response.Success(c, result, "Pipeline MinIO stats retrieved successfully")
|
|
}
|
|
|
|
func mapPipelineHTTPError(c echo.Context, err error, action string) error {
|
|
switch {
|
|
case errors.Is(err, ErrAINotConfigured):
|
|
return response.ServiceUnavailable(c, "AI service is not configured")
|
|
case errors.Is(err, ErrEmptyTenderBatch), errors.Is(err, ErrEmptyLanguages):
|
|
return response.BadRequest(c, "Invalid request", err.Error())
|
|
case errors.Is(err, ai_summarizer.ErrPipelineJobRunning):
|
|
return response.Conflict(c, "A pipeline job is already running")
|
|
case errors.Is(err, ai_summarizer.ErrObjectNotFound):
|
|
return response.NotFound(c, "Tender not found in AI service")
|
|
}
|
|
|
|
var upstream *ai_summarizer.APIStatusError
|
|
if errors.As(err, &upstream) {
|
|
details := upstream.Body
|
|
if details == "" {
|
|
details = fmt.Sprintf("upstream status %d", upstream.StatusCode)
|
|
}
|
|
|
|
switch upstream.StatusCode {
|
|
case http.StatusNotFound:
|
|
return response.NotFound(c, "Tender not found in AI service")
|
|
case http.StatusUnprocessableEntity:
|
|
return response.ValidationError(c, "AI service rejected the request", details)
|
|
case http.StatusBadGateway:
|
|
return c.JSON(http.StatusBadGateway, response.APIResponse{
|
|
Success: false,
|
|
Message: action + " failed",
|
|
Error: &response.APIError{
|
|
Code: "UPSTREAM_AI_ERROR",
|
|
Message: "Opplens AI service returned status 502",
|
|
Details: details,
|
|
},
|
|
})
|
|
default:
|
|
return c.JSON(http.StatusBadGateway, response.APIResponse{
|
|
Success: false,
|
|
Message: action + " failed",
|
|
Error: &response.APIError{
|
|
Code: "UPSTREAM_AI_ERROR",
|
|
Message: fmt.Sprintf("Opplens AI service returned status %d", upstream.StatusCode),
|
|
Details: details,
|
|
},
|
|
})
|
|
}
|
|
}
|
|
|
|
return response.InternalServerError(c, action+" failed")
|
|
}
|