Implement AI pipeline operations in the admin API
- Added new routes and handlers for AI pipeline operations, including scraping documents, batch summarization, translation, and syncing with the Opplens AI service. - Introduced request forms for handling tender references and batch operations. - Enhanced the AI service with methods for triggering batch operations and managing pipeline runs. - Updated Swagger documentation to reflect the new AI pipeline endpoints and their functionalities. This update integrates comprehensive AI pipeline capabilities into the tender management system, improving operational efficiency and user experience.
This commit is contained in:
Mazyar
@@ -0,0 +1,14 @@
|
||||
package ai_pipeline
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
// ErrAINotConfigured is returned when the Opplens AI client is not configured.
|
||||
ErrAINotConfigured = errors.New("ai_pipeline: AI service not configured")
|
||||
|
||||
// ErrEmptyTenderBatch is returned when a batch request has no tenders.
|
||||
ErrEmptyTenderBatch = errors.New("ai_pipeline: at least one tender is required")
|
||||
|
||||
// ErrEmptyLanguages is returned when languages are required but missing.
|
||||
ErrEmptyLanguages = errors.New("ai_pipeline: at least one language is required")
|
||||
)
|
||||
@@ -0,0 +1,73 @@
|
||||
package ai_pipeline
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"tm/pkg/ai_summarizer"
|
||||
)
|
||||
|
||||
// TenderRefForm identifies one tender in batch and pipeline requests.
|
||||
type TenderRefForm struct {
|
||||
ContractFolderID string `json:"contract_folder_id" valid:"required"`
|
||||
NoticePublicationID string `json:"notice_publication_id" valid:"required"`
|
||||
}
|
||||
|
||||
// ScrapeDocumentsForm is the request for POST /admin/v1/ai-pipeline/scrape/documents.
|
||||
type ScrapeDocumentsForm struct {
|
||||
ContractFolderID string `json:"contract_folder_id" valid:"required"`
|
||||
NoticePublicationID string `json:"notice_publication_id" valid:"required"`
|
||||
}
|
||||
|
||||
// TenderBatchForm is the request body for batch scrape and summarize endpoints.
|
||||
type TenderBatchForm struct {
|
||||
Tenders []TenderRefForm `json:"tenders" valid:"required"`
|
||||
}
|
||||
|
||||
// TranslateBatchForm is the request body for POST /admin/v1/ai-pipeline/translate/batch.
|
||||
type TranslateBatchForm struct {
|
||||
Tenders []TenderRefForm `json:"tenders" valid:"required"`
|
||||
Languages []string `json:"languages" valid:"required"`
|
||||
}
|
||||
|
||||
// PipelineRunForm is the request body for POST /admin/v1/ai-pipeline/run.
|
||||
type PipelineRunForm struct {
|
||||
ContractFolderID string `json:"contract_folder_id" valid:"required"`
|
||||
NoticePublicationID string `json:"notice_publication_id" valid:"required"`
|
||||
Languages []string `json:"languages" valid:"required"`
|
||||
}
|
||||
|
||||
// PipelineRunBatchForm is the request body for POST /admin/v1/ai-pipeline/run/batch.
|
||||
type PipelineRunBatchForm struct {
|
||||
Tenders []TenderRefForm `json:"tenders" valid:"required"`
|
||||
Languages []string `json:"languages" valid:"required"`
|
||||
}
|
||||
|
||||
// PipelineWindowQueryForm is the query form for report and stats endpoints.
|
||||
type PipelineWindowQueryForm struct {
|
||||
Window string `query:"window" valid:"optional,in(last_hour|last_24h|today|yesterday|last_7_days|last_30_days|all|daily)"`
|
||||
}
|
||||
|
||||
func toTenderRefs(forms []TenderRefForm) []ai_summarizer.TenderRef {
|
||||
refs := make([]ai_summarizer.TenderRef, 0, len(forms))
|
||||
for _, form := range forms {
|
||||
refs = append(refs, ai_summarizer.NewTenderRef(form.ContractFolderID, form.NoticePublicationID))
|
||||
}
|
||||
return refs
|
||||
}
|
||||
|
||||
func normalizeLanguages(languages []string) []string {
|
||||
out := make([]string, 0, len(languages))
|
||||
seen := make(map[string]struct{}, len(languages))
|
||||
for _, language := range languages {
|
||||
lang := strings.ToLower(strings.TrimSpace(language))
|
||||
if lang == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[lang]; dup {
|
||||
continue
|
||||
}
|
||||
seen[lang] = struct{}{}
|
||||
out = append(out, lang)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
package ai_pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"tm/pkg/ai_summarizer"
|
||||
"tm/pkg/response"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
// Handler handles admin HTTP requests for Opplens AI pipeline operations.
|
||||
type Handler struct {
|
||||
service Service
|
||||
}
|
||||
|
||||
// NewHandler creates a new AI pipeline handler.
|
||||
func NewHandler(service Service) *Handler {
|
||||
return &Handler{service: service}
|
||||
}
|
||||
|
||||
// ScrapeDocuments scrapes documents for one tender via the Opplens AI service.
|
||||
// @Summary Scrape tender documents
|
||||
// @Description Download all documents for one tender through the Opplens AI service (synchronous, cached)
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body ScrapeDocumentsForm true "Tender identifiers"
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.ScrapeDocumentsResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 404 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/scrape/documents [post]
|
||||
func (h *Handler) ScrapeDocuments(c echo.Context) error {
|
||||
form, err := response.Parse[ScrapeDocumentsForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.ScrapeDocuments(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Scrape documents")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Documents scraped successfully")
|
||||
}
|
||||
|
||||
// ScrapeDocumentsBatch enqueues background document scraping for many tenders.
|
||||
// @Summary Batch scrape tender documents
|
||||
// @Description Enqueue background document scraping for multiple tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body TenderBatchForm true "Tenders to scrape"
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/scrape/documents/batch [post]
|
||||
func (h *Handler) ScrapeDocumentsBatch(c echo.Context) error {
|
||||
form, err := response.Parse[TenderBatchForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.ScrapeDocumentsBatch(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Scrape documents batch")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Scrape batch accepted")
|
||||
}
|
||||
|
||||
// SummarizeBatch enqueues background summarization for many tenders.
|
||||
// @Summary Batch summarize tenders
|
||||
// @Description Enqueue background AI summarization for multiple tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body TenderBatchForm true "Tenders to summarize"
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/summarize/batch [post]
|
||||
func (h *Handler) SummarizeBatch(c echo.Context) error {
|
||||
form, err := response.Parse[TenderBatchForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.SummarizeBatch(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Summarize batch")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Summarize batch accepted")
|
||||
}
|
||||
|
||||
// TranslateBatch enqueues background translation for many tenders.
|
||||
// @Summary Batch translate tenders
|
||||
// @Description Enqueue background AI translation for multiple tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body TranslateBatchForm true "Tenders and target languages"
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/translate/batch [post]
|
||||
func (h *Handler) TranslateBatch(c echo.Context) error {
|
||||
form, err := response.Parse[TranslateBatchForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.TranslateBatch(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Translate batch")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Translate batch accepted")
|
||||
}
|
||||
|
||||
// Sync pulls the tender list from this backend into the Opplens AI service.
|
||||
// @Summary Sync tenders to AI pipeline
|
||||
// @Description Pull the tender list from the backend and store it in the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineSyncResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/sync [post]
|
||||
func (h *Handler) Sync(c echo.Context) error {
|
||||
result, err := h.service.Sync(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline sync")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline sync completed successfully")
|
||||
}
|
||||
|
||||
// Run runs the full AI pipeline for one tender (scrape → translate → summarize).
|
||||
// @Summary Run full AI pipeline for one tender
|
||||
// @Description Run scrape, translate, and summarize for one tender via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body PipelineRunForm true "Tender and languages"
|
||||
// @Success 200 {object} response.APIResponse{data=object}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 404 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/run [post]
|
||||
func (h *Handler) Run(c echo.Context) error {
|
||||
form, err := response.Parse[PipelineRunForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.Run(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline run")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline run completed successfully")
|
||||
}
|
||||
|
||||
// RunBatch enqueues the full AI pipeline for many tenders.
|
||||
// @Summary Batch run full AI pipeline
|
||||
// @Description Enqueue scrape, translate, and summarize for multiple tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body PipelineRunBatchForm true "Tenders and languages"
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/run/batch [post]
|
||||
func (h *Handler) RunBatch(c echo.Context) error {
|
||||
form, err := response.Parse[PipelineRunBatchForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.RunBatch(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline run batch")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Pipeline run batch accepted")
|
||||
}
|
||||
|
||||
// Analyze triggers analysis across stored tenders in the Opplens AI service.
|
||||
// @Summary Trigger AI analysis pipeline
|
||||
// @Description Trigger agentic analysis across stored tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineActionResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/analyze [post]
|
||||
func (h *Handler) Analyze(c echo.Context) error {
|
||||
result, err := h.service.Analyze(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline analyze")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline analyze triggered successfully")
|
||||
}
|
||||
|
||||
// DailyRun syncs tenders and runs the full pipeline on newly synced tenders.
|
||||
// @Summary Start daily AI pipeline run
|
||||
// @Description Sync tenders, then run the full pipeline on newly synced tenders (background, single-flight)
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.PipelineStartedResponse}
|
||||
// @Failure 409 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/daily-run [post]
|
||||
func (h *Handler) DailyRun(c echo.Context) error {
|
||||
result, err := h.service.DailyRun(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline daily-run")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Pipeline daily-run started")
|
||||
}
|
||||
|
||||
// Auto syncs tenders and completes all missing AI work across stored tenders.
|
||||
// @Summary Start auto AI pipeline run
|
||||
// @Description Sync tenders, then complete all missing scrape/translate/summarize work (background, single-flight, idempotent)
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.PipelineStartedResponse}
|
||||
// @Failure 409 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/auto [post]
|
||||
func (h *Handler) Auto(c echo.Context) error {
|
||||
result, err := h.service.Auto(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline auto")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Pipeline auto started")
|
||||
}
|
||||
|
||||
// GetLastRun returns the result of the last daily-run.
|
||||
// @Summary Get last daily-run report
|
||||
// @Description Retrieve the report for the last pipeline daily-run from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/last-run [get]
|
||||
func (h *Handler) GetLastRun(c echo.Context) error {
|
||||
result, err := h.service.GetLastRun(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get last pipeline run")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Last pipeline run retrieved successfully")
|
||||
}
|
||||
|
||||
// GetLastAutoRun returns the result of the last auto run.
|
||||
// @Summary Get last auto-run report
|
||||
// @Description Retrieve the report for the last pipeline auto run from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/last-auto-run [get]
|
||||
func (h *Handler) GetLastAutoRun(c echo.Context) error {
|
||||
result, err := h.service.GetLastAutoRun(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get last pipeline auto run")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Last pipeline auto run retrieved successfully")
|
||||
}
|
||||
|
||||
// GetReport returns an event-log summary for a time window.
|
||||
// @Summary Get AI pipeline event report
|
||||
// @Description Retrieve an event-log summary for a time window from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Param window query string false "Time window" Enums(last_hour,last_24h,today,yesterday,last_7_days,last_30_days,all,daily) default(last_24h)
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/report [get]
|
||||
func (h *Handler) GetReport(c echo.Context) error {
|
||||
form, err := response.Parse[PipelineWindowQueryForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid query parameters", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.GetReport(c.Request().Context(), form.Window)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get pipeline report")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline report retrieved successfully")
|
||||
}
|
||||
|
||||
// GetStats returns event-log and storage inventory for a time window.
|
||||
// @Summary Get AI pipeline stats
|
||||
// @Description Retrieve event-log summary and MinIO inventory for a time window from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Param window query string false "Time window" Enums(last_hour,last_24h,today,yesterday,last_7_days,last_30_days,all,daily) default(last_24h)
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineStatsResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/stats [get]
|
||||
func (h *Handler) GetStats(c echo.Context) error {
|
||||
form, err := response.Parse[PipelineWindowQueryForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid query parameters", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.GetStats(c.Request().Context(), form.Window)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get pipeline stats")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline stats retrieved successfully")
|
||||
}
|
||||
|
||||
// GetMinioStats returns storage inventory from the Opplens AI service.
|
||||
// @Summary Get AI pipeline MinIO stats
|
||||
// @Description Retrieve storage inventory only from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineMinioStatsResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/minio-stats [get]
|
||||
func (h *Handler) GetMinioStats(c echo.Context) error {
|
||||
result, err := h.service.GetMinioStats(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get pipeline minio stats")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline MinIO stats retrieved successfully")
|
||||
}
|
||||
|
||||
func mapPipelineHTTPError(c echo.Context, err error, action string) error {
|
||||
switch {
|
||||
case errors.Is(err, ErrAINotConfigured):
|
||||
return response.ServiceUnavailable(c, "AI service is not configured")
|
||||
case errors.Is(err, ErrEmptyTenderBatch), errors.Is(err, ErrEmptyLanguages):
|
||||
return response.BadRequest(c, "Invalid request", err.Error())
|
||||
case errors.Is(err, ai_summarizer.ErrPipelineJobRunning):
|
||||
return response.Conflict(c, "A pipeline job is already running")
|
||||
case errors.Is(err, ai_summarizer.ErrObjectNotFound):
|
||||
return response.NotFound(c, "Tender not found in AI service")
|
||||
default:
|
||||
if errors.Is(err, ai_summarizer.ErrAPINonSuccess) {
|
||||
return c.JSON(http.StatusBadGateway, response.APIResponse{
|
||||
Success: false,
|
||||
Message: action + " failed: upstream AI service error",
|
||||
})
|
||||
}
|
||||
return response.InternalServerError(c, action+" failed")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,400 @@
|
||||
package ai_pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"tm/pkg/ai_summarizer"
|
||||
"tm/pkg/logger"
|
||||
)
|
||||
|
||||
// Client defines Opplens AI operations exposed to the admin API.
|
||||
type Client interface {
|
||||
ScrapeDocuments(ctx context.Context, reqBody ai_summarizer.ScrapeDocumentsRequest) (*ai_summarizer.ScrapeDocumentsResponse, error)
|
||||
ScrapeDocumentsBatch(ctx context.Context, tenders []ai_summarizer.TenderRef) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
TriggerSummarizeBatch(ctx context.Context, tenders []ai_summarizer.TenderRef) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
TriggerTranslateBatch(ctx context.Context, tenders []ai_summarizer.TenderRef, languages []string) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
PipelineSync(ctx context.Context) (*ai_summarizer.PipelineSyncResponse, error)
|
||||
PipelineRun(ctx context.Context, reqBody ai_summarizer.PipelineRunRequest) (map[string]interface{}, error)
|
||||
PipelineRunBatch(ctx context.Context, tenders []ai_summarizer.TenderRef, languages []string) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
PipelineAnalyze(ctx context.Context) (*ai_summarizer.PipelineActionResponse, error)
|
||||
PipelineDailyRun(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||
PipelineAuto(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||
GetPipelineLastRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetPipelineLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetPipelineReport(ctx context.Context, window string) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetPipelineStats(ctx context.Context, window string) (*ai_summarizer.PipelineStatsResponse, error)
|
||||
GetPipelineMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error)
|
||||
}
|
||||
|
||||
// Service proxies admin pipeline operations to the Opplens AI service.
|
||||
type Service interface {
|
||||
ScrapeDocuments(ctx context.Context, form ScrapeDocumentsForm) (*ai_summarizer.ScrapeDocumentsResponse, error)
|
||||
ScrapeDocumentsBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
SummarizeBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
TranslateBatch(ctx context.Context, form TranslateBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
Sync(ctx context.Context) (*ai_summarizer.PipelineSyncResponse, error)
|
||||
Run(ctx context.Context, form PipelineRunForm) (map[string]interface{}, error)
|
||||
RunBatch(ctx context.Context, form PipelineRunBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
Analyze(ctx context.Context) (*ai_summarizer.PipelineActionResponse, error)
|
||||
DailyRun(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||
Auto(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||
GetLastRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetReport(ctx context.Context, window string) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetStats(ctx context.Context, window string) (*ai_summarizer.PipelineStatsResponse, error)
|
||||
GetMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error)
|
||||
}
|
||||
|
||||
type service struct {
|
||||
client Client
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
// NewService creates an AI pipeline admin service.
|
||||
func NewService(client Client, log logger.Logger) Service {
|
||||
return &service{
|
||||
client: client,
|
||||
logger: log,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *service) requireClient() error {
|
||||
if s.client == nil {
|
||||
return ErrAINotConfigured
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *service) ScrapeDocuments(ctx context.Context, form ScrapeDocumentsForm) (*ai_summarizer.ScrapeDocumentsResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin scrape documents requested", map[string]interface{}{
|
||||
"contract_folder_id": form.ContractFolderID,
|
||||
"notice_publication_id": form.NoticePublicationID,
|
||||
})
|
||||
|
||||
resp, err := s.client.ScrapeDocuments(ctx, ai_summarizer.ScrapeDocumentsRequest{
|
||||
ContractFolderID: form.ContractFolderID,
|
||||
NoticePublicationID: form.NoticePublicationID,
|
||||
})
|
||||
if err != nil {
|
||||
s.logger.Error("Admin scrape documents failed", map[string]interface{}{
|
||||
"contract_folder_id": form.ContractFolderID,
|
||||
"notice_publication_id": form.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("scrape documents failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) ScrapeDocumentsBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs, err := validateTenderBatch(form.Tenders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin scrape documents batch requested", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
})
|
||||
|
||||
resp, err := s.client.ScrapeDocumentsBatch(ctx, refs)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin scrape documents batch failed", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("scrape documents batch failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) SummarizeBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs, err := validateTenderBatch(form.Tenders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin summarize batch requested", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
})
|
||||
|
||||
resp, err := s.client.TriggerSummarizeBatch(ctx, refs)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin summarize batch failed", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("summarize batch failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) TranslateBatch(ctx context.Context, form TranslateBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs, err := validateTenderBatch(form.Tenders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
languages := normalizeLanguages(form.Languages)
|
||||
if len(languages) == 0 {
|
||||
return nil, ErrEmptyLanguages
|
||||
}
|
||||
|
||||
s.logger.Info("Admin translate batch requested", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"languages": languages,
|
||||
})
|
||||
|
||||
resp, err := s.client.TriggerTranslateBatch(ctx, refs, languages)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin translate batch failed", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"languages": languages,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("translate batch failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) Sync(ctx context.Context) (*ai_summarizer.PipelineSyncResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline sync requested", map[string]interface{}{})
|
||||
|
||||
resp, err := s.client.PipelineSync(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline sync failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline sync failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) Run(ctx context.Context, form PipelineRunForm) (map[string]interface{}, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
languages := normalizeLanguages(form.Languages)
|
||||
if len(languages) == 0 {
|
||||
return nil, ErrEmptyLanguages
|
||||
}
|
||||
|
||||
req := ai_summarizer.PipelineRunRequest{
|
||||
ContractFolderID: form.ContractFolderID,
|
||||
NoticePublicationID: form.NoticePublicationID,
|
||||
Languages: languages,
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline run requested", map[string]interface{}{
|
||||
"contract_folder_id": req.ContractFolderID,
|
||||
"notice_publication_id": req.NoticePublicationID,
|
||||
"languages": languages,
|
||||
})
|
||||
|
||||
resp, err := s.client.PipelineRun(ctx, req)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline run failed", map[string]interface{}{
|
||||
"contract_folder_id": req.ContractFolderID,
|
||||
"notice_publication_id": req.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline run failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) RunBatch(ctx context.Context, form PipelineRunBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs, err := validateTenderBatch(form.Tenders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
languages := normalizeLanguages(form.Languages)
|
||||
if len(languages) == 0 {
|
||||
return nil, ErrEmptyLanguages
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline run batch requested", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"languages": languages,
|
||||
})
|
||||
|
||||
resp, err := s.client.PipelineRunBatch(ctx, refs, languages)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline run batch failed", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"languages": languages,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline run batch failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) Analyze(ctx context.Context) (*ai_summarizer.PipelineActionResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline analyze requested", map[string]interface{}{})
|
||||
|
||||
resp, err := s.client.PipelineAnalyze(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline analyze failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline analyze failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) DailyRun(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline daily-run requested", map[string]interface{}{})
|
||||
|
||||
resp, err := s.client.PipelineDailyRun(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline daily-run failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline daily-run failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) Auto(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline auto requested", map[string]interface{}{})
|
||||
|
||||
resp, err := s.client.PipelineAuto(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline auto failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline auto failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetLastRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineLastRun(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline last-run failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline last-run failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineLastAutoRun(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline last-auto-run failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline last-auto-run failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetReport(ctx context.Context, window string) (*ai_summarizer.PipelineReportResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineReport(ctx, window)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline report failed", map[string]interface{}{
|
||||
"window": window,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline report failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetStats(ctx context.Context, window string) (*ai_summarizer.PipelineStatsResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineStats(ctx, window)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline stats failed", map[string]interface{}{
|
||||
"window": window,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline stats failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineMinioStats(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline minio stats failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline minio stats failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func validateTenderBatch(forms []TenderRefForm) ([]ai_summarizer.TenderRef, error) {
|
||||
if len(forms) == 0 {
|
||||
return nil, ErrEmptyTenderBatch
|
||||
}
|
||||
return toTenderRefs(forms), nil
|
||||
}
|
||||
Reference in New Issue
Block a user