API for AI document scraper integration

2026-04-28 16:53:36 +03:30
parent 4c5355ddf8
commit 81191656df
7 changed files with 372 additions and 16 deletions
@@ -0,0 +1,27 @@
+package document_scraper
+
+// DocumentScraperListRequest represents the request to list tenders pending document scraping
+type DocumentScraperListRequest struct {
+	Limit  int `query:"limit" valid:"optional,range(1|100)" default:"20"`
+	Offset int `query:"offset" valid:"optional,min(0)" default:"0"`
+}
+
+// DocumentScraperGetRequest represents the request to get a specific tender by notice publication ID
+type DocumentScraperGetRequest struct {
+	NoticeID string `param:"notice_id" valid:"required~Notice ID is required"`
+}
+
+// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service
+type DocumentScraperTenderResponse struct {
+	ID                  string      `json:"id"`
+	NoticePublicationID string      `json:"notice_publication_id"`
+	DocumentURL         string      `json:"document_url"`
+	Title               string      `json:"title"`
+	Description         string      `json:"description"`
+	ParsedXML           interface{} `json:"parsed_xml"`
+}
+
+// DocumentScraperListResponse represents the paginated list response
+type DocumentScraperListResponse struct {
+	Tenders []DocumentScraperTenderResponse `json:"tenders"`
+}
@@ -0,0 +1,95 @@
+package document_scraper
+
+import (
+	"strconv"
+	"tm/pkg/logger"
+	"tm/pkg/response"
+
+	"github.com/labstack/echo/v4"
+)
+
+// Handler handles HTTP requests for the document scraper API
+type Handler struct {
+	service Service
+	logger  logger.Logger
+}
+
+// NewHandler creates a new document scraper handler
+func NewHandler(service Service, logger logger.Logger) *Handler {
+	return &Handler{
+		service: service,
+		logger:  logger,
+	}
+}
+
+// ListPendingTenders retrieves tenders that need document scraping
+// @Summary List pending tenders for document scraping
+// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description, and parsed XML
+// @Tags Document-Scraper
+// @Produce json
+// @Param limit query int false "Number of items per page (default: 20, max: 100)"
+// @Param offset query int false "Number of items to skip (default: 0)"
+// @Success 200 {object} response.APIResponse{data=DocumentScraperListResponse} "List of pending tenders"
+// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
+// @Failure 500 {object} response.APIResponse "Internal server error"
+// @Security ApiKeyAuth
+// @Router /api/v1/scraper/tenders [get]
+func (h *Handler) ListPendingTenders(c echo.Context) error {
+	// Parse pagination parameters
+	limit, err := strconv.Atoi(c.QueryParam("limit"))
+	if err != nil || limit <= 0 {
+		limit = 20
+	}
+	if limit > 100 {
+		limit = 100
+	}
+
+	offset, err := strconv.Atoi(c.QueryParam("offset"))
+	if err != nil || offset < 0 {
+		offset = 0
+	}
+
+	tenders, meta, err := h.service.ListPendingTenders(c.Request().Context(), limit, offset)
+	if err != nil {
+		h.logger.Error("Failed to list pending tenders", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return response.InternalServerError(c, "Failed to list pending tenders")
+	}
+
+	return response.SuccessWithMeta(c, tenders, meta, "Pending tenders retrieved successfully")
+}
+
+// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
+// @Summary Get tender by notice ID for document scraping
+// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description, and parsed XML
+// @Tags Document-Scraper
+// @Produce json
+// @Param notice_id path string true "Notice Publication ID"
+// @Success 200 {object} response.APIResponse{data=DocumentScraperTenderResponse} "Tender details"
+// @Failure 400 {object} response.APIResponse "Invalid notice ID"
+// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
+// @Failure 404 {object} response.APIResponse "Tender not found"
+// @Failure 500 {object} response.APIResponse "Internal server error"
+// @Security ApiKeyAuth
+// @Router /api/v1/scraper/tenders/{notice_id} [get]
+func (h *Handler) GetTenderByNoticeID(c echo.Context) error {
+	noticeID := c.Param("notice_id")
+	if noticeID == "" {
+		return response.BadRequest(c, "Notice ID is required", "notice_id parameter cannot be empty")
+	}
+
+	tender, err := h.service.GetTenderByNoticeID(c.Request().Context(), noticeID)
+	if err != nil {
+		if err.Error() == "tender not found" {
+			return response.NotFound(c, "Tender not found")
+		}
+		h.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
+			"notice_id": noticeID,
+			"error":     err.Error(),
+		})
+		return response.InternalServerError(c, "Failed to get tender")
+	}
+
+	return response.Success(c, tender, "Tender retrieved successfully")
+}
@@ -0,0 +1,35 @@
+package document_scraper
+
+import (
+	"strings"
+	"tm/pkg/response"
+
+	"github.com/labstack/echo/v4"
+)
+
+// APIKeyMiddleware validates the API key for service-to-service authentication
+func APIKeyMiddleware(apiKey string) echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			// Extract API key from X-API-Key header
+			apiKeyHeader := c.Request().Header.Get("X-API-Key")
+			if apiKeyHeader == "" {
+				// Also check Authorization header with "ApiKey <key>" format
+				authHeader := c.Request().Header.Get("Authorization")
+				if strings.HasPrefix(authHeader, "ApiKey ") {
+					apiKeyHeader = strings.TrimPrefix(authHeader, "ApiKey ")
+				}
+			}
+
+			if apiKeyHeader == "" {
+				return response.Unauthorized(c, "API key required. Use X-API-Key header or Authorization: ApiKey <key>")
+			}
+
+			if apiKeyHeader != apiKey {
+				return response.Unauthorized(c, "Invalid API key")
+			}
+
+			return next(c)
+		}
+	}
+}
@@ -0,0 +1,175 @@
+package document_scraper
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"tm/internal/notice"
+	"tm/internal/tender"
+	"tm/pkg/logger"
+	"tm/pkg/response"
+	"tm/ted"
+)
+
+// Service defines business logic for the document scraper API
+type Service interface {
+	// ListPendingTenders retrieves tenders that need document scraping
+	ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error)
+	// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
+	GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error)
+}
+
+type service struct {
+	tenderRepo tender.TenderRepository
+	noticeRepo notice.Repository
+	tedParser  *ted.TEDParser
+	logger     logger.Logger
+}
+
+// NewService creates a new document scraper service
+func NewService(tenderRepo tender.TenderRepository, noticeRepo notice.Repository, logger logger.Logger) Service {
+	return &service{
+		tenderRepo: tenderRepo,
+		noticeRepo: noticeRepo,
+		tedParser:  ted.NewTEDParser(),
+		logger:     logger,
+	}
+}
+
+// parseXMLToJSON converts raw XML content into a structured JSON object
+func (s *service) parseXMLToJSON(xmlContent string) interface{} {
+	if xmlContent == "" {
+		return nil
+	}
+
+	parsedDoc, _, _, err := s.tedParser.ParseXML([]byte(xmlContent))
+	if err != nil {
+		s.logger.Warn("Failed to parse XML to JSON, returning raw XML string", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return xmlContent
+	}
+
+	// Convert to a generic map to clean up the JSON output
+	jsonBytes, err := json.Marshal(parsedDoc)
+	if err != nil {
+		s.logger.Warn("Failed to marshal parsed XML to JSON, returning raw XML string", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return xmlContent
+	}
+
+	var result interface{}
+	if err := json.Unmarshal(jsonBytes, &result); err != nil {
+		s.logger.Warn("Failed to unmarshal parsed XML, returning raw XML string", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return xmlContent
+	}
+
+	return result
+}
+
+// buildTenderResponse creates a DocumentScraperTenderResponse from a tender and its notice
+func (s *service) buildTenderResponse(ctx context.Context, t *tender.Tender) *DocumentScraperTenderResponse {
+	parsedXML := interface{}(nil)
+	documentURL := t.DocumentURI
+	if documentURL == "" {
+		documentURL = t.TenderURL
+	}
+
+	// Try to get the parsed XML from the corresponding notice
+	n, err := s.noticeRepo.GetByContractNoticeID(ctx, t.ContractNoticeID)
+	if err != nil {
+		s.logger.Warn("Failed to get notice for parsed XML", map[string]interface{}{
+			"contract_notice_id": t.ContractNoticeID,
+			"error":              err.Error(),
+		})
+	} else if n != nil {
+		parsedXML = s.parseXMLToJSON(n.ContentXML)
+		if documentURL == "" {
+			documentURL = n.DocumentURI
+		}
+	}
+
+	return &DocumentScraperTenderResponse{
+		ID:                  t.ID.Hex(),
+		NoticePublicationID: t.NoticePublicationID,
+		DocumentURL:         documentURL,
+		Title:               t.Title,
+		Description:         t.Description,
+		ParsedXML:           parsedXML,
+	}
+}
+
+// ListPendingTenders retrieves tenders that have not yet been scraped for documents
+func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) {
+	s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{
+		"limit":  limit,
+		"offset": offset,
+	})
+
+	tenders, totalCount, err := s.tenderRepo.GetUnScrapedTenders(ctx, limit, offset)
+	if err != nil {
+		s.logger.Error("Failed to list pending tenders", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
+	}
+
+	// Convert to response
+	result := &DocumentScraperListResponse{
+		Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
+	}
+
+	for i := range tenders {
+		result.Tenders[i] = *s.buildTenderResponse(ctx, &tenders[i])
+	}
+
+	// Build pagination metadata
+	meta := &response.Meta{
+		Total:  int(totalCount),
+		Limit:  limit,
+		Offset: offset,
+	}
+	if meta.Limit > 0 {
+		meta.Page = (offset / meta.Limit) + 1
+		meta.Pages = (int(totalCount) + meta.Limit - 1) / meta.Limit
+	}
+
+	s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
+		"count":       len(tenders),
+		"total_count": totalCount,
+	})
+
+	return result, meta, nil
+}
+
+// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
+func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
+	s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
+		"notice_id": noticeID,
+	})
+
+	t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID)
+	if err != nil {
+		s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
+			"notice_id": noticeID,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to get tender: %w", err)
+	}
+
+	if t == nil {
+		return nil, fmt.Errorf("tender not found")
+	}
+
+	result := s.buildTenderResponse(ctx, t)
+
+	s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{
+		"notice_id": noticeID,
+		"tender_id": t.ID.Hex(),
+	})
+
+	return result, nil
+}