API for AI document scraper integration

cmd/web/bootstrap/config.go

@@ -7,20 +7,21 @@ import (
 
 // Config holds configuration for the web command
 type Config struct {
-	Server       config.ServerConfig
+	Server          config.ServerConfig
-	Database     config.DatabaseConfig
+	Database        config.DatabaseConfig
-	Cache        config.CacheConfig
+	Cache           config.CacheConfig
-	Logging      config.LoggingConfig
+	Logging         config.LoggingConfig
-	RateLimit    config.RateLimitConfig
+	RateLimit       config.RateLimitConfig
-	Assets       config.AssetsConfig
+	Assets          config.AssetsConfig
-	Notification config.NotificationConfig
+	Notification    config.NotificationConfig
-	Ollama       config.OllamaConfig
+	Ollama          config.OllamaConfig
-	HCaptcha     config.HCaptchaConfig
+	HCaptcha        config.HCaptchaConfig
-	GLM          GLMConfig
+	GLM             GLMConfig
-	Scraper      ScraperConfig
+	Scraper         ScraperConfig
-	TED          TEDConfig
+	TED             TEDConfig
-	UserAuth     AuthConfig
+	UserAuth        AuthConfig
-	CustomerAuth AuthConfig
+	CustomerAuth    AuthConfig
+	DocumentScraper DocumentScraperConfig
 }
 
 type GLMConfig struct {
@@ -82,3 +83,8 @@ type JWTConfig struct {
 	AccessExpiresIn  int    `env:"USER_AUTH_ACCESS_EXPIRES_IN"`
 	RefreshExpiresIn int    `env:"USER_AUTH_REFRESH_EXPIRES_IN"`
 }
+
+// DocumentScraperConfig holds configuration for the external document scraper service API
+type DocumentScraperConfig struct {
+	APIKey string `env:"DOCUMENT_SCRAPER_API_KEY" envDefault:""`
+}

cmd/web/main.go

@@ -100,6 +100,7 @@ import (
 	"tm/internal/company_category"
 	"tm/internal/contact"
 	"tm/internal/customer"
+	"tm/internal/document_scraper"
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/kanban"
@@ -258,8 +259,9 @@ func main() {
 	tedConfig := buildTEDConfig(&conf.TED)
 	scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig)
 	kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger)
+	documentScraperService := document_scraper.NewService(tenderRepository, noticeRepository, logger)
 	logger.Info("Services initialized successfully", map[string]interface{}{
-		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban"},
+		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper"},
 	})
 
 	// Initialize handlers with services
@@ -278,8 +280,9 @@ func main() {
 	scraperHandler := scraper.NewHandler(scraperService)
 	kanbanHandler := kanban.NewHandler(kanbanService)
 	fileStoreHandler := filestore.NewHandler(fileStoreService, logger)
+	documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger)
 	logger.Info("Handlers initialized successfully", map[string]interface{}{
-		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore"},
+		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper"},
 	})
 
 	// Initialize HTTP server
@@ -288,6 +291,7 @@ func main() {
 	// Register routes
 	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler, kanbanHandler, fileStoreHandler, xssPolicy)
 	router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, fileStoreHandler, xssPolicy)
+	router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey)
 
 	// Start server
 	serverAddr := fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port)

cmd/web/router/routes.go

@@ -7,6 +7,7 @@ import (
 	"tm/internal/company_category"
 	"tm/internal/contact"
 	"tm/internal/customer"
+	"tm/internal/document_scraper"
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/kanban"
@@ -317,3 +318,16 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende
 		publicFilesGP.DELETE("/:file_id", fileStoreHandler.Delete)
 	}
 }
+
+func RegisterDocumentScraperRoutes(e *echo.Echo, docScraperHandler *document_scraper.Handler, apiKey string) {
+	v1 := e.Group("/api/v1/scraper")
+
+	// Apply API key middleware to all document scraper routes
+	v1.Use(document_scraper.APIKeyMiddleware(apiKey))
+
+	// Document Scraper Routes
+	{
+		v1.GET("/tenders", docScraperHandler.ListPendingTenders)
+		v1.GET("/tenders/:notice_id", docScraperHandler.GetTenderByNoticeID)
+	}
+}

internal/document_scraper/form.go

@@ -0,0 +1,27 @@
+package document_scraper
+
+// DocumentScraperListRequest represents the request to list tenders pending document scraping
+type DocumentScraperListRequest struct {
+	Limit  int `query:"limit" valid:"optional,range(1|100)" default:"20"`
+	Offset int `query:"offset" valid:"optional,min(0)" default:"0"`
+}
+
+// DocumentScraperGetRequest represents the request to get a specific tender by notice publication ID
+type DocumentScraperGetRequest struct {
+	NoticeID string `param:"notice_id" valid:"required~Notice ID is required"`
+}
+
+// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service
+type DocumentScraperTenderResponse struct {
+	ID                  string      `json:"id"`
+	NoticePublicationID string      `json:"notice_publication_id"`
+	DocumentURL         string      `json:"document_url"`
+	Title               string      `json:"title"`
+	Description         string      `json:"description"`
+	ParsedXML           interface{} `json:"parsed_xml"`
+}
+
+// DocumentScraperListResponse represents the paginated list response
+type DocumentScraperListResponse struct {
+	Tenders []DocumentScraperTenderResponse `json:"tenders"`
+}

internal/document_scraper/handler.go

@@ -0,0 +1,95 @@
+package document_scraper
+
+import (
+	"strconv"
+	"tm/pkg/logger"
+	"tm/pkg/response"
+
+	"github.com/labstack/echo/v4"
+)
+
+// Handler handles HTTP requests for the document scraper API
+type Handler struct {
+	service Service
+	logger  logger.Logger
+}
+
+// NewHandler creates a new document scraper handler
+func NewHandler(service Service, logger logger.Logger) *Handler {
+	return &Handler{
+		service: service,
+		logger:  logger,
+	}
+}
+
+// ListPendingTenders retrieves tenders that need document scraping
+// @Summary List pending tenders for document scraping
+// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description, and parsed XML
+// @Tags Document-Scraper
+// @Produce json
+// @Param limit query int false "Number of items per page (default: 20, max: 100)"
+// @Param offset query int false "Number of items to skip (default: 0)"
+// @Success 200 {object} response.APIResponse{data=DocumentScraperListResponse} "List of pending tenders"
+// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
+// @Failure 500 {object} response.APIResponse "Internal server error"
+// @Security ApiKeyAuth
+// @Router /api/v1/scraper/tenders [get]
+func (h *Handler) ListPendingTenders(c echo.Context) error {
+	// Parse pagination parameters
+	limit, err := strconv.Atoi(c.QueryParam("limit"))
+	if err != nil || limit <= 0 {
+		limit = 20
+	}
+	if limit > 100 {
+		limit = 100
+	}
+
+	offset, err := strconv.Atoi(c.QueryParam("offset"))
+	if err != nil || offset < 0 {
+		offset = 0
+	}
+
+	tenders, meta, err := h.service.ListPendingTenders(c.Request().Context(), limit, offset)
+	if err != nil {
+		h.logger.Error("Failed to list pending tenders", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return response.InternalServerError(c, "Failed to list pending tenders")
+	}
+
+	return response.SuccessWithMeta(c, tenders, meta, "Pending tenders retrieved successfully")
+}
+
+// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
+// @Summary Get tender by notice ID for document scraping
+// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description, and parsed XML
+// @Tags Document-Scraper
+// @Produce json
+// @Param notice_id path string true "Notice Publication ID"
+// @Success 200 {object} response.APIResponse{data=DocumentScraperTenderResponse} "Tender details"
+// @Failure 400 {object} response.APIResponse "Invalid notice ID"
+// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
+// @Failure 404 {object} response.APIResponse "Tender not found"
+// @Failure 500 {object} response.APIResponse "Internal server error"
+// @Security ApiKeyAuth
+// @Router /api/v1/scraper/tenders/{notice_id} [get]
+func (h *Handler) GetTenderByNoticeID(c echo.Context) error {
+	noticeID := c.Param("notice_id")
+	if noticeID == "" {
+		return response.BadRequest(c, "Notice ID is required", "notice_id parameter cannot be empty")
+	}
+
+	tender, err := h.service.GetTenderByNoticeID(c.Request().Context(), noticeID)
+	if err != nil {
+		if err.Error() == "tender not found" {
+			return response.NotFound(c, "Tender not found")
+		}
+		h.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
+			"notice_id": noticeID,
+			"error":     err.Error(),
+		})
+		return response.InternalServerError(c, "Failed to get tender")
+	}
+
+	return response.Success(c, tender, "Tender retrieved successfully")
+}

internal/document_scraper/middleware.go

@@ -0,0 +1,35 @@
+package document_scraper
+
+import (
+	"strings"
+	"tm/pkg/response"
+
+	"github.com/labstack/echo/v4"
+)
+
+// APIKeyMiddleware validates the API key for service-to-service authentication
+func APIKeyMiddleware(apiKey string) echo.MiddlewareFunc {
+	return func(next echo.HandlerFunc) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			// Extract API key from X-API-Key header
+			apiKeyHeader := c.Request().Header.Get("X-API-Key")
+			if apiKeyHeader == "" {
+				// Also check Authorization header with "ApiKey <key>" format
+				authHeader := c.Request().Header.Get("Authorization")
+				if strings.HasPrefix(authHeader, "ApiKey ") {
+					apiKeyHeader = strings.TrimPrefix(authHeader, "ApiKey ")
+				}
+			}
+
+			if apiKeyHeader == "" {
+				return response.Unauthorized(c, "API key required. Use X-API-Key header or Authorization: ApiKey <key>")
+			}
+
+			if apiKeyHeader != apiKey {
+				return response.Unauthorized(c, "Invalid API key")
+			}
+
+			return next(c)
+		}
+	}
+}

internal/document_scraper/service.go

@@ -0,0 +1,175 @@
+package document_scraper
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"tm/internal/notice"
+	"tm/internal/tender"
+	"tm/pkg/logger"
+	"tm/pkg/response"
+	"tm/ted"
+)
+
+// Service defines business logic for the document scraper API
+type Service interface {
+	// ListPendingTenders retrieves tenders that need document scraping
+	ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error)
+	// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
+	GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error)
+}
+
+type service struct {
+	tenderRepo tender.TenderRepository
+	noticeRepo notice.Repository
+	tedParser  *ted.TEDParser
+	logger     logger.Logger
+}
+
+// NewService creates a new document scraper service
+func NewService(tenderRepo tender.TenderRepository, noticeRepo notice.Repository, logger logger.Logger) Service {
+	return &service{
+		tenderRepo: tenderRepo,
+		noticeRepo: noticeRepo,
+		tedParser:  ted.NewTEDParser(),
+		logger:     logger,
+	}
+}
+
+// parseXMLToJSON converts raw XML content into a structured JSON object
+func (s *service) parseXMLToJSON(xmlContent string) interface{} {
+	if xmlContent == "" {
+		return nil
+	}
+
+	parsedDoc, _, _, err := s.tedParser.ParseXML([]byte(xmlContent))
+	if err != nil {
+		s.logger.Warn("Failed to parse XML to JSON, returning raw XML string", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return xmlContent
+	}
+
+	// Convert to a generic map to clean up the JSON output
+	jsonBytes, err := json.Marshal(parsedDoc)
+	if err != nil {
+		s.logger.Warn("Failed to marshal parsed XML to JSON, returning raw XML string", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return xmlContent
+	}
+
+	var result interface{}
+	if err := json.Unmarshal(jsonBytes, &result); err != nil {
+		s.logger.Warn("Failed to unmarshal parsed XML, returning raw XML string", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return xmlContent
+	}
+
+	return result
+}
+
+// buildTenderResponse creates a DocumentScraperTenderResponse from a tender and its notice
+func (s *service) buildTenderResponse(ctx context.Context, t *tender.Tender) *DocumentScraperTenderResponse {
+	parsedXML := interface{}(nil)
+	documentURL := t.DocumentURI
+	if documentURL == "" {
+		documentURL = t.TenderURL
+	}
+
+	// Try to get the parsed XML from the corresponding notice
+	n, err := s.noticeRepo.GetByContractNoticeID(ctx, t.ContractNoticeID)
+	if err != nil {
+		s.logger.Warn("Failed to get notice for parsed XML", map[string]interface{}{
+			"contract_notice_id": t.ContractNoticeID,
+			"error":              err.Error(),
+		})
+	} else if n != nil {
+		parsedXML = s.parseXMLToJSON(n.ContentXML)
+		if documentURL == "" {
+			documentURL = n.DocumentURI
+		}
+	}
+
+	return &DocumentScraperTenderResponse{
+		ID:                  t.ID.Hex(),
+		NoticePublicationID: t.NoticePublicationID,
+		DocumentURL:         documentURL,
+		Title:               t.Title,
+		Description:         t.Description,
+		ParsedXML:           parsedXML,
+	}
+}
+
+// ListPendingTenders retrieves tenders that have not yet been scraped for documents
+func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) {
+	s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{
+		"limit":  limit,
+		"offset": offset,
+	})
+
+	tenders, totalCount, err := s.tenderRepo.GetUnScrapedTenders(ctx, limit, offset)
+	if err != nil {
+		s.logger.Error("Failed to list pending tenders", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
+	}
+
+	// Convert to response
+	result := &DocumentScraperListResponse{
+		Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
+	}
+
+	for i := range tenders {
+		result.Tenders[i] = *s.buildTenderResponse(ctx, &tenders[i])
+	}
+
+	// Build pagination metadata
+	meta := &response.Meta{
+		Total:  int(totalCount),
+		Limit:  limit,
+		Offset: offset,
+	}
+	if meta.Limit > 0 {
+		meta.Page = (offset / meta.Limit) + 1
+		meta.Pages = (int(totalCount) + meta.Limit - 1) / meta.Limit
+	}
+
+	s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
+		"count":       len(tenders),
+		"total_count": totalCount,
+	})
+
+	return result, meta, nil
+}
+
+// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
+func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
+	s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
+		"notice_id": noticeID,
+	})
+
+	t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID)
+	if err != nil {
+		s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
+			"notice_id": noticeID,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to get tender: %w", err)
+	}
+
+	if t == nil {
+		return nil, fmt.Errorf("tender not found")
+	}
+
+	result := s.buildTenderResponse(ctx, t)
+
+	s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{
+		"notice_id": noticeID,
+		"tender_id": t.ID.Hex(),
+	})
+
+	return result, nil
+}