API for AI document scraper integration
This commit is contained in:
m.nazemi
+20
-14
@@ -7,20 +7,21 @@ import (
|
||||
|
||||
// Config holds configuration for the web command
|
||||
type Config struct {
|
||||
Server config.ServerConfig
|
||||
Database config.DatabaseConfig
|
||||
Cache config.CacheConfig
|
||||
Logging config.LoggingConfig
|
||||
RateLimit config.RateLimitConfig
|
||||
Assets config.AssetsConfig
|
||||
Notification config.NotificationConfig
|
||||
Ollama config.OllamaConfig
|
||||
HCaptcha config.HCaptchaConfig
|
||||
GLM GLMConfig
|
||||
Scraper ScraperConfig
|
||||
TED TEDConfig
|
||||
UserAuth AuthConfig
|
||||
CustomerAuth AuthConfig
|
||||
Server config.ServerConfig
|
||||
Database config.DatabaseConfig
|
||||
Cache config.CacheConfig
|
||||
Logging config.LoggingConfig
|
||||
RateLimit config.RateLimitConfig
|
||||
Assets config.AssetsConfig
|
||||
Notification config.NotificationConfig
|
||||
Ollama config.OllamaConfig
|
||||
HCaptcha config.HCaptchaConfig
|
||||
GLM GLMConfig
|
||||
Scraper ScraperConfig
|
||||
TED TEDConfig
|
||||
UserAuth AuthConfig
|
||||
CustomerAuth AuthConfig
|
||||
DocumentScraper DocumentScraperConfig
|
||||
}
|
||||
|
||||
type GLMConfig struct {
|
||||
@@ -82,3 +83,8 @@ type JWTConfig struct {
|
||||
AccessExpiresIn int `env:"USER_AUTH_ACCESS_EXPIRES_IN"`
|
||||
RefreshExpiresIn int `env:"USER_AUTH_REFRESH_EXPIRES_IN"`
|
||||
}
|
||||
|
||||
// DocumentScraperConfig holds configuration for the external document scraper service API
|
||||
type DocumentScraperConfig struct {
|
||||
APIKey string `env:"DOCUMENT_SCRAPER_API_KEY" envDefault:""`
|
||||
}
|
||||
|
||||
+6
-2
@@ -100,6 +100,7 @@ import (
|
||||
"tm/internal/company_category"
|
||||
"tm/internal/contact"
|
||||
"tm/internal/customer"
|
||||
"tm/internal/document_scraper"
|
||||
"tm/internal/feedback"
|
||||
"tm/internal/inquiry"
|
||||
"tm/internal/kanban"
|
||||
@@ -258,8 +259,9 @@ func main() {
|
||||
tedConfig := buildTEDConfig(&conf.TED)
|
||||
scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig)
|
||||
kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger)
|
||||
documentScraperService := document_scraper.NewService(tenderRepository, noticeRepository, logger)
|
||||
logger.Info("Services initialized successfully", map[string]interface{}{
|
||||
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban"},
|
||||
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper"},
|
||||
})
|
||||
|
||||
// Initialize handlers with services
|
||||
@@ -278,8 +280,9 @@ func main() {
|
||||
scraperHandler := scraper.NewHandler(scraperService)
|
||||
kanbanHandler := kanban.NewHandler(kanbanService)
|
||||
fileStoreHandler := filestore.NewHandler(fileStoreService, logger)
|
||||
documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger)
|
||||
logger.Info("Handlers initialized successfully", map[string]interface{}{
|
||||
"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore"},
|
||||
"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper"},
|
||||
})
|
||||
|
||||
// Initialize HTTP server
|
||||
@@ -288,6 +291,7 @@ func main() {
|
||||
// Register routes
|
||||
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler, kanbanHandler, fileStoreHandler, xssPolicy)
|
||||
router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, fileStoreHandler, xssPolicy)
|
||||
router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey)
|
||||
|
||||
// Start server
|
||||
serverAddr := fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"tm/internal/company_category"
|
||||
"tm/internal/contact"
|
||||
"tm/internal/customer"
|
||||
"tm/internal/document_scraper"
|
||||
"tm/internal/feedback"
|
||||
"tm/internal/inquiry"
|
||||
"tm/internal/kanban"
|
||||
@@ -317,3 +318,16 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende
|
||||
publicFilesGP.DELETE("/:file_id", fileStoreHandler.Delete)
|
||||
}
|
||||
}
|
||||
|
||||
func RegisterDocumentScraperRoutes(e *echo.Echo, docScraperHandler *document_scraper.Handler, apiKey string) {
|
||||
v1 := e.Group("/api/v1/scraper")
|
||||
|
||||
// Apply API key middleware to all document scraper routes
|
||||
v1.Use(document_scraper.APIKeyMiddleware(apiKey))
|
||||
|
||||
// Document Scraper Routes
|
||||
{
|
||||
v1.GET("/tenders", docScraperHandler.ListPendingTenders)
|
||||
v1.GET("/tenders/:notice_id", docScraperHandler.GetTenderByNoticeID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
package document_scraper
|
||||
|
||||
// DocumentScraperListRequest represents the request to list tenders pending document scraping
|
||||
type DocumentScraperListRequest struct {
|
||||
Limit int `query:"limit" valid:"optional,range(1|100)" default:"20"`
|
||||
Offset int `query:"offset" valid:"optional,min(0)" default:"0"`
|
||||
}
|
||||
|
||||
// DocumentScraperGetRequest represents the request to get a specific tender by notice publication ID
|
||||
type DocumentScraperGetRequest struct {
|
||||
NoticeID string `param:"notice_id" valid:"required~Notice ID is required"`
|
||||
}
|
||||
|
||||
// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service
|
||||
type DocumentScraperTenderResponse struct {
|
||||
ID string `json:"id"`
|
||||
NoticePublicationID string `json:"notice_publication_id"`
|
||||
DocumentURL string `json:"document_url"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
ParsedXML interface{} `json:"parsed_xml"`
|
||||
}
|
||||
|
||||
// DocumentScraperListResponse represents the paginated list response
|
||||
type DocumentScraperListResponse struct {
|
||||
Tenders []DocumentScraperTenderResponse `json:"tenders"`
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package document_scraper
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"tm/pkg/logger"
|
||||
"tm/pkg/response"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
// Handler handles HTTP requests for the document scraper API
|
||||
type Handler struct {
|
||||
service Service
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
// NewHandler creates a new document scraper handler
|
||||
func NewHandler(service Service, logger logger.Logger) *Handler {
|
||||
return &Handler{
|
||||
service: service,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// ListPendingTenders retrieves tenders that need document scraping
|
||||
// @Summary List pending tenders for document scraping
|
||||
// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description, and parsed XML
|
||||
// @Tags Document-Scraper
|
||||
// @Produce json
|
||||
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
||||
// @Param offset query int false "Number of items to skip (default: 0)"
|
||||
// @Success 200 {object} response.APIResponse{data=DocumentScraperListResponse} "List of pending tenders"
|
||||
// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
|
||||
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||
// @Security ApiKeyAuth
|
||||
// @Router /api/v1/scraper/tenders [get]
|
||||
func (h *Handler) ListPendingTenders(c echo.Context) error {
|
||||
// Parse pagination parameters
|
||||
limit, err := strconv.Atoi(c.QueryParam("limit"))
|
||||
if err != nil || limit <= 0 {
|
||||
limit = 20
|
||||
}
|
||||
if limit > 100 {
|
||||
limit = 100
|
||||
}
|
||||
|
||||
offset, err := strconv.Atoi(c.QueryParam("offset"))
|
||||
if err != nil || offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
tenders, meta, err := h.service.ListPendingTenders(c.Request().Context(), limit, offset)
|
||||
if err != nil {
|
||||
h.logger.Error("Failed to list pending tenders", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return response.InternalServerError(c, "Failed to list pending tenders")
|
||||
}
|
||||
|
||||
return response.SuccessWithMeta(c, tenders, meta, "Pending tenders retrieved successfully")
|
||||
}
|
||||
|
||||
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
|
||||
// @Summary Get tender by notice ID for document scraping
|
||||
// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description, and parsed XML
|
||||
// @Tags Document-Scraper
|
||||
// @Produce json
|
||||
// @Param notice_id path string true "Notice Publication ID"
|
||||
// @Success 200 {object} response.APIResponse{data=DocumentScraperTenderResponse} "Tender details"
|
||||
// @Failure 400 {object} response.APIResponse "Invalid notice ID"
|
||||
// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
|
||||
// @Failure 404 {object} response.APIResponse "Tender not found"
|
||||
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||
// @Security ApiKeyAuth
|
||||
// @Router /api/v1/scraper/tenders/{notice_id} [get]
|
||||
func (h *Handler) GetTenderByNoticeID(c echo.Context) error {
|
||||
noticeID := c.Param("notice_id")
|
||||
if noticeID == "" {
|
||||
return response.BadRequest(c, "Notice ID is required", "notice_id parameter cannot be empty")
|
||||
}
|
||||
|
||||
tender, err := h.service.GetTenderByNoticeID(c.Request().Context(), noticeID)
|
||||
if err != nil {
|
||||
if err.Error() == "tender not found" {
|
||||
return response.NotFound(c, "Tender not found")
|
||||
}
|
||||
h.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
|
||||
"notice_id": noticeID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return response.InternalServerError(c, "Failed to get tender")
|
||||
}
|
||||
|
||||
return response.Success(c, tender, "Tender retrieved successfully")
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package document_scraper
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"tm/pkg/response"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
// APIKeyMiddleware validates the API key for service-to-service authentication
|
||||
func APIKeyMiddleware(apiKey string) echo.MiddlewareFunc {
|
||||
return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
// Extract API key from X-API-Key header
|
||||
apiKeyHeader := c.Request().Header.Get("X-API-Key")
|
||||
if apiKeyHeader == "" {
|
||||
// Also check Authorization header with "ApiKey <key>" format
|
||||
authHeader := c.Request().Header.Get("Authorization")
|
||||
if strings.HasPrefix(authHeader, "ApiKey ") {
|
||||
apiKeyHeader = strings.TrimPrefix(authHeader, "ApiKey ")
|
||||
}
|
||||
}
|
||||
|
||||
if apiKeyHeader == "" {
|
||||
return response.Unauthorized(c, "API key required. Use X-API-Key header or Authorization: ApiKey <key>")
|
||||
}
|
||||
|
||||
if apiKeyHeader != apiKey {
|
||||
return response.Unauthorized(c, "Invalid API key")
|
||||
}
|
||||
|
||||
return next(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package document_scraper
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"tm/internal/notice"
|
||||
"tm/internal/tender"
|
||||
"tm/pkg/logger"
|
||||
"tm/pkg/response"
|
||||
"tm/ted"
|
||||
)
|
||||
|
||||
// Service defines business logic for the document scraper API
|
||||
type Service interface {
|
||||
// ListPendingTenders retrieves tenders that need document scraping
|
||||
ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error)
|
||||
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
|
||||
GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error)
|
||||
}
|
||||
|
||||
type service struct {
|
||||
tenderRepo tender.TenderRepository
|
||||
noticeRepo notice.Repository
|
||||
tedParser *ted.TEDParser
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
// NewService creates a new document scraper service
|
||||
func NewService(tenderRepo tender.TenderRepository, noticeRepo notice.Repository, logger logger.Logger) Service {
|
||||
return &service{
|
||||
tenderRepo: tenderRepo,
|
||||
noticeRepo: noticeRepo,
|
||||
tedParser: ted.NewTEDParser(),
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// parseXMLToJSON converts raw XML content into a structured JSON object
|
||||
func (s *service) parseXMLToJSON(xmlContent string) interface{} {
|
||||
if xmlContent == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
parsedDoc, _, _, err := s.tedParser.ParseXML([]byte(xmlContent))
|
||||
if err != nil {
|
||||
s.logger.Warn("Failed to parse XML to JSON, returning raw XML string", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return xmlContent
|
||||
}
|
||||
|
||||
// Convert to a generic map to clean up the JSON output
|
||||
jsonBytes, err := json.Marshal(parsedDoc)
|
||||
if err != nil {
|
||||
s.logger.Warn("Failed to marshal parsed XML to JSON, returning raw XML string", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return xmlContent
|
||||
}
|
||||
|
||||
var result interface{}
|
||||
if err := json.Unmarshal(jsonBytes, &result); err != nil {
|
||||
s.logger.Warn("Failed to unmarshal parsed XML, returning raw XML string", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return xmlContent
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// buildTenderResponse creates a DocumentScraperTenderResponse from a tender and its notice
|
||||
func (s *service) buildTenderResponse(ctx context.Context, t *tender.Tender) *DocumentScraperTenderResponse {
|
||||
parsedXML := interface{}(nil)
|
||||
documentURL := t.DocumentURI
|
||||
if documentURL == "" {
|
||||
documentURL = t.TenderURL
|
||||
}
|
||||
|
||||
// Try to get the parsed XML from the corresponding notice
|
||||
n, err := s.noticeRepo.GetByContractNoticeID(ctx, t.ContractNoticeID)
|
||||
if err != nil {
|
||||
s.logger.Warn("Failed to get notice for parsed XML", map[string]interface{}{
|
||||
"contract_notice_id": t.ContractNoticeID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
} else if n != nil {
|
||||
parsedXML = s.parseXMLToJSON(n.ContentXML)
|
||||
if documentURL == "" {
|
||||
documentURL = n.DocumentURI
|
||||
}
|
||||
}
|
||||
|
||||
return &DocumentScraperTenderResponse{
|
||||
ID: t.ID.Hex(),
|
||||
NoticePublicationID: t.NoticePublicationID,
|
||||
DocumentURL: documentURL,
|
||||
Title: t.Title,
|
||||
Description: t.Description,
|
||||
ParsedXML: parsedXML,
|
||||
}
|
||||
}
|
||||
|
||||
// ListPendingTenders retrieves tenders that have not yet been scraped for documents
|
||||
func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) {
|
||||
s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{
|
||||
"limit": limit,
|
||||
"offset": offset,
|
||||
})
|
||||
|
||||
tenders, totalCount, err := s.tenderRepo.GetUnScrapedTenders(ctx, limit, offset)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to list pending tenders", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
|
||||
}
|
||||
|
||||
// Convert to response
|
||||
result := &DocumentScraperListResponse{
|
||||
Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
|
||||
}
|
||||
|
||||
for i := range tenders {
|
||||
result.Tenders[i] = *s.buildTenderResponse(ctx, &tenders[i])
|
||||
}
|
||||
|
||||
// Build pagination metadata
|
||||
meta := &response.Meta{
|
||||
Total: int(totalCount),
|
||||
Limit: limit,
|
||||
Offset: offset,
|
||||
}
|
||||
if meta.Limit > 0 {
|
||||
meta.Page = (offset / meta.Limit) + 1
|
||||
meta.Pages = (int(totalCount) + meta.Limit - 1) / meta.Limit
|
||||
}
|
||||
|
||||
s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
|
||||
"count": len(tenders),
|
||||
"total_count": totalCount,
|
||||
})
|
||||
|
||||
return result, meta, nil
|
||||
}
|
||||
|
||||
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
|
||||
func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
|
||||
s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
|
||||
"notice_id": noticeID,
|
||||
})
|
||||
|
||||
t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
|
||||
"notice_id": noticeID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to get tender: %w", err)
|
||||
}
|
||||
|
||||
if t == nil {
|
||||
return nil, fmt.Errorf("tender not found")
|
||||
}
|
||||
|
||||
result := s.buildTenderResponse(ctx, t)
|
||||
|
||||
s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{
|
||||
"notice_id": noticeID,
|
||||
"tender_id": t.ID.Hex(),
|
||||
})
|
||||
|
||||
return result, nil
|
||||
}
|
||||
Reference in New Issue
Block a user