diff --git a/cmd/web/bootstrap/config.go b/cmd/web/bootstrap/config.go index e6ab8f9..9727d78 100644 --- a/cmd/web/bootstrap/config.go +++ b/cmd/web/bootstrap/config.go @@ -7,20 +7,21 @@ import ( // Config holds configuration for the web command type Config struct { - Server config.ServerConfig - Database config.DatabaseConfig - Cache config.CacheConfig - Logging config.LoggingConfig - RateLimit config.RateLimitConfig - Assets config.AssetsConfig - Notification config.NotificationConfig - Ollama config.OllamaConfig - HCaptcha config.HCaptchaConfig - GLM GLMConfig - Scraper ScraperConfig - TED TEDConfig - UserAuth AuthConfig - CustomerAuth AuthConfig + Server config.ServerConfig + Database config.DatabaseConfig + Cache config.CacheConfig + Logging config.LoggingConfig + RateLimit config.RateLimitConfig + Assets config.AssetsConfig + Notification config.NotificationConfig + Ollama config.OllamaConfig + HCaptcha config.HCaptchaConfig + GLM GLMConfig + Scraper ScraperConfig + TED TEDConfig + UserAuth AuthConfig + CustomerAuth AuthConfig + DocumentScraper DocumentScraperConfig } type GLMConfig struct { @@ -82,3 +83,8 @@ type JWTConfig struct { AccessExpiresIn int `env:"USER_AUTH_ACCESS_EXPIRES_IN"` RefreshExpiresIn int `env:"USER_AUTH_REFRESH_EXPIRES_IN"` } + +// DocumentScraperConfig holds configuration for the external document scraper service API +type DocumentScraperConfig struct { + APIKey string `env:"DOCUMENT_SCRAPER_API_KEY" envDefault:""` +} diff --git a/cmd/web/main.go b/cmd/web/main.go index d4eaccf..b371583 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -100,6 +100,7 @@ import ( "tm/internal/company_category" "tm/internal/contact" "tm/internal/customer" + "tm/internal/document_scraper" "tm/internal/feedback" "tm/internal/inquiry" "tm/internal/kanban" @@ -258,8 +259,9 @@ func main() { tedConfig := buildTEDConfig(&conf.TED) scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig) kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger) + documentScraperService := document_scraper.NewService(tenderRepository, noticeRepository, logger) logger.Info("Services initialized successfully", map[string]interface{}{ - "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban"}, + "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper"}, }) // Initialize handlers with services @@ -278,8 +280,9 @@ func main() { scraperHandler := scraper.NewHandler(scraperService) kanbanHandler := kanban.NewHandler(kanbanService) fileStoreHandler := filestore.NewHandler(fileStoreService, logger) + documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger) logger.Info("Handlers initialized successfully", map[string]interface{}{ - "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore"}, + "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper"}, }) // Initialize HTTP server @@ -288,6 +291,7 @@ func main() { // Register routes router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler, kanbanHandler, fileStoreHandler, xssPolicy) router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, fileStoreHandler, xssPolicy) + router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey) // Start server serverAddr := fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port) diff --git a/cmd/web/router/routes.go b/cmd/web/router/routes.go index 5cc6479..273ecd5 100644 --- a/cmd/web/router/routes.go +++ b/cmd/web/router/routes.go @@ -7,6 +7,7 @@ import ( "tm/internal/company_category" "tm/internal/contact" "tm/internal/customer" + "tm/internal/document_scraper" "tm/internal/feedback" "tm/internal/inquiry" "tm/internal/kanban" @@ -317,3 +318,16 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende publicFilesGP.DELETE("/:file_id", fileStoreHandler.Delete) } } + +func RegisterDocumentScraperRoutes(e *echo.Echo, docScraperHandler *document_scraper.Handler, apiKey string) { + v1 := e.Group("/api/v1/scraper") + + // Apply API key middleware to all document scraper routes + v1.Use(document_scraper.APIKeyMiddleware(apiKey)) + + // Document Scraper Routes + { + v1.GET("/tenders", docScraperHandler.ListPendingTenders) + v1.GET("/tenders/:notice_id", docScraperHandler.GetTenderByNoticeID) + } +} diff --git a/internal/document_scraper/form.go b/internal/document_scraper/form.go new file mode 100644 index 0000000..096d95c --- /dev/null +++ b/internal/document_scraper/form.go @@ -0,0 +1,27 @@ +package document_scraper + +// DocumentScraperListRequest represents the request to list tenders pending document scraping +type DocumentScraperListRequest struct { + Limit int `query:"limit" valid:"optional,range(1|100)" default:"20"` + Offset int `query:"offset" valid:"optional,min(0)" default:"0"` +} + +// DocumentScraperGetRequest represents the request to get a specific tender by notice publication ID +type DocumentScraperGetRequest struct { + NoticeID string `param:"notice_id" valid:"required~Notice ID is required"` +} + +// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service +type DocumentScraperTenderResponse struct { + ID string `json:"id"` + NoticePublicationID string `json:"notice_publication_id"` + DocumentURL string `json:"document_url"` + Title string `json:"title"` + Description string `json:"description"` + ParsedXML interface{} `json:"parsed_xml"` +} + +// DocumentScraperListResponse represents the paginated list response +type DocumentScraperListResponse struct { + Tenders []DocumentScraperTenderResponse `json:"tenders"` +} diff --git a/internal/document_scraper/handler.go b/internal/document_scraper/handler.go new file mode 100644 index 0000000..143929d --- /dev/null +++ b/internal/document_scraper/handler.go @@ -0,0 +1,95 @@ +package document_scraper + +import ( + "strconv" + "tm/pkg/logger" + "tm/pkg/response" + + "github.com/labstack/echo/v4" +) + +// Handler handles HTTP requests for the document scraper API +type Handler struct { + service Service + logger logger.Logger +} + +// NewHandler creates a new document scraper handler +func NewHandler(service Service, logger logger.Logger) *Handler { + return &Handler{ + service: service, + logger: logger, + } +} + +// ListPendingTenders retrieves tenders that need document scraping +// @Summary List pending tenders for document scraping +// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description, and parsed XML +// @Tags Document-Scraper +// @Produce json +// @Param limit query int false "Number of items per page (default: 20, max: 100)" +// @Param offset query int false "Number of items to skip (default: 0)" +// @Success 200 {object} response.APIResponse{data=DocumentScraperListResponse} "List of pending tenders" +// @Failure 401 {object} response.APIResponse "Invalid or missing API key" +// @Failure 500 {object} response.APIResponse "Internal server error" +// @Security ApiKeyAuth +// @Router /api/v1/scraper/tenders [get] +func (h *Handler) ListPendingTenders(c echo.Context) error { + // Parse pagination parameters + limit, err := strconv.Atoi(c.QueryParam("limit")) + if err != nil || limit <= 0 { + limit = 20 + } + if limit > 100 { + limit = 100 + } + + offset, err := strconv.Atoi(c.QueryParam("offset")) + if err != nil || offset < 0 { + offset = 0 + } + + tenders, meta, err := h.service.ListPendingTenders(c.Request().Context(), limit, offset) + if err != nil { + h.logger.Error("Failed to list pending tenders", map[string]interface{}{ + "error": err.Error(), + }) + return response.InternalServerError(c, "Failed to list pending tenders") + } + + return response.SuccessWithMeta(c, tenders, meta, "Pending tenders retrieved successfully") +} + +// GetTenderByNoticeID retrieves a specific tender by its notice publication ID +// @Summary Get tender by notice ID for document scraping +// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description, and parsed XML +// @Tags Document-Scraper +// @Produce json +// @Param notice_id path string true "Notice Publication ID" +// @Success 200 {object} response.APIResponse{data=DocumentScraperTenderResponse} "Tender details" +// @Failure 400 {object} response.APIResponse "Invalid notice ID" +// @Failure 401 {object} response.APIResponse "Invalid or missing API key" +// @Failure 404 {object} response.APIResponse "Tender not found" +// @Failure 500 {object} response.APIResponse "Internal server error" +// @Security ApiKeyAuth +// @Router /api/v1/scraper/tenders/{notice_id} [get] +func (h *Handler) GetTenderByNoticeID(c echo.Context) error { + noticeID := c.Param("notice_id") + if noticeID == "" { + return response.BadRequest(c, "Notice ID is required", "notice_id parameter cannot be empty") + } + + tender, err := h.service.GetTenderByNoticeID(c.Request().Context(), noticeID) + if err != nil { + if err.Error() == "tender not found" { + return response.NotFound(c, "Tender not found") + } + h.logger.Error("Failed to get tender by notice ID", map[string]interface{}{ + "notice_id": noticeID, + "error": err.Error(), + }) + return response.InternalServerError(c, "Failed to get tender") + } + + return response.Success(c, tender, "Tender retrieved successfully") +} diff --git a/internal/document_scraper/middleware.go b/internal/document_scraper/middleware.go new file mode 100644 index 0000000..9fc4f2b --- /dev/null +++ b/internal/document_scraper/middleware.go @@ -0,0 +1,35 @@ +package document_scraper + +import ( + "strings" + "tm/pkg/response" + + "github.com/labstack/echo/v4" +) + +// APIKeyMiddleware validates the API key for service-to-service authentication +func APIKeyMiddleware(apiKey string) echo.MiddlewareFunc { + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + // Extract API key from X-API-Key header + apiKeyHeader := c.Request().Header.Get("X-API-Key") + if apiKeyHeader == "" { + // Also check Authorization header with "ApiKey " format + authHeader := c.Request().Header.Get("Authorization") + if strings.HasPrefix(authHeader, "ApiKey ") { + apiKeyHeader = strings.TrimPrefix(authHeader, "ApiKey ") + } + } + + if apiKeyHeader == "" { + return response.Unauthorized(c, "API key required. Use X-API-Key header or Authorization: ApiKey ") + } + + if apiKeyHeader != apiKey { + return response.Unauthorized(c, "Invalid API key") + } + + return next(c) + } + } +} diff --git a/internal/document_scraper/service.go b/internal/document_scraper/service.go new file mode 100644 index 0000000..d3243e2 --- /dev/null +++ b/internal/document_scraper/service.go @@ -0,0 +1,175 @@ +package document_scraper + +import ( + "context" + "encoding/json" + "fmt" + "tm/internal/notice" + "tm/internal/tender" + "tm/pkg/logger" + "tm/pkg/response" + "tm/ted" +) + +// Service defines business logic for the document scraper API +type Service interface { + // ListPendingTenders retrieves tenders that need document scraping + ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) + // GetTenderByNoticeID retrieves a specific tender by its notice publication ID + GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) +} + +type service struct { + tenderRepo tender.TenderRepository + noticeRepo notice.Repository + tedParser *ted.TEDParser + logger logger.Logger +} + +// NewService creates a new document scraper service +func NewService(tenderRepo tender.TenderRepository, noticeRepo notice.Repository, logger logger.Logger) Service { + return &service{ + tenderRepo: tenderRepo, + noticeRepo: noticeRepo, + tedParser: ted.NewTEDParser(), + logger: logger, + } +} + +// parseXMLToJSON converts raw XML content into a structured JSON object +func (s *service) parseXMLToJSON(xmlContent string) interface{} { + if xmlContent == "" { + return nil + } + + parsedDoc, _, _, err := s.tedParser.ParseXML([]byte(xmlContent)) + if err != nil { + s.logger.Warn("Failed to parse XML to JSON, returning raw XML string", map[string]interface{}{ + "error": err.Error(), + }) + return xmlContent + } + + // Convert to a generic map to clean up the JSON output + jsonBytes, err := json.Marshal(parsedDoc) + if err != nil { + s.logger.Warn("Failed to marshal parsed XML to JSON, returning raw XML string", map[string]interface{}{ + "error": err.Error(), + }) + return xmlContent + } + + var result interface{} + if err := json.Unmarshal(jsonBytes, &result); err != nil { + s.logger.Warn("Failed to unmarshal parsed XML, returning raw XML string", map[string]interface{}{ + "error": err.Error(), + }) + return xmlContent + } + + return result +} + +// buildTenderResponse creates a DocumentScraperTenderResponse from a tender and its notice +func (s *service) buildTenderResponse(ctx context.Context, t *tender.Tender) *DocumentScraperTenderResponse { + parsedXML := interface{}(nil) + documentURL := t.DocumentURI + if documentURL == "" { + documentURL = t.TenderURL + } + + // Try to get the parsed XML from the corresponding notice + n, err := s.noticeRepo.GetByContractNoticeID(ctx, t.ContractNoticeID) + if err != nil { + s.logger.Warn("Failed to get notice for parsed XML", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "error": err.Error(), + }) + } else if n != nil { + parsedXML = s.parseXMLToJSON(n.ContentXML) + if documentURL == "" { + documentURL = n.DocumentURI + } + } + + return &DocumentScraperTenderResponse{ + ID: t.ID.Hex(), + NoticePublicationID: t.NoticePublicationID, + DocumentURL: documentURL, + Title: t.Title, + Description: t.Description, + ParsedXML: parsedXML, + } +} + +// ListPendingTenders retrieves tenders that have not yet been scraped for documents +func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) { + s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{ + "limit": limit, + "offset": offset, + }) + + tenders, totalCount, err := s.tenderRepo.GetUnScrapedTenders(ctx, limit, offset) + if err != nil { + s.logger.Error("Failed to list pending tenders", map[string]interface{}{ + "error": err.Error(), + }) + return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err) + } + + // Convert to response + result := &DocumentScraperListResponse{ + Tenders: make([]DocumentScraperTenderResponse, len(tenders)), + } + + for i := range tenders { + result.Tenders[i] = *s.buildTenderResponse(ctx, &tenders[i]) + } + + // Build pagination metadata + meta := &response.Meta{ + Total: int(totalCount), + Limit: limit, + Offset: offset, + } + if meta.Limit > 0 { + meta.Page = (offset / meta.Limit) + 1 + meta.Pages = (int(totalCount) + meta.Limit - 1) / meta.Limit + } + + s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{ + "count": len(tenders), + "total_count": totalCount, + }) + + return result, meta, nil +} + +// GetTenderByNoticeID retrieves a specific tender by its notice publication ID +func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) { + s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{ + "notice_id": noticeID, + }) + + t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID) + if err != nil { + s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{ + "notice_id": noticeID, + "error": err.Error(), + }) + return nil, fmt.Errorf("failed to get tender: %w", err) + } + + if t == nil { + return nil, fmt.Errorf("tender not found") + } + + result := s.buildTenderResponse(ctx, t) + + s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{ + "notice_id": noticeID, + "tender_id": t.ID.Hex(), + }) + + return result, nil +}