diff --git a/cmd/web/main.go b/cmd/web/main.go index b371583..cbace38 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -259,7 +259,7 @@ func main() { tedConfig := buildTEDConfig(&conf.TED) scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig) kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger) - documentScraperService := document_scraper.NewService(tenderRepository, noticeRepository, logger) + documentScraperService := document_scraper.NewService(tenderRepository, logger) logger.Info("Services initialized successfully", map[string]interface{}{ "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper"}, }) diff --git a/internal/document_scraper/form.go b/internal/document_scraper/form.go index 096d95c..e329474 100644 --- a/internal/document_scraper/form.go +++ b/internal/document_scraper/form.go @@ -13,12 +13,11 @@ type DocumentScraperGetRequest struct { // DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service type DocumentScraperTenderResponse struct { - ID string `json:"id"` - NoticePublicationID string `json:"notice_publication_id"` - DocumentURL string `json:"document_url"` - Title string `json:"title"` - Description string `json:"description"` - ParsedXML interface{} `json:"parsed_xml"` + ID string `json:"id"` + NoticePublicationID string `json:"notice_publication_id"` + DocumentURL string `json:"document_url"` + Title string `json:"title"` + Description string `json:"description"` } // DocumentScraperListResponse represents the paginated list response diff --git a/internal/document_scraper/handler.go b/internal/document_scraper/handler.go index 143929d..d7630da 100644 --- a/internal/document_scraper/handler.go +++ b/internal/document_scraper/handler.go @@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler { // ListPendingTenders retrieves tenders that need document scraping // @Summary List pending tenders for document scraping -// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description, and parsed XML +// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description // @Tags Document-Scraper // @Produce json // @Param limit query int false "Number of items per page (default: 20, max: 100)" @@ -62,7 +62,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error { // GetTenderByNoticeID retrieves a specific tender by its notice publication ID // @Summary Get tender by notice ID for document scraping -// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description, and parsed XML +// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description // @Tags Document-Scraper // @Produce json // @Param notice_id path string true "Notice Publication ID" diff --git a/internal/document_scraper/service.go b/internal/document_scraper/service.go index d3243e2..47ac9e1 100644 --- a/internal/document_scraper/service.go +++ b/internal/document_scraper/service.go @@ -2,9 +2,7 @@ package document_scraper import ( "context" - "encoding/json" "fmt" - "tm/internal/notice" "tm/internal/tender" "tm/pkg/logger" "tm/pkg/response" @@ -21,84 +19,32 @@ type Service interface { type service struct { tenderRepo tender.TenderRepository - noticeRepo notice.Repository tedParser *ted.TEDParser logger logger.Logger } // NewService creates a new document scraper service -func NewService(tenderRepo tender.TenderRepository, noticeRepo notice.Repository, logger logger.Logger) Service { +func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Service { return &service{ tenderRepo: tenderRepo, - noticeRepo: noticeRepo, tedParser: ted.NewTEDParser(), logger: logger, } } -// parseXMLToJSON converts raw XML content into a structured JSON object -func (s *service) parseXMLToJSON(xmlContent string) interface{} { - if xmlContent == "" { - return nil - } - - parsedDoc, _, _, err := s.tedParser.ParseXML([]byte(xmlContent)) - if err != nil { - s.logger.Warn("Failed to parse XML to JSON, returning raw XML string", map[string]interface{}{ - "error": err.Error(), - }) - return xmlContent - } - - // Convert to a generic map to clean up the JSON output - jsonBytes, err := json.Marshal(parsedDoc) - if err != nil { - s.logger.Warn("Failed to marshal parsed XML to JSON, returning raw XML string", map[string]interface{}{ - "error": err.Error(), - }) - return xmlContent - } - - var result interface{} - if err := json.Unmarshal(jsonBytes, &result); err != nil { - s.logger.Warn("Failed to unmarshal parsed XML, returning raw XML string", map[string]interface{}{ - "error": err.Error(), - }) - return xmlContent - } - - return result -} - -// buildTenderResponse creates a DocumentScraperTenderResponse from a tender and its notice -func (s *service) buildTenderResponse(ctx context.Context, t *tender.Tender) *DocumentScraperTenderResponse { - parsedXML := interface{}(nil) +// buildTenderResponse creates a DocumentScraperTenderResponse from a tender +func (s *service) buildTenderResponse(_ context.Context, t *tender.Tender) *DocumentScraperTenderResponse { documentURL := t.DocumentURI if documentURL == "" { documentURL = t.TenderURL } - // Try to get the parsed XML from the corresponding notice - n, err := s.noticeRepo.GetByContractNoticeID(ctx, t.ContractNoticeID) - if err != nil { - s.logger.Warn("Failed to get notice for parsed XML", map[string]interface{}{ - "contract_notice_id": t.ContractNoticeID, - "error": err.Error(), - }) - } else if n != nil { - parsedXML = s.parseXMLToJSON(n.ContentXML) - if documentURL == "" { - documentURL = n.DocumentURI - } - } - return &DocumentScraperTenderResponse{ ID: t.ID.Hex(), NoticePublicationID: t.NoticePublicationID, DocumentURL: documentURL, Title: t.Title, Description: t.Description, - ParsedXML: parsedXML, } }