Document scraper integration field fix

cmd/web/main.go

@@ -259,7 +259,7 @@ func main() {
 	tedConfig := buildTEDConfig(&conf.TED)
 	scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig)
 	kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger)
-	documentScraperService := document_scraper.NewService(tenderRepository, noticeRepository, logger)
+	documentScraperService := document_scraper.NewService(tenderRepository, logger)
 	logger.Info("Services initialized successfully", map[string]interface{}{
 		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper"},
 	})

internal/document_scraper/form.go

@@ -18,7 +18,6 @@ type DocumentScraperTenderResponse struct {
 	DocumentURL         string `json:"document_url"`
 	Title               string `json:"title"`
 	Description         string `json:"description"`
-	ParsedXML           interface{} `json:"parsed_xml"`
 }
 
 // DocumentScraperListResponse represents the paginated list response

internal/document_scraper/handler.go

@@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler {
 
 // ListPendingTenders retrieves tenders that need document scraping
 // @Summary List pending tenders for document scraping
-// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description, and parsed XML
+// @Description Retrieve tenders that have not yet been scraped for documents, including notice ID, title, description
 // @Tags Document-Scraper
 // @Produce json
 // @Param limit query int false "Number of items per page (default: 20, max: 100)"
@@ -62,7 +62,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
 
 // GetTenderByNoticeID retrieves a specific tender by its notice publication ID
 // @Summary Get tender by notice ID for document scraping
-// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description, and parsed XML
+// @Description Retrieve a specific tender by its notice publication ID, including notice ID, title, description
 // @Tags Document-Scraper
 // @Produce json
 // @Param notice_id path string true "Notice Publication ID"

internal/document_scraper/service.go

@@ -2,9 +2,7 @@ package document_scraper
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
-	"tm/internal/notice"
 	"tm/internal/tender"
 	"tm/pkg/logger"
 	"tm/pkg/response"
@@ -21,84 +19,32 @@ type Service interface {
 
 type service struct {
 	tenderRepo tender.TenderRepository
-	noticeRepo notice.Repository
 	tedParser  *ted.TEDParser
 	logger     logger.Logger
 }
 
 // NewService creates a new document scraper service
-func NewService(tenderRepo tender.TenderRepository, noticeRepo notice.Repository, logger logger.Logger) Service {
+func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Service {
 	return &service{
 		tenderRepo: tenderRepo,
-		noticeRepo: noticeRepo,
 		tedParser:  ted.NewTEDParser(),
 		logger:     logger,
 	}
 }
 
-// parseXMLToJSON converts raw XML content into a structured JSON object
+// buildTenderResponse creates a DocumentScraperTenderResponse from a tender
-func (s *service) parseXMLToJSON(xmlContent string) interface{} {
+func (s *service) buildTenderResponse(_ context.Context, t *tender.Tender) *DocumentScraperTenderResponse {
-	if xmlContent == "" {
-		return nil
-	}
-
-	parsedDoc, _, _, err := s.tedParser.ParseXML([]byte(xmlContent))
-	if err != nil {
-		s.logger.Warn("Failed to parse XML to JSON, returning raw XML string", map[string]interface{}{
-			"error": err.Error(),
-		})
-		return xmlContent
-	}
-
-	// Convert to a generic map to clean up the JSON output
-	jsonBytes, err := json.Marshal(parsedDoc)
-	if err != nil {
-		s.logger.Warn("Failed to marshal parsed XML to JSON, returning raw XML string", map[string]interface{}{
-			"error": err.Error(),
-		})
-		return xmlContent
-	}
-
-	var result interface{}
-	if err := json.Unmarshal(jsonBytes, &result); err != nil {
-		s.logger.Warn("Failed to unmarshal parsed XML, returning raw XML string", map[string]interface{}{
-			"error": err.Error(),
-		})
-		return xmlContent
-	}
-
-	return result
-}
-
-// buildTenderResponse creates a DocumentScraperTenderResponse from a tender and its notice
-func (s *service) buildTenderResponse(ctx context.Context, t *tender.Tender) *DocumentScraperTenderResponse {
-	parsedXML := interface{}(nil)
 	documentURL := t.DocumentURI
 	if documentURL == "" {
 		documentURL = t.TenderURL
 	}
 
-	// Try to get the parsed XML from the corresponding notice
-	n, err := s.noticeRepo.GetByContractNoticeID(ctx, t.ContractNoticeID)
-	if err != nil {
-		s.logger.Warn("Failed to get notice for parsed XML", map[string]interface{}{
-			"contract_notice_id": t.ContractNoticeID,
-			"error":              err.Error(),
-		})
-	} else if n != nil {
-		parsedXML = s.parseXMLToJSON(n.ContentXML)
-		if documentURL == "" {
-			documentURL = n.DocumentURI
-		}
-	}
-
 	return &DocumentScraperTenderResponse{
 		ID:                  t.ID.Hex(),
 		NoticePublicationID: t.NoticePublicationID,
 		DocumentURL:         documentURL,
 		Title:               t.Title,
 		Description:         t.Description,
-		ParsedXML:           parsedXML,
 	}
 }