API for AI document scraper integration

internal/document_scraper/service.go

@@ -0,0 +1,175 @@
+package document_scraper
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"tm/internal/notice"
+	"tm/internal/tender"
+	"tm/pkg/logger"
+	"tm/pkg/response"
+	"tm/ted"
+)
+
+// Service defines business logic for the document scraper API
+type Service interface {
+	// ListPendingTenders retrieves tenders that need document scraping
+	ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error)
+	// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
+	GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error)
+}
+
+type service struct {
+	tenderRepo tender.TenderRepository
+	noticeRepo notice.Repository
+	tedParser  *ted.TEDParser
+	logger     logger.Logger
+}
+
+// NewService creates a new document scraper service
+func NewService(tenderRepo tender.TenderRepository, noticeRepo notice.Repository, logger logger.Logger) Service {
+	return &service{
+		tenderRepo: tenderRepo,
+		noticeRepo: noticeRepo,
+		tedParser:  ted.NewTEDParser(),
+		logger:     logger,
+	}
+}
+
+// parseXMLToJSON converts raw XML content into a structured JSON object
+func (s *service) parseXMLToJSON(xmlContent string) interface{} {
+	if xmlContent == "" {
+		return nil
+	}
+
+	parsedDoc, _, _, err := s.tedParser.ParseXML([]byte(xmlContent))
+	if err != nil {
+		s.logger.Warn("Failed to parse XML to JSON, returning raw XML string", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return xmlContent
+	}
+
+	// Convert to a generic map to clean up the JSON output
+	jsonBytes, err := json.Marshal(parsedDoc)
+	if err != nil {
+		s.logger.Warn("Failed to marshal parsed XML to JSON, returning raw XML string", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return xmlContent
+	}
+
+	var result interface{}
+	if err := json.Unmarshal(jsonBytes, &result); err != nil {
+		s.logger.Warn("Failed to unmarshal parsed XML, returning raw XML string", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return xmlContent
+	}
+
+	return result
+}
+
+// buildTenderResponse creates a DocumentScraperTenderResponse from a tender and its notice
+func (s *service) buildTenderResponse(ctx context.Context, t *tender.Tender) *DocumentScraperTenderResponse {
+	parsedXML := interface{}(nil)
+	documentURL := t.DocumentURI
+	if documentURL == "" {
+		documentURL = t.TenderURL
+	}
+
+	// Try to get the parsed XML from the corresponding notice
+	n, err := s.noticeRepo.GetByContractNoticeID(ctx, t.ContractNoticeID)
+	if err != nil {
+		s.logger.Warn("Failed to get notice for parsed XML", map[string]interface{}{
+			"contract_notice_id": t.ContractNoticeID,
+			"error":              err.Error(),
+		})
+	} else if n != nil {
+		parsedXML = s.parseXMLToJSON(n.ContentXML)
+		if documentURL == "" {
+			documentURL = n.DocumentURI
+		}
+	}
+
+	return &DocumentScraperTenderResponse{
+		ID:                  t.ID.Hex(),
+		NoticePublicationID: t.NoticePublicationID,
+		DocumentURL:         documentURL,
+		Title:               t.Title,
+		Description:         t.Description,
+		ParsedXML:           parsedXML,
+	}
+}
+
+// ListPendingTenders retrieves tenders that have not yet been scraped for documents
+func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) {
+	s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{
+		"limit":  limit,
+		"offset": offset,
+	})
+
+	tenders, totalCount, err := s.tenderRepo.GetUnScrapedTenders(ctx, limit, offset)
+	if err != nil {
+		s.logger.Error("Failed to list pending tenders", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
+	}
+
+	// Convert to response
+	result := &DocumentScraperListResponse{
+		Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
+	}
+
+	for i := range tenders {
+		result.Tenders[i] = *s.buildTenderResponse(ctx, &tenders[i])
+	}
+
+	// Build pagination metadata
+	meta := &response.Meta{
+		Total:  int(totalCount),
+		Limit:  limit,
+		Offset: offset,
+	}
+	if meta.Limit > 0 {
+		meta.Page = (offset / meta.Limit) + 1
+		meta.Pages = (int(totalCount) + meta.Limit - 1) / meta.Limit
+	}
+
+	s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
+		"count":       len(tenders),
+		"total_count": totalCount,
+	})
+
+	return result, meta, nil
+}
+
+// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
+func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
+	s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
+		"notice_id": noticeID,
+	})
+
+	t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID)
+	if err != nil {
+		s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
+			"notice_id": noticeID,
+			"error":     err.Error(),
+		})
+		return nil, fmt.Errorf("failed to get tender: %w", err)
+	}
+
+	if t == nil {
+		return nil, fmt.Errorf("tender not found")
+	}
+
+	result := s.buildTenderResponse(ctx, t)
+
+	s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{
+		"notice_id": noticeID,
+		"tender_id": t.ID.Hex(),
+	})
+
+	return result, nil
+}