Implement TED Scraper and Configuration Management

- Introduced a new TED scraper in `cmd/scraper` to handle downloading and parsing TED XML files. - Added configuration management for the scraper, including a new `Config` struct and YAML configuration file. - Created necessary handler, service, and repository layers for tender management, adhering to Clean Architecture principles. - Implemented comprehensive logging and error handling throughout the scraper functionality. - Updated API routes to include tender management operations, enhancing the overall system capabilities.
2025-08-13 16:52:24 +03:30
parent b7fa012d13
commit 96c21b8b78
16 changed files with 6970 additions and 7 deletions
@@ -0,0 +1,292 @@
+package ted
+
+import (
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"strings"
+	"tm/pkg/xmlparser"
+)
+
+// TEDParser handles parsing of TED XML files
+type TEDParser struct {
+	contractNoticeParser            xmlparser.Parser[ContractNotice]
+	contractAwardNoticeParser       xmlparser.Parser[ContractAwardNotice]
+	priorInformationNoticeParser    xmlparser.Parser[PriorInformationNotice]
+	designContestParser             xmlparser.Parser[DesignContest]
+	qualificationSystemNoticeParser xmlparser.Parser[QualificationSystemNotice]
+	concessionNoticeParser          xmlparser.Parser[ConcessionNotice]
+	planningNoticeParser            xmlparser.Parser[PlanningNotice]
+	competitionNoticeParser         xmlparser.Parser[CompetitionNotice]
+	resultNoticeParser              xmlparser.Parser[ResultNotice]
+	changeNoticeParser              xmlparser.Parser[ChangeNotice]
+	subcontractNoticeParser         xmlparser.Parser[SubcontractNotice]
+}
+
+// NewTEDParser creates a new TED parser
+func NewTEDParser() *TEDParser {
+	options := xmlparser.DefaultParserOptions()
+	options.MaxSize = 50 * 1024 * 1024   // 50MB limit for TED XML files
+	options.IgnoreUnknownElements = true // TED XML has many optional fields
+
+	return &TEDParser{
+		contractNoticeParser:            xmlparser.NewParser[ContractNotice](options),
+		contractAwardNoticeParser:       xmlparser.NewParser[ContractAwardNotice](options),
+		priorInformationNoticeParser:    xmlparser.NewParser[PriorInformationNotice](options),
+		designContestParser:             xmlparser.NewParser[DesignContest](options),
+		qualificationSystemNoticeParser: xmlparser.NewParser[QualificationSystemNotice](options),
+		concessionNoticeParser:          xmlparser.NewParser[ConcessionNotice](options),
+		planningNoticeParser:            xmlparser.NewParser[PlanningNotice](options),
+		competitionNoticeParser:         xmlparser.NewParser[CompetitionNotice](options),
+		resultNoticeParser:              xmlparser.NewParser[ResultNotice](options),
+		changeNoticeParser:              xmlparser.NewParser[ChangeNotice](options),
+		subcontractNoticeParser:         xmlparser.NewParser[SubcontractNotice](options),
+	}
+}
+
+// DetectDocumentType detects the type of TED document from XML data
+func (p *TEDParser) DetectDocumentType(xmlData []byte) (DocumentType, error) {
+	// Fast string-based detection for common patterns
+	xmlStr := string(xmlData)
+
+	if strings.Contains(xmlStr, "<ContractNotice") {
+		return DocumentTypeContractNotice, nil
+	}
+	if strings.Contains(xmlStr, "<ContractAwardNotice") {
+		return DocumentTypeContractAwardNotice, nil
+	}
+	if strings.Contains(xmlStr, "<PriorInformationNotice") {
+		return DocumentTypePriorInformationNotice, nil
+	}
+	if strings.Contains(xmlStr, "<DesignContest") {
+		return DocumentTypeDesignContest, nil
+	}
+	if strings.Contains(xmlStr, "<QualificationSystemNotice") {
+		return DocumentTypeQualificationSystemNotice, nil
+	}
+	if strings.Contains(xmlStr, "<ConcessionNotice") {
+		return DocumentTypeConcessionNotice, nil
+	}
+	if strings.Contains(xmlStr, "<PlanningNotice") {
+		return DocumentTypePlanningNotice, nil
+	}
+	if strings.Contains(xmlStr, "<CompetitionNotice") {
+		return DocumentTypeCompetitionNotice, nil
+	}
+	if strings.Contains(xmlStr, "<ResultNotice") {
+		return DocumentTypeResultNotice, nil
+	}
+	if strings.Contains(xmlStr, "<ChangeNotice") {
+		return DocumentTypeChangeNotice, nil
+	}
+	if strings.Contains(xmlStr, "<SubcontractNotice") {
+		return DocumentTypeSubcontractNotice, nil
+	}
+
+	// Fallback to XML token parsing for more complex detection
+	decoder := xml.NewDecoder(bytes.NewReader(xmlData))
+	for {
+		token, err := decoder.Token()
+		if err != nil {
+			break
+		}
+		if startElement, ok := token.(xml.StartElement); ok {
+			switch startElement.Name.Local {
+			case "ContractNotice":
+				return DocumentTypeContractNotice, nil
+			case "ContractAwardNotice":
+				return DocumentTypeContractAwardNotice, nil
+			case "PriorInformationNotice":
+				return DocumentTypePriorInformationNotice, nil
+			case "DesignContest":
+				return DocumentTypeDesignContest, nil
+			case "QualificationSystemNotice":
+				return DocumentTypeQualificationSystemNotice, nil
+			case "ConcessionNotice":
+				return DocumentTypeConcessionNotice, nil
+			case "PlanningNotice":
+				return DocumentTypePlanningNotice, nil
+			case "CompetitionNotice":
+				return DocumentTypeCompetitionNotice, nil
+			case "ResultNotice":
+				return DocumentTypeResultNotice, nil
+			case "ChangeNotice":
+				return DocumentTypeChangeNotice, nil
+			case "SubcontractNotice":
+				return DocumentTypeSubcontractNotice, nil
+			}
+		}
+	}
+
+	return "", fmt.Errorf("unknown document type")
+}
+
+// ParseXML automatically detects and parses any supported TED XML document type
+func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
+	docType, err := p.DetectDocumentType(xmlData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to detect document type: %w", err)
+	}
+
+	parsedDoc := &ParsedDocument{Type: docType}
+
+	switch docType {
+	case DocumentTypeContractNotice:
+		parsed, err := p.contractNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse contract notice: %w", err)
+		}
+		parsedDoc.ContractNotice = parsed
+
+	case DocumentTypeContractAwardNotice:
+		parsed, err := p.contractAwardNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse contract award notice: %w", err)
+		}
+		parsedDoc.ContractAwardNotice = parsed
+
+	case DocumentTypePriorInformationNotice:
+		parsed, err := p.priorInformationNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse prior information notice: %w", err)
+		}
+		parsedDoc.PriorInformationNotice = parsed
+
+	case DocumentTypeDesignContest:
+		parsed, err := p.designContestParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse design contest: %w", err)
+		}
+		parsedDoc.DesignContest = parsed
+
+	case DocumentTypeQualificationSystemNotice:
+		parsed, err := p.qualificationSystemNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse qualification system notice: %w", err)
+		}
+		parsedDoc.QualificationSystemNotice = parsed
+
+	case DocumentTypeConcessionNotice:
+		parsed, err := p.concessionNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse concession notice: %w", err)
+		}
+		parsedDoc.ConcessionNotice = parsed
+
+	case DocumentTypePlanningNotice:
+		parsed, err := p.planningNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse planning notice: %w", err)
+		}
+		parsedDoc.PlanningNotice = parsed
+
+	case DocumentTypeCompetitionNotice:
+		parsed, err := p.competitionNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse competition notice: %w", err)
+		}
+		parsedDoc.CompetitionNotice = parsed
+
+	case DocumentTypeResultNotice:
+		parsed, err := p.resultNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse result notice: %w", err)
+		}
+		parsedDoc.ResultNotice = parsed
+
+	case DocumentTypeChangeNotice:
+		parsed, err := p.changeNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse change notice: %w", err)
+		}
+		parsedDoc.ChangeNotice = parsed
+
+	case DocumentTypeSubcontractNotice:
+		parsed, err := p.subcontractNoticeParser.ParseBytes(xmlData)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse subcontract notice: %w", err)
+		}
+		parsedDoc.SubcontractNotice = parsed
+
+	// Handle legacy document types that should be parsed as ContractNotice
+	case DocumentTypeCorrigendum, DocumentTypeModificationNotice:
+		// Transform to ContractNotice format and parse
+		transformedXML := p.transformToContractNotice(xmlData, string(docType))
+		parsed, err := p.contractNoticeParser.ParseBytes(transformedXML)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse %s as contract notice: %w", docType, err)
+		}
+		parsedDoc.ContractNotice = parsed
+
+	default:
+		return nil, fmt.Errorf("unsupported document type: %s", docType)
+	}
+
+	return parsedDoc, nil
+}
+
+// transformToContractNotice transforms various notice types to ContractNotice format
+func (p *TEDParser) transformToContractNotice(xmlData []byte, docType string) []byte {
+	xmlStr := string(xmlData)
+
+	// Replace opening tag
+	xmlStr = strings.Replace(xmlStr, "<"+docType, "<ContractNotice", 1)
+
+	// Replace closing tag
+	xmlStr = strings.Replace(xmlStr, "</"+docType+">", "</ContractNotice>", 1)
+
+	return []byte(xmlStr)
+}
+
+// transformPriorInformationNoticeToContractNotice transforms a PriorInformationNotice XML to ContractNotice format
+func (p *TEDParser) transformPriorInformationNoticeToContractNotice(xmlData []byte) []byte {
+	return p.transformToContractNotice(xmlData, "PriorInformationNotice")
+}
+
+// Legacy methods for backward compatibility
+// ParseXMLAsContractNotice parses XML specifically as ContractNotice (legacy method)
+func (p *TEDParser) ParseXMLAsContractNotice(xmlData []byte) (*ContractNotice, error) {
+	return p.contractNoticeParser.ParseBytes(xmlData)
+}
+
+// ParseXMLAsContractAwardNotice parses XML specifically as ContractAwardNotice
+func (p *TEDParser) ParseXMLAsContractAwardNotice(xmlData []byte) (*ContractAwardNotice, error) {
+	return p.contractAwardNoticeParser.ParseBytes(xmlData)
+}
+
+// Specific parser methods for all notice types
+func (p *TEDParser) ParseXMLAsPriorInformationNotice(xmlData []byte) (*PriorInformationNotice, error) {
+	return p.priorInformationNoticeParser.ParseBytes(xmlData)
+}
+
+func (p *TEDParser) ParseXMLAsDesignContest(xmlData []byte) (*DesignContest, error) {
+	return p.designContestParser.ParseBytes(xmlData)
+}
+
+func (p *TEDParser) ParseXMLAsQualificationSystemNotice(xmlData []byte) (*QualificationSystemNotice, error) {
+	return p.qualificationSystemNoticeParser.ParseBytes(xmlData)
+}
+
+func (p *TEDParser) ParseXMLAsConcessionNotice(xmlData []byte) (*ConcessionNotice, error) {
+	return p.concessionNoticeParser.ParseBytes(xmlData)
+}
+
+func (p *TEDParser) ParseXMLAsPlanningNotice(xmlData []byte) (*PlanningNotice, error) {
+	return p.planningNoticeParser.ParseBytes(xmlData)
+}
+
+func (p *TEDParser) ParseXMLAsCompetitionNotice(xmlData []byte) (*CompetitionNotice, error) {
+	return p.competitionNoticeParser.ParseBytes(xmlData)
+}
+
+func (p *TEDParser) ParseXMLAsResultNotice(xmlData []byte) (*ResultNotice, error) {
+	return p.resultNoticeParser.ParseBytes(xmlData)
+}
+
+func (p *TEDParser) ParseXMLAsChangeNotice(xmlData []byte) (*ChangeNotice, error) {
+	return p.changeNoticeParser.ParseBytes(xmlData)
+}
+
+func (p *TEDParser) ParseXMLAsSubcontractNotice(xmlData []byte) (*SubcontractNotice, error) {
+	return p.subcontractNoticeParser.ParseBytes(xmlData)
+}