n.nakhostin
96d3b4fb9c
- Updated the Tender entity to remove redundant comments on Unix timestamp fields, enhancing code readability. - Renamed the AwardedEntity type to Awarded for improved clarity and consistency in naming conventions. - Removed the transformPriorInformationNoticeToContractNotice method from TEDParser, streamlining the parser's functionality and reducing legacy code.
437 lines
14 KiB
Go
437 lines
14 KiB
Go
package ted
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"strings"
|
|
"tm/pkg/xmlparser"
|
|
)
|
|
|
|
// TEDParser handles parsing of TED XML files
|
|
type TEDParser struct {
|
|
contractNoticeParser xmlparser.Parser[ContractNotice]
|
|
contractAwardNoticeParser xmlparser.Parser[ContractAwardNotice]
|
|
priorInformationNoticeParser xmlparser.Parser[PriorInformationNotice]
|
|
designContestParser xmlparser.Parser[DesignContest]
|
|
qualificationSystemNoticeParser xmlparser.Parser[QualificationSystemNotice]
|
|
concessionNoticeParser xmlparser.Parser[ConcessionNotice]
|
|
planningNoticeParser xmlparser.Parser[PlanningNotice]
|
|
competitionNoticeParser xmlparser.Parser[CompetitionNotice]
|
|
resultNoticeParser xmlparser.Parser[ResultNotice]
|
|
changeNoticeParser xmlparser.Parser[ChangeNotice]
|
|
subcontractNoticeParser xmlparser.Parser[SubcontractNotice]
|
|
}
|
|
|
|
// NewTEDParser creates a new TED parser
|
|
func NewTEDParser() *TEDParser {
|
|
options := xmlparser.DefaultParserOptions()
|
|
options.MaxSize = 50 * 1024 * 1024 // 50MB limit for TED XML files
|
|
options.IgnoreUnknownElements = true // TED XML has many optional fields
|
|
|
|
return &TEDParser{
|
|
contractNoticeParser: xmlparser.NewParser[ContractNotice](options),
|
|
contractAwardNoticeParser: xmlparser.NewParser[ContractAwardNotice](options),
|
|
priorInformationNoticeParser: xmlparser.NewParser[PriorInformationNotice](options),
|
|
designContestParser: xmlparser.NewParser[DesignContest](options),
|
|
qualificationSystemNoticeParser: xmlparser.NewParser[QualificationSystemNotice](options),
|
|
concessionNoticeParser: xmlparser.NewParser[ConcessionNotice](options),
|
|
planningNoticeParser: xmlparser.NewParser[PlanningNotice](options),
|
|
competitionNoticeParser: xmlparser.NewParser[CompetitionNotice](options),
|
|
resultNoticeParser: xmlparser.NewParser[ResultNotice](options),
|
|
changeNoticeParser: xmlparser.NewParser[ChangeNotice](options),
|
|
subcontractNoticeParser: xmlparser.NewParser[SubcontractNotice](options),
|
|
}
|
|
}
|
|
|
|
// DetectDocumentType detects the type of TED document from XML data
|
|
func (p *TEDParser) DetectDocumentType(xmlData []byte) (DocumentType, error) {
|
|
// Fast string-based detection for common patterns
|
|
xmlStr := string(xmlData)
|
|
|
|
if strings.Contains(xmlStr, "<ContractNotice") {
|
|
return DocumentTypeContractNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<ContractAwardNotice") {
|
|
return DocumentTypeContractAwardNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<PriorInformationNotice") {
|
|
return DocumentTypePriorInformationNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<DesignContest") {
|
|
return DocumentTypeDesignContest, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<QualificationSystemNotice") {
|
|
return DocumentTypeQualificationSystemNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<ConcessionNotice") {
|
|
return DocumentTypeConcessionNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<PlanningNotice") {
|
|
return DocumentTypePlanningNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<CompetitionNotice") {
|
|
return DocumentTypeCompetitionNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<ResultNotice") {
|
|
return DocumentTypeResultNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<ChangeNotice") {
|
|
return DocumentTypeChangeNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<SubcontractNotice") {
|
|
return DocumentTypeSubcontractNotice, nil
|
|
}
|
|
|
|
// Fallback to XML token parsing for more complex detection
|
|
decoder := xml.NewDecoder(bytes.NewReader(xmlData))
|
|
for {
|
|
token, err := decoder.Token()
|
|
if err != nil {
|
|
break
|
|
}
|
|
if startElement, ok := token.(xml.StartElement); ok {
|
|
switch startElement.Name.Local {
|
|
case "ContractNotice":
|
|
return DocumentTypeContractNotice, nil
|
|
case "ContractAwardNotice":
|
|
return DocumentTypeContractAwardNotice, nil
|
|
case "PriorInformationNotice":
|
|
return DocumentTypePriorInformationNotice, nil
|
|
case "DesignContest":
|
|
return DocumentTypeDesignContest, nil
|
|
case "QualificationSystemNotice":
|
|
return DocumentTypeQualificationSystemNotice, nil
|
|
case "ConcessionNotice":
|
|
return DocumentTypeConcessionNotice, nil
|
|
case "PlanningNotice":
|
|
return DocumentTypePlanningNotice, nil
|
|
case "CompetitionNotice":
|
|
return DocumentTypeCompetitionNotice, nil
|
|
case "ResultNotice":
|
|
return DocumentTypeResultNotice, nil
|
|
case "ChangeNotice":
|
|
return DocumentTypeChangeNotice, nil
|
|
case "SubcontractNotice":
|
|
return DocumentTypeSubcontractNotice, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
return "", fmt.Errorf("unknown document type")
|
|
}
|
|
|
|
// DetectNoticeStatus detects the status of a TED notice from XML data
|
|
func (p *TEDParser) DetectNoticeStatus(xmlData []byte) NoticeStatus {
|
|
// Convert to string for fast text search
|
|
xmlStr := string(xmlData)
|
|
|
|
// Check for common status indicators in XML content
|
|
// Many TED XMLs contain status information in extensions or specific tags
|
|
|
|
// Check for cancellation indicators
|
|
if strings.Contains(xmlStr, "<CancellationReason") ||
|
|
strings.Contains(xmlStr, "cancelled") ||
|
|
strings.Contains(xmlStr, "canceled") ||
|
|
strings.Contains(xmlStr, "CANCELLED") ||
|
|
strings.Contains(xmlStr, "CANCELED") {
|
|
return NoticeStatusCancelled
|
|
}
|
|
|
|
// Check for award indicators
|
|
if strings.Contains(xmlStr, "<AwardDate") ||
|
|
strings.Contains(xmlStr, "<TenderResult") ||
|
|
strings.Contains(xmlStr, "<SettledContract") ||
|
|
strings.Contains(xmlStr, "<ContractAwardNotice") ||
|
|
strings.Contains(xmlStr, "awarded") ||
|
|
strings.Contains(xmlStr, "AWARDED") {
|
|
return NoticeStatusAwarded
|
|
}
|
|
|
|
// Check for modification indicators
|
|
if strings.Contains(xmlStr, "<ChangeNotice") ||
|
|
strings.Contains(xmlStr, "<ChangeReason") ||
|
|
strings.Contains(xmlStr, "<Modification") ||
|
|
strings.Contains(xmlStr, "modified") ||
|
|
strings.Contains(xmlStr, "MODIFIED") {
|
|
return NoticeStatusModified
|
|
}
|
|
|
|
// Check for suspension indicators
|
|
if strings.Contains(xmlStr, "<SuspensionReason") ||
|
|
strings.Contains(xmlStr, "suspended") ||
|
|
strings.Contains(xmlStr, "SUSPENDED") {
|
|
return NoticeStatusSuspended
|
|
}
|
|
|
|
// Check for closed indicators
|
|
if strings.Contains(xmlStr, "closed") ||
|
|
strings.Contains(xmlStr, "CLOSED") ||
|
|
strings.Contains(xmlStr, "<ClosedDate") {
|
|
return NoticeStatusClosed
|
|
}
|
|
|
|
// Check for draft indicators
|
|
if strings.Contains(xmlStr, "draft") ||
|
|
strings.Contains(xmlStr, "DRAFT") {
|
|
return NoticeStatusDraft
|
|
}
|
|
|
|
// Check for explicit status in XML using XML parsing
|
|
status := p.extractStatusFromXML(xmlData)
|
|
if status != "" {
|
|
switch strings.ToLower(status) {
|
|
case "draft":
|
|
return NoticeStatusDraft
|
|
case "published":
|
|
return NoticeStatusPublished
|
|
case "cancelled", "canceled":
|
|
return NoticeStatusCancelled
|
|
case "awarded":
|
|
return NoticeStatusAwarded
|
|
case "closed":
|
|
return NoticeStatusClosed
|
|
case "modified":
|
|
return NoticeStatusModified
|
|
case "suspended":
|
|
return NoticeStatusSuspended
|
|
}
|
|
}
|
|
|
|
// Default to Published for most TED notices
|
|
return NoticeStatusPublished
|
|
}
|
|
|
|
// extractStatusFromXML extracts status information using XML parsing
|
|
func (p *TEDParser) extractStatusFromXML(xmlData []byte) string {
|
|
decoder := xml.NewDecoder(bytes.NewReader(xmlData))
|
|
|
|
for {
|
|
token, err := decoder.Token()
|
|
if err != nil {
|
|
break
|
|
}
|
|
|
|
if startElement, ok := token.(xml.StartElement); ok {
|
|
// Look for common status element names
|
|
switch startElement.Name.Local {
|
|
case "NoticeStatus", "Status", "StatusCode", "DocumentStatus":
|
|
// Read the content of this element
|
|
if content, err := p.readElementContent(decoder); err == nil {
|
|
return strings.TrimSpace(content)
|
|
}
|
|
}
|
|
|
|
// Check attributes for status information
|
|
for _, attr := range startElement.Attr {
|
|
if strings.Contains(strings.ToLower(attr.Name.Local), "status") {
|
|
return strings.TrimSpace(attr.Value)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// readElementContent reads the text content of the current XML element
|
|
func (p *TEDParser) readElementContent(decoder *xml.Decoder) (string, error) {
|
|
var content strings.Builder
|
|
|
|
for {
|
|
token, err := decoder.Token()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
switch t := token.(type) {
|
|
case xml.CharData:
|
|
content.Write(t)
|
|
case xml.EndElement:
|
|
return content.String(), nil
|
|
case xml.StartElement:
|
|
// Skip nested elements
|
|
decoder.Skip()
|
|
}
|
|
}
|
|
}
|
|
|
|
// ParseXML automatically detects and parses any supported TED XML document type
|
|
func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
|
|
docType, err := p.DetectDocumentType(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to detect document type: %w", err)
|
|
}
|
|
|
|
// Detect notice status
|
|
noticeStatus := p.DetectNoticeStatus(xmlData)
|
|
|
|
parsedDoc := &ParsedDocument{
|
|
Type: docType,
|
|
Status: noticeStatus,
|
|
}
|
|
|
|
switch docType {
|
|
case DocumentTypeContractNotice:
|
|
parsed, err := p.contractNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse contract notice: %w", err)
|
|
}
|
|
parsedDoc.ContractNotice = parsed
|
|
// Override status based on document content
|
|
if parsed != nil {
|
|
parsedDoc.Status = parsed.GetNoticeStatus()
|
|
}
|
|
|
|
case DocumentTypeContractAwardNotice:
|
|
parsed, err := p.contractAwardNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse contract award notice: %w", err)
|
|
}
|
|
parsedDoc.ContractAwardNotice = parsed
|
|
// Override status based on document content
|
|
if parsed != nil {
|
|
parsedDoc.Status = parsed.GetNoticeStatus()
|
|
}
|
|
|
|
case DocumentTypePriorInformationNotice:
|
|
parsed, err := p.priorInformationNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse prior information notice: %w", err)
|
|
}
|
|
parsedDoc.PriorInformationNotice = parsed
|
|
|
|
case DocumentTypeDesignContest:
|
|
parsed, err := p.designContestParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse design contest: %w", err)
|
|
}
|
|
parsedDoc.DesignContest = parsed
|
|
|
|
case DocumentTypeQualificationSystemNotice:
|
|
parsed, err := p.qualificationSystemNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse qualification system notice: %w", err)
|
|
}
|
|
parsedDoc.QualificationSystemNotice = parsed
|
|
|
|
case DocumentTypeConcessionNotice:
|
|
parsed, err := p.concessionNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse concession notice: %w", err)
|
|
}
|
|
parsedDoc.ConcessionNotice = parsed
|
|
|
|
case DocumentTypePlanningNotice:
|
|
parsed, err := p.planningNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse planning notice: %w", err)
|
|
}
|
|
parsedDoc.PlanningNotice = parsed
|
|
|
|
case DocumentTypeCompetitionNotice:
|
|
parsed, err := p.competitionNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse competition notice: %w", err)
|
|
}
|
|
parsedDoc.CompetitionNotice = parsed
|
|
|
|
case DocumentTypeResultNotice:
|
|
parsed, err := p.resultNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse result notice: %w", err)
|
|
}
|
|
parsedDoc.ResultNotice = parsed
|
|
|
|
case DocumentTypeChangeNotice:
|
|
parsed, err := p.changeNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse change notice: %w", err)
|
|
}
|
|
parsedDoc.ChangeNotice = parsed
|
|
|
|
case DocumentTypeSubcontractNotice:
|
|
parsed, err := p.subcontractNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse subcontract notice: %w", err)
|
|
}
|
|
parsedDoc.SubcontractNotice = parsed
|
|
|
|
// Handle legacy document types that should be parsed as ContractNotice
|
|
case DocumentTypeCorrigendum, DocumentTypeModificationNotice:
|
|
// Transform to ContractNotice format and parse
|
|
transformedXML := p.transformToContractNotice(xmlData, string(docType))
|
|
parsed, err := p.contractNoticeParser.ParseBytes(transformedXML)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse %s as contract notice: %w", docType, err)
|
|
}
|
|
parsedDoc.ContractNotice = parsed
|
|
|
|
default:
|
|
return nil, fmt.Errorf("unsupported document type: %s", docType)
|
|
}
|
|
|
|
return parsedDoc, nil
|
|
}
|
|
|
|
// transformToContractNotice transforms various notice types to ContractNotice format
|
|
func (p *TEDParser) transformToContractNotice(xmlData []byte, docType string) []byte {
|
|
xmlStr := string(xmlData)
|
|
|
|
// Replace opening tag
|
|
xmlStr = strings.Replace(xmlStr, "<"+docType, "<ContractNotice", 1)
|
|
|
|
// Replace closing tag
|
|
xmlStr = strings.Replace(xmlStr, "</"+docType+">", "</ContractNotice>", 1)
|
|
|
|
return []byte(xmlStr)
|
|
}
|
|
|
|
// Legacy methods for backward compatibility
|
|
// ParseXMLAsContractNotice parses XML specifically as ContractNotice (legacy method)
|
|
func (p *TEDParser) ParseXMLAsContractNotice(xmlData []byte) (*ContractNotice, error) {
|
|
return p.contractNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
// ParseXMLAsContractAwardNotice parses XML specifically as ContractAwardNotice
|
|
func (p *TEDParser) ParseXMLAsContractAwardNotice(xmlData []byte) (*ContractAwardNotice, error) {
|
|
return p.contractAwardNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
// Specific parser methods for all notice types
|
|
func (p *TEDParser) ParseXMLAsPriorInformationNotice(xmlData []byte) (*PriorInformationNotice, error) {
|
|
return p.priorInformationNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsDesignContest(xmlData []byte) (*DesignContest, error) {
|
|
return p.designContestParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsQualificationSystemNotice(xmlData []byte) (*QualificationSystemNotice, error) {
|
|
return p.qualificationSystemNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsConcessionNotice(xmlData []byte) (*ConcessionNotice, error) {
|
|
return p.concessionNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsPlanningNotice(xmlData []byte) (*PlanningNotice, error) {
|
|
return p.planningNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsCompetitionNotice(xmlData []byte) (*CompetitionNotice, error) {
|
|
return p.competitionNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsResultNotice(xmlData []byte) (*ResultNotice, error) {
|
|
return p.resultNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsChangeNotice(xmlData []byte) (*ChangeNotice, error) {
|
|
return p.changeNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsSubcontractNotice(xmlData []byte) (*SubcontractNotice, error) {
|
|
return p.subcontractNoticeParser.ParseBytes(xmlData)
|
|
}
|