n.nakhostin
f57e53bd9b
- Added new fields to the Tender entity for handling cancellation, award, and suspension details, improving the entity's capability to manage tender statuses. - Implemented a DetectNoticeStatus method in the TEDParser to identify the status of TED notices from XML data, enhancing the parser's functionality. - Updated the ParseXML method to include notice status detection, ensuring accurate status representation in parsed documents. - Introduced upsert logic in the TED scraper to handle tender records more effectively, allowing for updates or creation based on existing ContractIDs. - Enhanced logging and error handling throughout the scraping process to improve traceability and robustness. - Added a comprehensive usage example document to illustrate the new features and usage patterns for the TED XML parser and scraper.
442 lines
14 KiB
Go
442 lines
14 KiB
Go
package ted
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"strings"
|
|
"tm/pkg/xmlparser"
|
|
)
|
|
|
|
// TEDParser handles parsing of TED XML files
|
|
type TEDParser struct {
|
|
contractNoticeParser xmlparser.Parser[ContractNotice]
|
|
contractAwardNoticeParser xmlparser.Parser[ContractAwardNotice]
|
|
priorInformationNoticeParser xmlparser.Parser[PriorInformationNotice]
|
|
designContestParser xmlparser.Parser[DesignContest]
|
|
qualificationSystemNoticeParser xmlparser.Parser[QualificationSystemNotice]
|
|
concessionNoticeParser xmlparser.Parser[ConcessionNotice]
|
|
planningNoticeParser xmlparser.Parser[PlanningNotice]
|
|
competitionNoticeParser xmlparser.Parser[CompetitionNotice]
|
|
resultNoticeParser xmlparser.Parser[ResultNotice]
|
|
changeNoticeParser xmlparser.Parser[ChangeNotice]
|
|
subcontractNoticeParser xmlparser.Parser[SubcontractNotice]
|
|
}
|
|
|
|
// NewTEDParser creates a new TED parser
|
|
func NewTEDParser() *TEDParser {
|
|
options := xmlparser.DefaultParserOptions()
|
|
options.MaxSize = 50 * 1024 * 1024 // 50MB limit for TED XML files
|
|
options.IgnoreUnknownElements = true // TED XML has many optional fields
|
|
|
|
return &TEDParser{
|
|
contractNoticeParser: xmlparser.NewParser[ContractNotice](options),
|
|
contractAwardNoticeParser: xmlparser.NewParser[ContractAwardNotice](options),
|
|
priorInformationNoticeParser: xmlparser.NewParser[PriorInformationNotice](options),
|
|
designContestParser: xmlparser.NewParser[DesignContest](options),
|
|
qualificationSystemNoticeParser: xmlparser.NewParser[QualificationSystemNotice](options),
|
|
concessionNoticeParser: xmlparser.NewParser[ConcessionNotice](options),
|
|
planningNoticeParser: xmlparser.NewParser[PlanningNotice](options),
|
|
competitionNoticeParser: xmlparser.NewParser[CompetitionNotice](options),
|
|
resultNoticeParser: xmlparser.NewParser[ResultNotice](options),
|
|
changeNoticeParser: xmlparser.NewParser[ChangeNotice](options),
|
|
subcontractNoticeParser: xmlparser.NewParser[SubcontractNotice](options),
|
|
}
|
|
}
|
|
|
|
// DetectDocumentType detects the type of TED document from XML data
|
|
func (p *TEDParser) DetectDocumentType(xmlData []byte) (DocumentType, error) {
|
|
// Fast string-based detection for common patterns
|
|
xmlStr := string(xmlData)
|
|
|
|
if strings.Contains(xmlStr, "<ContractNotice") {
|
|
return DocumentTypeContractNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<ContractAwardNotice") {
|
|
return DocumentTypeContractAwardNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<PriorInformationNotice") {
|
|
return DocumentTypePriorInformationNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<DesignContest") {
|
|
return DocumentTypeDesignContest, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<QualificationSystemNotice") {
|
|
return DocumentTypeQualificationSystemNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<ConcessionNotice") {
|
|
return DocumentTypeConcessionNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<PlanningNotice") {
|
|
return DocumentTypePlanningNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<CompetitionNotice") {
|
|
return DocumentTypeCompetitionNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<ResultNotice") {
|
|
return DocumentTypeResultNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<ChangeNotice") {
|
|
return DocumentTypeChangeNotice, nil
|
|
}
|
|
if strings.Contains(xmlStr, "<SubcontractNotice") {
|
|
return DocumentTypeSubcontractNotice, nil
|
|
}
|
|
|
|
// Fallback to XML token parsing for more complex detection
|
|
decoder := xml.NewDecoder(bytes.NewReader(xmlData))
|
|
for {
|
|
token, err := decoder.Token()
|
|
if err != nil {
|
|
break
|
|
}
|
|
if startElement, ok := token.(xml.StartElement); ok {
|
|
switch startElement.Name.Local {
|
|
case "ContractNotice":
|
|
return DocumentTypeContractNotice, nil
|
|
case "ContractAwardNotice":
|
|
return DocumentTypeContractAwardNotice, nil
|
|
case "PriorInformationNotice":
|
|
return DocumentTypePriorInformationNotice, nil
|
|
case "DesignContest":
|
|
return DocumentTypeDesignContest, nil
|
|
case "QualificationSystemNotice":
|
|
return DocumentTypeQualificationSystemNotice, nil
|
|
case "ConcessionNotice":
|
|
return DocumentTypeConcessionNotice, nil
|
|
case "PlanningNotice":
|
|
return DocumentTypePlanningNotice, nil
|
|
case "CompetitionNotice":
|
|
return DocumentTypeCompetitionNotice, nil
|
|
case "ResultNotice":
|
|
return DocumentTypeResultNotice, nil
|
|
case "ChangeNotice":
|
|
return DocumentTypeChangeNotice, nil
|
|
case "SubcontractNotice":
|
|
return DocumentTypeSubcontractNotice, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
return "", fmt.Errorf("unknown document type")
|
|
}
|
|
|
|
// DetectNoticeStatus detects the status of a TED notice from XML data
|
|
func (p *TEDParser) DetectNoticeStatus(xmlData []byte) NoticeStatus {
|
|
// Convert to string for fast text search
|
|
xmlStr := string(xmlData)
|
|
|
|
// Check for common status indicators in XML content
|
|
// Many TED XMLs contain status information in extensions or specific tags
|
|
|
|
// Check for cancellation indicators
|
|
if strings.Contains(xmlStr, "<CancellationReason") ||
|
|
strings.Contains(xmlStr, "cancelled") ||
|
|
strings.Contains(xmlStr, "canceled") ||
|
|
strings.Contains(xmlStr, "CANCELLED") ||
|
|
strings.Contains(xmlStr, "CANCELED") {
|
|
return NoticeStatusCancelled
|
|
}
|
|
|
|
// Check for award indicators
|
|
if strings.Contains(xmlStr, "<AwardDate") ||
|
|
strings.Contains(xmlStr, "<TenderResult") ||
|
|
strings.Contains(xmlStr, "<SettledContract") ||
|
|
strings.Contains(xmlStr, "<ContractAwardNotice") ||
|
|
strings.Contains(xmlStr, "awarded") ||
|
|
strings.Contains(xmlStr, "AWARDED") {
|
|
return NoticeStatusAwarded
|
|
}
|
|
|
|
// Check for modification indicators
|
|
if strings.Contains(xmlStr, "<ChangeNotice") ||
|
|
strings.Contains(xmlStr, "<ChangeReason") ||
|
|
strings.Contains(xmlStr, "<Modification") ||
|
|
strings.Contains(xmlStr, "modified") ||
|
|
strings.Contains(xmlStr, "MODIFIED") {
|
|
return NoticeStatusModified
|
|
}
|
|
|
|
// Check for suspension indicators
|
|
if strings.Contains(xmlStr, "<SuspensionReason") ||
|
|
strings.Contains(xmlStr, "suspended") ||
|
|
strings.Contains(xmlStr, "SUSPENDED") {
|
|
return NoticeStatusSuspended
|
|
}
|
|
|
|
// Check for closed indicators
|
|
if strings.Contains(xmlStr, "closed") ||
|
|
strings.Contains(xmlStr, "CLOSED") ||
|
|
strings.Contains(xmlStr, "<ClosedDate") {
|
|
return NoticeStatusClosed
|
|
}
|
|
|
|
// Check for draft indicators
|
|
if strings.Contains(xmlStr, "draft") ||
|
|
strings.Contains(xmlStr, "DRAFT") {
|
|
return NoticeStatusDraft
|
|
}
|
|
|
|
// Check for explicit status in XML using XML parsing
|
|
status := p.extractStatusFromXML(xmlData)
|
|
if status != "" {
|
|
switch strings.ToLower(status) {
|
|
case "draft":
|
|
return NoticeStatusDraft
|
|
case "published":
|
|
return NoticeStatusPublished
|
|
case "cancelled", "canceled":
|
|
return NoticeStatusCancelled
|
|
case "awarded":
|
|
return NoticeStatusAwarded
|
|
case "closed":
|
|
return NoticeStatusClosed
|
|
case "modified":
|
|
return NoticeStatusModified
|
|
case "suspended":
|
|
return NoticeStatusSuspended
|
|
}
|
|
}
|
|
|
|
// Default to Published for most TED notices
|
|
return NoticeStatusPublished
|
|
}
|
|
|
|
// extractStatusFromXML extracts status information using XML parsing
|
|
func (p *TEDParser) extractStatusFromXML(xmlData []byte) string {
|
|
decoder := xml.NewDecoder(bytes.NewReader(xmlData))
|
|
|
|
for {
|
|
token, err := decoder.Token()
|
|
if err != nil {
|
|
break
|
|
}
|
|
|
|
if startElement, ok := token.(xml.StartElement); ok {
|
|
// Look for common status element names
|
|
switch startElement.Name.Local {
|
|
case "NoticeStatus", "Status", "StatusCode", "DocumentStatus":
|
|
// Read the content of this element
|
|
if content, err := p.readElementContent(decoder); err == nil {
|
|
return strings.TrimSpace(content)
|
|
}
|
|
}
|
|
|
|
// Check attributes for status information
|
|
for _, attr := range startElement.Attr {
|
|
if strings.Contains(strings.ToLower(attr.Name.Local), "status") {
|
|
return strings.TrimSpace(attr.Value)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// readElementContent reads the text content of the current XML element
|
|
func (p *TEDParser) readElementContent(decoder *xml.Decoder) (string, error) {
|
|
var content strings.Builder
|
|
|
|
for {
|
|
token, err := decoder.Token()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
switch t := token.(type) {
|
|
case xml.CharData:
|
|
content.Write(t)
|
|
case xml.EndElement:
|
|
return content.String(), nil
|
|
case xml.StartElement:
|
|
// Skip nested elements
|
|
decoder.Skip()
|
|
}
|
|
}
|
|
}
|
|
|
|
// ParseXML automatically detects and parses any supported TED XML document type
|
|
func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
|
|
docType, err := p.DetectDocumentType(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to detect document type: %w", err)
|
|
}
|
|
|
|
// Detect notice status
|
|
noticeStatus := p.DetectNoticeStatus(xmlData)
|
|
|
|
parsedDoc := &ParsedDocument{
|
|
Type: docType,
|
|
Status: noticeStatus,
|
|
}
|
|
|
|
switch docType {
|
|
case DocumentTypeContractNotice:
|
|
parsed, err := p.contractNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse contract notice: %w", err)
|
|
}
|
|
parsedDoc.ContractNotice = parsed
|
|
// Override status based on document content
|
|
if parsed != nil {
|
|
parsedDoc.Status = parsed.GetNoticeStatus()
|
|
}
|
|
|
|
case DocumentTypeContractAwardNotice:
|
|
parsed, err := p.contractAwardNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse contract award notice: %w", err)
|
|
}
|
|
parsedDoc.ContractAwardNotice = parsed
|
|
// Override status based on document content
|
|
if parsed != nil {
|
|
parsedDoc.Status = parsed.GetNoticeStatus()
|
|
}
|
|
|
|
case DocumentTypePriorInformationNotice:
|
|
parsed, err := p.priorInformationNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse prior information notice: %w", err)
|
|
}
|
|
parsedDoc.PriorInformationNotice = parsed
|
|
|
|
case DocumentTypeDesignContest:
|
|
parsed, err := p.designContestParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse design contest: %w", err)
|
|
}
|
|
parsedDoc.DesignContest = parsed
|
|
|
|
case DocumentTypeQualificationSystemNotice:
|
|
parsed, err := p.qualificationSystemNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse qualification system notice: %w", err)
|
|
}
|
|
parsedDoc.QualificationSystemNotice = parsed
|
|
|
|
case DocumentTypeConcessionNotice:
|
|
parsed, err := p.concessionNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse concession notice: %w", err)
|
|
}
|
|
parsedDoc.ConcessionNotice = parsed
|
|
|
|
case DocumentTypePlanningNotice:
|
|
parsed, err := p.planningNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse planning notice: %w", err)
|
|
}
|
|
parsedDoc.PlanningNotice = parsed
|
|
|
|
case DocumentTypeCompetitionNotice:
|
|
parsed, err := p.competitionNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse competition notice: %w", err)
|
|
}
|
|
parsedDoc.CompetitionNotice = parsed
|
|
|
|
case DocumentTypeResultNotice:
|
|
parsed, err := p.resultNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse result notice: %w", err)
|
|
}
|
|
parsedDoc.ResultNotice = parsed
|
|
|
|
case DocumentTypeChangeNotice:
|
|
parsed, err := p.changeNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse change notice: %w", err)
|
|
}
|
|
parsedDoc.ChangeNotice = parsed
|
|
|
|
case DocumentTypeSubcontractNotice:
|
|
parsed, err := p.subcontractNoticeParser.ParseBytes(xmlData)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse subcontract notice: %w", err)
|
|
}
|
|
parsedDoc.SubcontractNotice = parsed
|
|
|
|
// Handle legacy document types that should be parsed as ContractNotice
|
|
case DocumentTypeCorrigendum, DocumentTypeModificationNotice:
|
|
// Transform to ContractNotice format and parse
|
|
transformedXML := p.transformToContractNotice(xmlData, string(docType))
|
|
parsed, err := p.contractNoticeParser.ParseBytes(transformedXML)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse %s as contract notice: %w", docType, err)
|
|
}
|
|
parsedDoc.ContractNotice = parsed
|
|
|
|
default:
|
|
return nil, fmt.Errorf("unsupported document type: %s", docType)
|
|
}
|
|
|
|
return parsedDoc, nil
|
|
}
|
|
|
|
// transformToContractNotice transforms various notice types to ContractNotice format
|
|
func (p *TEDParser) transformToContractNotice(xmlData []byte, docType string) []byte {
|
|
xmlStr := string(xmlData)
|
|
|
|
// Replace opening tag
|
|
xmlStr = strings.Replace(xmlStr, "<"+docType, "<ContractNotice", 1)
|
|
|
|
// Replace closing tag
|
|
xmlStr = strings.Replace(xmlStr, "</"+docType+">", "</ContractNotice>", 1)
|
|
|
|
return []byte(xmlStr)
|
|
}
|
|
|
|
// transformPriorInformationNoticeToContractNotice transforms a PriorInformationNotice XML to ContractNotice format
|
|
func (p *TEDParser) transformPriorInformationNoticeToContractNotice(xmlData []byte) []byte {
|
|
return p.transformToContractNotice(xmlData, "PriorInformationNotice")
|
|
}
|
|
|
|
// Legacy methods for backward compatibility
|
|
// ParseXMLAsContractNotice parses XML specifically as ContractNotice (legacy method)
|
|
func (p *TEDParser) ParseXMLAsContractNotice(xmlData []byte) (*ContractNotice, error) {
|
|
return p.contractNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
// ParseXMLAsContractAwardNotice parses XML specifically as ContractAwardNotice
|
|
func (p *TEDParser) ParseXMLAsContractAwardNotice(xmlData []byte) (*ContractAwardNotice, error) {
|
|
return p.contractAwardNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
// Specific parser methods for all notice types
|
|
func (p *TEDParser) ParseXMLAsPriorInformationNotice(xmlData []byte) (*PriorInformationNotice, error) {
|
|
return p.priorInformationNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsDesignContest(xmlData []byte) (*DesignContest, error) {
|
|
return p.designContestParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsQualificationSystemNotice(xmlData []byte) (*QualificationSystemNotice, error) {
|
|
return p.qualificationSystemNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsConcessionNotice(xmlData []byte) (*ConcessionNotice, error) {
|
|
return p.concessionNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsPlanningNotice(xmlData []byte) (*PlanningNotice, error) {
|
|
return p.planningNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsCompetitionNotice(xmlData []byte) (*CompetitionNotice, error) {
|
|
return p.competitionNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsResultNotice(xmlData []byte) (*ResultNotice, error) {
|
|
return p.resultNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsChangeNotice(xmlData []byte) (*ChangeNotice, error) {
|
|
return p.changeNoticeParser.ParseBytes(xmlData)
|
|
}
|
|
|
|
func (p *TEDParser) ParseXMLAsSubcontractNotice(xmlData []byte) (*SubcontractNotice, error) {
|
|
return p.subcontractNoticeParser.ParseBytes(xmlData)
|
|
}
|