n.nakhostin
0dd877a46d
- Updated the tender repository to create a unique index on tender_id, enhancing database query performance. - Introduced new TED mapping functions to convert parsed TED documents into tender entities, improving integration with the tender management system. - Added error handling for contract notice processing in the TED scraper, ensuring robust logging and error management. - Removed deprecated eform structures to streamline the codebase and focus on essential components.
88 lines
3.1 KiB
Go
88 lines
3.1 KiB
Go
package ted
|
|
|
|
import (
|
|
"encoding/xml"
|
|
"fmt"
|
|
"strings"
|
|
"tm/pkg/xmlparser"
|
|
)
|
|
|
|
type TEDParser struct {
|
|
ContractNotice xmlparser.Parser[ContractNotice]
|
|
ContractAwardNotice xmlparser.Parser[ContractAwardNotice]
|
|
PriorInformationNotice xmlparser.Parser[PriorInformationNotice]
|
|
// BusinessRegistrationInformationNotice xmlparser.Parser[BusinessRegistrationInformationNotice]
|
|
}
|
|
|
|
// NewTEDParser creates a new TED parser
|
|
func NewTEDParser() *TEDParser {
|
|
options := xmlparser.DefaultParserOptions()
|
|
options.MaxSize = 50 * 1024 * 1024 // 50MB limit for TED XML files
|
|
options.IgnoreUnknownElements = true // TED XML has many optional fields
|
|
|
|
contractNoticeParser := xmlparser.NewParser[ContractNotice](options)
|
|
contractAwardNoticeParser := xmlparser.NewParser[ContractAwardNotice](options)
|
|
priorInformationNoticeParser := xmlparser.NewParser[PriorInformationNotice](options)
|
|
// businessRegistrationInformationNoticeParser := xmlparser.NewParser[BusinessRegistrationInformationNotice](options)
|
|
|
|
return &TEDParser{
|
|
ContractNotice: contractNoticeParser,
|
|
ContractAwardNotice: contractAwardNoticeParser,
|
|
PriorInformationNotice: priorInformationNoticeParser,
|
|
// BusinessRegistrationInformationNotice: businessRegistrationInformationNoticeParser,
|
|
}
|
|
}
|
|
|
|
// ParseXML automatically detects and parses any supported TED XML document type
|
|
func (p *TEDParser) ParseXML(content []byte) (*ParsedDocument, string, DocumentType, error) {
|
|
// First, detect the root element type
|
|
rootElement, err := detectRootElement(content)
|
|
if err != nil {
|
|
return nil, "", DocumentType(""), fmt.Errorf("failed to detect root element: %w", err)
|
|
}
|
|
|
|
parsedDoc := &ParsedDocument{Type: DocumentType(rootElement)}
|
|
|
|
switch parsedDoc.Type {
|
|
case DocumentTypeContractNotice:
|
|
notice, err := p.ContractNotice.ParseBytes(content)
|
|
if err != nil {
|
|
return nil, "", DocumentType(""), fmt.Errorf("failed to parse contract notice: %w", err)
|
|
}
|
|
parsedDoc.ContractNotice = notice
|
|
return parsedDoc, notice.GetID(), parsedDoc.Type, nil
|
|
|
|
case DocumentTypeContractAwardNotice:
|
|
notice, err := p.ContractAwardNotice.ParseBytes(content)
|
|
if err != nil {
|
|
return nil, "", DocumentType(""), fmt.Errorf("failed to parse contract award notice: %w", err)
|
|
}
|
|
parsedDoc.ContractAwardNotice = notice
|
|
return parsedDoc, notice.GetID(), parsedDoc.Type, nil
|
|
|
|
case DocumentTypePriorInformationNotice:
|
|
notice, err := p.PriorInformationNotice.ParseBytes(content)
|
|
if err != nil {
|
|
return nil, "", DocumentType(""), fmt.Errorf("failed to parse prior information notice: %w", err)
|
|
}
|
|
parsedDoc.PriorInformationNotice = notice
|
|
return parsedDoc, notice.GetID(), parsedDoc.Type, nil
|
|
default:
|
|
return nil, "", DocumentType(""), fmt.Errorf("unsupported document type: %s", parsedDoc.Type)
|
|
}
|
|
}
|
|
|
|
// detectRootElement detects the root XML element name
|
|
func detectRootElement(content []byte) (string, error) {
|
|
decoder := xml.NewDecoder(strings.NewReader(string(content)))
|
|
for {
|
|
token, err := decoder.Token()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if startElement, ok := token.(xml.StartElement); ok {
|
|
return startElement.Name.Local, nil
|
|
}
|
|
}
|
|
}
|