05c7eae8a2
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
173 lines
5.2 KiB
Go
173 lines
5.2 KiB
Go
package ted
|
|
|
|
import (
|
|
"encoding/xml"
|
|
"fmt"
|
|
"strings"
|
|
"tm/pkg/notification"
|
|
"tm/pkg/xmlparser"
|
|
)
|
|
|
|
type ParserType string
|
|
|
|
const (
|
|
ParserTypeContractNotice ParserType = "ContractNotice"
|
|
ParserTypeContractAwardNotice ParserType = "ContractAwardNotice"
|
|
ParserTypePriorInformationNotice ParserType = "PriorInformationNotice"
|
|
ParserTypeBusinessRegistrationInformationNotice ParserType = "BusinessRegistrationInformationNotice"
|
|
)
|
|
|
|
type TEDParser struct {
|
|
Notification notification.SDK
|
|
NoticeParser xmlparser.Parser[ContractNotice]
|
|
}
|
|
|
|
// NewTEDParser creates a new TED parser
|
|
func NewTEDParser() *TEDParser {
|
|
options := xmlparser.DefaultParserOptions()
|
|
options.MaxSize = 50 * 1024 * 1024 // 50MB limit for TED XML files
|
|
options.IgnoreUnknownElements = true // TED XML has many optional fields
|
|
|
|
parser := xmlparser.NewParser[ContractNotice](options)
|
|
return &TEDParser{
|
|
NoticeParser: parser,
|
|
}
|
|
}
|
|
|
|
// ParseAnyNotice parses XML content and detects the notice type automatically
|
|
func (tp *TEDParser) Parse(content []byte) (interface{}, string, ParserType, error) {
|
|
// First, detect the root element type
|
|
rootElement, err := detectRootElement(content)
|
|
if err != nil {
|
|
return nil, "", ParserType(""), fmt.Errorf("failed to detect root element: %w", err)
|
|
}
|
|
|
|
// Parse based on the detected type
|
|
switch ParserType(rootElement) {
|
|
case ParserTypeContractNotice:
|
|
notice, err := tp.parseContractNotice(content)
|
|
return notice, notice.GetID(), ParserTypeContractNotice, err
|
|
case ParserTypeContractAwardNotice:
|
|
notice, err := tp.parseContractAwardNotice(content)
|
|
return notice, notice.GetID(), ParserTypeContractAwardNotice, err
|
|
case ParserTypePriorInformationNotice:
|
|
notice, err := tp.parsePriorInformationNotice(content)
|
|
return notice, notice.GetID(), ParserTypePriorInformationNotice, err
|
|
case ParserTypeBusinessRegistrationInformationNotice:
|
|
notice, err := tp.parseBusinessRegistrationInformationNotice(content)
|
|
return notice, notice.GetID(), ParserTypeBusinessRegistrationInformationNotice, err
|
|
default:
|
|
return nil, "", ParserType(""), fmt.Errorf("unsupported notice type: %s", rootElement)
|
|
}
|
|
}
|
|
|
|
// detectRootElement detects the root XML element name
|
|
func detectRootElement(content []byte) (string, error) {
|
|
decoder := xml.NewDecoder(strings.NewReader(string(content)))
|
|
for {
|
|
token, err := decoder.Token()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if startElement, ok := token.(xml.StartElement); ok {
|
|
return startElement.Name.Local, nil
|
|
}
|
|
}
|
|
}
|
|
|
|
// parseContractNotice parses a ContractNotice
|
|
func (tp *TEDParser) parseContractNotice(content []byte) (*ContractNotice, error) {
|
|
options := xmlparser.DefaultParserOptions()
|
|
options.MaxSize = 50 * 1024 * 1024
|
|
options.IgnoreUnknownElements = true
|
|
|
|
parser := xmlparser.NewParser[ContractNotice](options)
|
|
notice, err := parser.ParseBytes(content)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return notice, nil
|
|
}
|
|
|
|
// parseContractAwardNotice parses a ContractAwardNotice
|
|
func (tp *TEDParser) parseContractAwardNotice(content []byte) (*ContractAwardNotice, error) {
|
|
options := xmlparser.DefaultParserOptions()
|
|
options.MaxSize = 50 * 1024 * 1024
|
|
options.IgnoreUnknownElements = true
|
|
|
|
parser := xmlparser.NewParser[ContractAwardNotice](options)
|
|
notice, err := parser.ParseBytes(content)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return notice, nil
|
|
}
|
|
|
|
// parsePriorInformationNotice parses a PriorInformationNotice
|
|
func (tp *TEDParser) parsePriorInformationNotice(content []byte) (*PriorInformationNotice, error) {
|
|
options := xmlparser.DefaultParserOptions()
|
|
options.MaxSize = 50 * 1024 * 1024
|
|
options.IgnoreUnknownElements = true
|
|
|
|
parser := xmlparser.NewParser[PriorInformationNotice](options)
|
|
notice, err := parser.ParseBytes(content)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return notice, nil
|
|
}
|
|
|
|
// parseBusinessRegistrationInformationNotice parses a BusinessRegistrationInformationNotice
|
|
func (tp *TEDParser) parseBusinessRegistrationInformationNotice(content []byte) (*BusinessRegistrationInformationNotice, error) {
|
|
options := xmlparser.DefaultParserOptions()
|
|
options.MaxSize = 50 * 1024 * 1024
|
|
options.IgnoreUnknownElements = true
|
|
|
|
parser := xmlparser.NewParser[BusinessRegistrationInformationNotice](options)
|
|
notice, err := parser.ParseBytes(content)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return notice, nil
|
|
}
|
|
|
|
// GetID returns the contract notice ID
|
|
func (cn ContractNotice) GetID() string {
|
|
return cn.ID
|
|
}
|
|
|
|
// GetID returns the contract award notice ID
|
|
func (can ContractAwardNotice) GetID() string {
|
|
return can.ID
|
|
}
|
|
|
|
// GetID returns the prior information notice ID
|
|
func (pin PriorInformationNotice) GetID() string {
|
|
return pin.ID
|
|
}
|
|
|
|
// GetID returns the business registration information notice ID
|
|
func (brin BusinessRegistrationInformationNotice) GetID() string {
|
|
return brin.ID
|
|
}
|
|
|
|
// Validate validates the parsed model
|
|
func (cn ContractNotice) Validate() error {
|
|
return nil
|
|
}
|
|
|
|
// Validate validates the parsed model
|
|
func (can ContractAwardNotice) Validate() error {
|
|
return nil
|
|
}
|
|
|
|
// Validate validates the parsed model
|
|
func (pin PriorInformationNotice) Validate() error {
|
|
return nil
|
|
}
|
|
|
|
// Validate validates the parsed model
|
|
func (brin BusinessRegistrationInformationNotice) Validate() error {
|
|
return nil
|
|
}
|