Files
tm_back/ted/parser.go
T
n.nakhostin 05c7eae8a2 Add TED Scraper Configuration and Cron Scheduler Implementation
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters.
- Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling.
- Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality.
- Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling.
- Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
2025-09-30 16:03:53 +03:30

173 lines
5.2 KiB
Go

package ted
import (
"encoding/xml"
"fmt"
"strings"
"tm/pkg/notification"
"tm/pkg/xmlparser"
)
type ParserType string
const (
ParserTypeContractNotice ParserType = "ContractNotice"
ParserTypeContractAwardNotice ParserType = "ContractAwardNotice"
ParserTypePriorInformationNotice ParserType = "PriorInformationNotice"
ParserTypeBusinessRegistrationInformationNotice ParserType = "BusinessRegistrationInformationNotice"
)
type TEDParser struct {
Notification notification.SDK
NoticeParser xmlparser.Parser[ContractNotice]
}
// NewTEDParser creates a new TED parser
func NewTEDParser() *TEDParser {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024 // 50MB limit for TED XML files
options.IgnoreUnknownElements = true // TED XML has many optional fields
parser := xmlparser.NewParser[ContractNotice](options)
return &TEDParser{
NoticeParser: parser,
}
}
// ParseAnyNotice parses XML content and detects the notice type automatically
func (tp *TEDParser) Parse(content []byte) (interface{}, string, ParserType, error) {
// First, detect the root element type
rootElement, err := detectRootElement(content)
if err != nil {
return nil, "", ParserType(""), fmt.Errorf("failed to detect root element: %w", err)
}
// Parse based on the detected type
switch ParserType(rootElement) {
case ParserTypeContractNotice:
notice, err := tp.parseContractNotice(content)
return notice, notice.GetID(), ParserTypeContractNotice, err
case ParserTypeContractAwardNotice:
notice, err := tp.parseContractAwardNotice(content)
return notice, notice.GetID(), ParserTypeContractAwardNotice, err
case ParserTypePriorInformationNotice:
notice, err := tp.parsePriorInformationNotice(content)
return notice, notice.GetID(), ParserTypePriorInformationNotice, err
case ParserTypeBusinessRegistrationInformationNotice:
notice, err := tp.parseBusinessRegistrationInformationNotice(content)
return notice, notice.GetID(), ParserTypeBusinessRegistrationInformationNotice, err
default:
return nil, "", ParserType(""), fmt.Errorf("unsupported notice type: %s", rootElement)
}
}
// detectRootElement detects the root XML element name
func detectRootElement(content []byte) (string, error) {
decoder := xml.NewDecoder(strings.NewReader(string(content)))
for {
token, err := decoder.Token()
if err != nil {
return "", err
}
if startElement, ok := token.(xml.StartElement); ok {
return startElement.Name.Local, nil
}
}
}
// parseContractNotice parses a ContractNotice
func (tp *TEDParser) parseContractNotice(content []byte) (*ContractNotice, error) {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024
options.IgnoreUnknownElements = true
parser := xmlparser.NewParser[ContractNotice](options)
notice, err := parser.ParseBytes(content)
if err != nil {
return nil, err
}
return notice, nil
}
// parseContractAwardNotice parses a ContractAwardNotice
func (tp *TEDParser) parseContractAwardNotice(content []byte) (*ContractAwardNotice, error) {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024
options.IgnoreUnknownElements = true
parser := xmlparser.NewParser[ContractAwardNotice](options)
notice, err := parser.ParseBytes(content)
if err != nil {
return nil, err
}
return notice, nil
}
// parsePriorInformationNotice parses a PriorInformationNotice
func (tp *TEDParser) parsePriorInformationNotice(content []byte) (*PriorInformationNotice, error) {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024
options.IgnoreUnknownElements = true
parser := xmlparser.NewParser[PriorInformationNotice](options)
notice, err := parser.ParseBytes(content)
if err != nil {
return nil, err
}
return notice, nil
}
// parseBusinessRegistrationInformationNotice parses a BusinessRegistrationInformationNotice
func (tp *TEDParser) parseBusinessRegistrationInformationNotice(content []byte) (*BusinessRegistrationInformationNotice, error) {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024
options.IgnoreUnknownElements = true
parser := xmlparser.NewParser[BusinessRegistrationInformationNotice](options)
notice, err := parser.ParseBytes(content)
if err != nil {
return nil, err
}
return notice, nil
}
// GetID returns the contract notice ID
func (cn ContractNotice) GetID() string {
return cn.ID
}
// GetID returns the contract award notice ID
func (can ContractAwardNotice) GetID() string {
return can.ID
}
// GetID returns the prior information notice ID
func (pin PriorInformationNotice) GetID() string {
return pin.ID
}
// GetID returns the business registration information notice ID
func (brin BusinessRegistrationInformationNotice) GetID() string {
return brin.ID
}
// Validate validates the parsed model
func (cn ContractNotice) Validate() error {
return nil
}
// Validate validates the parsed model
func (can ContractAwardNotice) Validate() error {
return nil
}
// Validate validates the parsed model
func (pin PriorInformationNotice) Validate() error {
return nil
}
// Validate validates the parsed model
func (brin BusinessRegistrationInformationNotice) Validate() error {
return nil
}