Remove TED XML Parser and Scraper Implementation

- Deleted the TED XML parser and scraper implementation files, including parser.go, scraper.go, and ted.go, to streamline the codebase and eliminate unused components.
- Commented out the initialization code in main.go to prevent execution while maintaining the structure for future reference.
- Removed the README.md for the TED package, which contained documentation for the now-deleted parser and scraper functionalities.
This commit is contained in:
n.nakhostin
2025-09-28 14:12:27 +03:30
parent fa288fb3fa
commit 0c50935c42
6 changed files with 146 additions and 5292 deletions
+123 -123
View File
@@ -1,140 +1,140 @@
package bootstrap
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"time"
"tm/internal/tender"
"tm/pkg/config"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/ted"
)
// import (
// "context"
// "fmt"
// "os"
// "os/signal"
// "syscall"
// "time"
// "tm/internal/tender"
// "tm/pkg/config"
// "tm/pkg/logger"
// "tm/pkg/mongo"
// "tm/ted"
// )
// Init Application Configuration
func InitConfig() (*Config, error) {
conf, err := config.LoadConfig(".", &Config{})
if err != nil {
return nil, fmt.Errorf("failed to load config: %v", err)
}
// // Init Application Configuration
// func InitConfig() (*Config, error) {
// conf, err := config.LoadConfig(".", &Config{})
// if err != nil {
// return nil, fmt.Errorf("failed to load config: %v", err)
// }
return conf, nil
}
// return conf, nil
// }
// Init Logger Service
func InitLogger(conf config.LoggingConfig) logger.Logger {
return logger.NewLogger(&logger.Config{
Level: conf.Level,
Format: conf.Format,
Output: conf.Output,
File: logger.FileConfig{
Path: conf.File.Path,
MaxSize: conf.File.MaxSize,
MaxBackups: conf.File.MaxBackups,
MaxAge: conf.File.MaxAge,
Compress: conf.File.Compress,
},
})
}
// // Init Logger Service
// func InitLogger(conf config.LoggingConfig) logger.Logger {
// return logger.NewLogger(&logger.Config{
// Level: conf.Level,
// Format: conf.Format,
// Output: conf.Output,
// File: logger.FileConfig{
// Path: conf.File.Path,
// MaxSize: conf.File.MaxSize,
// MaxBackups: conf.File.MaxBackups,
// MaxAge: conf.File.MaxAge,
// Compress: conf.File.Compress,
// },
// })
// }
// Init MongoDB Connection Manager
func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
// Convert infra.MongoConfig to mongo.ConnectionConfig
connectionConfig := mongo.ConnectionConfig{
URI: conf.URI,
Database: conf.Name,
MaxPoolSize: uint64(conf.MaxPoolSize),
MinPoolSize: 5, // Default minimum pool size
MaxConnIdleTime: 30 * time.Minute, // Default idle time
ConnectTimeout: conf.Timeout,
ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
}
// // Init MongoDB Connection Manager
// func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
// // Convert infra.MongoConfig to mongo.ConnectionConfig
// connectionConfig := mongo.ConnectionConfig{
// URI: conf.URI,
// Database: conf.Name,
// MaxPoolSize: uint64(conf.MaxPoolSize),
// MinPoolSize: 5, // Default minimum pool size
// MaxConnIdleTime: 30 * time.Minute, // Default idle time
// ConnectTimeout: conf.Timeout,
// ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
// }
// Create MongoDB connection manager
connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
if err != nil {
log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
"error": err.Error(),
"uri": conf.URI,
"db": conf.Name,
})
panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
}
// // Create MongoDB connection manager
// connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
// if err != nil {
// log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
// "error": err.Error(),
// "uri": conf.URI,
// "db": conf.Name,
// })
// panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
// }
log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
"database": conf.Name,
"uri": conf.URI,
"pool_size": conf.MaxPoolSize,
})
// log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
// "database": conf.Name,
// "uri": conf.URI,
// "pool_size": conf.MaxPoolSize,
// })
return connectionManager
}
// return connectionManager
// }
// init TED scraper
func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger) {
// Initialize tender repository
tenderRepo := tender.NewRepository(mongoManager, appLogger)
// // init TED scraper
// func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger) {
// // Initialize tender repository
// tenderRepo := tender.NewRepository(mongoManager, appLogger)
// Initialize TED scraper
tedScraper := ted.NewTEDScraper(
&ted.Config{
BaseURL: config.TED.BaseURL,
Timeout: config.TED.Timeout,
MaxRetries: config.TED.MaxRetries,
RetryDelay: config.TED.RetryDelay,
UserAgent: config.TED.UserAgent,
MaxConcurrency: config.TED.MaxConcurrency,
DownloadDir: config.TED.DownloadDir,
CleanupAfter: config.TED.CleanupAfter,
ScrapingInterval: config.TED.ScrapingInterval,
},
appLogger,
mongoManager,
tenderRepo,
)
// // Initialize TED scraper
// tedScraper := ted.NewTEDScraper(
// &ted.Config{
// BaseURL: config.TED.BaseURL,
// Timeout: config.TED.Timeout,
// MaxRetries: config.TED.MaxRetries,
// RetryDelay: config.TED.RetryDelay,
// UserAgent: config.TED.UserAgent,
// MaxConcurrency: config.TED.MaxConcurrency,
// DownloadDir: config.TED.DownloadDir,
// CleanupAfter: config.TED.CleanupAfter,
// ScrapingInterval: config.TED.ScrapingInterval,
// },
// appLogger,
// mongoManager,
// tenderRepo,
// )
// Create context with cancellation
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// // Create context with cancellation
// ctx, cancel := context.WithCancel(context.Background())
// defer cancel()
// Set up signal handling for graceful shutdown
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
// // Set up signal handling for graceful shutdown
// signalChan := make(chan os.Signal, 1)
// signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
// Start the scraper in a goroutine
scraperDone := make(chan error, 1)
go func() {
appLogger.Info("Starting TED scraper", map[string]interface{}{
"interval": config.TED.ScrapingInterval.String(),
})
scraperDone <- tedScraper.RunPeriodicScraping(ctx)
}()
// // Start the scraper in a goroutine
// scraperDone := make(chan error, 1)
// go func() {
// appLogger.Info("Starting TED scraper", map[string]interface{}{
// "interval": config.TED.ScrapingInterval.String(),
// })
// scraperDone <- tedScraper.RunPeriodicScraping(ctx)
// }()
// Wait for shutdown signal or scraper completion
select {
case <-signalChan:
appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{})
cancel()
// // Wait for shutdown signal or scraper completion
// select {
// case <-signalChan:
// appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{})
// cancel()
// Wait for scraper to finish gracefully with timeout
shutdownTimeout := time.NewTimer(30 * time.Second)
select {
case <-scraperDone:
appLogger.Info("Scraper stopped gracefully", map[string]interface{}{})
case <-shutdownTimeout.C:
appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{})
}
// // Wait for scraper to finish gracefully with timeout
// shutdownTimeout := time.NewTimer(30 * time.Second)
// select {
// case <-scraperDone:
// appLogger.Info("Scraper stopped gracefully", map[string]interface{}{})
// case <-shutdownTimeout.C:
// appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{})
// }
case err := <-scraperDone:
if err != nil {
appLogger.Error("Scraper finished with error", map[string]interface{}{
"error": err.Error(),
})
os.Exit(1)
}
appLogger.Info("Scraper finished successfully", map[string]interface{}{})
}
}
// case err := <-scraperDone:
// if err != nil {
// appLogger.Error("Scraper finished with error", map[string]interface{}{
// "error": err.Error(),
// })
// os.Exit(1)
// }
// appLogger.Info("Scraper finished successfully", map[string]interface{}{})
// }
// }
+27 -33
View File
@@ -1,37 +1,31 @@
package main
import (
"context"
"time"
"tm/cmd/scraper/bootstrap"
)
func main() {
// Load configuration
config, err := bootstrap.InitConfig()
if err != nil {
panic(err)
}
// Initialize logger
appLogger := bootstrap.InitLogger(config.Logging)
appLogger.Info("Starting TED scraper application", map[string]interface{}{
"version": "1.0.0",
})
// Initialize MongoDB connection
mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := mongoManager.Close(ctx); err != nil {
appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
"error": err.Error(),
})
}
}()
bootstrap.InitTEDScraper(*config, mongoManager, appLogger)
appLogger.Info("TED scraper application stopped", map[string]interface{}{})
// // Load configuration
// config, err := bootstrap.InitConfig()
// if err != nil {
// panic(err)
// }
// // Initialize logger
// appLogger := bootstrap.InitLogger(config.Logging)
// appLogger.Info("Starting TED scraper application", map[string]interface{}{
// "version": "1.0.0",
// })
// // Initialize MongoDB connection
// mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
// defer func() {
// ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
// defer cancel()
// if err := mongoManager.Close(ctx); err != nil {
// appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
// "error": err.Error(),
// })
// }
// }()
// bootstrap.InitTEDScraper(*config, mongoManager, appLogger)
// appLogger.Info("TED scraper application stopped", map[string]interface{}{})
}
-396
View File
@@ -1,396 +0,0 @@
# TED XML Parser - Complete Model
This package provides comprehensive Go structures for parsing all TED (Tenders Electronic Daily) XML documents, supporting the complete range of eForms notice types and legacy TED XML formats.
## 🚀 Features
-**Complete TED XML Support**: All notice subtypes from eForms SDK 1.3 to 1.13
-**Automatic Document Detection**: Intelligent parsing with type detection
-**Full Field Coverage**: Comprehensive mapping of all TED XML fields
-**eForms Compliance**: Full support for eForms extensions and organizations
-**Legacy Compatibility**: Backward compatibility with old TED XML formats
-**Validation & Utility Methods**: Built-in validation and data extraction helpers
-**Award Results**: Complete lot tender and award information
-**Multi-language Support**: Proper handling of multilingual content
-**High Performance**: Optimized parsing with 50MB file support
## 📋 Supported Document Types
### Core Notice Types
- **ContractNotice** - Standard tender notices for procurement opportunities
- **ContractAwardNotice** - Award notices announcing procurement results
- **PriorInformationNotice** - Early market engagement and planning notices
- **ResultNotice** - eForms-specific result notices
### Specialized Notice Types
- **DesignContest** - Design competition notices with jury and reward information
- **QualificationSystemNotice** - Qualification system establishment notices
- **ConcessionNotice** - Concession contract notices with revenue terms
- **PlanningNotice** - Procurement planning and budget notices
- **CompetitionNotice** - Competition procedure notices
- **ChangeNotice** - Notice modifications and corrections (eForms)
- **SubcontractNotice** - Subcontracting opportunity notices
### Legacy Document Types
- **Corrigendum** - Corrections (parsed as ContractNotice)
- **ModificationNotice** - Modifications (parsed as ContractNotice)
## 🏗️ Complete Data Model
### Business Groups Covered
#### **BG-1: Notice**
- Complete notice metadata (ID, version, dates, language)
- Notice type codes and regulatory domains
- Publication information and gazette references
- UBL extensions with eForms data
#### **BG-2: Competition**
- Procedure codes and submission methods
- Tender submission deadlines and periods
- Open tender events and auction terms
- Process justification and government agreements
#### **BG-3: Tender**
- Tender reference and identification
- Tender variants and ranking information
- Legal monetary totals and payable amounts
- Subcontracting terms and descriptions
#### **BG-4: Lot**
- Lot identification and titles
- Procurement project details per lot
- Tendering terms and processes per lot
- Award criteria and selection requirements
#### **BG-5: Part**
- Part identification and classification
- Place of performance and location details
- Contract duration and planned periods
- Additional commodity classifications
#### **BG-6: Contracting Entity**
- Contracting party identification
- Buyer profile URIs and contact information
- Contracting party types and activities
- Service provider party details
#### **BG-7: Procedure**
- Procurement procedures and frameworks
- Procedure identifiers (UUID v4)
- Framework agreement terms
- Cross-border procurement indicators
#### **BG-8: Organization**
- Complete organization details and contacts
- Party identification and legal entities
- Postal addresses with country codes
- Natural person indicators
#### **BG-9: Document**
- Document references and attachments
- External references with URIs and hashes
- Document types and status codes
- Validity periods and language specifications
#### **BG-10: Lot Result**
- Tender result codes and award decisions
- Received submission statistics
- Winner identification and tender details
- Contract references and settlement information
#### **BG-11: Winner**
- Winning tenderer identification
- Winner organization details
- SME indicator and natural person flags
- Country of origin information
#### **BG-12: Settled Contract**
- Contract identification and titles
- Issue dates and signatory parties
- Contract values and currency information
- Performance period details
#### **BG-13: Subcontracting**
- Subcontracting terms and percentages
- Subcontractor identification
- Value thresholds and descriptions
- Unknown price indicators
### Extended Structures
#### **Design Contest Specific**
- **ContestDesignJury**: Jury decisions and member information
- **ContestDesignReward**: Reward structures and benefit values
- **JudgingCriteria**: Evaluation criteria and descriptions
#### **Qualification System Specific**
- **QualificationApplication**: Application procedures and periods
- **TendererQualificationRequest**: Qualification requirements
- **ApplicationPeriod**: Application start/end dates and times
#### **Concession Specific**
- **ConcessionRevenue**: Revenue types and amounts
- **ConcessionTerm**: Concession values and duration
- **FrameworkAgreement**: Framework terms and participants
#### **Planning Specific**
- **BudgetAccountLine**: Budget codes and amounts
- **BudgetAccount**: Account classification schemes
- **PlannedPeriod**: Planning timeline information
#### **Award Results Enhanced**
- **EnhancedTenderResult**: Comprehensive award information
- **AwardedTenderedProject**: Project award details
- **WinningTenderLot**: Lot-specific winners
- **WinningTender**: Individual tender results
- **ReceivedSubmissionsStatistics**: Submission counts and statistics
## 🔧 Usage Examples
### Basic Document Parsing
```go
package main
import (
"fmt"
"io/ioutil"
"log"
"tm/ted"
)
func main() {
// Create comprehensive TED parser
parser := ted.NewTEDParser()
// Read any TED XML file
xmlData, err := ioutil.ReadFile("ted_document.xml")
if err != nil {
log.Fatal(err)
}
// Automatic detection and parsing
parsedDoc, err := parser.ParseXML(xmlData)
if err != nil {
log.Fatal(err)
}
// Get document information
document := parsedDoc.GetDocument()
fmt.Printf("Document Type: %s\n", parsedDoc.Type)
fmt.Printf("Document ID: %s\n", document.GetID())
fmt.Printf("Notice Type: %s\n", document.GetNoticeType())
fmt.Printf("Language: %s\n", document.GetLanguage())
// Validate document
if err := document.Validate(); err != nil {
log.Printf("Validation error: %v", err)
}
}
```
### Advanced Usage by Document Type
```go
// Handle specific document types
switch parsedDoc.Type {
case ted.DocumentTypeContractNotice:
cn := parsedDoc.ContractNotice
fmt.Printf("Tender Deadline: %s\n", cn.GetTenderDeadline())
amount, currency := cn.GetBudget()
fmt.Printf("Estimated Value: %s %s\n", amount, currency)
case ted.DocumentTypeContractAwardNotice:
can := parsedDoc.ContractAwardNotice
fmt.Printf("Award Date: %s\n", can.GetAwardDate())
amount, currency := can.GetTotalAwardValue()
fmt.Printf("Total Award: %s %s\n", amount, currency)
case ted.DocumentTypePriorInformationNotice:
pin := parsedDoc.PriorInformationNotice
fmt.Printf("Prior Information ID: %s\n", pin.GetID())
case ted.DocumentTypeDesignContest:
dc := parsedDoc.DesignContest
fmt.Printf("Design Contest ID: %s\n", dc.GetID())
if dc.ContestDesignJury != nil {
fmt.Printf("Jury Decision: %s\n", dc.ContestDesignJury.JuryDecision)
}
case ted.DocumentTypeQualificationSystemNotice:
qsn := parsedDoc.QualificationSystemNotice
fmt.Printf("Qualification System ID: %s\n", qsn.GetID())
case ted.DocumentTypeConcessionNotice:
cn := parsedDoc.ConcessionNotice
fmt.Printf("Concession ID: %s\n", cn.GetID())
if cn.ConcessionRevenue != nil {
fmt.Printf("Revenue Type: %s\n", cn.ConcessionRevenue.RevenueType)
}
case ted.DocumentTypePlanningNotice:
pn := parsedDoc.PlanningNotice
fmt.Printf("Planning ID: %s\n", pn.GetID())
fmt.Printf("Budget Lines: %d\n", len(pn.BudgetAccountLine))
case ted.DocumentTypeResultNotice:
rn := parsedDoc.ResultNotice
fmt.Printf("Result ID: %s\n", rn.GetID())
fmt.Printf("Tender Results: %d\n", len(rn.TenderResult))
case ted.DocumentTypeChangeNotice:
cn := parsedDoc.ChangeNotice
fmt.Printf("Change ID: %s\n", cn.GetID())
fmt.Printf("Change Reasons: %d\n", len(cn.ChangeReason))
case ted.DocumentTypeSubcontractNotice:
sn := parsedDoc.SubcontractNotice
fmt.Printf("Subcontract ID: %s\n", sn.GetID())
fmt.Printf("Subcontract Calls: %d\n", len(sn.SubcontractCall))
}
```
### Working with Organizations and Extensions
```go
// Extract organization information (works for all notice types)
if doc := parsedDoc.GetDocument(); doc != nil {
// Type assertion for specific methods
if notice, ok := doc.(*ted.ContractNotice); ok {
orgs := notice.GetOrganizations()
if orgs != nil {
fmt.Printf("Organizations: %d\n", len(orgs.Organization))
for _, org := range orgs.Organization {
if org.Company != nil && org.Company.PartyName != nil {
fmt.Printf("- %s\n", org.Company.PartyName.Name)
}
}
}
// Get publication information
pub := notice.GetPublicationInfo()
if pub != nil {
fmt.Printf("Publication ID: %s\n", pub.NoticePublicationID)
fmt.Printf("OJS ID: %s\n", pub.GazetteID)
fmt.Printf("Publication Date: %s\n", pub.PublicationDate)
}
}
}
```
## 🔍 Document Type Detection
The parser includes intelligent document type detection:
```go
// Detect document type without parsing
docType, err := parser.DetectDocumentType(xmlData)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Detected document type: %s\n", docType)
// Parse specific type if known
switch docType {
case ted.DocumentTypeContractNotice:
notice, err := parser.ParseXMLAsContractNotice(xmlData)
case ted.DocumentTypePriorInformationNotice:
notice, err := parser.ParseXMLAsPriorInformationNotice(xmlData)
case ted.DocumentTypeDesignContest:
notice, err := parser.ParseXMLAsDesignContest(xmlData)
// ... other specific parsers available
}
```
## ✅ Comprehensive Validation
All notice types include thorough validation:
```go
// Validate any notice type
err := notice.Validate()
if err != nil {
log.Printf("Validation failed: %v", err)
}
```
### Validation Coverage
- Required fields (ID, ContractFolderID, IssueDate)
- Date format validation (multiple ISO 8601 formats)
- Notice type code validation
- Language code validation (3-letter ISO codes)
- Currency code validation (3-letter ISO codes)
- Duration unit code validation
- Business rule validation per notice type
## 🎯 Complete Field Coverage
### Core Metadata Fields
- UBLVersionID, CustomizationID, ID, ContractFolderID
- IssueDate, IssueTime, VersionID, RegulatoryDomain
- NoticeTypeCode, NoticeLanguageCode
- All namespace declarations and XML attributes
### Business Information
- Contracting party details and buyer profiles
- Tendering terms, processes, and submission requirements
- Procurement projects with lots and parts
- Award criteria and selection requirements
- Legal monetary totals with currency support
- Duration measures with unit codes
### eForms Extensions
- Complete eForms extension support
- Organizations with party identification
- Publication information and gazette details
- Notice results with lot awards
- Selection criteria and official languages
- Award criterion parameters
### Enhanced Structures
- Subcontracting terms with enhanced details
- Framework agreements and parent contracts
- Document references with validity periods
- Address information with country codes
- Contact details and electronic communication
## 🧪 Tested Compatibility
This comprehensive parser has been tested with:
- ✅ All eForms SDK versions 1.3 to 1.13
- ✅ Legacy TED XML formats (R2.0.8, R2.0.9)
- ✅ Multiple notice subtypes (20+ tested)
- ✅ Complex organization structures (10+ organizations)
- ✅ Large procurement lots (50+ lots tested)
- ✅ Multiple currencies and value formats
- ✅ Multilingual content and character encoding
- ✅ Large XML files (up to 50MB)
- ✅ Real TED database documents
## 📊 Performance Characteristics
- **Memory Efficient**: Streaming XML parsing with configurable limits
- **Fast Detection**: String-based detection with XML fallback
- **Scalable**: Handles large files and complex structures
- **Concurrent Safe**: Thread-safe parser instances
- **Error Resilient**: Graceful handling of malformed or incomplete XML
## 🔗 Dependencies
- Go 1.19+
- Standard library: `encoding/xml`, `time`, `fmt`
- Custom XML parser: `tm/pkg/xmlparser`
## 📈 Future Extensions
This model is designed to be forward-compatible with:
- New eForms SDK versions
- Additional notice subtypes
- Extended business groups
- Enhanced validation rules
- Performance optimizations
The comprehensive nature of this TED XML model ensures complete coverage of all procurement notice types and supports the full lifecycle of European public procurement processes.
-436
View File
@@ -1,436 +0,0 @@
package ted
import (
"bytes"
"encoding/xml"
"fmt"
"strings"
"tm/pkg/xmlparser"
)
// TEDParser handles parsing of TED XML files
type TEDParser struct {
contractNoticeParser xmlparser.Parser[ContractNotice]
contractAwardNoticeParser xmlparser.Parser[ContractAwardNotice]
priorInformationNoticeParser xmlparser.Parser[PriorInformationNotice]
designContestParser xmlparser.Parser[DesignContest]
qualificationSystemNoticeParser xmlparser.Parser[QualificationSystemNotice]
concessionNoticeParser xmlparser.Parser[ConcessionNotice]
planningNoticeParser xmlparser.Parser[PlanningNotice]
competitionNoticeParser xmlparser.Parser[CompetitionNotice]
resultNoticeParser xmlparser.Parser[ResultNotice]
changeNoticeParser xmlparser.Parser[ChangeNotice]
subcontractNoticeParser xmlparser.Parser[SubcontractNotice]
}
// NewTEDParser creates a new TED parser
func NewTEDParser() *TEDParser {
options := xmlparser.DefaultParserOptions()
options.MaxSize = 50 * 1024 * 1024 // 50MB limit for TED XML files
options.IgnoreUnknownElements = true // TED XML has many optional fields
return &TEDParser{
contractNoticeParser: xmlparser.NewParser[ContractNotice](options),
contractAwardNoticeParser: xmlparser.NewParser[ContractAwardNotice](options),
priorInformationNoticeParser: xmlparser.NewParser[PriorInformationNotice](options),
designContestParser: xmlparser.NewParser[DesignContest](options),
qualificationSystemNoticeParser: xmlparser.NewParser[QualificationSystemNotice](options),
concessionNoticeParser: xmlparser.NewParser[ConcessionNotice](options),
planningNoticeParser: xmlparser.NewParser[PlanningNotice](options),
competitionNoticeParser: xmlparser.NewParser[CompetitionNotice](options),
resultNoticeParser: xmlparser.NewParser[ResultNotice](options),
changeNoticeParser: xmlparser.NewParser[ChangeNotice](options),
subcontractNoticeParser: xmlparser.NewParser[SubcontractNotice](options),
}
}
// DetectDocumentType detects the type of TED document from XML data
func (p *TEDParser) DetectDocumentType(xmlData []byte) (DocumentType, error) {
// Fast string-based detection for common patterns
xmlStr := string(xmlData)
if strings.Contains(xmlStr, "<ContractNotice") {
return DocumentTypeContractNotice, nil
}
if strings.Contains(xmlStr, "<ContractAwardNotice") {
return DocumentTypeContractAwardNotice, nil
}
if strings.Contains(xmlStr, "<PriorInformationNotice") {
return DocumentTypePriorInformationNotice, nil
}
if strings.Contains(xmlStr, "<DesignContest") {
return DocumentTypeDesignContest, nil
}
if strings.Contains(xmlStr, "<QualificationSystemNotice") {
return DocumentTypeQualificationSystemNotice, nil
}
if strings.Contains(xmlStr, "<ConcessionNotice") {
return DocumentTypeConcessionNotice, nil
}
if strings.Contains(xmlStr, "<PlanningNotice") {
return DocumentTypePlanningNotice, nil
}
if strings.Contains(xmlStr, "<CompetitionNotice") {
return DocumentTypeCompetitionNotice, nil
}
if strings.Contains(xmlStr, "<ResultNotice") {
return DocumentTypeResultNotice, nil
}
if strings.Contains(xmlStr, "<ChangeNotice") {
return DocumentTypeChangeNotice, nil
}
if strings.Contains(xmlStr, "<SubcontractNotice") {
return DocumentTypeSubcontractNotice, nil
}
// Fallback to XML token parsing for more complex detection
decoder := xml.NewDecoder(bytes.NewReader(xmlData))
for {
token, err := decoder.Token()
if err != nil {
break
}
if startElement, ok := token.(xml.StartElement); ok {
switch startElement.Name.Local {
case "ContractNotice":
return DocumentTypeContractNotice, nil
case "ContractAwardNotice":
return DocumentTypeContractAwardNotice, nil
case "PriorInformationNotice":
return DocumentTypePriorInformationNotice, nil
case "DesignContest":
return DocumentTypeDesignContest, nil
case "QualificationSystemNotice":
return DocumentTypeQualificationSystemNotice, nil
case "ConcessionNotice":
return DocumentTypeConcessionNotice, nil
case "PlanningNotice":
return DocumentTypePlanningNotice, nil
case "CompetitionNotice":
return DocumentTypeCompetitionNotice, nil
case "ResultNotice":
return DocumentTypeResultNotice, nil
case "ChangeNotice":
return DocumentTypeChangeNotice, nil
case "SubcontractNotice":
return DocumentTypeSubcontractNotice, nil
}
}
}
return "", fmt.Errorf("unknown document type")
}
// DetectNoticeStatus detects the status of a TED notice from XML data
func (p *TEDParser) DetectNoticeStatus(xmlData []byte) NoticeStatus {
// Convert to string for fast text search
xmlStr := string(xmlData)
// Check for common status indicators in XML content
// Many TED XMLs contain status information in extensions or specific tags
// Check for cancellation indicators
if strings.Contains(xmlStr, "<CancellationReason") ||
strings.Contains(xmlStr, "cancelled") ||
strings.Contains(xmlStr, "canceled") ||
strings.Contains(xmlStr, "CANCELLED") ||
strings.Contains(xmlStr, "CANCELED") {
return NoticeStatusCancelled
}
// Check for award indicators
if strings.Contains(xmlStr, "<AwardDate") ||
strings.Contains(xmlStr, "<TenderResult") ||
strings.Contains(xmlStr, "<SettledContract") ||
strings.Contains(xmlStr, "<ContractAwardNotice") ||
strings.Contains(xmlStr, "awarded") ||
strings.Contains(xmlStr, "AWARDED") {
return NoticeStatusAwarded
}
// Check for modification indicators
if strings.Contains(xmlStr, "<ChangeNotice") ||
strings.Contains(xmlStr, "<ChangeReason") ||
strings.Contains(xmlStr, "<Modification") ||
strings.Contains(xmlStr, "modified") ||
strings.Contains(xmlStr, "MODIFIED") {
return NoticeStatusModified
}
// Check for suspension indicators
if strings.Contains(xmlStr, "<SuspensionReason") ||
strings.Contains(xmlStr, "suspended") ||
strings.Contains(xmlStr, "SUSPENDED") {
return NoticeStatusSuspended
}
// Check for closed indicators
if strings.Contains(xmlStr, "closed") ||
strings.Contains(xmlStr, "CLOSED") ||
strings.Contains(xmlStr, "<ClosedDate") {
return NoticeStatusClosed
}
// Check for draft indicators
if strings.Contains(xmlStr, "draft") ||
strings.Contains(xmlStr, "DRAFT") {
return NoticeStatusDraft
}
// Check for explicit status in XML using XML parsing
status := p.extractStatusFromXML(xmlData)
if status != "" {
switch strings.ToLower(status) {
case "draft":
return NoticeStatusDraft
case "published":
return NoticeStatusPublished
case "cancelled", "canceled":
return NoticeStatusCancelled
case "awarded":
return NoticeStatusAwarded
case "closed":
return NoticeStatusClosed
case "modified":
return NoticeStatusModified
case "suspended":
return NoticeStatusSuspended
}
}
// Default to Published for most TED notices
return NoticeStatusPublished
}
// extractStatusFromXML extracts status information using XML parsing
func (p *TEDParser) extractStatusFromXML(xmlData []byte) string {
decoder := xml.NewDecoder(bytes.NewReader(xmlData))
for {
token, err := decoder.Token()
if err != nil {
break
}
if startElement, ok := token.(xml.StartElement); ok {
// Look for common status element names
switch startElement.Name.Local {
case "NoticeStatus", "Status", "StatusCode", "DocumentStatus":
// Read the content of this element
if content, err := p.readElementContent(decoder); err == nil {
return strings.TrimSpace(content)
}
}
// Check attributes for status information
for _, attr := range startElement.Attr {
if strings.Contains(strings.ToLower(attr.Name.Local), "status") {
return strings.TrimSpace(attr.Value)
}
}
}
}
return ""
}
// readElementContent reads the text content of the current XML element
func (p *TEDParser) readElementContent(decoder *xml.Decoder) (string, error) {
var content strings.Builder
for {
token, err := decoder.Token()
if err != nil {
return "", err
}
switch t := token.(type) {
case xml.CharData:
content.Write(t)
case xml.EndElement:
return content.String(), nil
case xml.StartElement:
// Skip nested elements
decoder.Skip()
}
}
}
// ParseXML automatically detects and parses any supported TED XML document type
func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
docType, err := p.DetectDocumentType(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to detect document type: %w", err)
}
// Detect notice status
noticeStatus := p.DetectNoticeStatus(xmlData)
parsedDoc := &ParsedDocument{
Type: docType,
Status: noticeStatus,
}
switch docType {
case DocumentTypeContractNotice:
parsed, err := p.contractNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse contract notice: %w", err)
}
parsedDoc.ContractNotice = parsed
// Override status based on document content
if parsed != nil {
parsedDoc.Status = parsed.GetNoticeStatus()
}
case DocumentTypeContractAwardNotice:
parsed, err := p.contractAwardNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse contract award notice: %w", err)
}
parsedDoc.ContractAwardNotice = parsed
// Override status based on document content
if parsed != nil {
parsedDoc.Status = parsed.GetNoticeStatus()
}
case DocumentTypePriorInformationNotice:
parsed, err := p.priorInformationNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse prior information notice: %w", err)
}
parsedDoc.PriorInformationNotice = parsed
case DocumentTypeDesignContest:
parsed, err := p.designContestParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse design contest: %w", err)
}
parsedDoc.DesignContest = parsed
case DocumentTypeQualificationSystemNotice:
parsed, err := p.qualificationSystemNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse qualification system notice: %w", err)
}
parsedDoc.QualificationSystemNotice = parsed
case DocumentTypeConcessionNotice:
parsed, err := p.concessionNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse concession notice: %w", err)
}
parsedDoc.ConcessionNotice = parsed
case DocumentTypePlanningNotice:
parsed, err := p.planningNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse planning notice: %w", err)
}
parsedDoc.PlanningNotice = parsed
case DocumentTypeCompetitionNotice:
parsed, err := p.competitionNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse competition notice: %w", err)
}
parsedDoc.CompetitionNotice = parsed
case DocumentTypeResultNotice:
parsed, err := p.resultNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse result notice: %w", err)
}
parsedDoc.ResultNotice = parsed
case DocumentTypeChangeNotice:
parsed, err := p.changeNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse change notice: %w", err)
}
parsedDoc.ChangeNotice = parsed
case DocumentTypeSubcontractNotice:
parsed, err := p.subcontractNoticeParser.ParseBytes(xmlData)
if err != nil {
return nil, fmt.Errorf("failed to parse subcontract notice: %w", err)
}
parsedDoc.SubcontractNotice = parsed
// Handle legacy document types that should be parsed as ContractNotice
case DocumentTypeCorrigendum, DocumentTypeModificationNotice:
// Transform to ContractNotice format and parse
transformedXML := p.transformToContractNotice(xmlData, string(docType))
parsed, err := p.contractNoticeParser.ParseBytes(transformedXML)
if err != nil {
return nil, fmt.Errorf("failed to parse %s as contract notice: %w", docType, err)
}
parsedDoc.ContractNotice = parsed
default:
return nil, fmt.Errorf("unsupported document type: %s", docType)
}
return parsedDoc, nil
}
// transformToContractNotice transforms various notice types to ContractNotice format
func (p *TEDParser) transformToContractNotice(xmlData []byte, docType string) []byte {
xmlStr := string(xmlData)
// Replace opening tag
xmlStr = strings.Replace(xmlStr, "<"+docType, "<ContractNotice", 1)
// Replace closing tag
xmlStr = strings.Replace(xmlStr, "</"+docType+">", "</ContractNotice>", 1)
return []byte(xmlStr)
}
// Legacy methods for backward compatibility
// ParseXMLAsContractNotice parses XML specifically as ContractNotice (legacy method)
func (p *TEDParser) ParseXMLAsContractNotice(xmlData []byte) (*ContractNotice, error) {
return p.contractNoticeParser.ParseBytes(xmlData)
}
// ParseXMLAsContractAwardNotice parses XML specifically as ContractAwardNotice
func (p *TEDParser) ParseXMLAsContractAwardNotice(xmlData []byte) (*ContractAwardNotice, error) {
return p.contractAwardNoticeParser.ParseBytes(xmlData)
}
// Specific parser methods for all notice types
func (p *TEDParser) ParseXMLAsPriorInformationNotice(xmlData []byte) (*PriorInformationNotice, error) {
return p.priorInformationNoticeParser.ParseBytes(xmlData)
}
func (p *TEDParser) ParseXMLAsDesignContest(xmlData []byte) (*DesignContest, error) {
return p.designContestParser.ParseBytes(xmlData)
}
func (p *TEDParser) ParseXMLAsQualificationSystemNotice(xmlData []byte) (*QualificationSystemNotice, error) {
return p.qualificationSystemNoticeParser.ParseBytes(xmlData)
}
func (p *TEDParser) ParseXMLAsConcessionNotice(xmlData []byte) (*ConcessionNotice, error) {
return p.concessionNoticeParser.ParseBytes(xmlData)
}
func (p *TEDParser) ParseXMLAsPlanningNotice(xmlData []byte) (*PlanningNotice, error) {
return p.planningNoticeParser.ParseBytes(xmlData)
}
func (p *TEDParser) ParseXMLAsCompetitionNotice(xmlData []byte) (*CompetitionNotice, error) {
return p.competitionNoticeParser.ParseBytes(xmlData)
}
func (p *TEDParser) ParseXMLAsResultNotice(xmlData []byte) (*ResultNotice, error) {
return p.resultNoticeParser.ParseBytes(xmlData)
}
func (p *TEDParser) ParseXMLAsChangeNotice(xmlData []byte) (*ChangeNotice, error) {
return p.changeNoticeParser.ParseBytes(xmlData)
}
func (p *TEDParser) ParseXMLAsSubcontractNotice(xmlData []byte) (*SubcontractNotice, error) {
return p.subcontractNoticeParser.ParseBytes(xmlData)
}
-2067
View File
File diff suppressed because it is too large Load Diff
-2241
View File
File diff suppressed because it is too large Load Diff