diff --git a/cmd/scraper/README.md b/cmd/scraper/README.md new file mode 100644 index 0000000..ab1f6aa --- /dev/null +++ b/cmd/scraper/README.md @@ -0,0 +1,349 @@ +# TED XML Scraper + +A comprehensive TED (Tenders Electronic Daily) XML scraper that downloads and parses daily calendar XML files, converting them to tender entities and storing them in MongoDB. + +## Features + +- **Daily Scheduled Scraping**: Runs automatically at 10:00 AM daily using cron jobs +- **TED XML Format Support**: Handles eForms XML, Contract Notices, and TED Calendar formats +- **XML Parser Integration**: Uses the custom `xmlparser` package with generic type support +- **Robust Error Handling**: Comprehensive error handling, retry mechanisms, and logging +- **Data Conversion**: Converts TED XML structures to internal tender entity models +- **MongoDB Integration**: Stores processed data with proper indexing and validation +- **Progress Tracking**: Job tracking, status monitoring, and statistics +- **Configurable**: Environment variables and YAML configuration support + +## Architecture + +``` +cmd/scraper/ +├── main.go # Application entry point +├── config.yaml # Default configuration +├── bootstrap/ +│ ├── bootstrap.go # Application initialization +│ └── config.go # Configuration structures +└── README.md # This file + +internal/scraper/ +├── service.go # Core scraping service logic +├── repository.go # MongoDB operations for scraping jobs +└── scheduler.go # Cron job scheduling + +pkg/xmlparser/ted/ +├── models.go # TED XML data structures +└── converter.go # XML to entity conversion +``` + +## Installation + +1. **Prerequisites**: + - Go 1.21+ + - MongoDB 4.4+ + +2. **Dependencies**: + ```bash + cd cmd/scraper + go mod tidy + ``` + +3. **Configuration**: + Create a `config.yaml` file or set environment variables: + ```yaml + database: + mongodb: + uri: "mongodb://localhost:27017" + name: "tender_management" + + ted: + base_url: "https://ted.europa.eu" + timeout: 30s + max_retries: 3 + ``` + +## Usage + +### Running the Scraper + +```bash +# Run with default configuration +go run main.go + +# Run with custom config file +CONFIG_PATH=./custom-config.yaml go run main.go + +# Run with environment variables +TED_BASE_URL=https://ted.europa.eu go run main.go +``` + +### Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `TED_BASE_URL` | `https://ted.europa.eu` | TED website base URL | +| `TED_TIMEOUT` | `30s` | HTTP request timeout | +| `TED_MAX_RETRIES` | `3` | Maximum retry attempts | +| `TED_RETRY_DELAY` | `5s` | Delay between retries | +| `TED_USER_AGENT` | `TM-TED-Scraper/1.0` | HTTP User-Agent header | +| `TED_MAX_CONCURRENCY` | `5` | Maximum concurrent downloads | +| `TED_DOWNLOAD_DIR` | `./downloads` | Download directory | +| `TED_CLEANUP_AFTER` | `24h` | File cleanup interval | + +### Manual Trigger + +To manually trigger a scraping job outside the schedule: + +```bash +# Using the scheduler API (if implemented) +curl -X POST http://localhost:8080/api/scraper/trigger + +# Or restart the application to trigger immediate run +``` + +## How It Works + +### 1. Daily Calendar Download + +The scraper downloads TED daily packages using this URL pattern: +``` +https://ted.europa.eu/packages/daily/{yyyynnnnn} +``` + +Where `{yyyynnnnn}` is the year + 5-digit day of year (e.g., `202500001` for January 1st, 2025). + +### 2. XML Parsing + +The scraper supports multiple TED XML formats: + +- **Contract Notice**: Individual tender announcements +- **Tender Calendar**: Daily collections of multiple notices +- **eForms**: New European standard format +- **Legacy TED Schema**: Older format support + +### 3. Data Conversion + +XML data is converted to internal tender entities with: + +- **Field Mapping**: TED XML fields → Tender entity fields +- **Data Validation**: Required field validation and sanitization +- **Unix Timestamps**: All dates converted to Unix timestamps +- **Text Normalization**: Clean up whitespace and formatting +- **URL Generation**: Generate TED website URLs for notices + +### 4. Storage + +Processed data is stored in MongoDB with: + +- **Indexes**: Optimized for search and filtering +- **Deduplication**: Based on contract notice ID +- **Audit Trail**: Creation and update timestamps +- **Error Tracking**: Processing errors and warnings + +## TED XML Format Analysis + +### Contract Notice Structure + +```xml + + 123456-2025 + folder-123 + 2025-01-09+02:00 + cn-standard + ENG + + +``` + +### Key XML Elements Mapped + +| XML Element | Tender Field | Description | +|-------------|--------------|-------------| +| `ID` | `contract_notice_id` | Unique notice identifier | +| `ContractFolderID` | `contract_folder_id` | Contract folder reference | +| `IssueDate` | `issue_date` | Notice publication date | +| `ProcurementProject/Name` | `title` | Tender title | +| `ProcurementProject/Description` | `description` | Tender description | +| `ContractingParty` | `buyer_organization` | Buyer/contracting authority | +| `TenderSubmissionDeadlinePeriod` | `tender_deadline` | Submission deadline | + +## Scheduling + +### Cron Schedule + +- **Main Job**: `0 0 10 * * *` (10:00 AM daily UTC) +- **Cleanup Job**: `0 0 2 * * *` (2:00 AM daily UTC) + +### Job Configuration + +```go +schedulerConfig := scraper.SchedulerConfig{ + CronExpression: "0 0 10 * * *", // 10:00 AM daily (seconds, minutes, hours, day, month, weekday) + Timezone: "UTC", + JobTimeout: 2 * time.Hour, + RetryAttempts: 3, + RetryDelay: 30 * time.Minute, +} +``` + +### Retry Logic + +- **Automatic Retries**: Up to 3 attempts on failure +- **Exponential Backoff**: 30-minute delay between retries +- **Context Timeout**: 2-hour maximum per job +- **Graceful Shutdown**: Waits for running jobs to complete + +## Monitoring + +### Job Status + +Track scraping job progress and status: + +```json +{ + "id": "job-123", + "type": "ted_scraper", + "status": "running", + "progress": { + "processed_count": 150, + "total_count": 500, + "success_count": 145, + "error_count": 5, + "percentage": 30.0, + "current_step": "processing_tenders" + } +} +``` + +### Statistics + +```json +{ + "period": "Last 7 days", + "total_jobs": 7, + "completed_jobs": 6, + "failed_jobs": 1, + "total_processed": 3500, + "total_created": 2100, + "total_updated": 1200, + "total_errors": 200 +} +``` + +## Error Handling + +### Error Types + +1. **Download Errors**: Network issues, server errors +2. **Parsing Errors**: Invalid XML, schema mismatches +3. **Validation Errors**: Missing required fields +4. **Database Errors**: Connection issues, write failures + +### Error Recovery + +- **Retry Mechanism**: Automatic retries with backoff +- **Partial Success**: Continue processing valid notices +- **Error Logging**: Detailed error tracking and reporting +- **Graceful Degradation**: Skip problematic entries + +### Example Error Response + +```json +{ + "job_id": "job-123", + "status": "completed", + "results": { + "total_processed": 100, + "total_created": 85, + "total_errors": 15, + "processing_errors": [ + "notice 23: validation failed: title is required", + "notice 45: failed to parse deadline: invalid format" + ] + } +} +``` + +## Best Practices + +### Performance + +- **Batch Processing**: Process tenders in configurable batches (default: 10) +- **Connection Pooling**: MongoDB connection management +- **Memory Management**: Stream large XML files +- **Concurrent Downloads**: Configurable concurrency limits + +### Data Quality + +- **Validation**: Comprehensive field validation +- **Sanitization**: Text cleaning and normalization +- **Deduplication**: Prevent duplicate tender entries +- **Audit Trail**: Track all changes and updates + +### Reliability + +- **Health Checks**: Monitor service health +- **Graceful Shutdown**: Clean exit on signals +- **Resource Cleanup**: Automatic file cleanup +- **State Persistence**: Track scraping state across restarts + +## Development + +### Adding New XML Formats + +1. **Extend TED Models**: Add new XML structures in `pkg/xmlparser/ted/models.go` +2. **Update Converter**: Add conversion logic in `pkg/xmlparser/ted/converter.go` +3. **Add Parser Support**: Register new parsers in service +4. **Test Thoroughly**: Add comprehensive test cases + +### Custom Validation + +```go +// Add custom validators in converter.go +func (c *Converter) ValidateCustomField(value string) error { + if len(value) > 1000 { + return fmt.Errorf("field too long: %d characters", len(value)) + } + return nil +} +``` + +### Extending Scheduling + +```go +// Add new scheduled jobs in scheduler.go +_, err = s.cron.AddFunc("0 */6 * * *", s.runStatisticsJob) // Every 6 hours +``` + +## Troubleshooting + +### Common Issues + +1. **Connection Errors**: Check MongoDB connection and credentials +2. **Parsing Failures**: Verify TED XML format changes +3. **Schedule Issues**: Check timezone and cron expression +4. **Memory Usage**: Monitor for large XML files + +### Debug Mode + +Enable debug logging: + +```yaml +logging: + level: "debug" +``` + +### Manual Testing + +```bash +# Test XML parsing +go run main.go --test-xml ./sample.xml + +# Test specific date +go run main.go --test-date 2025-01-09 + +# Validate configuration +go run main.go --validate-config +``` + +## License + +This project is part of the Tender Management system and follows the same licensing terms. diff --git a/cmd/scraper/bootstrap/bootstrap.go b/cmd/scraper/bootstrap/bootstrap.go index dfa875f..2fa2a2c 100644 --- a/cmd/scraper/bootstrap/bootstrap.go +++ b/cmd/scraper/bootstrap/bootstrap.go @@ -1,140 +1,184 @@ package bootstrap -// import ( -// "context" -// "fmt" -// "os" -// "os/signal" -// "syscall" -// "time" -// "tm/internal/tender" -// "tm/pkg/config" -// "tm/pkg/logger" -// "tm/pkg/mongo" -// "tm/ted" -// ) +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + "tm/internal/tender" + "tm/pkg/config" + "tm/pkg/logger" + "tm/pkg/mongo" + "tm/pkg/notification" + "tm/pkg/schedule" + "tm/ted" +) -// // Init Application Configuration -// func InitConfig() (*Config, error) { -// conf, err := config.LoadConfig(".", &Config{}) -// if err != nil { -// return nil, fmt.Errorf("failed to load config: %v", err) -// } +// Init Application Configuration +func InitConfig() (*Config, error) { + conf, err := config.LoadConfig(".", &Config{}) + if err != nil { + return nil, fmt.Errorf("failed to load config: %v", err) + } -// return conf, nil -// } + return conf, nil +} -// // Init Logger Service -// func InitLogger(conf config.LoggingConfig) logger.Logger { -// return logger.NewLogger(&logger.Config{ -// Level: conf.Level, -// Format: conf.Format, -// Output: conf.Output, -// File: logger.FileConfig{ -// Path: conf.File.Path, -// MaxSize: conf.File.MaxSize, -// MaxBackups: conf.File.MaxBackups, -// MaxAge: conf.File.MaxAge, -// Compress: conf.File.Compress, -// }, -// }) -// } +// Init Logger Service +func InitLogger(conf config.LoggingConfig) logger.Logger { + return logger.NewLogger(&logger.Config{ + Level: conf.Level, + Format: conf.Format, + Output: conf.Output, + File: logger.FileConfig{ + Path: conf.File.Path, + MaxSize: conf.File.MaxSize, + MaxBackups: conf.File.MaxBackups, + MaxAge: conf.File.MaxAge, + Compress: conf.File.Compress, + }, + }) +} -// // Init MongoDB Connection Manager -// func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager { -// // Convert infra.MongoConfig to mongo.ConnectionConfig -// connectionConfig := mongo.ConnectionConfig{ -// URI: conf.URI, -// Database: conf.Name, -// MaxPoolSize: uint64(conf.MaxPoolSize), -// MinPoolSize: 5, // Default minimum pool size -// MaxConnIdleTime: 30 * time.Minute, // Default idle time -// ConnectTimeout: conf.Timeout, -// ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout -// } +// Init MongoDB Connection Manager +func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager { + // Convert infra.MongoConfig to mongo.ConnectionConfig + connectionConfig := mongo.ConnectionConfig{ + URI: conf.URI, + Database: conf.Name, + MaxPoolSize: uint64(conf.MaxPoolSize), + MinPoolSize: 5, // Default minimum pool size + MaxConnIdleTime: 30 * time.Minute, // Default idle time + ConnectTimeout: conf.Timeout, + ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout + } -// // Create MongoDB connection manager -// connectionManager, err := mongo.NewConnectionManager(connectionConfig, log) -// if err != nil { -// log.Error("Failed to initialize MongoDB connection", map[string]interface{}{ -// "error": err.Error(), -// "uri": conf.URI, -// "db": conf.Name, -// }) -// panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err)) -// } + // Create MongoDB connection manager + connectionManager, err := mongo.NewConnectionManager(connectionConfig, log) + if err != nil { + log.Error("Failed to initialize MongoDB connection", map[string]interface{}{ + "error": err.Error(), + "uri": conf.URI, + "db": conf.Name, + }) + panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err)) + } -// log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{ -// "database": conf.Name, -// "uri": conf.URI, -// "pool_size": conf.MaxPoolSize, -// }) + log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{ + "database": conf.Name, + "uri": conf.URI, + "pool_size": conf.MaxPoolSize, + }) -// return connectionManager -// } + return connectionManager +} -// // init TED scraper -// func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger) { -// // Initialize tender repository -// tenderRepo := tender.NewRepository(mongoManager, appLogger) +// init TED scraper +func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) { + // Initialize tender repository + tenderRepo := tender.NewRepository(mongoManager, appLogger) -// // Initialize TED scraper -// tedScraper := ted.NewTEDScraper( -// &ted.Config{ -// BaseURL: config.TED.BaseURL, -// Timeout: config.TED.Timeout, -// MaxRetries: config.TED.MaxRetries, -// RetryDelay: config.TED.RetryDelay, -// UserAgent: config.TED.UserAgent, -// MaxConcurrency: config.TED.MaxConcurrency, -// DownloadDir: config.TED.DownloadDir, -// CleanupAfter: config.TED.CleanupAfter, -// ScrapingInterval: config.TED.ScrapingInterval, -// }, -// appLogger, -// mongoManager, -// tenderRepo, -// ) + // Initialize TED scraper + tedScraper := ted.NewTEDScraper( + &ted.Config{ + BaseURL: config.TED.BaseURL, + Timeout: config.TED.Timeout, + MaxRetries: config.TED.MaxRetries, + RetryDelay: config.TED.RetryDelay, + UserAgent: config.TED.UserAgent, + MaxConcurrency: config.TED.MaxConcurrency, + DownloadDir: config.TED.DownloadDir, + CleanupAfter: config.TED.CleanupAfter, + ScrapingInterval: config.TED.ScrapingInterval, + }, + appLogger, + mongoManager, + tenderRepo, + notify, + ) -// // Create context with cancellation -// ctx, cancel := context.WithCancel(context.Background()) -// defer cancel() + // start TED scraper job + schedule.NewCronScheduler(appLogger, mongoManager, tenderRepo).AddJob(schedule.Job{ + Name: "TED Scraper Job", + Func: func() { + _ = tedScraper.Run(context.Background()) + }, + Expr: config.TED.ScrapingInterval, + }) -// // Set up signal handling for graceful shutdown -// signalChan := make(chan os.Signal, 1) -// signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + // Create context with cancellation + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() -// // Start the scraper in a goroutine -// scraperDone := make(chan error, 1) -// go func() { -// appLogger.Info("Starting TED scraper", map[string]interface{}{ -// "interval": config.TED.ScrapingInterval.String(), -// }) -// scraperDone <- tedScraper.RunPeriodicScraping(ctx) -// }() + // Set up signal handling for graceful shutdown + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) -// // Wait for shutdown signal or scraper completion -// select { -// case <-signalChan: -// appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{}) -// cancel() + // Start the scraper in a goroutine + scraperDone := make(chan error, 1) + go func() { + appLogger.Info("Starting TED scraper", map[string]interface{}{ + "interval": config.TED.ScrapingInterval, + }) + scraperDone <- tedScraper.Run(ctx) + }() -// // Wait for scraper to finish gracefully with timeout -// shutdownTimeout := time.NewTimer(30 * time.Second) -// select { -// case <-scraperDone: -// appLogger.Info("Scraper stopped gracefully", map[string]interface{}{}) -// case <-shutdownTimeout.C: -// appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{}) -// } + // Wait for shutdown signal or scraper completion + select { + case <-signalChan: + appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{}) + cancel() -// case err := <-scraperDone: -// if err != nil { -// appLogger.Error("Scraper finished with error", map[string]interface{}{ -// "error": err.Error(), -// }) -// os.Exit(1) -// } -// appLogger.Info("Scraper finished successfully", map[string]interface{}{}) -// } -// } + // Wait for scraper to finish gracefully with timeout + shutdownTimeout := time.NewTimer(30 * time.Second) + select { + case <-scraperDone: + appLogger.Info("Scraper stopped gracefully", map[string]interface{}{}) + case <-shutdownTimeout.C: + appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{}) + } + + case err := <-scraperDone: + + if err != nil { + appLogger.Error("Scraper finished with error", map[string]interface{}{ + "error": err.Error(), + }) + os.Exit(1) + } + appLogger.Info("Scraper finished successfully", map[string]interface{}{}) + } +} + +// Init Notification Service +func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK { + notificationConfig := ¬ification.Config{ + BaseURL: conf.BaseURL, + Timeout: conf.Timeout, + RetryAttempts: conf.RetryAttempts, + RetryDelay: conf.RetryDelay, + EnableLogging: conf.EnableLogging, + UserAgent: conf.UserAgent, + } + + notificationSDK, err := notification.New(notificationConfig, log) + if err != nil { + log.Error("Failed to initialize Notification SDK", map[string]interface{}{ + "error": err.Error(), + }) + fmt.Printf("Failed to initialize Notification SDK: %v\n", err.Error()) + } + + log.Info("Notification SDK initialized successfully", map[string]interface{}{ + "base_url": conf.BaseURL, + "timeout": conf.Timeout, + "retry_attempts": conf.RetryAttempts, + "retry_delay": conf.RetryDelay, + "enable_logging": conf.EnableLogging, + "user_agent": conf.UserAgent, + }) + + return *notificationSDK +} diff --git a/cmd/scraper/bootstrap/config.go b/cmd/scraper/bootstrap/config.go index 1a866a6..7766ae2 100644 --- a/cmd/scraper/bootstrap/config.go +++ b/cmd/scraper/bootstrap/config.go @@ -7,19 +7,20 @@ import ( // Config defines the scraper application configuration type Config struct { - Database config.DatabaseConfig - Logging config.LoggingConfig - TED ScraperConfig + Database config.DatabaseConfig + Logging config.LoggingConfig + TED ScraperConfig + Notification config.NotificationConfig } type ScraperConfig struct { - BaseURL string `env:"TED_BASE_URL"` - Timeout time.Duration `env:"TED_TIMEOUT"` - MaxRetries int `env:"TED_MAX_RETRIES"` - RetryDelay time.Duration `env:"TED_RETRY_DELAY"` - UserAgent string `env:"TED_USER_AGENT"` - MaxConcurrency int `env:"TED_MAX_CONCURRENCY"` - DownloadDir string `env:"TED_DOWNLOAD_DIR"` - CleanupAfter time.Duration `env:"TED_CLEANUP_AFTER"` - ScrapingInterval time.Duration `env:"TED_SCRAPING_INTERVAL"` + BaseURL string `env:"TED_BASE_URL" envDefault:"https://ted.europa.eu"` + Timeout time.Duration `env:"TED_TIMEOUT" envDefault:"30s"` + MaxRetries int `env:"TED_MAX_RETRIES" envDefault:"3"` + RetryDelay time.Duration `env:"TED_RETRY_DELAY" envDefault:"5s"` + UserAgent string `env:"TED_USER_AGENT" envDefault:"TM-TED-Scraper/1.0"` + MaxConcurrency int `env:"TED_MAX_CONCURRENCY" envDefault:"5"` + DownloadDir string `env:"TED_DOWNLOAD_DIR" envDefault:"./downloads"` + CleanupAfter time.Duration `env:"TED_CLEANUP_AFTER" envDefault:"24h"` + ScrapingInterval string `env:"TED_SCRAPING_INTERVAL" envDefault:"* * 10 * * *"` } diff --git a/cmd/scraper/config.yaml b/cmd/scraper/config.yaml new file mode 100644 index 0000000..39a628d --- /dev/null +++ b/cmd/scraper/config.yaml @@ -0,0 +1,29 @@ +# TED Scraper Configuration +database: + mongodb: + uri: "mongodb://localhost:27017" + name: "tender_management" + max_pool_size: 100 + timeout: 30s + +logging: + level: "info" + format: "json" + output: "stdout" + file: + path: "./logs/scraper.log" + max_size: 100 + max_backups: 10 + max_age: 30 + compress: true + +ted: + base_url: "https://ted.europa.eu" + timeout: 30s + max_retries: 3 + retry_delay: 5s + user_agent: "TM-TED-Scraper/1.0" + max_concurrency: 5 + download_dir: "./downloads" + cleanup_after: 24h + scraping_interval: 24h diff --git a/cmd/scraper/main.go b/cmd/scraper/main.go index e8bb8ba..4bf7395 100644 --- a/cmd/scraper/main.go +++ b/cmd/scraper/main.go @@ -1,31 +1,54 @@ package main +import ( + "context" + "os" + "os/signal" + "syscall" + "time" + "tm/cmd/scraper/bootstrap" +) + func main() { - // // Load configuration - // config, err := bootstrap.InitConfig() - // if err != nil { - // panic(err) - // } + // Load configuration + config, err := bootstrap.InitConfig() + if err != nil { + panic(err) + } - // // Initialize logger - // appLogger := bootstrap.InitLogger(config.Logging) - // appLogger.Info("Starting TED scraper application", map[string]interface{}{ - // "version": "1.0.0", - // }) + // Initialize logger + appLogger := bootstrap.InitLogger(config.Logging) + appLogger.Info("Starting TED scraper application", map[string]interface{}{ + "version": "1.0.0", + }) - // // Initialize MongoDB connection - // mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger) - // defer func() { - // ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - // defer cancel() - // if err := mongoManager.Close(ctx); err != nil { - // appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{ - // "error": err.Error(), - // }) - // } - // }() + // Initialize notification service - // bootstrap.InitTEDScraper(*config, mongoManager, appLogger) + // Initialize MongoDB connection + mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger) + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := mongoManager.Close(ctx); err != nil { + appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{ + "error": err.Error(), + }) + } + }() - // appLogger.Info("TED scraper application stopped", map[string]interface{}{}) + notificationService := bootstrap.InitNotificationService(config.Notification, appLogger) + bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService) + + // Set up signal handling for graceful shutdown + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + + // Wait for shutdown signal + <-signalChan + appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{}) + + // Stop the scheduler + // scheduler.Stop() + + appLogger.Info("TED scraper application stopped", map[string]interface{}{}) } diff --git a/go.mod b/go.mod index 438eab5..23b995c 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/golang-jwt/jwt/v5 v5.3.0 github.com/labstack/echo/v4 v4.13.4 github.com/redis/go-redis/v9 v9.14.0 + github.com/robfig/cron/v3 v3.0.1 github.com/stretchr/testify v1.10.0 github.com/swaggo/echo-swagger v1.4.1 github.com/swaggo/swag v1.16.6 diff --git a/go.sum b/go.sum index 476cb85..dbec14f 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redis/go-redis/v9 v9.14.0 h1:u4tNCjXOyzfgeLN+vAZaW1xUooqWDqVEsZN0U01jfAE= github.com/redis/go-redis/v9 v9.14.0/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= diff --git a/internal/scraping/scraping.go b/internal/scraping/scraping.go deleted file mode 100644 index 0332651..0000000 --- a/internal/scraping/scraping.go +++ /dev/null @@ -1,153 +0,0 @@ -package scraping - -import "time" - -// ScrapingJobType represents the type of scraping job -type ScrapingJobType string - -const ( - ScrapingJobTypeTEDScraper ScrapingJobType = "ted_scraper" - ScrapingJobTypeManual ScrapingJobType = "manual" - ScrapingJobTypeAPI ScrapingJobType = "api" - ScrapingJobTypeBulkImport ScrapingJobType = "bulk_import" -) - -// ScrapingJobStatus represents the status of a scraping job -type ScrapingJobStatus string - -const ( - ScrapingJobStatusPending ScrapingJobStatus = "pending" - ScrapingJobStatusRunning ScrapingJobStatus = "running" - ScrapingJobStatusCompleted ScrapingJobStatus = "completed" - ScrapingJobStatusFailed ScrapingJobStatus = "failed" - ScrapingJobStatusCancelled ScrapingJobStatus = "cancelled" -) - -// JobProgress represents the progress of a scraping job -type JobProgress struct { - ProcessedCount int `bson:"processed_count" json:"processed_count"` - TotalCount int `bson:"total_count" json:"total_count"` - SuccessCount int `bson:"success_count" json:"success_count"` - ErrorCount int `bson:"error_count" json:"error_count"` - Percentage float64 `bson:"percentage" json:"percentage"` - CurrentStep string `bson:"current_step" json:"current_step"` -} - -// JobResults represents the results of a scraping job -type JobResults struct { - TotalProcessed int `bson:"total_processed" json:"total_processed"` - TotalCreated int `bson:"total_created" json:"total_created"` - TotalUpdated int `bson:"total_updated" json:"total_updated"` - TotalSkipped int `bson:"total_skipped" json:"total_skipped"` - TotalErrors int `bson:"total_errors" json:"total_errors"` - ProcessingErrors []string `bson:"processing_errors,omitempty" json:"processing_errors,omitempty"` - Summary string `bson:"summary,omitempty" json:"summary,omitempty"` -} - -// ScrapingJob represents a scraping job entity -type ScrapingJob struct { - ID string `bson:"_id,omitempty" json:"id"` - Type ScrapingJobType `bson:"type" json:"type"` - Status ScrapingJobStatus `bson:"status" json:"status"` - StartedAt int64 `bson:"started_at" json:"started_at"` // Unix seconds - CompletedAt *int64 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` // Unix seconds - Duration *int64 `bson:"duration,omitempty" json:"duration,omitempty"` // Duration in seconds - Progress JobProgress `bson:"progress" json:"progress"` - Results JobResults `bson:"results" json:"results"` - Errors []string `bson:"errors,omitempty" json:"errors,omitempty"` - Config map[string]interface{} `bson:"config,omitempty" json:"config,omitempty"` - CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds - UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds -} - -// ScrapingState represents the state of the scraping process -type ScrapingState struct { - ID string `bson:"_id,omitempty" json:"id"` - LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"` - LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"` - LastRunAt int64 `bson:"last_run_at" json:"last_run_at"` // Unix seconds - NextRunAt int64 `bson:"next_run_at" json:"next_run_at"` // Unix seconds - IsRunning bool `bson:"is_running" json:"is_running"` - CurrentJobID string `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"` - CreatedAt int64 `bson:"created_at" json:"created_at"` // Unix seconds - UpdatedAt int64 `bson:"updated_at" json:"updated_at"` // Unix seconds -} - -// GetJobID returns the scraping job ID -func (j *ScrapingJob) GetJobID() string { - return j.ID -} - -// IsActive returns true if the job is actively running -func (j *ScrapingJob) IsActive() bool { - return j.Status == ScrapingJobStatusRunning || j.Status == ScrapingJobStatusPending -} - -// IsCompleted returns true if the job has completed (successfully or with failure) -func (j *ScrapingJob) IsCompleted() bool { - return j.Status == ScrapingJobStatusCompleted || j.Status == ScrapingJobStatusFailed || j.Status == ScrapingJobStatusCancelled -} - -// GetProgressPercentage calculates and returns the progress percentage -func (j *ScrapingJob) GetProgressPercentage() float64 { - if j.Progress.TotalCount == 0 { - return 0 - } - return float64(j.Progress.ProcessedCount) / float64(j.Progress.TotalCount) * 100 -} - -// UpdateProgress updates the job progress -func (j *ScrapingJob) UpdateProgress(processed, total, success, errors int, step string) { - j.Progress.ProcessedCount = processed - j.Progress.TotalCount = total - j.Progress.SuccessCount = success - j.Progress.ErrorCount = errors - j.Progress.CurrentStep = step - j.Progress.Percentage = j.GetProgressPercentage() - j.UpdatedAt = time.Now().Unix() -} - -// AddError adds an error to the job -func (j *ScrapingJob) AddError(error string) { - if j.Errors == nil { - j.Errors = make([]string, 0) - } - j.Errors = append(j.Errors, error) - j.UpdatedAt = time.Now().Unix() -} - -// MarkCompleted marks the job as completed -func (j *ScrapingJob) MarkCompleted(status ScrapingJobStatus, results JobResults) { - j.Status = status - j.Results = results - completedAt := time.Now().Unix() - j.CompletedAt = &completedAt - if j.StartedAt > 0 { - duration := completedAt - j.StartedAt - j.Duration = &duration - } - j.UpdatedAt = completedAt -} - -// IsStateRunning returns true if the scraping state indicates a job is running -func (s *ScrapingState) IsStateRunning() bool { - return s.IsRunning -} - -// MarkAsRunning marks the state as running with a job ID -func (s *ScrapingState) MarkAsRunning(jobID string) { - s.IsRunning = true - s.CurrentJobID = jobID - s.LastRunAt = time.Now().Unix() - s.UpdatedAt = time.Now().Unix() -} - -// MarkAsCompleted marks the state as completed -func (s *ScrapingState) MarkAsCompleted(year, number int) { - s.IsRunning = false - s.CurrentJobID = "" - s.LastProcessedYear = year - s.LastProcessedNumber = number - s.NextRunAt = time.Now().Add(4 * time.Hour).Unix() // Schedule next run in 4 hours - s.UpdatedAt = time.Now().Unix() -} diff --git a/pkg/schedule/cron.go b/pkg/schedule/cron.go new file mode 100644 index 0000000..2f9220d --- /dev/null +++ b/pkg/schedule/cron.go @@ -0,0 +1,144 @@ +package schedule + +import ( + "time" + "tm/internal/tender" + "tm/pkg/logger" + "tm/pkg/mongo" + + "github.com/robfig/cron/v3" +) + +// CronScheduler handles scheduled tasks for TED operations +type CronScheduler struct { + cron *cron.Cron + mongoManager *mongo.ConnectionManager + tenderRepo tender.TenderRepository + logger logger.Logger +} + +// Logger interface for structured logging +type Logger interface { + Info(msg string, fields map[string]interface{}) + Error(msg string, fields map[string]interface{}) + Debug(msg string, fields map[string]interface{}) +} + +type Job struct { + Name string + Func func() + Expr string +} + +// NewCronScheduler creates a new cron scheduler instance +func NewCronScheduler(logger logger.Logger, mongoManager *mongo.ConnectionManager, tenderRepo tender.TenderRepository) *CronScheduler { + // Create cron with timezone support and seconds precision + c := cron.New(cron.WithLocation(time.UTC), cron.WithSeconds()) + + return &CronScheduler{ + cron: c, + mongoManager: mongoManager, + tenderRepo: tenderRepo, + logger: logger, + } +} + +// Start starts the cron scheduler +func (cs *CronScheduler) Start() { + cs.logger.Info("Starting TED cron scheduler", map[string]interface{}{}) + cs.cron.Start() +} + +// Stop stops the cron scheduler gracefully +func (cs *CronScheduler) Stop() { + cs.logger.Info("Stopping TED cron scheduler", map[string]interface{}{}) + ctx := cs.cron.Stop() + <-ctx.Done() +} + +// ScheduleDailyJob schedules a job to run daily at 10:00 AM UTC +func (cs *CronScheduler) AddJob(job Job) error { + // Cron expression: "0 0 10 * * *" means: + // Second: 0 + // Minute: 0 + // Hour: 10 (10 AM) + // Day of month: * (every day) + // Month: * (every month) + // Day of week: * (every day of week) + _, err := cs.cron.AddFunc(job.Expr, func() { + now := time.Now().Local() + cs.logger.Info("Starting daily TED job", map[string]interface{}{ + "job_name": job.Name, + "cron_expression": job.Expr, + "time": time.Now().Format(time.RFC3339), + }) + + job.Func() + + cs.logger.Info("Completed job successfully", map[string]interface{}{ + "time": now.Format(time.RFC3339), + "job_name": job.Name, + "cron_expression": job.Expr, + }) + }) + + if err != nil { + cs.logger.Error("Failed to schedule daily job", map[string]interface{}{ + "error": err.Error(), + "job_name": job.Name, + "cron_expression": job.Expr, + "time": time.Now().Format(time.RFC3339), + }) + return err + } + + cs.logger.Info("Daily job scheduled successfully", map[string]interface{}{ + "job_name": job.Name, + "cron_expression": job.Expr, + "time": time.Now().Format(time.RFC3339), + }) + + return nil +} + +// ExampleTEDScrapingJob is an example job function that can be scheduled +func ExampleTEDScrapingJob() { + // now := time.Now().Local() + // // Example: Get current date for TED scraping + // currentDate := time.Now().Format("02/01/2006") + // currentYear := time.Now().Year() + + // // This is just an example - replace with actual TED scraping logic + // ojs, err := GetOJS(currentYear, currentDate) + // if err != nil { + // log.Printf("Error getting OJS: %v", err) + // return + // } + + // log.Printf("Successfully retrieved OJS: %s for date: %s", ojs, currentDate) + + // Add your actual TED scraping/processing logic here + + // ojs, err := GetOJS(now.Year(), now.Format("02/01/2006")) + // if err != nil { + // cs.logger.Error("Failed to get OJS", map[string]interface{}{ + // "error": err.Error(), + // }) + // return + // } + + // scraper := NewTEDScraper(cs.config, cs.logger, cs.mongoManager, cs.tenderRepo) + // result, err := scraper.DownloadDailyPackage(context.Background(), ojs) + // if err != nil { + // cs.logger.Error("Failed to download daily package", map[string]interface{}{ + // "error": err.Error(), + // }) + // return + // } + + // cs.logger.Info("Downloaded daily package successfully", map[string]interface{}{ + // "result": result, + // "time": now.Format(time.RFC3339), + // "ojs": ojs, + // }) +} diff --git a/ted/calendar.go b/ted/calendar.go new file mode 100644 index 0000000..3cfd3e8 --- /dev/null +++ b/ted/calendar.go @@ -0,0 +1,46 @@ +package ted + +import ( + "bytes" + "encoding/csv" + "fmt" + "io" + "net/http" + "strings" +) + +const ( + calendarUrl = "https://ted.europa.eu/en/release-calendar/-/download/file/CSV/%d" +) + +// Get OJS Number +func GetOJS(year int, date string) (string, error) { + var ( + ojs string + ) + url := fmt.Sprintf(calendarUrl, year) + + resp, err := http.Get(url) + if err != nil { + return ojs, err + } + defer resp.Body.Close() + + // read CSV + body, _ := io.ReadAll(resp.Body) + reader := csv.NewReader(bytes.NewReader(body)) + records, _ := reader.ReadAll() + + for _, row := range records[1:] { + rowDate := strings.TrimSpace(row[1]) + if rowDate == strings.TrimSpace(date) { + ojs = fmt.Sprintf("%v00%v", year, strings.TrimSpace(fmt.Sprint(row[0]))) + break + } + } + if ojs == "" { + return ojs, fmt.Errorf("can not find ojs number") + } + + return ojs, nil +} diff --git a/ted/eform.go b/ted/eform.go new file mode 100644 index 0000000..c478bb9 --- /dev/null +++ b/ted/eform.go @@ -0,0 +1,126 @@ +package ted + +import ( + "encoding/xml" + "tm/ted/eform" +) + +// ===================================================== +// ROOT NOTICE STRUCTURE +// ===================================================== + +// Notice represents the UBL ContractNotice structure used by TED +type ContractNotice struct { + XMLName xml.Name `xml:"ContractNotice"` + UBLVersionID string `xml:"UBLVersionID"` + CustomizationID string `xml:"CustomizationID"` + ID string `xml:"ID"` + ContractFolderID string `xml:"ContractFolderID"` + IssueDate string `xml:"IssueDate"` + IssueTime string `xml:"IssueTime"` + VersionID string `xml:"VersionID"` + RegulatoryDomain string `xml:"RegulatoryDomain"` + NoticeTypeCode string `xml:"NoticeTypeCode"` + NoticeLanguageCode string `xml:"NoticeLanguageCode"` + UBLExtensions eform.UBLExtensions `xml:"UBLExtensions,omitempty"` + ContractingParty eform.ContractingParty `xml:"ContractingParty"` + TenderingTerms eform.TenderingTerms `xml:"TenderingTerms,omitempty"` + TenderingProcess eform.TenderingProcess `xml:"TenderingProcess,omitempty"` + ProcurementProject eform.ProcurementProject `xml:"ProcurementProject"` + ProcurementProjectLot []eform.ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"` +} + +// ContractAwardNotice represents the UBL ContractAwardNotice structure +type ContractAwardNotice struct { + XMLName xml.Name `xml:"ContractAwardNotice"` + UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"` + CustomizationID string `json:"customization_id" xml:"CustomizationID"` + ID string `json:"id" xml:"ID"` + ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"` + IssueDate string `json:"issue_date" xml:"IssueDate"` + IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"` + VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"` + RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"` + NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"` + NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"` + ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"` + TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"` + TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"` + ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"` + ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"` + UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` +} + +// PriorInformationNotice represents the UBL PriorInformationNotice structure +type PriorInformationNotice struct { + XMLName xml.Name `xml:"PriorInformationNotice"` + UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"` + CustomizationID string `json:"customization_id" xml:"CustomizationID"` + ID string `json:"id" xml:"ID"` + ContractFolderID string `json:"contract_folder_id" xml:"ContractFolderID"` + IssueDate string `json:"issue_date" xml:"IssueDate"` + IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"` + VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"` + RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"` + NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"` + NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"` + ContractingParty eform.ContractingParty `json:"contracting_party" xml:"ContractingParty"` + TenderingTerms eform.TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"` + TenderingProcess eform.TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"` + ProcurementProject eform.ProcurementProject `json:"procurement_project" xml:"ProcurementProject"` + ProcurementProjectLot []eform.ProcurementProjectLot `json:"procurement_project_lots,omitempty" xml:"ProcurementProjectLot,omitempty"` + UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` +} + +// BusinessRegistrationInformationNotice represents the UBL BusinessRegistrationInformationNotice structure +type BusinessRegistrationInformationNotice struct { + XMLName xml.Name `xml:"BusinessRegistrationInformationNotice"` + UBLVersionID string `json:"ubl_version_id" xml:"UBLVersionID"` + CustomizationID string `json:"customization_id" xml:"CustomizationID"` + ID string `json:"id" xml:"ID"` + IssueDate string `json:"issue_date" xml:"IssueDate"` + IssueTime string `json:"issue_time,omitempty" xml:"IssueTime,omitempty"` + VersionID string `json:"version_id,omitempty" xml:"VersionID,omitempty"` + RegulatoryDomain string `json:"regulatory_domain,omitempty" xml:"RegulatoryDomain,omitempty"` + NoticeTypeCode string `json:"notice_type_code" xml:"NoticeTypeCode"` + NoticeLanguageCode string `json:"notice_language_code" xml:"NoticeLanguageCode"` + UBLExtensions eform.UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` + AdditionalNoticeLanguage []eform.AdditionalNoticeLanguage `json:"additional_notice_language,omitempty" xml:"AdditionalNoticeLanguage,omitempty"` + SenderParty eform.SenderParty `json:"sender_party,omitempty" xml:"SenderParty,omitempty"` + BusinessParty eform.BusinessParty `json:"business_party" xml:"BusinessParty"` + AdditionalDocumentReference []eform.AdditionalDocumentReference `json:"additional_document_reference,omitempty" xml:"AdditionalDocumentReference,omitempty"` + BusinessCapability []eform.BusinessCapability `json:"business_capability,omitempty" xml:"BusinessCapability,omitempty"` + NoticePurpose eform.NoticePurpose `json:"notice_purpose,omitempty" xml:"NoticePurpose,omitempty"` +} + +// ===================================================== +// BUSINESS LOGIC HELPER METHODS +// ===================================================== + +// GetMainBuyer returns the lead/main buyer organization +func (n *ContractNotice) GetMainBuyer() *eform.Organization { + if len(n.UBLExtensions.UBLExtension) > 0 { + orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization + for i := range orgs { + for _, role := range orgs[i].Roles { + if role == eform.RoleBuyer || role == eform.RoleLeadBuyer { + return &orgs[i] + } + } + } + } + return nil +} + +// GetOrganizationByID retrieves organization by ID +func (n *ContractNotice) GetOrganizationByID(id string) *eform.Organization { + if len(n.UBLExtensions.UBLExtension) > 0 { + orgs := n.UBLExtensions.UBLExtension[0].ExtensionContent.EformsExtension.Organizations.Organization + for i := range orgs { + if orgs[i].ID == id { + return &orgs[i] + } + } + } + return nil +} diff --git a/ted/eform/award_criteria.go b/ted/eform/award_criteria.go new file mode 100644 index 0000000..3b426f1 --- /dev/null +++ b/ted/eform/award_criteria.go @@ -0,0 +1,54 @@ +package eform + +// ===================================================== +// AWARD CRITERIA +// ===================================================== + +// AwardCriteria defines how tenders will be evaluated +type AwardCriteria struct { + Type AwardCriteriaType `json:"type" xml:"type" validate:"required"` + PriceWeight *float64 `json:"price_weight,omitempty" xml:"price_weight,omitempty" validate:"omitempty,min=0,max=100"` + CriteriaStatement string `json:"criteria_statement,omitempty" xml:"criteria_statement,omitempty" validate:"max=10000"` + Criteria []AwardCriterion `json:"criteria,omitempty" xml:"criteria,omitempty" validate:"dive"` + CostCalculation string `json:"cost_calculation,omitempty" xml:"cost_calculation,omitempty" validate:"max=10000"` +} + +// AwardCriteriaType represents the evaluation approach +type AwardCriteriaType string + +const ( + AwardTypePrice AwardCriteriaType = "price" + AwardTypeCost AwardCriteriaType = "cost" + AwardTypeQualPrice AwardCriteriaType = "qual-price" +) + +// AwardCriterion represents an individual evaluation criterion +type AwardCriterion struct { + ID string `json:"id" xml:"id" validate:"required,acIDFormat"` + Name string `json:"name" xml:"name" validate:"required,max=200"` + Description string `json:"description,omitempty" xml:"description,omitempty" validate:"max=2000"` + Weight float64 `json:"weight" xml:"weight" validate:"required,min=0,max=100"` + Type AwardCriterionType `json:"type" xml:"type" validate:"required"` + SubCriteria []AwardCriterion `json:"sub_criteria,omitempty" xml:"sub_criteria,omitempty" validate:"dive"` +} + +// AwardCriterionType represents the type of criterion +type AwardCriterionType string + +const ( + CriterionQuality AwardCriterionType = "quality" + CriterionCost AwardCriterionType = "cost" + CriterionPrice AwardCriterionType = "price" + CriterionSocial AwardCriterionType = "social" + CriterionEnvironmental AwardCriterionType = "environmental" + CriterionInnovative AwardCriterionType = "innovative" +) + +// CalculateTotalAwardCriteriaWeight validates criteria weights sum to 100 +func (ac *AwardCriteria) CalculateTotalAwardCriteriaWeight() float64 { + var total float64 + for _, criterion := range ac.Criteria { + total += criterion.Weight + } + return total +} diff --git a/ted/eform/business_registration.go b/ted/eform/business_registration.go new file mode 100644 index 0000000..da5318c --- /dev/null +++ b/ted/eform/business_registration.go @@ -0,0 +1,77 @@ +package eform + +import "encoding/xml" + +// ===================================================== +// BUSINESS REGISTRATION INFORMATION NOTICE +// ===================================================== + +// AdditionalNoticeLanguage represents additional notice language +type AdditionalNoticeLanguage struct { + XMLName xml.Name `xml:"AdditionalNoticeLanguage"` + ID string `json:"id" xml:"ID"` +} + +// SenderParty represents the sender party +type SenderParty struct { + XMLName xml.Name `xml:"SenderParty"` + Contact Contact `json:"contact,omitempty" xml:"Contact,omitempty"` +} + +// BusinessParty represents the business party +type BusinessParty struct { + XMLName xml.Name `xml:"BusinessParty"` + PostalAddress PostalAddress `json:"postal_address,omitempty" xml:"PostalAddress,omitempty"` + PartyLegalEntity []PartyLegalEntityEnhanced `json:"party_legal_entity,omitempty" xml:"PartyLegalEntity,omitempty"` + Contact Contact `json:"contact,omitempty" xml:"Contact,omitempty"` + WebsiteURI string `json:"website_uri,omitempty" xml:"WebsiteURI,omitempty"` +} + +// AdditionalDocumentReference represents additional document reference +type AdditionalDocumentReference struct { + XMLName xml.Name `xml:"AdditionalDocumentReference"` + ID string `json:"id" xml:"ID"` + IssueDate string `json:"issue_date,omitempty" xml:"IssueDate,omitempty"` + ReferencedDocumentInternalAddress string `json:"referenced_document_internal_address,omitempty" xml:"ReferencedDocumentInternalAddress,omitempty"` + DocumentDescription string `json:"document_description,omitempty" xml:"DocumentDescription,omitempty"` + Attachment Attachment `json:"attachment,omitempty" xml:"Attachment,omitempty"` +} + +// BusinessCapability represents business capability +type BusinessCapability struct { + XMLName xml.Name `xml:"BusinessCapability"` + CapabilityTypeCode string `json:"capability_type_code" xml:"CapabilityTypeCode"` +} + +// NoticePurpose represents notice purpose +type NoticePurpose struct { + XMLName xml.Name `xml:"NoticePurpose"` + PurposeCode string `json:"purpose_code" xml:"PurposeCode"` +} + +// ===================================================== +// ENHANCED PARTY LEGAL ENTITY FOR BUSINESS REGISTRATION +// ===================================================== + +// PartyLegalEntityEnhanced represents enhanced party legal entity for business registration +type PartyLegalEntityEnhanced struct { + XMLName xml.Name `xml:"PartyLegalEntity"` + RegistrationName string `json:"registration_name,omitempty" xml:"RegistrationName,omitempty"` + CompanyID string `json:"company_id,omitempty" xml:"CompanyID,omitempty"` + RegistrationDate string `json:"registration_date,omitempty" xml:"RegistrationDate,omitempty"` + CorporateRegistrationScheme CorporateRegistrationScheme `json:"corporate_registration_scheme,omitempty" xml:"CorporateRegistrationScheme,omitempty"` +} + +// CorporateRegistrationScheme represents corporate registration scheme +type CorporateRegistrationScheme struct { + XMLName xml.Name `xml:"CorporateRegistrationScheme"` + JurisdictionRegionAddress JurisdictionRegionAddress `json:"jurisdiction_region_address,omitempty" xml:"JurisdictionRegionAddress,omitempty"` +} + +// JurisdictionRegionAddress represents jurisdiction region address +type JurisdictionRegionAddress struct { + XMLName xml.Name `xml:"JurisdictionRegionAddress"` + CityName string `json:"city_name,omitempty" xml:"CityName,omitempty"` + PostalZone string `json:"postal_zone,omitempty" xml:"PostalZone,omitempty"` + Country Country `json:"country,omitempty" xml:"Country,omitempty"` +} diff --git a/ted/eform/constant.go b/ted/eform/constant.go new file mode 100644 index 0000000..b830bad --- /dev/null +++ b/ted/eform/constant.go @@ -0,0 +1,96 @@ +package eform + +// ===================================================== +// ENUMERATIONS AND CONSTANTS +// ===================================================== + +// Common CPV Code Prefixes (for validation) +const ( + CPVWorks = "45" // Construction works + CPVServices = "50" // Services (general range 50-99 with exceptions) +) + +// Common NUTS Code Structure +// NUTS codes are hierarchical: CC1234 +// CC = Country code (2 chars) +// 1 = NUTS 1 level +// 12 = NUTS 2 level +// 123 = NUTS 3 level + +// ISO 4217 Currency Codes (common ones) +const ( + CurrencyEUR = "EUR" // Euro + CurrencyUSD = "USD" // US Dollar + CurrencyGBP = "GBP" // British Pound + CurrencySEK = "SEK" // Swedish Krona + CurrencyDKK = "DKK" // Danish Krone + CurrencyPLN = "PLN" // Polish Zloty + CurrencyCZK = "CZK" // Czech Koruna + CurrencyHUF = "HUF" // Hungarian Forint + CurrencyRON = "RON" // Romanian Leu + CurrencyBGN = "BGN" // Bulgarian Lev + CurrencyHRK = "HRK" // Croatian Kuna +) + +// ISO 3166-1 Alpha-2 Country Codes (EU/EEA) +const ( + CountryAT = "AT" // Austria + CountryBE = "BE" // Belgium + CountryBG = "BG" // Bulgaria + CountryCY = "CY" // Cyprus + CountryCZ = "CZ" // Czech Republic + CountryDE = "DE" // Germany + CountryDK = "DK" // Denmark + CountryEE = "EE" // Estonia + CountryES = "ES" // Spain + CountryFI = "FI" // Finland + CountryFR = "FR" // France + CountryGR = "GR" // Greece + CountryHR = "HR" // Croatia + CountryHU = "HU" // Hungary + CountryIE = "IE" // Ireland + CountryIT = "IT" // Italy + CountryLT = "LT" // Lithuania + CountryLU = "LU" // Luxembourg + CountryLV = "LV" // Latvia + CountryMT = "MT" // Malta + CountryNL = "NL" // Netherlands + CountryPL = "PL" // Poland + CountryPT = "PT" // Portugal + CountryRO = "RO" // Romania + CountrySE = "SE" // Sweden + CountrySI = "SI" // Slovenia + CountrySK = "SK" // Slovakia + CountryIS = "IS" // Iceland (EEA) + CountryLI = "LI" // Liechtenstein (EEA) + CountryNO = "NO" // Norway (EEA) + CountryGB = "GB" // United Kingdom +) + +// ISO 639-1 Language Codes (EU official) +const ( + LangBG = "BG" // Bulgarian + LangCS = "CS" // Czech + LangDA = "DA" // Danish + LangDE = "DE" // German + LangEL = "EL" // Greek + LangEN = "EN" // English + LangES = "ES" // Spanish + LangET = "ET" // Estonian + LangFI = "FI" // Finnish + LangFR = "FR" // French + LangGA = "GA" // Irish + LangHR = "HR" // Croatian + LangHU = "HU" // Hungarian + LangIT = "IT" // Italian + LangLT = "LT" // Lithuanian + LangLV = "LV" // Latvian + LangMT = "MT" // Maltese + LangNL = "NL" // Dutch + LangPL = "PL" // Polish + LangPT = "PT" // Portuguese + LangRO = "RO" // Romanian + LangSK = "SK" // Slovak + LangSL = "SL" // Slovenian + LangSV = "SV" // Swedish +) diff --git a/ted/eform/contracting_party.go b/ted/eform/contracting_party.go new file mode 100644 index 0000000..a074147 --- /dev/null +++ b/ted/eform/contracting_party.go @@ -0,0 +1,82 @@ +package eform + +import "encoding/xml" + +// ===================================================== +// CONTRACTING PARTY +// ===================================================== + +// ContractingParty represents the contracting party +type ContractingParty struct { + XMLName xml.Name `xml:"ContractingParty"` + BuyerProfileURI string `xml:"BuyerProfileURI,omitempty"` + ContractingPartyType ContractingPartyType `xml:"ContractingPartyType,omitempty"` + ContractingActivity ContractingActivity `xml:"ContractingActivity,omitempty"` + Party Party `xml:"Party,omitempty"` +} + +// ContractingPartyType contains party type information +type ContractingPartyType struct { + XMLName xml.Name `xml:"ContractingPartyType"` + PartyTypeCode string `xml:"PartyTypeCode"` +} + +// ContractingActivity represents the contracting activity +type ContractingActivity struct { + XMLName xml.Name `xml:"ContractingActivity"` + ActivityTypeCode string `xml:"ActivityTypeCode"` +} + +// Party represents a party +type Party struct { + XMLName xml.Name `xml:"Party"` + PartyIdentification []PartyIdentification `xml:"PartyIdentification,omitempty"` + PartyName []PartyName `xml:"PartyName,omitempty"` + PostalAddress PostalAddress `xml:"PostalAddress,omitempty"` + PartyLegalEntity PartyLegalEntity `xml:"PartyLegalEntity,omitempty"` + Contact Contact `xml:"Contact,omitempty"` + WebsiteURI string `xml:"WebsiteURI,omitempty"` +} + +// PartyIdentification represents party identification +type PartyIdentification struct { + XMLName xml.Name `xml:"PartyIdentification"` + ID string `xml:"ID"` +} + +// PartyName represents party name +type PartyName struct { + XMLName xml.Name `xml:"PartyName"` + Name string `xml:"Name"` +} + +// PostalAddress represents postal address +type PostalAddress struct { + XMLName xml.Name `xml:"PostalAddress"` + StreetName string `xml:"StreetName,omitempty"` + CityName string `xml:"CityName,omitempty"` + PostalZone string `xml:"PostalZone,omitempty"` + CountrySubentityCode string `xml:"CountrySubentityCode,omitempty"` + Country Country `xml:"Country,omitempty"` +} + +// Country represents country information +type Country struct { + XMLName xml.Name `xml:"Country"` + IdentificationCode string `xml:"IdentificationCode"` +} + +// PartyLegalEntity represents party legal entity +type PartyLegalEntity struct { + XMLName xml.Name `xml:"PartyLegalEntity"` + CompanyID string `xml:"CompanyID"` +} + +// Contact represents contact information +type Contact struct { + XMLName xml.Name `xml:"Contact"` + Name string `xml:"Name,omitempty"` + Telephone string `xml:"Telephone,omitempty"` + Telefax string `xml:"Telefax,omitempty"` + ElectronicMail string `xml:"ElectronicMail,omitempty"` +} diff --git a/ted/eform/document.go b/ted/eform/document.go new file mode 100644 index 0000000..e04b30b --- /dev/null +++ b/ted/eform/document.go @@ -0,0 +1,31 @@ +package eform + +// ===================================================== +// DOCUMENT STRUCTURE +// ===================================================== + +// Document represents attached documentation +type Document struct { + ID string `json:"id" xml:"id" validate:"required"` + Type DocumentType `json:"type" xml:"type" validate:"required"` + Title string `json:"title" xml:"title" validate:"required,max=500"` + Description string `json:"description,omitempty" xml:"description,omitempty" validate:"max=2000"` + URL string `json:"url,omitempty" xml:"url,omitempty" validate:"omitempty,url"` + RestrictedAccess bool `json:"restricted_access" xml:"restricted_access"` + Language LanguageCode `json:"language" xml:"language" validate:"required,iso639_1"` +} + +// DocumentType represents the type of document +type DocumentType string + +const ( + DocTypeProcurementDocs DocumentType = "procurement-docs" + DocTypeAdditionalInfo DocumentType = "additional-info" + DocTypeTenderSpecs DocumentType = "tender-specs" + DocTypeTenderForm DocumentType = "tender-form" + DocTypeContractDraft DocumentType = "contract-draft" + DocTypeEvaluationCriteria DocumentType = "evaluation-criteria" + DocTypeBillsQuantities DocumentType = "bills-quantities" + DocTypeIllustrations DocumentType = "illustrations" + DocTypeOther DocumentType = "other" +) diff --git a/ted/eform/lot.go b/ted/eform/lot.go new file mode 100644 index 0000000..0815989 --- /dev/null +++ b/ted/eform/lot.go @@ -0,0 +1,62 @@ +package eform + +// ===================================================== +// LOT STRUCTURE +// ===================================================== + +// Lot represents an individual procurement lot +type Lot struct { + ID string `json:"id" xml:"id" validate:"required,lotIDFormat"` + InternalID string `json:"internal_id,omitempty" xml:"internal_id,omitempty" validate:"max=100"` + Title string `json:"title" xml:"title" validate:"required,max=500"` + Description string `json:"description" xml:"description" validate:"required,max=10000"` + MainCPVCode string `json:"main_cpv_code" xml:"main_cpv_code" validate:"required,cpvCode"` + AdditionalCPVCodes []string `json:"additional_cpv_codes,omitempty" xml:"additional_cpv_codes,omitempty" validate:"dive,cpvCode"` + ContractNature ContractNature `json:"contract_nature" xml:"contract_nature" validate:"required"` + EstimatedValue *Money `json:"estimated_value,omitempty" xml:"estimated_value,omitempty"` + PlacesOfPerformance []Place `json:"places_of_performance" xml:"places_of_performance" validate:"required,min=1,dive"` + Duration *Duration `json:"duration,omitempty" xml:"duration,omitempty"` + Renewable bool `json:"renewable" xml:"renewable"` + RenewalDescription string `json:"renewal_description,omitempty" xml:"renewal_description,omitempty" validate:"required_if=Renewable true"` + OptionsDescription string `json:"options_description,omitempty" xml:"options_description,omitempty" validate:"max=10000"` + VariantsAccepted bool `json:"variants_accepted" xml:"variants_accepted"` + MaxLotsPerTenderer *int `json:"max_lots_per_tenderer,omitempty" xml:"max_lots_per_tenderer,omitempty" validate:"omitempty,min=1"` + MaxLotsAwarded *int `json:"max_lots_awarded,omitempty" xml:"max_lots_awarded,omitempty" validate:"omitempty,min=1"` + LotCombinations []string `json:"lot_combinations,omitempty" xml:"lot_combinations,omitempty"` + SubcontractingObligation SubcontractingObligation `json:"subcontracting_obligation" xml:"subcontracting_obligation"` + SubcontractingPercentage *float64 `json:"subcontracting_percentage,omitempty" xml:"subcontracting_percentage,omitempty" validate:"omitempty,min=0,max=100"` + TechnicalSpecifications TechnicalSpecifications `json:"technical_specifications" xml:"technical_specifications" validate:"required"` + AwardCriteria AwardCriteria `json:"award_criteria" xml:"award_criteria" validate:"required"` + TenderingTerms TenderingTerms `json:"tendering_terms" xml:"tendering_terms" validate:"required"` + Result *LotResult `json:"result,omitempty" xml:"result,omitempty"` +} + +// SubcontractingObligation represents subcontracting requirements +type SubcontractingObligation string + +const ( + SubcontractingNone SubcontractingObligation = "none" + SubcontractingMinimum SubcontractingObligation = "minimum" + SubcontractingRequired SubcontractingObligation = "required" +) + +// Place represents a place of performance +type Place struct { + StreetAddress string `json:"street_address,omitempty" xml:"street_address,omitempty" validate:"max=200"` + Locality string `json:"locality,omitempty" xml:"locality,omitempty" validate:"max=100"` + PostalCode string `json:"postal_code,omitempty" xml:"postal_code,omitempty" validate:"max=20"` + CountrySubdivision string `json:"country_subdivision,omitempty" xml:"country_subdivision,omitempty" validate:"max=100"` + CountryCode string `json:"country_code" xml:"country_code" validate:"required,iso3166_1_alpha2"` + NUTSCodes []string `json:"nuts_codes" xml:"nuts_codes" validate:"required,min=1,dive,nutsCode"` + Description string `json:"description,omitempty" xml:"description,omitempty" validate:"max=1000"` +} + +// HasSubcontractingObligation checks if subcontracting is required +func (l *Lot) HasSubcontractingObligation() bool { + return l.SubcontractingObligation != SubcontractingNone +} + +// IsReservedContract checks if lot is reserved for specific participants +func (l *Lot) IsReservedContract() bool { + return l.TechnicalSpecifications.ReservedExecution != ReservedNone +} diff --git a/ted/eform/lot_result.go b/ted/eform/lot_result.go new file mode 100644 index 0000000..4717671 --- /dev/null +++ b/ted/eform/lot_result.go @@ -0,0 +1,54 @@ +package eform + +import "time" + +// ===================================================== +// LOT RESULT STRUCTURE +// ===================================================== + +// LotResult represents the outcome of lot procurement +type LotResult struct { + ID string `json:"id" xml:"id" validate:"required,resIDFormat"` + Status ResultStatus `json:"status" xml:"status" validate:"required"` + StatusDescription string `json:"status_description,omitempty" xml:"status_description,omitempty" validate:"max=10000"` + DecisionDate *time.Time `json:"decision_date,omitempty" xml:"decision_date,omitempty"` + TendersReceived int `json:"tenders_received" xml:"tenders_received" validate:"min=0"` + TendersSME int `json:"tenders_sme" xml:"tenders_sme" validate:"min=0"` + TendersElectronic int `json:"tenders_electronic" xml:"tenders_electronic" validate:"min=0"` + TendersOtherEU int `json:"tenders_other_eu" xml:"tenders_other_eu" validate:"min=0"` + TendersNonEU int `json:"tenders_non_eu" xml:"tenders_non_eu" validate:"min=0"` + AwardedValue *Money `json:"awarded_value,omitempty" xml:"awarded_value,omitempty"` + SubcontractingShare *float64 `json:"subcontracting_share,omitempty" xml:"subcontracting_share,omitempty" validate:"omitempty,min=0,max=100"` + SubcontractingValue *Money `json:"subcontracting_value,omitempty" xml:"subcontracting_value,omitempty"` + ConcessionRevenue *Money `json:"concession_revenue,omitempty" xml:"concession_revenue,omitempty"` + Tenders []Tender `json:"tenders,omitempty" xml:"tenders,omitempty" validate:"dive"` + SettledContract *SettledContract `json:"settled_contract,omitempty" xml:"settled_contract,omitempty"` + NonAwardJustification string `json:"non_award_justification,omitempty" xml:"non_award_justification,omitempty" validate:"max=10000"` +} + +// ResultStatus represents the lot result status +type ResultStatus string + +const ( + ResultOpen ResultStatus = "open" + ResultClose ResultStatus = "close" + ResultWinner ResultStatus = "winn" + ResultNoWinner ResultStatus = "no-winn" + ResultSelected ResultStatus = "selec" + ResultClosedNoWin ResultStatus = "clos-nw" +) + +// HasWinner checks if lot has a winning tender +func (lr *LotResult) HasWinner() bool { + return lr.Status == ResultWinner +} + +// GetWinningTender returns the winning tender if exists +func (lr *LotResult) GetWinningTender() *Tender { + for i := range lr.Tenders { + if lr.Tenders[i].IsWinner { + return &lr.Tenders[i] + } + } + return nil +} diff --git a/ted/eform/modification.go b/ted/eform/modification.go new file mode 100644 index 0000000..7dd6513 --- /dev/null +++ b/ted/eform/modification.go @@ -0,0 +1,53 @@ +package eform + +import "time" + +// ===================================================== +// MODIFICATION STRUCTURE +// ===================================================== + +// Modification represents changes to existing contracts +type Modification struct { + ID string `json:"id" xml:"id" validate:"required,modIDFormat"` + OriginalContractID string `json:"original_contract_id" xml:"original_contract_id" validate:"required,conIDFormat"` + Reason ModificationReason `json:"reason" xml:"reason" validate:"required"` + ReasonDescription string `json:"reason_description" xml:"reason_description" validate:"required,max=10000"` + ValueBefore Money `json:"value_before" xml:"value_before" validate:"required"` + ValueAfter Money `json:"value_after" xml:"value_after" validate:"required"` + ValueChange Money `json:"value_change" xml:"value_change"` + PercentageChange float64 `json:"percentage_change" xml:"percentage_change"` + DurationBefore *Duration `json:"duration_before,omitempty" xml:"duration_before,omitempty"` + DurationAfter *Duration `json:"duration_after,omitempty" xml:"duration_after,omitempty"` + ModificationDescription string `json:"modification_description" xml:"modification_description" validate:"required,max=10000"` + ModificationDate time.Time `json:"modification_date" xml:"modification_date" validate:"required"` + ContractorChange bool `json:"contractor_change" xml:"contractor_change"` + NewContractorIDs []string `json:"new_contractor_ids,omitempty" xml:"new_contractor_ids,omitempty"` +} + +// ModificationReason represents the reason for contract modification +type ModificationReason string + +const ( + ModReasonAdditionalWorks ModificationReason = "add-wks" + ModReasonUnforeseen ModificationReason = "unforeseen" + ModReasonDesignDefect ModificationReason = "design-defect" + ModReasonAdditionalNeed ModificationReason = "additional-need" + ModReasonRepeatSimilar ModificationReason = "repeat-similar" + ModReasonRatePriceUpdate ModificationReason = "rate-price-update" + ModReasonSuspendCall ModificationReason = "suspend-call" + ModReasonOther ModificationReason = "other" +) + +// CalculateModificationChange calculates value change for modifications +func (m *Modification) CalculateModificationChange() { + if m.ValueBefore.Currency == m.ValueAfter.Currency { + change := m.ValueAfter.Amount - m.ValueBefore.Amount + m.ValueChange = Money{ + Amount: change, + Currency: m.ValueBefore.Currency, + } + if m.ValueBefore.Amount != 0 { + m.PercentageChange = (change / m.ValueBefore.Amount) * 100 + } + } +} diff --git a/ted/eform/notice_change.go b/ted/eform/notice_change.go new file mode 100644 index 0000000..f8d2277 --- /dev/null +++ b/ted/eform/notice_change.go @@ -0,0 +1,16 @@ +package eform + +// ===================================================== +// CHANGE/CORRECTION STRUCTURE +// ===================================================== + +// NoticeChange represents changes or corrections to notices +type NoticeChange struct { + ChangeID string `json:"change_id" xml:"change_id" validate:"required"` + SectionReference string `json:"section_reference" xml:"section_reference" validate:"required"` + LotReference string `json:"lot_reference,omitempty" xml:"lot_reference,omitempty"` + Label string `json:"label" xml:"label" validate:"required,max=200"` + OldValue string `json:"old_value,omitempty" xml:"old_value,omitempty"` + NewValue string `json:"new_value" xml:"new_value" validate:"required"` + ChangeDescription string `json:"change_description,omitempty" xml:"change_description,omitempty" validate:"max=2000"` +} diff --git a/ted/eform/notice_metadata.go b/ted/eform/notice_metadata.go new file mode 100644 index 0000000..34107c4 --- /dev/null +++ b/ted/eform/notice_metadata.go @@ -0,0 +1,144 @@ +package eform + +import "time" + +// ===================================================== +// NOTICE METADATA (Legacy - keeping for compatibility) +// ===================================================== + +// NoticeMetadata contains identification and publication information +type NoticeMetadata struct { + NoticeID string `json:"notice_id" xml:"notice_id" validate:"required,noticeIDFormat"` + NoticeType NoticeType `json:"notice_type" xml:"notice_type" validate:"required"` + NoticeVersion string `json:"notice_version" xml:"notice_version" validate:"required,semver"` + PublicationDate *time.Time `json:"publication_date,omitempty" xml:"publication_date,omitempty"` + DispatchDate time.Time `json:"dispatch_date" xml:"dispatch_date" validate:"required"` + Language LanguageCode `json:"language" xml:"language" validate:"required,iso639_1"` + TranslationLanguages []LanguageCode `json:"translation_languages,omitempty" xml:"translation_languages,omitempty" validate:"dive,iso639_1"` + LegalBasis LegalBasis `json:"legal_basis" xml:"legal_basis" validate:"required"` + PreviousNoticeID string `json:"previous_notice_id,omitempty" xml:"previous_notice_id,omitempty" validate:"omitempty,noticeIDFormat"` + ChangedNoticeID string `json:"changed_notice_id,omitempty" xml:"changed_notice_id,omitempty" validate:"omitempty,noticeIDFormat"` + CorrectionPrevious bool `json:"correction_previous" xml:"correction_previous"` +} + +// NoticeType represents the eForms subtype identifier (1-40, E1-E5) +type NoticeType string + +const ( + // Planning notices (1-6) + NoticeTypePIN1 NoticeType = "1" // Prior Information Notice - standard + NoticeTypePIN2 NoticeType = "2" // Prior Information Notice - utilities + NoticeTypePIN3 NoticeType = "3" // Prior Information Notice - social + NoticeTypePINCallComp4 NoticeType = "4" // PIN as call for competition - standard + NoticeTypePINCallComp5 NoticeType = "5" // PIN as call for competition - utilities + NoticeTypePINCallComp6 NoticeType = "6" // PIN as call for competition - social + + // Competition notices (7-14) + NoticeTypeCN7 NoticeType = "7" // Contract Notice - open/restricted + NoticeTypeCN8 NoticeType = "8" // Contract Notice - utilities + NoticeTypeCN9 NoticeType = "9" // Contract Notice - negotiated + NoticeTypeCN10 NoticeType = "10" // Contract Notice - competitive dialogue + NoticeTypeCN11 NoticeType = "11" // Contract Notice - innovation partnership + NoticeTypeCN12 NoticeType = "12" // Contract Notice - social services + NoticeTypeCN13 NoticeType = "13" // Contract Notice - concessions + NoticeTypeCN14 NoticeType = "14" // Contract Notice - design contest + + // Direct award notices (15-24) + NoticeTypeCANDirect15 NoticeType = "15" // CAN without prior - direct award + NoticeTypeCANDirect16 NoticeType = "16" // CAN without prior - utilities + NoticeTypeCANDirect17 NoticeType = "17" // CAN without prior - social + NoticeTypeCANDirect18 NoticeType = "18" // CAN without prior - negotiated + NoticeTypeCANDirect19 NoticeType = "19" // CAN without prior - concessions + NoticeTypeCANDirect20 NoticeType = "20" // Voluntary ex-ante transparency - direct award + NoticeTypeCANDirect21 NoticeType = "21" // Voluntary ex-ante transparency - utilities + NoticeTypeCANDirect22 NoticeType = "22" // Voluntary ex-ante transparency - social + NoticeTypeCANDirect23 NoticeType = "23" // Voluntary ex-ante transparency - concessions + NoticeTypeCANDirect24 NoticeType = "24" // Voluntary ex-ante transparency - defence + + // Result notices (25-35) + NoticeTypeCAN25 NoticeType = "25" // Contract Award Notice - standard + NoticeTypeCAN26 NoticeType = "26" // Contract Award Notice - utilities + NoticeTypeCAN27 NoticeType = "27" // Contract Award Notice - social + NoticeTypeCAN28 NoticeType = "28" // Contract Award Notice - concessions + NoticeTypeCAN29 NoticeType = "29" // Contract Award Notice - design contest results + NoticeTypeCAN30 NoticeType = "30" // Contract Award Notice - defence + NoticeTypeCAN31 NoticeType = "31" // Contract Award Notice - framework single + NoticeTypeCAN32 NoticeType = "32" // Contract Award Notice - DPS award + NoticeTypeCAN33 NoticeType = "33" // Contract Award Notice - modification + NoticeTypeCAN34 NoticeType = "34" // Contract Award Notice - completion + NoticeTypeCAN35 NoticeType = "35" // Contract Award Notice - quarterly DPS + + // Modification notices (36-40) + NoticeTypeMod36 NoticeType = "36" // Contract Modification Notice - standard + NoticeTypeMod37 NoticeType = "37" // Contract Modification Notice - utilities + NoticeTypeMod38 NoticeType = "38" // Contract Modification Notice - social + NoticeTypeMod39 NoticeType = "39" // Contract Modification Notice - concessions + NoticeTypeMod40 NoticeType = "40" // Contract Modification Notice - defence + + // Voluntary notices (E1-E5) + NoticeTypeVoluntaryE1 NoticeType = "E1" // Voluntary notice - below threshold + NoticeTypeVoluntaryE2 NoticeType = "E2" // Voluntary notice - modification + NoticeTypeVoluntaryE3 NoticeType = "E3" // Voluntary notice - contract completion + NoticeTypeVoluntaryE4 NoticeType = "E4" // Voluntary notice - concession award + NoticeTypeVoluntaryE5 NoticeType = "E5" // Voluntary notice - qualification system +) + +// LegalBasis represents applicable EU directives +type LegalBasis string + +const ( + LegalBasisClassicDir LegalBasis = "32014L0024" // Directive 2014/24/EU (classic) + LegalBasisUtilitiesDir LegalBasis = "32014L0025" // Directive 2014/25/EU (utilities) + LegalBasisConcessionsDir LegalBasis = "32014L0023" // Directive 2014/23/EU (concessions) + LegalBasisDefenceDir LegalBasis = "32009L0081" // Directive 2009/81/EC (defence) + LegalBasisFinancialReg LegalBasis = "32018R1046" // Financial Regulation (EU institutions) +) + +// LanguageCode represents ISO 639-1 language codes +type LanguageCode string + +// IsPlanningNotice checks if notice is a planning/PIN notice +func (nt NoticeType) IsPlanningNotice() bool { + planningTypes := []NoticeType{ + NoticeTypePIN1, NoticeTypePIN2, NoticeTypePIN3, + NoticeTypePINCallComp4, NoticeTypePINCallComp5, NoticeTypePINCallComp6, + } + for _, t := range planningTypes { + if nt == t { + return true + } + } + return false +} + +// IsCompetitionNotice checks if notice is a competition/CN notice +func (nt NoticeType) IsCompetitionNotice() bool { + return nt >= NoticeTypeCN7 && nt <= NoticeTypeCN14 +} + +// IsResultNotice checks if notice is a result/CAN notice +func (nt NoticeType) IsResultNotice() bool { + return (nt >= NoticeTypeCANDirect15 && nt <= NoticeTypeCANDirect24) || + (nt >= NoticeTypeCAN25 && nt <= NoticeTypeCAN35) +} + +// IsModificationNotice checks if notice is a modification notice +func (nt NoticeType) IsModificationNotice() bool { + return nt >= NoticeTypeMod36 && nt <= NoticeTypeMod40 +} + +// IsVoluntaryNotice checks if notice is voluntary +func (nt NoticeType) IsVoluntaryNotice() bool { + voluntaryTypes := []NoticeType{ + NoticeTypeVoluntaryE1, NoticeTypeVoluntaryE2, NoticeTypeVoluntaryE3, + NoticeTypeVoluntaryE4, NoticeTypeVoluntaryE5, + NoticeTypeCANDirect20, NoticeTypeCANDirect21, NoticeTypeCANDirect22, + NoticeTypeCANDirect23, NoticeTypeCANDirect24, + } + for _, t := range voluntaryTypes { + if nt == t { + return true + } + } + return false +} diff --git a/ted/eform/organization.go b/ted/eform/organization.go new file mode 100644 index 0000000..c50856e --- /dev/null +++ b/ted/eform/organization.go @@ -0,0 +1,97 @@ +package eform + +// ===================================================== +// ORGANIZATION STRUCTURE +// ===================================================== + +// Organization represents any party involved in the procurement +type Organization struct { + ID string `json:"id" xml:"id" validate:"required,orgIDFormat"` + Roles []OrganizationRole `json:"roles" xml:"roles" validate:"required,min=1,dive"` + LegalName string `json:"legal_name" xml:"legal_name" validate:"required,max=500"` + LegalType *OrganizationType `json:"legal_type,omitempty" xml:"legal_type,omitempty"` + RegistrationID string `json:"registration_id,omitempty" xml:"registration_id,omitempty"` + VATNumber string `json:"vat_number,omitempty" xml:"vat_number,omitempty" validate:"omitempty,vatFormat"` + NationalID string `json:"national_id,omitempty" xml:"national_id,omitempty"` + Address Address `json:"address" xml:"address" validate:"required"` + ContactPoint ContactPoint `json:"contact_point" xml:"contact_point" validate:"required"` + BuyerProfile string `json:"buyer_profile,omitempty" xml:"buyer_profile,omitempty" validate:"omitempty,url"` + ProcurementActivity *ActivityType `json:"procurement_activity,omitempty" xml:"procurement_activity,omitempty"` + SME bool `json:"sme" xml:"sme"` +} + +// OrganizationRole defines the role of an organization in the procurement +type OrganizationRole string + +const ( + RoleBuyer OrganizationRole = "buyer" + RoleServiceProvider OrganizationRole = "service-provider" + RoleTEDeSender OrganizationRole = "ted-esen" + RoleWinner OrganizationRole = "winner" + RoleSubcontractor OrganizationRole = "subcontractor" + RoleReviewBody OrganizationRole = "review-body" + RoleReviewOrg OrganizationRole = "review-org" + RoleMediationBody OrganizationRole = "mediation-body" + RoleLeadBuyer OrganizationRole = "lead-buyer" + RoleCentralBuyer OrganizationRole = "central-buyer" + RoleAcquirer OrganizationRole = "acquirer" +) + +// OrganizationType represents the legal type of organization +type OrganizationType string + +const ( + OrgTypeBodyPublicLaw OrganizationType = "body-pl" + OrgTypeCentralGov OrganizationType = "cga" + OrgTypeLocalAuthority OrganizationType = "la" + OrgTypeSubsidized OrganizationType = "org-sub" + OrgTypePublicUndertaking OrganizationType = "pub-undert" + OrgTypeEUInstitution OrganizationType = "eu-ins-bod-ag" + OrgTypeOther OrganizationType = "org-other" +) + +// ActivityType represents main procurement activity +type ActivityType string + +const ( + ActivityAuthority ActivityType = "auth-oth" + ActivityDefence ActivityType = "def" + ActivityEconomicAffairs ActivityType = "econ-aff" + ActivityEducation ActivityType = "edu" + ActivityEnvironment ActivityType = "env" + ActivityGeneralPublic ActivityType = "gen-pub" + ActivityHealth ActivityType = "health" + ActivityHousingCommunity ActivityType = "hous-com" + ActivityPublicOrderSafety ActivityType = "pub-os" + ActivityRecreationCulture ActivityType = "rec-com" + ActivitySocialProtection ActivityType = "soc-pro" + ActivityWater ActivityType = "water" + ActivityAirport ActivityType = "airport" + ActivityElectricity ActivityType = "electricity" + ActivityExplorationExtract ActivityType = "exploration-extraction" + ActivityGasHeat ActivityType = "gas-heat" + ActivityPort ActivityType = "port" + ActivityPost ActivityType = "post" + ActivityRailway ActivityType = "rail" + ActivityUrbanRailway ActivityType = "urban-railway" + ActivityWaterUtility ActivityType = "water" +) + +// Address represents physical address information +type Address struct { + StreetAddress string `json:"street_address" xml:"street_address" validate:"required,max=200"` + Locality string `json:"locality" xml:"locality" validate:"required,max=100"` + PostalCode string `json:"postal_code" xml:"postal_code" validate:"required,max=20"` + CountrySubdivision string `json:"country_subdivision,omitempty" xml:"country_subdivision,omitempty" validate:"max=100"` + CountryCode string `json:"country_code" xml:"country_code" validate:"required,iso3166_1_alpha2"` + NUTSCodes []string `json:"nuts_codes,omitempty" xml:"nuts_codes,omitempty" validate:"dive,nutsCode"` +} + +// ContactPoint represents contact information +type ContactPoint struct { + Name string `json:"name,omitempty" xml:"name,omitempty" validate:"max=200"` + Email string `json:"email" xml:"email" validate:"required,email"` + Telephone string `json:"telephone,omitempty" xml:"telephone,omitempty" validate:"max=30"` + Fax string `json:"fax,omitempty" xml:"fax,omitempty" validate:"max=30"` + URL string `json:"url,omitempty" xml:"url,omitempty" validate:"omitempty,url"` +} diff --git a/ted/eform/procedure.go b/ted/eform/procedure.go new file mode 100644 index 0000000..b03d306 --- /dev/null +++ b/ted/eform/procedure.go @@ -0,0 +1,244 @@ +package eform + +// ===================================================== +// PROCEDURE STRUCTURE +// ===================================================== + +// Procedure represents the procurement procedure +type Procedure struct { + ID string `json:"id" xml:"id" validate:"required,procIDFormat"` + Title string `json:"title" xml:"title" validate:"required,max=500"` + Description string `json:"description" xml:"description" validate:"required,max=10000"` + ProcedureType ProcedureType `json:"procedure_type" xml:"procedure_type" validate:"required"` + AcceleratedProcedure bool `json:"accelerated_procedure" xml:"accelerated_procedure"` + AcceleratedJustification string `json:"accelerated_justification,omitempty" xml:"accelerated_justification,omitempty" validate:"required_if=AcceleratedProcedure true"` + FrameworkAgreement bool `json:"framework_agreement" xml:"framework_agreement"` + FAMaxParticipants *int `json:"fa_max_participants,omitempty" xml:"fa_max_participants,omitempty" validate:"omitempty,min=1"` + FADuration *Duration `json:"fa_duration,omitempty" xml:"fa_duration,omitempty"` + DynamicPurchasingSystem bool `json:"dynamic_purchasing_system" xml:"dynamic_purchasing_system"` + ElectronicAuction bool `json:"electronic_auction" xml:"electronic_auction"` + JointProcurement bool `json:"joint_procurement" xml:"joint_procurement"` + CentralPurchasing bool `json:"central_purchasing" xml:"central_purchasing"` + GPA bool `json:"gpa" xml:"gpa"` + InternationalAgreements []string `json:"international_agreements,omitempty" xml:"international_agreements,omitempty"` + MainCPVCode string `json:"main_cpv_code" xml:"main_cpv_code" validate:"required,cpvCode"` + AdditionalCPVCodes []string `json:"additional_cpv_codes,omitempty" xml:"additional_cpv_codes,omitempty" validate:"dive,cpvCode"` + ContractNature ContractNature `json:"contract_nature" xml:"contract_nature" validate:"required"` + EstimatedValue *Money `json:"estimated_value,omitempty" xml:"estimated_value,omitempty"` + EstimatedValueRange *ValueRange `json:"estimated_value_range,omitempty" xml:"estimated_value_range,omitempty"` + ProcurementDocumentsURL string `json:"procurement_documents_url,omitempty" xml:"procurement_documents_url,omitempty" validate:"omitempty,url"` + ProcurementDocsRestricted bool `json:"procurement_docs_restricted" xml:"procurement_docs_restricted"` + SubmissionElectronic SubmissionMethod `json:"submission_electronic" xml:"submission_electronic" validate:"required"` + SubmissionLanguages []LanguageCode `json:"submission_languages" xml:"submission_languages" validate:"required,min=1,dive,iso639_1"` + TenderValidityDuration *Duration `json:"tender_validity_duration,omitempty" xml:"tender_validity_duration,omitempty"` + Lots []Lot `json:"lots,omitempty" xml:"lots,omitempty" validate:"dive"` + SecondStage *SecondStage `json:"second_stage,omitempty" xml:"second_stage,omitempty"` + Recurrence bool `json:"recurrence" xml:"recurrence"` + RecurrenceTiming string `json:"recurrence_timing,omitempty" xml:"recurrence_timing,omitempty" validate:"max=1000"` +} + +// ProcedureType represents the type of procurement procedure +type ProcedureType string + +const ( + ProcTypeOpen ProcedureType = "open" + ProcTypeRestricted ProcedureType = "restricted" + ProcTypeCompetitiveDialogue ProcedureType = "comp-dial" + ProcTypeInnovation ProcedureType = "innovation" + ProcTypeNegotiatedWithCall ProcedureType = "neg-w-call" + ProcTypeNegotiatedWoCall ProcedureType = "neg-wo-call" + ProcTypeCompWithNeg ProcedureType = "comp-w-n" + ProcTypeAwardWoCall ProcedureType = "pt-award-wo-call" + ProcTypeOtherSingle ProcedureType = "oth-single" + ProcTypeOtherMultiple ProcedureType = "oth-mult" +) + +// ContractNature represents the type of contract +type ContractNature string + +const ( + ContractNatureWorks ContractNature = "works" + ContractNatureSupplies ContractNature = "supplies" + ContractNatureServices ContractNature = "services" + ContractNatureCombined ContractNature = "combined" +) + +// SubmissionMethod represents how tenders can be submitted +type SubmissionMethod string + +const ( + SubmissionAllowed SubmissionMethod = "allowed" + SubmissionRequired SubmissionMethod = "required" + SubmissionNotAllowed SubmissionMethod = "not-allowed" +) + +// Duration represents a time duration +type Duration struct { + Type DurationType `json:"type" xml:"type" validate:"required"` + Value *int `json:"value,omitempty" xml:"value,omitempty" validate:"omitempty,min=1"` +} + +// DurationType represents the unit of duration +type DurationType string + +const ( + DurationMonths DurationType = "months" + DurationDays DurationType = "days" + DurationYears DurationType = "years" + DurationUnlimited DurationType = "unlimited" +) + +// Money represents a monetary value +type Money struct { + Amount float64 `json:"amount" xml:"amount" validate:"required,min=0"` + Currency string `json:"currency" xml:"currency" validate:"required,iso4217"` +} + +// ValueRange represents a range of monetary values +type ValueRange struct { + MinValue Money `json:"min_value" xml:"min_value" validate:"required"` + MaxValue Money `json:"max_value" xml:"max_value" validate:"required,gtfield=MinValue.Amount"` +} + +// SecondStage represents second stage of two-stage procedures +type SecondStage struct { + MinCandidates *int `json:"min_candidates,omitempty" xml:"min_candidates,omitempty" validate:"omitempty,min=1"` + MaxCandidates *int `json:"max_candidates,omitempty" xml:"max_candidates,omitempty" validate:"omitempty,min=1"` + ObjectiveCriteria string `json:"objective_criteria,omitempty" xml:"objective_criteria,omitempty" validate:"max=10000"` +} + +// IsAboveThreshold checks if procedure value is above EU thresholds +func (p *Procedure) IsAboveThreshold() bool { + if p.EstimatedValue == nil { + return false + } + + // EU thresholds 2024-2025 (in EUR) + thresholds := map[ContractNature]float64{ + ContractNatureWorks: 5382000, + ContractNatureSupplies: 143000, // central government + ContractNatureServices: 143000, // central government + } + + threshold, exists := thresholds[p.ContractNature] + if !exists { + return false + } + + // Convert to EUR if needed + valueEUR := p.EstimatedValue.Amount + if p.EstimatedValue.Currency != CurrencyEUR { + // In real implementation, would use exchange rates + // This is simplified + return true // Assume above threshold for non-EUR + } + + return valueEUR >= threshold +} + +// GetTotalEstimatedValue calculates total value from lots +func (p *Procedure) GetTotalEstimatedValue() *Money { + if len(p.Lots) == 0 { + return p.EstimatedValue + } + + var total float64 + var currency string + + for _, lot := range p.Lots { + if lot.EstimatedValue != nil { + if currency == "" { + currency = lot.EstimatedValue.Currency + } + if lot.EstimatedValue.Currency == currency { + total += lot.EstimatedValue.Amount + } + } + } + + if currency != "" { + return &Money{ + Amount: total, + Currency: currency, + } + } + + return p.EstimatedValue +} + +// IsDividedIntoLots checks if procedure has lots +func (p *Procedure) IsDividedIntoLots() bool { + return len(p.Lots) > 0 +} + +// GetLotByID retrieves lot by ID +func (p *Procedure) GetLotByID(id string) *Lot { + for i := range p.Lots { + if p.Lots[i].ID == id { + return &p.Lots[i] + } + } + return nil +} + +// IsFrameworkAgreement checks if this is a framework agreement +func (p *Procedure) IsFrameworkAgreement() bool { + return p.FrameworkAgreement +} + +// IsDynamicPurchasingSystem checks if this is a DPS +func (p *Procedure) IsDynamicPurchasingSystem() bool { + return p.DynamicPurchasingSystem +} + +// RequiresTwoStage checks if procedure is two-stage +func (p *Procedure) RequiresTwoStage() bool { + twoStageProcs := []ProcedureType{ + ProcTypeRestricted, + ProcTypeCompetitiveDialogue, + ProcTypeInnovation, + ProcTypeNegotiatedWithCall, + } + + for _, t := range twoStageProcs { + if p.ProcedureType == t { + return true + } + } + return false +} + +// GetAwardedContracts returns all awarded contracts from procedure +func (p *Procedure) GetAwardedContracts() []SettledContract { + var contracts []SettledContract + + for _, lot := range p.Lots { + if lot.Result != nil && lot.Result.SettledContract != nil { + contracts = append(contracts, *lot.Result.SettledContract) + } + } + + return contracts +} + +// GetTotalAwardedValue calculates total value of all awarded contracts +func (p *Procedure) GetTotalAwardedValue() *Money { + contracts := p.GetAwardedContracts() + if len(contracts) == 0 { + return nil + } + + var total float64 + currency := contracts[0].ContractValue.Currency + + for _, contract := range contracts { + if contract.ContractValue.Currency == currency { + total += contract.ContractValue.Amount + } + } + + return &Money{ + Amount: total, + Currency: currency, + } +} diff --git a/ted/eform/procurement_project.go b/ted/eform/procurement_project.go new file mode 100644 index 0000000..0bdf7bd --- /dev/null +++ b/ted/eform/procurement_project.go @@ -0,0 +1,49 @@ +package eform + +import "encoding/xml" + +// ===================================================== +// PROCUREMENT PROJECT +// ===================================================== + +// ProcurementProject represents the procurement project +type ProcurementProject struct { + XMLName xml.Name `xml:"ProcurementProject"` + ID string `xml:"ID"` + Name string `xml:"Name"` + Description string `xml:"Description"` + ProcurementTypeCode string `xml:"ProcurementTypeCode"` + Note string `xml:"Note,omitempty"` + RequestedTenderTotal RequestedTenderTotal `xml:"RequestedTenderTotal,omitempty"` + MainCommodityClassification MainCommodityClassification `xml:"MainCommodityClassification,omitempty"` + RealizedLocation RealizedLocation `xml:"RealizedLocation,omitempty"` + PlannedPeriod PlannedPeriod `xml:"PlannedPeriod,omitempty"` +} + +// RequestedTenderTotal represents the estimated contract amount +type RequestedTenderTotal struct { + XMLName xml.Name `xml:"RequestedTenderTotal"` + EstimatedOverallContractAmount string `xml:"EstimatedOverallContractAmount"` +} + +// MainCommodityClassification represents the CPV code +type MainCommodityClassification struct { + ItemClassificationCode string `json:"item_classification_code" xml:"ItemClassificationCode"` +} + +// RealizedLocation represents the location where work will be performed +type RealizedLocation struct { + Address PostalAddress `xml:"PostalAddress"` + Region string `xml:"Region,omitempty"` +} + +// PlannedPeriod represents the planned duration of the contract +type PlannedPeriod struct { + DurationMeasure DurationMeasure `json:"duration_measure" xml:"DurationMeasure"` +} + +// DurationMeasure represents a duration with unit +type DurationMeasure struct { + UnitCode string `json:"unit_code" xml:"unitCode,attr"` + Value string `json:"value" xml:",chardata"` +} diff --git a/ted/eform/procurement_project_lot.go b/ted/eform/procurement_project_lot.go new file mode 100644 index 0000000..94e58b0 --- /dev/null +++ b/ted/eform/procurement_project_lot.go @@ -0,0 +1,13 @@ +package eform + +// ===================================================== +// PROCUREMENT PROJECT LOT +// ===================================================== + +// ProcurementProjectLot represents a procurement project lot +type ProcurementProjectLot struct { + ID string `json:"id" xml:"ID"` + TenderingTerms TenderingTerms `json:"tendering_terms,omitempty" xml:"TenderingTerms,omitempty"` + TenderingProcess TenderingProcess `json:"tendering_process,omitempty" xml:"TenderingProcess,omitempty"` + ProcurementProject ProcurementProject `json:"procurement_project,omitempty" xml:"ProcurementProject,omitempty"` +} diff --git a/ted/eform/review_info.go b/ted/eform/review_info.go new file mode 100644 index 0000000..03d26ac --- /dev/null +++ b/ted/eform/review_info.go @@ -0,0 +1,17 @@ +package eform + +// ===================================================== +// REVIEW/APPEAL INFORMATION +// ===================================================== + +// ReviewInfo represents information about review procedures +type ReviewInfo struct { + ReviewBody string `json:"review_body" xml:"review_body" validate:"required"` + ReviewBodyID string `json:"review_body_id" xml:"review_body_id" validate:"required,orgIDFormat"` + ReviewProcedure string `json:"review_procedure" xml:"review_procedure" validate:"required,max=10000"` + ReviewDeadline string `json:"review_deadline,omitempty" xml:"review_deadline,omitempty" validate:"max=1000"` + MediationBody string `json:"mediation_body,omitempty" xml:"mediation_body,omitempty"` + MediationBodyID string `json:"mediation_body_id,omitempty" xml:"mediation_body_id,omitempty" validate:"omitempty,orgIDFormat"` + ReviewInfoURL string `json:"review_info_url,omitempty" xml:"review_info_url,omitempty" validate:"omitempty,url"` + LodgingOfAppeals string `json:"lodging_of_appeals,omitempty" xml:"lodging_of_appeals,omitempty" validate:"max=10000"` +} diff --git a/ted/eform/settled_contract.go b/ted/eform/settled_contract.go new file mode 100644 index 0000000..a3ce3c3 --- /dev/null +++ b/ted/eform/settled_contract.go @@ -0,0 +1,26 @@ +package eform + +import "time" + +// ===================================================== +// SETTLED CONTRACT STRUCTURE +// ===================================================== + +// SettledContract represents awarded contract details +type SettledContract struct { + ID string `json:"id" xml:"id" validate:"required,conIDFormat"` + ContractNumber string `json:"contract_number,omitempty" xml:"contract_number,omitempty" validate:"max=100"` + Title string `json:"title" xml:"title" validate:"required,max=500"` + ConclusionDate time.Time `json:"conclusion_date" xml:"conclusion_date" validate:"required"` + ContractValue Money `json:"contract_value" xml:"contract_value" validate:"required"` + InitialValue *Money `json:"initial_value,omitempty" xml:"initial_value,omitempty"` + FrameworkMaxValue *Money `json:"framework_max_value,omitempty" xml:"framework_max_value,omitempty"` + StartDate *time.Time `json:"start_date,omitempty" xml:"start_date,omitempty"` + EndDate *time.Time `json:"end_date,omitempty" xml:"end_date,omitempty"` + Duration *Duration `json:"duration,omitempty" xml:"duration,omitempty"` + ContractorIDs []string `json:"contractor_ids" xml:"contractor_ids" validate:"required,min=1"` + Modifications []Modification `json:"modifications,omitempty" xml:"modifications,omitempty" validate:"dive"` + FinalValue *Money `json:"final_value,omitempty" xml:"final_value,omitempty"` + CompletionDate *time.Time `json:"completion_date,omitempty" xml:"completion_date,omitempty"` + GroupID string `json:"group_id,omitempty" xml:"group_id,omitempty"` +} diff --git a/ted/eform/technical_specifications.go b/ted/eform/technical_specifications.go new file mode 100644 index 0000000..95189eb --- /dev/null +++ b/ted/eform/technical_specifications.go @@ -0,0 +1,43 @@ +package eform + +// ===================================================== +// TECHNICAL SPECIFICATIONS +// ===================================================== + +// TechnicalSpecifications contains technical and eligibility requirements +type TechnicalSpecifications struct { + Description string `json:"description" xml:"description" validate:"required,max=10000"` + EligibilityCriteria string `json:"eligibility_criteria,omitempty" xml:"eligibility_criteria,omitempty" validate:"max=10000"` + TechnicalCapability string `json:"technical_capability,omitempty" xml:"technical_capability,omitempty" validate:"max=10000"` + EconomicCapability string `json:"economic_capability,omitempty" xml:"economic_capability,omitempty" validate:"max=10000"` + References bool `json:"references" xml:"references"` + ReferencesDescription string `json:"references_description,omitempty" xml:"references_description,omitempty" validate:"required_if=References true"` + SecurityClearance bool `json:"security_clearance" xml:"security_clearance"` + SecurityClearanceDescription string `json:"security_clearance_description,omitempty" xml:"security_clearance_description,omitempty" validate:"required_if=SecurityClearance true"` + ReservedExecution ReservedExecution `json:"reserved_execution" xml:"reserved_execution"` + StrategicProcurement []StrategicType `json:"strategic_procurement,omitempty" xml:"strategic_procurement,omitempty"` + AccessibilityCriteria bool `json:"accessibility_criteria" xml:"accessibility_criteria"` + AccessibilityDescription string `json:"accessibility_description,omitempty" xml:"accessibility_description,omitempty" validate:"max=10000"` + GreenProcurement bool `json:"green_procurement" xml:"green_procurement"` + SocialProcurement bool `json:"social_procurement" xml:"social_procurement"` + InnovativeProcurement bool `json:"innovative_procurement" xml:"innovative_procurement"` +} + +// ReservedExecution represents reserved participation types +type ReservedExecution string + +const ( + ReservedNone ReservedExecution = "none" + ReservedShelteredWS ReservedExecution = "sheltered-workshop" + ReservedPublicService ReservedExecution = "public-service-mission" +) + +// StrategicType represents strategic procurement objectives +type StrategicType string + +const ( + StrategicEnvironmental StrategicType = "environmental" + StrategicSocial StrategicType = "social" + StrategicInnovation StrategicType = "innovation" + StrategicOther StrategicType = "other" +) diff --git a/ted/eform/tender.go b/ted/eform/tender.go new file mode 100644 index 0000000..65ee548 --- /dev/null +++ b/ted/eform/tender.go @@ -0,0 +1,24 @@ +package eform + +// ===================================================== +// TENDER STRUCTURE +// ===================================================== + +// Tender represents information about individual bids +type Tender struct { + ID string `json:"id" xml:"id" validate:"required,tenIDFormat"` + TenderRank *int `json:"tender_rank,omitempty" xml:"tender_rank,omitempty" validate:"omitempty,min=1"` + IsWinner bool `json:"is_winner" xml:"is_winner"` + TendererIDs []string `json:"tenderer_ids" xml:"tenderer_ids" validate:"required,min=1"` + IsConsortium bool `json:"is_consortium" xml:"is_consortium"` + VariantTender bool `json:"variant_tender" xml:"variant_tender"` + ExcludedTender bool `json:"excluded_tender" xml:"excluded_tender"` + ExclusionReason string `json:"exclusion_reason,omitempty" xml:"exclusion_reason,omitempty" validate:"required_if=ExcludedTender true"` + TenderValue *Money `json:"tender_value,omitempty" xml:"tender_value,omitempty"` + SubcontractingPercentage *float64 `json:"subcontracting_percentage,omitempty" xml:"subcontracting_percentage,omitempty" validate:"omitempty,min=0,max=100"` + SMETender bool `json:"sme_tender" xml:"sme_tender"` + ConcessionValue *Money `json:"concession_value,omitempty" xml:"concession_value,omitempty"` + SubcontractorIDs []string `json:"subcontractor_ids,omitempty" xml:"subcontractor_ids,omitempty"` + AbnormallyLow bool `json:"abnormally_low" xml:"abnormally_low"` + AbnormallyLowExplanation string `json:"abnormally_low_explanation,omitempty" xml:"abnormally_low_explanation,omitempty" validate:"max=10000"` +} diff --git a/ted/eform/tendering_process.go b/ted/eform/tendering_process.go new file mode 100644 index 0000000..58f9264 --- /dev/null +++ b/ted/eform/tendering_process.go @@ -0,0 +1,28 @@ +package eform + +import "encoding/xml" + +// ===================================================== +// TENDERING PROCESS +// ===================================================== + +// TenderingProcess represents the tendering process +type TenderingProcess struct { + XMLName xml.Name `xml:"TenderingProcess"` + ProcedureCode string `xml:"ProcedureCode"` + ProcessJustification ProcessJustification `xml:"ProcessJustification,omitempty"` + TenderSubmissionDeadlinePeriod TenderSubmissionDeadlinePeriod `xml:"TenderSubmissionDeadlinePeriod,omitempty"` +} + +// ProcessJustification represents process justification +type ProcessJustification struct { + XMLName xml.Name `xml:"ProcessJustification"` + ProcessReasonCode string `xml:"ProcessReasonCode"` +} + +// TenderSubmissionDeadlinePeriod represents the deadline for tender submission +type TenderSubmissionDeadlinePeriod struct { + XMLName xml.Name `xml:"TenderSubmissionDeadlinePeriod"` + EndDate string `xml:"EndDate"` + EndTime string `xml:"EndTime,omitempty"` +} diff --git a/ted/eform/tendering_terms.go b/ted/eform/tendering_terms.go new file mode 100644 index 0000000..8cdeddb --- /dev/null +++ b/ted/eform/tendering_terms.go @@ -0,0 +1,85 @@ +package eform + +import ( + "encoding/xml" + "time" +) + +// ===================================================== +// TENDERING TERMS +// ===================================================== + +// TenderingTerms represents tendering terms +type TenderingTerms struct { + TendererQualificationRequest []TendererQualificationRequest `json:"tenderer_qualification_request,omitempty" xml:"TendererQualificationRequest,omitempty"` + AppealTerms AppealTerms `json:"appeal_terms,omitempty" xml:"AppealTerms,omitempty"` + CallForTendersDocumentReference CallForTendersDocumentReference `json:"call_for_tenders_document_reference,omitempty" xml:"CallForTendersDocumentReference,omitempty"` + TenderRecipientParty TenderRecipientParty `json:"tender_recipient_party,omitempty" xml:"TenderRecipientParty,omitempty"` + UBLExtensions UBLExtensions `json:"ubl_extensions,omitempty" xml:"UBLExtensions,omitempty"` +} + +// TendererQualificationRequest represents qualification requirements +type TendererQualificationRequest struct { + SpecificTendererRequirement []SpecificTendererRequirement `json:"specific_tenderer_requirement,omitempty" xml:"SpecificTendererRequirement,omitempty"` +} + +// SpecificTendererRequirement represents specific requirements +type SpecificTendererRequirement struct { + XMLName xml.Name `xml:"SpecificTendererRequirement"` + TendererRequirementTypeCode string `xml:"TendererRequirementTypeCode"` + Description string `xml:"Description,omitempty"` +} + +// AppealTerms represents appeal terms for review organization +type AppealTerms struct { + XMLName xml.Name `xml:"AppealTerms"` + AppealReceiverParty AppealReceiverParty `xml:"AppealReceiverParty"` +} + +// AppealReceiverParty represents the party that receives appeals +type AppealReceiverParty struct { + XMLName xml.Name `xml:"AppealReceiverParty"` + PartyIdentification PartyIdentification `xml:"PartyIdentification"` +} + +// CallForTendersDocumentReference represents document reference +type CallForTendersDocumentReference struct { + XMLName xml.Name `xml:"CallForTendersDocumentReference"` + ID string `xml:"ID"` + DocumentType string `xml:"DocumentType,omitempty"` + Attachment Attachment `xml:"Attachment"` + UBLExtensions UBLExtensions `xml:"UBLExtensions,omitempty"` +} + +// Attachment represents document attachment +type Attachment struct { + ExternalReference ExternalReference `json:"external_reference" xml:"ExternalReference"` +} + +// ExternalReference represents external document reference +type ExternalReference struct { + XMLName xml.Name `xml:"ExternalReference"` + URI string `xml:"URI"` +} + +// TenderRecipientParty represents the party that receives tenders +type TenderRecipientParty struct { + XMLName xml.Name `xml:"TenderRecipientParty"` + EndpointID string `xml:"EndpointID"` +} + +// EOSize represents economic operator size restrictions +type EOSize string + +const ( + EOSizeNone EOSize = "none" + EOSizeSME EOSize = "sme" + EOSizeNotSME EOSize = "not-sme" +) + +// TenderOpening represents tender opening information +type TenderOpening struct { + DateTime time.Time `json:"date_time" xml:"date_time" validate:"required"` + Place string `json:"place,omitempty" xml:"place,omitempty" validate:"max=500"` + Description string `json:"description,omitempty" xml:"description,omitempty" validate:"max=2000"` +} diff --git a/ted/eform/ubl.go b/ted/eform/ubl.go new file mode 100644 index 0000000..e3ffc63 --- /dev/null +++ b/ted/eform/ubl.go @@ -0,0 +1,71 @@ +package eform + +import "encoding/xml" + +// ===================================================== +// UBL EXTENSIONS +// ===================================================== + +// UBLExtensions represents the UBL extensions element +type UBLExtensions struct { + UBLExtension []UBLExtension `json:"ubl_extension,omitempty" xml:"UBLExtension,omitempty"` +} + +// UBLExtension represents a single UBL extension +type UBLExtension struct { + ExtensionContent ExtensionContent `json:"extension_content,omitempty" xml:"ExtensionContent,omitempty"` +} + +// ExtensionContent represents the extension content +type ExtensionContent struct { + EformsExtension EformsExtension `json:"eforms_extension,omitempty" xml:"EformsExtension,omitempty"` +} + +// EformsExtension represents the eforms extension data +type EformsExtension struct { + NoticeSubType NoticeSubType `json:"notice_sub_type,omitempty" xml:"NoticeSubType,omitempty"` + Organizations Organizations `json:"organizations,omitempty" xml:"Organizations,omitempty"` + Publication Publication `json:"publication,omitempty" xml:"Publication,omitempty"` + SelectionCriteria []SelectionCriteria `json:"selection_criteria,omitempty" xml:"SelectionCriteria,omitempty"` + OfficialLanguages OfficialLanguages `json:"official_languages,omitempty" xml:"OfficialLanguages,omitempty"` +} + +// NoticeSubType represents the notice subtype +type NoticeSubType struct { + XMLName xml.Name `xml:"NoticeSubType"` + SubTypeCode string `xml:"SubTypeCode"` +} + +// Organizations represents the organizations element +type Organizations struct { + XMLName xml.Name `xml:"Organizations"` + Organization []Organization `xml:"Organization,omitempty"` +} + +// Publication represents the publication information +type Publication struct { + XMLName xml.Name `xml:"Publication"` + NoticePublicationID string `xml:"NoticePublicationID"` + GazetteID string `xml:"GazetteID"` + PublicationDate string `xml:"PublicationDate"` +} + +// SelectionCriteria represents selection criteria for tenderers +type SelectionCriteria struct { + XMLName xml.Name `xml:"SelectionCriteria"` + TendererRequirementTypeCode string `xml:"TendererRequirementTypeCode"` + Description string `xml:"Description,omitempty"` + SecondStageIndicator bool `xml:"SecondStageIndicator,omitempty"` +} + +// OfficialLanguages represents official languages for the tender +type OfficialLanguages struct { + XMLName xml.Name `xml:"OfficialLanguages"` + Language []Language `xml:"Language,omitempty"` +} + +// Language represents a language code +type Language struct { + XMLName xml.Name `xml:"Language"` + ID string `xml:"ID"` +} diff --git a/ted/eform/validation.go b/ted/eform/validation.go new file mode 100644 index 0000000..421fa17 --- /dev/null +++ b/ted/eform/validation.go @@ -0,0 +1,19 @@ +package eform + +// ===================================================== +// VALIDATION HELPER TYPES +// ===================================================== + +// ValidationError represents a validation error +type ValidationError struct { + Field string `json:"field"` + Tag string `json:"tag"` + Value string `json:"value"` + Message string `json:"message"` +} + +// ValidationResult represents the result of validation +type ValidationResult struct { + Valid bool `json:"valid"` + Errors []ValidationError `json:"errors,omitempty"` +} diff --git a/ted/email_template.go b/ted/email_template.go new file mode 100644 index 0000000..b778e25 --- /dev/null +++ b/ted/email_template.go @@ -0,0 +1,508 @@ +package ted + +import ( + "fmt" + "strings" + "time" +) + +// FileProcessingEmailTemplateData represents the data needed for the file processing email template +type FileProcessingEmailTemplateData struct { + OJS string + ProcessedCount int + SuccessCount int + Errors []string + ProcessedAt int64 + CompanyName string + CompanyLogo string + CompanyURL string + SupportEmail string +} + +// GenerateFileProcessingEmailTemplate generates a beautiful HTML email template for file processing results +func GenerateFileProcessingEmailTemplate(data *FileProcessingEmailTemplateData) string { + // Format the processing date + processedAt := time.Unix(data.ProcessedAt, 0).Format("January 2, 2006 at 3:04 PM") + + // Calculate success rate + successRate := 0.0 + if data.ProcessedCount > 0 { + successRate = float64(data.SuccessCount) / float64(data.ProcessedCount) * 100 + } + + // Determine status color and message + var statusColor, statusMessage string + if len(data.Errors) == 0 && data.ProcessedCount > 0 { + statusColor = "#10b981" // Green + statusMessage = "All files processed successfully" + } else if data.SuccessCount > 0 { + statusColor = "#f59e0b" // Amber + statusMessage = "Partial success with some errors" + } else { + statusColor = "#ef4444" // Red + statusMessage = "Processing failed" + } + + // Generate errors section + var errorsHTML string + if len(data.Errors) > 0 { + errorsHTML = ` +
+

Errors Encountered

+
` + + for i, error := range data.Errors { + if i >= 10 { // Limit to first 10 errors + errorsHTML += fmt.Sprintf(` +
+ %d. + %s +
`, i+1, error) + } + } + + if len(data.Errors) > 10 { + errorsHTML += fmt.Sprintf(` +
+ ... and %d more errors +
`, len(data.Errors)-10) + } + + errorsHTML += ` +
+
` + } + + html := ` + + + + + + TED File Processing Report - {{COMPANY_NAME}} + + + +
+ +
+

TED File Processing Report

+

Automated processing results from the TED scraper system

+
+ + +
+
+ {{STATUS_MESSAGE}} +
+ +
+
OJS: {{OJS}}
+ +
+
+
{{PROCESSED_COUNT}}
+
Files Processed
+
+ +
+
{{SUCCESS_COUNT}}
+
Successfully Processed
+
+ +
+
{{SUCCESS_RATE}}%
+
Success Rate
+
+ +
+
{{ERROR_COUNT}}
+
Errors
+
+
+ + {{ERRORS_SECTION}} + +
+ Processed on: {{PROCESSED_AT}} +
+

This report was automatically generated by the TED Scraper System

+
+
+
+ + +
+

This is an automated message from the TED Scraper System

+

For support, contact: {{SUPPORT_EMAIL}}

+

{{COMPANY_NAME}}

+
+
+ +` + + // Replace placeholders with actual data + html = strings.ReplaceAll(html, "{{COMPANY_NAME}}", data.CompanyName) + html = strings.ReplaceAll(html, "{{OJS}}", data.OJS) + html = strings.ReplaceAll(html, "{{PROCESSED_COUNT}}", fmt.Sprintf("%d", data.ProcessedCount)) + html = strings.ReplaceAll(html, "{{SUCCESS_COUNT}}", fmt.Sprintf("%d", data.SuccessCount)) + html = strings.ReplaceAll(html, "{{SUCCESS_RATE}}", fmt.Sprintf("%.1f", successRate)) + html = strings.ReplaceAll(html, "{{ERROR_COUNT}}", fmt.Sprintf("%d", len(data.Errors))) + html = strings.ReplaceAll(html, "{{PROCESSED_AT}}", processedAt) + html = strings.ReplaceAll(html, "{{STATUS_COLOR}}", statusColor) + html = strings.ReplaceAll(html, "{{STATUS_MESSAGE}}", statusMessage) + html = strings.ReplaceAll(html, "{{ERRORS_SECTION}}", errorsHTML) + html = strings.ReplaceAll(html, "{{SUPPORT_EMAIL}}", data.SupportEmail) + html = strings.ReplaceAll(html, "{{COMPANY_URL}}", data.CompanyURL) + + return html +} + +// GenerateFileProcessingEmailMessage generates a simple text message for the email +func GenerateFileProcessingEmailMessage(data *FileProcessingEmailTemplateData) string { + processedAt := time.Unix(data.ProcessedAt, 0).Format("January 2, 2006 at 3:04 PM") + + // Calculate success rate + successRate := 0.0 + if data.ProcessedCount > 0 { + successRate = float64(data.SuccessCount) / float64(data.ProcessedCount) * 100 + } + + // Determine status message + var statusMessage string + if len(data.Errors) == 0 && data.ProcessedCount > 0 { + statusMessage = "✅ All files processed successfully" + } else if data.SuccessCount > 0 { + statusMessage = "⚠️ Partial success with some errors" + } else { + statusMessage = "❌ Processing failed" + } + + message := fmt.Sprintf(` +TED File Processing Report + +%s + +OJS: %s +Files Processed: %d +Successfully Processed: %d +Success Rate: %.1f%% +Errors: %d + +Processed on: %s + +`, statusMessage, data.OJS, data.ProcessedCount, data.SuccessCount, successRate, len(data.Errors), processedAt) + + // Add errors if any + if len(data.Errors) > 0 { + message += "Errors encountered:\n" + for i, error := range data.Errors { + if i >= 10 { // Limit to first 10 errors + break + } + message += fmt.Sprintf("%d. %s\n", i+1, error) + } + + if len(data.Errors) > 10 { + message += fmt.Sprintf("... and %d more errors\n", len(data.Errors)-10) + } + } + + message += "\nThis report was automatically generated by the TED Scraper System." + + return message +} + +// GenerateFileProcessingEmailSubject generates an appropriate subject line for the email +func GenerateFileProcessingEmailSubject(data *FileProcessingEmailTemplateData) string { + var status string + if len(data.Errors) == 0 && data.ProcessedCount > 0 { + status = "✅ Success" + } else if data.SuccessCount > 0 { + status = "⚠️ Partial Success" + } else { + status = "❌ Failed" + } + + return fmt.Sprintf("TED Processing Report - OJS %s - %s (%d/%d files)", + data.OJS, status, data.SuccessCount, data.ProcessedCount) +} diff --git a/ted/parser.go b/ted/parser.go new file mode 100644 index 0000000..b609491 --- /dev/null +++ b/ted/parser.go @@ -0,0 +1,172 @@ +package ted + +import ( + "encoding/xml" + "fmt" + "strings" + "tm/pkg/notification" + "tm/pkg/xmlparser" +) + +type ParserType string + +const ( + ParserTypeContractNotice ParserType = "ContractNotice" + ParserTypeContractAwardNotice ParserType = "ContractAwardNotice" + ParserTypePriorInformationNotice ParserType = "PriorInformationNotice" + ParserTypeBusinessRegistrationInformationNotice ParserType = "BusinessRegistrationInformationNotice" +) + +type TEDParser struct { + Notification notification.SDK + NoticeParser xmlparser.Parser[ContractNotice] +} + +// NewTEDParser creates a new TED parser +func NewTEDParser() *TEDParser { + options := xmlparser.DefaultParserOptions() + options.MaxSize = 50 * 1024 * 1024 // 50MB limit for TED XML files + options.IgnoreUnknownElements = true // TED XML has many optional fields + + parser := xmlparser.NewParser[ContractNotice](options) + return &TEDParser{ + NoticeParser: parser, + } +} + +// ParseAnyNotice parses XML content and detects the notice type automatically +func (tp *TEDParser) Parse(content []byte) (interface{}, string, ParserType, error) { + // First, detect the root element type + rootElement, err := detectRootElement(content) + if err != nil { + return nil, "", ParserType(""), fmt.Errorf("failed to detect root element: %w", err) + } + + // Parse based on the detected type + switch ParserType(rootElement) { + case ParserTypeContractNotice: + notice, err := tp.parseContractNotice(content) + return notice, notice.GetID(), ParserTypeContractNotice, err + case ParserTypeContractAwardNotice: + notice, err := tp.parseContractAwardNotice(content) + return notice, notice.GetID(), ParserTypeContractAwardNotice, err + case ParserTypePriorInformationNotice: + notice, err := tp.parsePriorInformationNotice(content) + return notice, notice.GetID(), ParserTypePriorInformationNotice, err + case ParserTypeBusinessRegistrationInformationNotice: + notice, err := tp.parseBusinessRegistrationInformationNotice(content) + return notice, notice.GetID(), ParserTypeBusinessRegistrationInformationNotice, err + default: + return nil, "", ParserType(""), fmt.Errorf("unsupported notice type: %s", rootElement) + } +} + +// detectRootElement detects the root XML element name +func detectRootElement(content []byte) (string, error) { + decoder := xml.NewDecoder(strings.NewReader(string(content))) + for { + token, err := decoder.Token() + if err != nil { + return "", err + } + if startElement, ok := token.(xml.StartElement); ok { + return startElement.Name.Local, nil + } + } +} + +// parseContractNotice parses a ContractNotice +func (tp *TEDParser) parseContractNotice(content []byte) (*ContractNotice, error) { + options := xmlparser.DefaultParserOptions() + options.MaxSize = 50 * 1024 * 1024 + options.IgnoreUnknownElements = true + + parser := xmlparser.NewParser[ContractNotice](options) + notice, err := parser.ParseBytes(content) + if err != nil { + return nil, err + } + return notice, nil +} + +// parseContractAwardNotice parses a ContractAwardNotice +func (tp *TEDParser) parseContractAwardNotice(content []byte) (*ContractAwardNotice, error) { + options := xmlparser.DefaultParserOptions() + options.MaxSize = 50 * 1024 * 1024 + options.IgnoreUnknownElements = true + + parser := xmlparser.NewParser[ContractAwardNotice](options) + notice, err := parser.ParseBytes(content) + if err != nil { + return nil, err + } + return notice, nil +} + +// parsePriorInformationNotice parses a PriorInformationNotice +func (tp *TEDParser) parsePriorInformationNotice(content []byte) (*PriorInformationNotice, error) { + options := xmlparser.DefaultParserOptions() + options.MaxSize = 50 * 1024 * 1024 + options.IgnoreUnknownElements = true + + parser := xmlparser.NewParser[PriorInformationNotice](options) + notice, err := parser.ParseBytes(content) + if err != nil { + return nil, err + } + return notice, nil +} + +// parseBusinessRegistrationInformationNotice parses a BusinessRegistrationInformationNotice +func (tp *TEDParser) parseBusinessRegistrationInformationNotice(content []byte) (*BusinessRegistrationInformationNotice, error) { + options := xmlparser.DefaultParserOptions() + options.MaxSize = 50 * 1024 * 1024 + options.IgnoreUnknownElements = true + + parser := xmlparser.NewParser[BusinessRegistrationInformationNotice](options) + notice, err := parser.ParseBytes(content) + if err != nil { + return nil, err + } + return notice, nil +} + +// GetID returns the contract notice ID +func (cn ContractNotice) GetID() string { + return cn.ID +} + +// GetID returns the contract award notice ID +func (can ContractAwardNotice) GetID() string { + return can.ID +} + +// GetID returns the prior information notice ID +func (pin PriorInformationNotice) GetID() string { + return pin.ID +} + +// GetID returns the business registration information notice ID +func (brin BusinessRegistrationInformationNotice) GetID() string { + return brin.ID +} + +// Validate validates the parsed model +func (cn ContractNotice) Validate() error { + return nil +} + +// Validate validates the parsed model +func (can ContractAwardNotice) Validate() error { + return nil +} + +// Validate validates the parsed model +func (pin PriorInformationNotice) Validate() error { + return nil +} + +// Validate validates the parsed model +func (brin BusinessRegistrationInformationNotice) Validate() error { + return nil +} diff --git a/ted/scraper.go b/ted/scraper.go new file mode 100644 index 0000000..33cb7c7 --- /dev/null +++ b/ted/scraper.go @@ -0,0 +1,439 @@ +package ted + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + "time" + "tm/internal/tender" + "tm/pkg/logger" + "tm/pkg/mongo" + "tm/pkg/notification" +) + +// ScraperConfig holds configuration for the TED scraper +type Config struct { + BaseURL string `mapstructure:"base_url"` + UserAgent string `mapstructure:"user_agent"` + DownloadDir string `mapstructure:"download_dir"` + MaxRetries int `mapstructure:"max_retries"` + MaxConcurrency int `mapstructure:"max_concurrency"` + Timeout time.Duration `mapstructure:"timeout"` + RetryDelay time.Duration `mapstructure:"retry_delay"` + CleanupAfter time.Duration `mapstructure:"cleanup_after"` + ScrapingInterval string `mapstructure:"scraping_interval"` +} + +// TEDScraper handles downloading and parsing TED XML files +type TEDScraper struct { + notify notification.SDK + config *Config + logger logger.Logger + httpClient *http.Client + xmlParser *TEDParser + mongoManager *mongo.ConnectionManager + tenderRepo tender.TenderRepository +} + +// NewTEDScraper creates a new TED scraper instance +func NewTEDScraper( + config *Config, + logger logger.Logger, + mongoManager *mongo.ConnectionManager, + tenderRepo tender.TenderRepository, + notify notification.SDK, +) *TEDScraper { + httpClient := &http.Client{ + Timeout: config.Timeout, + } + + return &TEDScraper{ + notify: notify, + config: config, + logger: logger, + httpClient: httpClient, + xmlParser: NewTEDParser(), + mongoManager: mongoManager, + tenderRepo: tenderRepo, + } +} + +// downloadDailyPackage downloads a file from a URL +func (s *TEDScraper) DownloadDailyPackage(ctx context.Context, ojs string) (*FileProcessingResult, error) { + var ( + result *FileProcessingResult + err error + ) + + url := fmt.Sprintf("%s/packages/daily/%s", s.config.BaseURL, ojs) + + s.logger.Info("Downloading TED file", map[string]interface{}{ + "url": url, + "ojs": ojs, + }) + + for attempt := 1; attempt <= s.config.MaxRetries; attempt++ { + result, err = s.downloadWithRetry(ctx, url, ojs) + if err == nil { + break + } + + if attempt < s.config.MaxRetries { + s.logger.Warn("Download attempt failed, retrying", map[string]interface{}{ + "attempt": attempt, + "error": err.Error(), + "url": url, + }) + + select { + case <-ctx.Done(): + return result, ctx.Err() + case <-time.After(s.config.RetryDelay): + // Continue to next attempt + } + } + } + + return result, err +} + +// FileProcessingResult represents the result of processing a single file +type FileProcessingResult struct { + OJS string `json:"ojs"` + ProcessedCount int `json:"processed_count"` + SuccessCount int `json:"success_count"` + Errors []string `json:"errors"` + ProcessedAt int64 `json:"processed_at"` +} + +func (s *TEDScraper) downloadWithRetry(ctx context.Context, url string, ojs string) (*FileProcessingResult, error) { + // Create a longer timeout context for processing large files + processCtx, cancel := context.WithTimeout(ctx, s.config.Timeout) + defer cancel() + + // Create HTTP request + req, err := http.NewRequestWithContext(processCtx, "GET", url, nil) + if err != nil { + return &FileProcessingResult{}, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", s.config.UserAgent) + + s.logger.Info("Starting download", map[string]interface{}{ + "url": url, + "ojs": ojs, + }) + + // Make request + resp, err := s.httpClient.Do(req) + if err != nil { + return &FileProcessingResult{}, fmt.Errorf("failed to download file: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return &FileProcessingResult{}, fmt.Errorf("HTTP error: %d %s", resp.StatusCode, resp.Status) + } + + s.logger.Info("Download completed, starting processing", map[string]interface{}{ + "url": url, + "ojs": ojs, + "content_length": resp.Header.Get("Content-Length"), + }) + + return s.tarGzFileProcessor(processCtx, resp.Body, ojs) +} + +// tarGzFileProcessor extracts and processes XML files from a tar.gz archive +func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, ojs string) (*FileProcessingResult, error) { + // Create gzip reader + gzipReader, err := gzip.NewReader(reader) + if err != nil { + return &FileProcessingResult{}, fmt.Errorf("failed to create gzip reader: %w", err) + } + defer gzipReader.Close() + + // Create tar reader + tarReader := tar.NewReader(gzipReader) + + // Process each file in the tar archive + fileCount := 0 + result := &FileProcessingResult{ + OJS: ojs, + ProcessedCount: 0, + SuccessCount: 0, + Errors: make([]string, 0), + ProcessedAt: time.Now().Unix(), + } + for { + // Check context for cancellation before each tar entry + select { + case <-ctx.Done(): + s.logger.Warn("Context cancelled during tar processing", map[string]interface{}{ + "processed_files": fileCount, + "error": ctx.Err().Error(), + }) + return &FileProcessingResult{}, ctx.Err() + default: + } + + header, err := tarReader.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + s.logger.Warn("Error reading tar entry", map[string]interface{}{ + "error": err.Error(), + "processed_files": fileCount, + }) + // If it's a timeout error, return it immediately + if ctx.Err() != nil { + return &FileProcessingResult{}, fmt.Errorf("timeout during tar processing: %w", err) + } + continue + } + fileCount++ + + // Skip directories and non-XML files + if header.Typeflag != tar.TypeReg { + continue + } + + fileName := filepath.Base(header.Name) + if !strings.HasSuffix(strings.ToLower(fileName), ".xml") { + continue + } + + // Log progress every 10 files or for first file + if result.ProcessedCount == 0 || result.ProcessedCount%10 == 0 { + s.logger.Info("Processing XML file from tar.gz", map[string]interface{}{ + "file_name": fileName, + "size": header.Size, + "processed_count": result.ProcessedCount, + "total_files_seen": fileCount, + "ojs": ojs, + }) + } else { + s.logger.Debug("Processing XML file from tar.gz", map[string]interface{}{ + "file_name": fileName, + "processed_count": result.ProcessedCount, + "ojs": ojs, + }) + } + + // Read file content + content, err := io.ReadAll(tarReader) + if err != nil { + errMsg := fmt.Sprintf("failed to read XML file %s: %v", fileName, err) + result.Errors = append(result.Errors, errMsg) + continue + } + + // Parse and process XML content + if err := s.processXMLContent(ctx, content, fileName, result); err != nil { + errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err) + result.Errors = append(result.Errors, errMsg) + } + + result.ProcessedCount++ + + // Check context for cancellation + select { + case <-ctx.Done(): + return result, ctx.Err() + default: + // Continue processing + } + } + + go s.notify.SendNotification( + context.Background(), + ¬ification.NotificationRequest{ + EventType: notification.EventTypeEmail, + Title: "TED scraper completed", + Message: GenerateFileProcessingEmailTemplate( + &FileProcessingEmailTemplateData{ + OJS: ojs, + ProcessedCount: result.ProcessedCount, + SuccessCount: result.SuccessCount, + Errors: result.Errors, + ProcessedAt: result.ProcessedAt, + }), + Priority: "important", + Methods: notification.NotificationMethods{ + Email: "nakhostin.nima1998@gmail.com", + }, + UserID: "ted-scraper", + Type: "alert", + }) + + return result, nil +} + +func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, result *FileProcessingResult) error { + s.logger.Debug("Processing XML content", map[string]interface{}{ + "file_name": fileName, + "content_size": len(content), + "first_1000_chars": string(content[:min(1000, len(content))]), + }) + + // Parse XML using the flexible TED parser that detects notice type + _, id, noticeType, err := s.xmlParser.Parse(content) + if err != nil { + s.logger.Error("XML parsing failed", map[string]interface{}{ + "file_name": fileName, + "error": err.Error(), + "content_preview": string(content[:min(500, len(content))]), + }) + return fmt.Errorf("failed to parse XML: %w", err) + } + + // Log detailed parsed information based on notice type + s.logger.Info("Successfully parsed notice", map[string]interface{}{ + "notice_id": id, + "notice_type": noticeType, + "file_name": fileName, + }) + + result.SuccessCount++ + return nil +} + +func (s *TEDScraper) Run(ctx context.Context) error { + now := time.Now().Local() + s.logger.Info("Running periodic scraping", map[string]interface{}{ + "time": now.Format(time.RFC3339), + }) + + // get ojs + ojs, err := GetOJS(now.Year(), now.Format("02/01/2006")) + if err != nil { + s.logger.Error("Failed to get OJS", map[string]interface{}{ + "error": err.Error(), + }) + return err + } + + // download daily package + result, err := s.DownloadDailyPackage(ctx, ojs) + if err != nil { + s.logger.Error("Failed to download daily package", map[string]interface{}{ + "error": err.Error(), + }) + return err + } + + // process daily package + s.logger.Info("Processed daily package successfully", map[string]interface{}{ + "result": result, + "time": now.Format(time.RFC3339), + "ojs": ojs, + }) + + return nil +} + +// parseDate parses various date formats used in TED XML +func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) { + if dateStr == "" { + return time.Time{}, fmt.Errorf("empty date string") + } + + // Common TED date formats + formats := []string{ + "2006-01-02+07:00", + "2006-01-02-07:00", + "2006-01-02T15:04:05+07:00", + "2006-01-02T15:04:05-07:00", + "2006-01-02T15:04:05Z", + "2006-01-02Z", + "2006-01-02", + time.RFC3339, + time.RFC3339Nano, + } + + for _, format := range formats { + if parsedTime, err := time.Parse(format, dateStr); err == nil { + return parsedTime, nil + } + } + + return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr) +} + +// parseDateToUnixMilli converts string date to Unix milliseconds (int64) +func (s *TEDScraper) parseDateToUnixMilli(dateStr string) int64 { + if dateStr == "" { + return 0 + } + + if parsedTime, err := s.parseDate(dateStr); err == nil { + return parsedTime.UnixMilli() + } + + return 0 +} + +// parseTimeToUnixMilli converts string time to Unix milliseconds (int64) +func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 { + if timeStr == "" { + return 0 + } + + // If time string is just time without date, use today's date + if !strings.Contains(timeStr, "T") && !strings.Contains(timeStr, " ") { + // Assume it's just a time like "15:04:05" + today := time.Now().Format("2006-01-02") + timeStr = fmt.Sprintf("%sT%s", today, timeStr) + } + + if parsedTime, err := s.parseDate(timeStr); err == nil { + return parsedTime.UnixMilli() + } + + return 0 +} + +// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation +func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string { + if unixMilli == 0 { + return "00000000" + } + + return time.UnixMilli(unixMilli).Format("20060102") +} + +func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time { + deadline := publicationDate + workingHoursAdded := 0 + + for workingHoursAdded < 48 { + deadline = deadline.Add(1 * time.Hour) + + if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday { + workingHoursAdded++ + } + } + + return deadline +} + +func (s *TEDScraper) calculateApplicationDeadline(tenderDeadline time.Time) time.Time { + applicationDeadline := tenderDeadline + workingDaysSubtracted := 0 + + for workingDaysSubtracted < 7 { + applicationDeadline = applicationDeadline.AddDate(0, 0, -1) + + if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday { + workingDaysSubtracted++ + } + } + + return applicationDeadline +}