n.nakhostin
96c21b8b78
- Introduced a new TED scraper in `cmd/scraper` to handle downloading and parsing TED XML files. - Added configuration management for the scraper, including a new `Config` struct and YAML configuration file. - Created necessary handler, service, and repository layers for tender management, adhering to Clean Architecture principles. - Implemented comprehensive logging and error handling throughout the scraper functionality. - Updated API routes to include tender management operations, enhancing the overall system capabilities.
51 lines
1.5 KiB
Go
51 lines
1.5 KiB
Go
package main
|
|
|
|
import (
|
|
"time"
|
|
"tm/pkg/config"
|
|
)
|
|
|
|
// Config defines the scraper application configuration
|
|
type Config struct {
|
|
Database config.DatabaseConfig `mapstructure:"database"`
|
|
Logging config.LoggingConfig `mapstructure:"logging"`
|
|
TED ScraperConfig `mapstructure:"ted"`
|
|
}
|
|
|
|
type ScraperConfig struct {
|
|
BaseURL string `mapstructure:"base_url"`
|
|
Timeout time.Duration `mapstructure:"timeout"`
|
|
MaxRetries int `mapstructure:"max_retries"`
|
|
RetryDelay time.Duration `mapstructure:"retry_delay"`
|
|
UserAgent string `mapstructure:"user_agent"`
|
|
MaxConcurrency int `mapstructure:"max_concurrency"`
|
|
DownloadDir string `mapstructure:"download_dir"`
|
|
CleanupAfter time.Duration `mapstructure:"cleanup_after"`
|
|
ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
|
|
}
|
|
|
|
// LoadConfig loads the configuration from file
|
|
func LoadConfig(path string) (*Config, error) {
|
|
cfg := &Config{
|
|
// Set default values
|
|
TED: ScraperConfig{
|
|
BaseURL: "https://ted.europa.eu/packages/daily",
|
|
Timeout: 30 * time.Second,
|
|
MaxRetries: 3,
|
|
RetryDelay: 5 * time.Second,
|
|
UserAgent: "TenderManagement-TED-Scraper/1.0",
|
|
MaxConcurrency: 3,
|
|
DownloadDir: "/tmp/ted_downloads",
|
|
CleanupAfter: 24 * time.Hour,
|
|
ScrapingInterval: 12 * time.Hour,
|
|
},
|
|
}
|
|
|
|
result, err := config.LoadConfig(path, cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return result, nil
|
|
}
|