Nima Nakhostin
10fd53af41
- Removed command line flag parsing for date range scraping in the scraper's main function. - Introduced configuration fields for one-time scraping and date range (FromDate, ToDate) in the ScraperConfig struct. - Updated the main function to utilize the new configuration fields, improving the clarity and maintainability of the scraping logic. - This change enhances the scraper's functionality by centralizing configuration management and reducing command line complexity.
31 lines
1.3 KiB
Go
31 lines
1.3 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"time"
|
|
"tm/pkg/config"
|
|
)
|
|
|
|
// Config defines the scraper application configuration
|
|
type Config struct {
|
|
Database config.DatabaseConfig
|
|
Logging config.LoggingConfig
|
|
TED ScraperConfig
|
|
Notification config.NotificationConfig
|
|
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
|
|
}
|
|
|
|
type ScraperConfig struct {
|
|
BaseURL string `env:"TED_BASE_URL" envDefault:"https://ted.europa.eu"`
|
|
Timeout time.Duration `env:"TED_TIMEOUT" envDefault:"30s"`
|
|
MaxRetries int `env:"TED_MAX_RETRIES" envDefault:"3"`
|
|
RetryDelay time.Duration `env:"TED_RETRY_DELAY" envDefault:"5s"`
|
|
UserAgent string `env:"TED_USER_AGENT" envDefault:"TM-TED-Scraper/1.0"`
|
|
MaxConcurrency int `env:"TED_MAX_CONCURRENCY" envDefault:"5"`
|
|
DownloadDir string `env:"TED_DOWNLOAD_DIR" envDefault:"./downloads"`
|
|
CleanupAfter time.Duration `env:"TED_CLEANUP_AFTER" envDefault:"24h"`
|
|
ScrapingInterval string `env:"TED_SCRAPING_INTERVAL" envDefault:"* * 10 * * *"`
|
|
OneTime bool `env:"TED_ONE_TIME" envDefault:"false"`
|
|
FromDate string `env:"TED_FROM_DATE" envDefault:"02/01/2025"`
|
|
ToDate string `env:"TED_TO_DATE" envDefault:"13/09/2025"`
|
|
}
|