Add TED Scraper Configuration and Cron Scheduler Implementation

- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters.
- Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling.
- Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality.
- Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling.
- Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.

cmd/scraper/bootstrap/config.go

@@ -7,19 +7,20 @@ import (
 
 // Config defines the scraper application configuration
 type Config struct {
-	Database config.DatabaseConfig
-	Logging  config.LoggingConfig
-	TED      ScraperConfig
+	Database     config.DatabaseConfig
+	Logging      config.LoggingConfig
+	TED          ScraperConfig
+	Notification config.NotificationConfig
 }
 
 type ScraperConfig struct {
-	BaseURL          string        `env:"TED_BASE_URL"`
-	Timeout          time.Duration `env:"TED_TIMEOUT"`
-	MaxRetries       int           `env:"TED_MAX_RETRIES"`
-	RetryDelay       time.Duration `env:"TED_RETRY_DELAY"`
-	UserAgent        string        `env:"TED_USER_AGENT"`
-	MaxConcurrency   int           `env:"TED_MAX_CONCURRENCY"`
-	DownloadDir      string        `env:"TED_DOWNLOAD_DIR"`
-	CleanupAfter     time.Duration `env:"TED_CLEANUP_AFTER"`
-	ScrapingInterval time.Duration `env:"TED_SCRAPING_INTERVAL"`
+	BaseURL          string        `env:"TED_BASE_URL" envDefault:"https://ted.europa.eu"`
+	Timeout          time.Duration `env:"TED_TIMEOUT" envDefault:"30s"`
+	MaxRetries       int           `env:"TED_MAX_RETRIES" envDefault:"3"`
+	RetryDelay       time.Duration `env:"TED_RETRY_DELAY" envDefault:"5s"`
+	UserAgent        string        `env:"TED_USER_AGENT" envDefault:"TM-TED-Scraper/1.0"`
+	MaxConcurrency   int           `env:"TED_MAX_CONCURRENCY" envDefault:"5"`
+	DownloadDir      string        `env:"TED_DOWNLOAD_DIR" envDefault:"./downloads"`
+	CleanupAfter     time.Duration `env:"TED_CLEANUP_AFTER" envDefault:"24h"`
+	ScrapingInterval string        `env:"TED_SCRAPING_INTERVAL" envDefault:"* * 10 * * *"`
 }