Refactor Configuration System to Use Environment Variables

- Removed YAML configuration files for the scraper and web commands, transitioning to an environment-based configuration system.
- Updated configuration structs to utilize `env` tags for loading values from `.env` files, enhancing security and flexibility.
- Implemented reflection-based parsing for environment variables, ensuring type-safe configuration loading.
- Revised documentation to reflect the new configuration approach, including usage examples and best practices for environment variable naming conventions.

cmd/scraper/bootstrap/config.go

@@ -7,44 +7,19 @@ import (
 
 // Config defines the scraper application configuration
 type Config struct {
-	Database config.DatabaseConfig `mapstructure:"database"`
-	Logging  config.LoggingConfig  `mapstructure:"logging"`
-	TED      ScraperConfig         `mapstructure:"ted"`
+	Database config.DatabaseConfig
+	Logging  config.LoggingConfig
+	TED      ScraperConfig
 }
 
 type ScraperConfig struct {
-	BaseURL          string        `mapstructure:"base_url"`
-	Timeout          time.Duration `mapstructure:"timeout"`
-	MaxRetries       int           `mapstructure:"max_retries"`
-	RetryDelay       time.Duration `mapstructure:"retry_delay"`
-	UserAgent        string        `mapstructure:"user_agent"`
-	MaxConcurrency   int           `mapstructure:"max_concurrency"`
-	DownloadDir      string        `mapstructure:"download_dir"`
-	CleanupAfter     time.Duration `mapstructure:"cleanup_after"`
-	ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
-}
-
-// LoadConfig loads the configuration from file
-func LoadConfig(path string) (*Config, error) {
-	cfg := &Config{
-		// Set default values
-		TED: ScraperConfig{
-			BaseURL:          "https://ted.europa.eu/packages/daily",
-			Timeout:          30 * time.Second,
-			MaxRetries:       3,
-			RetryDelay:       5 * time.Second,
-			UserAgent:        "TenderManagement-TED-Scraper/1.0",
-			MaxConcurrency:   3,
-			DownloadDir:      "/tmp/ted_downloads",
-			CleanupAfter:     24 * time.Hour,
-			ScrapingInterval: 12 * time.Hour,
-		},
-	}
-
-	result, err := config.LoadConfig(path, cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	return result, nil
+	BaseURL          string        `env:"TED_BASE_URL"`
+	Timeout          time.Duration `env:"TED_TIMEOUT"`
+	MaxRetries       int           `env:"TED_MAX_RETRIES"`
+	RetryDelay       time.Duration `env:"TED_RETRY_DELAY"`
+	UserAgent        string        `env:"TED_USER_AGENT"`
+	MaxConcurrency   int           `env:"TED_MAX_CONCURRENCY"`
+	DownloadDir      string        `env:"TED_DOWNLOAD_DIR"`
+	CleanupAfter     time.Duration `env:"TED_CLEANUP_AFTER"`
+	ScrapingInterval time.Duration `env:"TED_SCRAPING_INTERVAL"`
 }