Refactor Configuration System to Use Environment Variables

- Removed YAML configuration files for the scraper and web commands, transitioning to an environment-based configuration system.
- Updated configuration structs to utilize `env` tags for loading values from `.env` files, enhancing security and flexibility.
- Implemented reflection-based parsing for environment variables, ensuring type-safe configuration loading.
- Revised documentation to reflect the new configuration approach, including usage examples and best practices for environment variable naming conventions.

cmd/scraper/bootstrap/config.go

@@ -7,44 +7,19 @@ import (
 
 // Config defines the scraper application configuration
 type Config struct {
-	Database config.DatabaseConfig `mapstructure:"database"`
-	Logging  config.LoggingConfig  `mapstructure:"logging"`
-	TED      ScraperConfig         `mapstructure:"ted"`
+	Database config.DatabaseConfig
+	Logging  config.LoggingConfig
+	TED      ScraperConfig
 }
 
 type ScraperConfig struct {
-	BaseURL          string        `mapstructure:"base_url"`
-	Timeout          time.Duration `mapstructure:"timeout"`
-	MaxRetries       int           `mapstructure:"max_retries"`
-	RetryDelay       time.Duration `mapstructure:"retry_delay"`
-	UserAgent        string        `mapstructure:"user_agent"`
-	MaxConcurrency   int           `mapstructure:"max_concurrency"`
-	DownloadDir      string        `mapstructure:"download_dir"`
-	CleanupAfter     time.Duration `mapstructure:"cleanup_after"`
-	ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
-}
-
-// LoadConfig loads the configuration from file
-func LoadConfig(path string) (*Config, error) {
-	cfg := &Config{
-		// Set default values
-		TED: ScraperConfig{
-			BaseURL:          "https://ted.europa.eu/packages/daily",
-			Timeout:          30 * time.Second,
-			MaxRetries:       3,
-			RetryDelay:       5 * time.Second,
-			UserAgent:        "TenderManagement-TED-Scraper/1.0",
-			MaxConcurrency:   3,
-			DownloadDir:      "/tmp/ted_downloads",
-			CleanupAfter:     24 * time.Hour,
-			ScrapingInterval: 12 * time.Hour,
-		},
-	}
-
-	result, err := config.LoadConfig(path, cfg)
-	if err != nil {
-		return nil, err
-	}
-
-	return result, nil
+	BaseURL          string        `env:"TED_BASE_URL"`
+	Timeout          time.Duration `env:"TED_TIMEOUT"`
+	MaxRetries       int           `env:"TED_MAX_RETRIES"`
+	RetryDelay       time.Duration `env:"TED_RETRY_DELAY"`
+	UserAgent        string        `env:"TED_USER_AGENT"`
+	MaxConcurrency   int           `env:"TED_MAX_CONCURRENCY"`
+	DownloadDir      string        `env:"TED_DOWNLOAD_DIR"`
+	CleanupAfter     time.Duration `env:"TED_CLEANUP_AFTER"`
+	ScrapingInterval time.Duration `env:"TED_SCRAPING_INTERVAL"`
 }

cmd/scraper/config.yaml

@@ -1,28 +0,0 @@
-database:
-  mongodb:
-    uri: "mongodb://localhost:27017"
-    name: "tender_management"
-    timeout: 10s
-    max_pool_size: 50
-
-logging:
-  level: "info"
-  format: "json"
-  output: "file"  # stdout, stderr, file
-  file:
-    path: "./logs/scraper.log"
-    max_size: 100      # Max size in MB before rotation
-    max_backups: 5     # Number of backup files to keep
-    max_age: 30        # Max age in days to keep log files
-    compress: true     # Compress rotated files
-
-ted:
-  base_url: "https://ted.europa.eu/packages/daily"
-  timeout: 300s  # Increased to 5 minutes for large files
-  max_retries: 5
-  retry_delay: 10s
-  user_agent: "TenderManagement-TED-Scraper/1.0"
-  max_concurrency: 2  # Reduced to avoid overwhelming the server
-  download_dir: "./tmp/ted_downloads"
-  cleanup_after: 72h
-  scraping_interval: 12h  # Run scraping every 12 hours