Refactor Configuration System to Use Environment Variables
- Removed YAML configuration files for the scraper and web commands, transitioning to an environment-based configuration system. - Updated configuration structs to utilize `env` tags for loading values from `.env` files, enhancing security and flexibility. - Implemented reflection-based parsing for environment variables, ensuring type-safe configuration loading. - Revised documentation to reflect the new configuration approach, including usage examples and best practices for environment variable naming conventions.
This commit is contained in:
n.nakhostin
@@ -7,44 +7,19 @@ import (
|
||||
|
||||
// Config defines the scraper application configuration
|
||||
type Config struct {
|
||||
Database config.DatabaseConfig `mapstructure:"database"`
|
||||
Logging config.LoggingConfig `mapstructure:"logging"`
|
||||
TED ScraperConfig `mapstructure:"ted"`
|
||||
Database config.DatabaseConfig
|
||||
Logging config.LoggingConfig
|
||||
TED ScraperConfig
|
||||
}
|
||||
|
||||
type ScraperConfig struct {
|
||||
BaseURL string `mapstructure:"base_url"`
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
MaxRetries int `mapstructure:"max_retries"`
|
||||
RetryDelay time.Duration `mapstructure:"retry_delay"`
|
||||
UserAgent string `mapstructure:"user_agent"`
|
||||
MaxConcurrency int `mapstructure:"max_concurrency"`
|
||||
DownloadDir string `mapstructure:"download_dir"`
|
||||
CleanupAfter time.Duration `mapstructure:"cleanup_after"`
|
||||
ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
|
||||
}
|
||||
|
||||
// LoadConfig loads the configuration from file
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
cfg := &Config{
|
||||
// Set default values
|
||||
TED: ScraperConfig{
|
||||
BaseURL: "https://ted.europa.eu/packages/daily",
|
||||
Timeout: 30 * time.Second,
|
||||
MaxRetries: 3,
|
||||
RetryDelay: 5 * time.Second,
|
||||
UserAgent: "TenderManagement-TED-Scraper/1.0",
|
||||
MaxConcurrency: 3,
|
||||
DownloadDir: "/tmp/ted_downloads",
|
||||
CleanupAfter: 24 * time.Hour,
|
||||
ScrapingInterval: 12 * time.Hour,
|
||||
},
|
||||
}
|
||||
|
||||
result, err := config.LoadConfig(path, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
BaseURL string `env:"TED_BASE_URL"`
|
||||
Timeout time.Duration `env:"TED_TIMEOUT"`
|
||||
MaxRetries int `env:"TED_MAX_RETRIES"`
|
||||
RetryDelay time.Duration `env:"TED_RETRY_DELAY"`
|
||||
UserAgent string `env:"TED_USER_AGENT"`
|
||||
MaxConcurrency int `env:"TED_MAX_CONCURRENCY"`
|
||||
DownloadDir string `env:"TED_DOWNLOAD_DIR"`
|
||||
CleanupAfter time.Duration `env:"TED_CLEANUP_AFTER"`
|
||||
ScrapingInterval time.Duration `env:"TED_SCRAPING_INTERVAL"`
|
||||
}
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
database:
|
||||
mongodb:
|
||||
uri: "mongodb://localhost:27017"
|
||||
name: "tender_management"
|
||||
timeout: 10s
|
||||
max_pool_size: 50
|
||||
|
||||
logging:
|
||||
level: "info"
|
||||
format: "json"
|
||||
output: "file" # stdout, stderr, file
|
||||
file:
|
||||
path: "./logs/scraper.log"
|
||||
max_size: 100 # Max size in MB before rotation
|
||||
max_backups: 5 # Number of backup files to keep
|
||||
max_age: 30 # Max age in days to keep log files
|
||||
compress: true # Compress rotated files
|
||||
|
||||
ted:
|
||||
base_url: "https://ted.europa.eu/packages/daily"
|
||||
timeout: 300s # Increased to 5 minutes for large files
|
||||
max_retries: 5
|
||||
retry_delay: 10s
|
||||
user_agent: "TenderManagement-TED-Scraper/1.0"
|
||||
max_concurrency: 2 # Reduced to avoid overwhelming the server
|
||||
download_dir: "./tmp/ted_downloads"
|
||||
cleanup_after: 72h
|
||||
scraping_interval: 12h # Run scraping every 12 hours
|
||||
+13
-17
@@ -6,27 +6,23 @@ import (
|
||||
|
||||
// Config holds configuration for the web command
|
||||
type Config struct {
|
||||
Server config.ServerConfig `mapstructure:"server"`
|
||||
Database config.DatabaseConfig `mapstructure:"database"`
|
||||
Cache config.CacheConfig `mapstructure:"cache"`
|
||||
Logging config.LoggingConfig `mapstructure:"logging"`
|
||||
RateLimit config.RateLimitConfig `mapstructure:"rate_limiting"`
|
||||
UserAuth AuthConfig `mapstructure:"user_authorization"`
|
||||
CustomerAuth AuthConfig `mapstructure:"customer_authorization"`
|
||||
Assets AssetsConfig `mapstructure:"assets"`
|
||||
Server config.ServerConfig
|
||||
Database config.DatabaseConfig
|
||||
Cache config.CacheConfig
|
||||
Logging config.LoggingConfig
|
||||
RateLimit config.RateLimitConfig
|
||||
Assets config.AssetsConfig
|
||||
UserAuth AuthConfig
|
||||
CustomerAuth AuthConfig
|
||||
}
|
||||
|
||||
type AuthConfig struct {
|
||||
JWT JWTConfig `mapstructure:"jwt"`
|
||||
JWT JWTConfig
|
||||
}
|
||||
|
||||
type JWTConfig struct {
|
||||
AccessSecret string `mapstructure:"access_secret"`
|
||||
RefreshSecret string `mapstructure:"refresh_secret"`
|
||||
AccessExpiresIn int `mapstructure:"access_expires_in"`
|
||||
RefreshExpiresIn int `mapstructure:"refresh_expires_in"`
|
||||
}
|
||||
|
||||
type AssetsConfig struct {
|
||||
FlagsPath string `mapstructure:"flags_path"`
|
||||
AccessSecret string `env:"USER_AUTH_ACCESS_SECRET"`
|
||||
RefreshSecret string `env:"USER_AUTH_REFRESH_SECRET"`
|
||||
AccessExpiresIn int `env:"USER_AUTH_ACCESS_EXPIRES_IN"`
|
||||
RefreshExpiresIn int `env:"USER_AUTH_REFRESH_EXPIRES_IN"`
|
||||
}
|
||||
|
||||
@@ -1,53 +0,0 @@
|
||||
server:
|
||||
host: "0.0.0.0"
|
||||
port: 8082
|
||||
timeout: 30s
|
||||
read_timeout: 10s
|
||||
write_timeout: 10s
|
||||
|
||||
database:
|
||||
mongodb:
|
||||
uri: "mongodb://admin:admin1234@localhost:27017"
|
||||
name: "tender_management"
|
||||
timeout: 10s
|
||||
max_pool_size: 100
|
||||
|
||||
cache:
|
||||
redis:
|
||||
host: "localhost"
|
||||
port: 6379
|
||||
password: "admin1234"
|
||||
db: 0
|
||||
pool_size: 10
|
||||
|
||||
user_authorization:
|
||||
jwt:
|
||||
access_secret: "your-super-secret-access-token-key-here-make-it-long-and-secure"
|
||||
refresh_secret: "your-super-secret-refresh-token-key-here-make-it-long-and-secure"
|
||||
access_expires_in: 3600 # 1 hour in seconds
|
||||
refresh_expires_in: 2592000 # 30 days in seconds
|
||||
|
||||
customer_authorization:
|
||||
jwt:
|
||||
access_secret: "your-super-secret-access-token-key-here-make-it-long-and-secure"
|
||||
refresh_secret: "your-super-secret-refresh-token-key-here-make-it-long-and-secure"
|
||||
access_expires_in: 3600 # 1 hour in seconds
|
||||
refresh_expires_in: 2592000 # 30 days in seconds
|
||||
|
||||
logging:
|
||||
level: "info"
|
||||
format: "json"
|
||||
output: "file" # stdout, stderr, file
|
||||
file:
|
||||
path: "./logs/app.log"
|
||||
max_size: 100 # Max size in MB before rotation
|
||||
max_backups: 5 # Number of backup files to keep
|
||||
max_age: 30 # Max age in days to keep log files
|
||||
compress: true # Compress rotated files
|
||||
|
||||
rate_limiting:
|
||||
requests_per_minute: 100
|
||||
burst: 20
|
||||
|
||||
assets:
|
||||
flags_path: "./assets/flags"
|
||||
Reference in New Issue
Block a user