Refactor Configuration System to Use Environment Variables
- Removed YAML configuration files for the scraper and web commands, transitioning to an environment-based configuration system. - Updated configuration structs to utilize `env` tags for loading values from `.env` files, enhancing security and flexibility. - Implemented reflection-based parsing for environment variables, ensuring type-safe configuration loading. - Revised documentation to reflect the new configuration approach, including usage examples and best practices for environment variable naming conventions.
This commit is contained in:
@@ -7,44 +7,19 @@ import (
|
|||||||
|
|
||||||
// Config defines the scraper application configuration
|
// Config defines the scraper application configuration
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Database config.DatabaseConfig `mapstructure:"database"`
|
Database config.DatabaseConfig
|
||||||
Logging config.LoggingConfig `mapstructure:"logging"`
|
Logging config.LoggingConfig
|
||||||
TED ScraperConfig `mapstructure:"ted"`
|
TED ScraperConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
type ScraperConfig struct {
|
type ScraperConfig struct {
|
||||||
BaseURL string `mapstructure:"base_url"`
|
BaseURL string `env:"TED_BASE_URL"`
|
||||||
Timeout time.Duration `mapstructure:"timeout"`
|
Timeout time.Duration `env:"TED_TIMEOUT"`
|
||||||
MaxRetries int `mapstructure:"max_retries"`
|
MaxRetries int `env:"TED_MAX_RETRIES"`
|
||||||
RetryDelay time.Duration `mapstructure:"retry_delay"`
|
RetryDelay time.Duration `env:"TED_RETRY_DELAY"`
|
||||||
UserAgent string `mapstructure:"user_agent"`
|
UserAgent string `env:"TED_USER_AGENT"`
|
||||||
MaxConcurrency int `mapstructure:"max_concurrency"`
|
MaxConcurrency int `env:"TED_MAX_CONCURRENCY"`
|
||||||
DownloadDir string `mapstructure:"download_dir"`
|
DownloadDir string `env:"TED_DOWNLOAD_DIR"`
|
||||||
CleanupAfter time.Duration `mapstructure:"cleanup_after"`
|
CleanupAfter time.Duration `env:"TED_CLEANUP_AFTER"`
|
||||||
ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
|
ScrapingInterval time.Duration `env:"TED_SCRAPING_INTERVAL"`
|
||||||
}
|
|
||||||
|
|
||||||
// LoadConfig loads the configuration from file
|
|
||||||
func LoadConfig(path string) (*Config, error) {
|
|
||||||
cfg := &Config{
|
|
||||||
// Set default values
|
|
||||||
TED: ScraperConfig{
|
|
||||||
BaseURL: "https://ted.europa.eu/packages/daily",
|
|
||||||
Timeout: 30 * time.Second,
|
|
||||||
MaxRetries: 3,
|
|
||||||
RetryDelay: 5 * time.Second,
|
|
||||||
UserAgent: "TenderManagement-TED-Scraper/1.0",
|
|
||||||
MaxConcurrency: 3,
|
|
||||||
DownloadDir: "/tmp/ted_downloads",
|
|
||||||
CleanupAfter: 24 * time.Hour,
|
|
||||||
ScrapingInterval: 12 * time.Hour,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := config.LoadConfig(path, cfg)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return result, nil
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,28 +0,0 @@
|
|||||||
database:
|
|
||||||
mongodb:
|
|
||||||
uri: "mongodb://localhost:27017"
|
|
||||||
name: "tender_management"
|
|
||||||
timeout: 10s
|
|
||||||
max_pool_size: 50
|
|
||||||
|
|
||||||
logging:
|
|
||||||
level: "info"
|
|
||||||
format: "json"
|
|
||||||
output: "file" # stdout, stderr, file
|
|
||||||
file:
|
|
||||||
path: "./logs/scraper.log"
|
|
||||||
max_size: 100 # Max size in MB before rotation
|
|
||||||
max_backups: 5 # Number of backup files to keep
|
|
||||||
max_age: 30 # Max age in days to keep log files
|
|
||||||
compress: true # Compress rotated files
|
|
||||||
|
|
||||||
ted:
|
|
||||||
base_url: "https://ted.europa.eu/packages/daily"
|
|
||||||
timeout: 300s # Increased to 5 minutes for large files
|
|
||||||
max_retries: 5
|
|
||||||
retry_delay: 10s
|
|
||||||
user_agent: "TenderManagement-TED-Scraper/1.0"
|
|
||||||
max_concurrency: 2 # Reduced to avoid overwhelming the server
|
|
||||||
download_dir: "./tmp/ted_downloads"
|
|
||||||
cleanup_after: 72h
|
|
||||||
scraping_interval: 12h # Run scraping every 12 hours
|
|
||||||
+13
-17
@@ -6,27 +6,23 @@ import (
|
|||||||
|
|
||||||
// Config holds configuration for the web command
|
// Config holds configuration for the web command
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Server config.ServerConfig `mapstructure:"server"`
|
Server config.ServerConfig
|
||||||
Database config.DatabaseConfig `mapstructure:"database"`
|
Database config.DatabaseConfig
|
||||||
Cache config.CacheConfig `mapstructure:"cache"`
|
Cache config.CacheConfig
|
||||||
Logging config.LoggingConfig `mapstructure:"logging"`
|
Logging config.LoggingConfig
|
||||||
RateLimit config.RateLimitConfig `mapstructure:"rate_limiting"`
|
RateLimit config.RateLimitConfig
|
||||||
UserAuth AuthConfig `mapstructure:"user_authorization"`
|
Assets config.AssetsConfig
|
||||||
CustomerAuth AuthConfig `mapstructure:"customer_authorization"`
|
UserAuth AuthConfig
|
||||||
Assets AssetsConfig `mapstructure:"assets"`
|
CustomerAuth AuthConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
type AuthConfig struct {
|
type AuthConfig struct {
|
||||||
JWT JWTConfig `mapstructure:"jwt"`
|
JWT JWTConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
type JWTConfig struct {
|
type JWTConfig struct {
|
||||||
AccessSecret string `mapstructure:"access_secret"`
|
AccessSecret string `env:"USER_AUTH_ACCESS_SECRET"`
|
||||||
RefreshSecret string `mapstructure:"refresh_secret"`
|
RefreshSecret string `env:"USER_AUTH_REFRESH_SECRET"`
|
||||||
AccessExpiresIn int `mapstructure:"access_expires_in"`
|
AccessExpiresIn int `env:"USER_AUTH_ACCESS_EXPIRES_IN"`
|
||||||
RefreshExpiresIn int `mapstructure:"refresh_expires_in"`
|
RefreshExpiresIn int `env:"USER_AUTH_REFRESH_EXPIRES_IN"`
|
||||||
}
|
|
||||||
|
|
||||||
type AssetsConfig struct {
|
|
||||||
FlagsPath string `mapstructure:"flags_path"`
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,53 +0,0 @@
|
|||||||
server:
|
|
||||||
host: "0.0.0.0"
|
|
||||||
port: 8082
|
|
||||||
timeout: 30s
|
|
||||||
read_timeout: 10s
|
|
||||||
write_timeout: 10s
|
|
||||||
|
|
||||||
database:
|
|
||||||
mongodb:
|
|
||||||
uri: "mongodb://admin:admin1234@localhost:27017"
|
|
||||||
name: "tender_management"
|
|
||||||
timeout: 10s
|
|
||||||
max_pool_size: 100
|
|
||||||
|
|
||||||
cache:
|
|
||||||
redis:
|
|
||||||
host: "localhost"
|
|
||||||
port: 6379
|
|
||||||
password: "admin1234"
|
|
||||||
db: 0
|
|
||||||
pool_size: 10
|
|
||||||
|
|
||||||
user_authorization:
|
|
||||||
jwt:
|
|
||||||
access_secret: "your-super-secret-access-token-key-here-make-it-long-and-secure"
|
|
||||||
refresh_secret: "your-super-secret-refresh-token-key-here-make-it-long-and-secure"
|
|
||||||
access_expires_in: 3600 # 1 hour in seconds
|
|
||||||
refresh_expires_in: 2592000 # 30 days in seconds
|
|
||||||
|
|
||||||
customer_authorization:
|
|
||||||
jwt:
|
|
||||||
access_secret: "your-super-secret-access-token-key-here-make-it-long-and-secure"
|
|
||||||
refresh_secret: "your-super-secret-refresh-token-key-here-make-it-long-and-secure"
|
|
||||||
access_expires_in: 3600 # 1 hour in seconds
|
|
||||||
refresh_expires_in: 2592000 # 30 days in seconds
|
|
||||||
|
|
||||||
logging:
|
|
||||||
level: "info"
|
|
||||||
format: "json"
|
|
||||||
output: "file" # stdout, stderr, file
|
|
||||||
file:
|
|
||||||
path: "./logs/app.log"
|
|
||||||
max_size: 100 # Max size in MB before rotation
|
|
||||||
max_backups: 5 # Number of backup files to keep
|
|
||||||
max_age: 30 # Max age in days to keep log files
|
|
||||||
compress: true # Compress rotated files
|
|
||||||
|
|
||||||
rate_limiting:
|
|
||||||
requests_per_minute: 100
|
|
||||||
burst: 20
|
|
||||||
|
|
||||||
assets:
|
|
||||||
flags_path: "./assets/flags"
|
|
||||||
+145
-107
@@ -1,21 +1,22 @@
|
|||||||
# Modular Configuration System
|
# Environment-Based Configuration System
|
||||||
|
|
||||||
This package provides a modular configuration system for the Tender Management backend. It allows each command (web, scraper, worker, etc.) to have its own specific configuration while sharing common base configurations.
|
This package provides a configuration system for the Tender Management backend that loads configuration exclusively from `.env` files. It maintains the same structure as the previous YAML-based system while using environment variables for all configuration values.
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
### Base Configuration (`BaseConfig`)
|
### Base Configuration (`BaseConfig`)
|
||||||
|
|
||||||
The `BaseConfig` struct contains common configuration fields that are used across all commands:
|
The configuration system provides common configuration structs that are used across all commands:
|
||||||
|
|
||||||
- **Server**: HTTP server configuration (host, port, timeouts)
|
- **ServerConfig**: HTTP server configuration (host, port, timeouts)
|
||||||
- **Database**: MongoDB connection configuration
|
- **DatabaseConfig**: MongoDB connection configuration
|
||||||
- **Cache**: Redis connection configuration
|
- **CacheConfig**: Redis connection configuration
|
||||||
- **Logging**: Logging system configuration
|
- **LoggingConfig**: Logging system configuration
|
||||||
|
- **RateLimitConfig**: Rate limiting configuration
|
||||||
|
|
||||||
### Command-Specific Configuration
|
### Command-Specific Configuration
|
||||||
|
|
||||||
Each command can extend the `BaseConfig` by embedding it and adding its own specific configuration fields.
|
Each command can extend the base configuration by embedding the common configs and adding their own specific configuration fields.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
@@ -33,14 +34,17 @@ import (
|
|||||||
|
|
||||||
// Config holds configuration for your command
|
// Config holds configuration for your command
|
||||||
type Config struct {
|
type Config struct {
|
||||||
config.BaseConfig
|
config.ServerConfig
|
||||||
YourSpecific YourSpecificConfig `mapstructure:"your_specific"`
|
config.DatabaseConfig
|
||||||
|
config.CacheConfig
|
||||||
|
config.LoggingConfig
|
||||||
|
YourSpecific YourSpecificConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
type YourSpecificConfig struct {
|
type YourSpecificConfig struct {
|
||||||
SomeField string `mapstructure:"some_field"`
|
SomeField string `env:"SOME_FIELD"`
|
||||||
SomeTimeout time.Duration `mapstructure:"some_timeout"`
|
SomeTimeout time.Duration `env:"SOME_TIMEOUT"`
|
||||||
SomeNumber int `mapstructure:"some_number"`
|
SomeNumber int `env:"SOME_NUMBER"`
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -59,50 +63,45 @@ func initConfig() Config {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
### 3. Create YAML Configuration File
|
### 3. Create .env Configuration File
|
||||||
|
|
||||||
Create a `config.yaml` file in your command directory:
|
Create a `.env` file in your command directory:
|
||||||
|
|
||||||
```yaml
|
```env
|
||||||
# Base configuration (common to all commands)
|
# Server Configuration
|
||||||
server:
|
SERVER_HOST=localhost
|
||||||
host: "localhost"
|
SERVER_PORT=8080
|
||||||
port: 8080
|
SERVER_TIMEOUT=30s
|
||||||
timeout: 30s
|
SERVER_READ_TIMEOUT=10s
|
||||||
read_timeout: 10s
|
SERVER_WRITE_TIMEOUT=10s
|
||||||
write_timeout: 10s
|
|
||||||
|
|
||||||
database:
|
# Database Configuration
|
||||||
mongodb:
|
MONGODB_URI=mongodb://localhost:27017
|
||||||
uri: "mongodb://localhost:27017"
|
MONGODB_NAME=your_database
|
||||||
name: "your_database"
|
MONGODB_TIMEOUT=10s
|
||||||
timeout: 10s
|
MONGODB_MAX_POOL_SIZE=100
|
||||||
max_pool_size: 100
|
|
||||||
|
|
||||||
cache:
|
# Cache Configuration
|
||||||
redis:
|
REDIS_HOST=localhost
|
||||||
host: "localhost"
|
REDIS_PORT=6379
|
||||||
port: 6379
|
REDIS_PASSWORD=
|
||||||
password: ""
|
REDIS_DB=0
|
||||||
db: 0
|
REDIS_POOL_SIZE=10
|
||||||
pool_size: 10
|
|
||||||
|
|
||||||
logging:
|
# Logging Configuration
|
||||||
level: "info"
|
LOG_LEVEL=info
|
||||||
format: "json"
|
LOG_FORMAT=json
|
||||||
output: "stdout"
|
LOG_OUTPUT=stdout
|
||||||
file:
|
LOG_FILE_PATH=/var/log/tm/your-command.log
|
||||||
path: "/var/log/tm/your-command.log"
|
LOG_FILE_MAX_SIZE=100
|
||||||
max_size: 100
|
LOG_FILE_MAX_BACKUPS=5
|
||||||
max_backups: 5
|
LOG_FILE_MAX_AGE=30
|
||||||
max_age: 30
|
LOG_FILE_COMPRESS=true
|
||||||
compress: true
|
|
||||||
|
|
||||||
# Your command-specific configuration
|
# Your command-specific configuration
|
||||||
your_specific:
|
SOME_FIELD=value
|
||||||
some_field: "value"
|
SOME_TIMEOUT=30s
|
||||||
some_timeout: 30s
|
SOME_NUMBER=42
|
||||||
some_number: 42
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
@@ -111,12 +110,29 @@ your_specific:
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
type Config struct {
|
type Config struct {
|
||||||
config.BaseConfig
|
config.ServerConfig
|
||||||
Queue QueueConfig `mapstructure:"queue"`
|
config.DatabaseConfig
|
||||||
UserAuthorization AuthConfig `mapstructure:"user_authorization"`
|
config.CacheConfig
|
||||||
CustomerAuthorization AuthConfig `mapstructure:"customer_authorization"`
|
config.LoggingConfig
|
||||||
AI AIConfig `mapstructure:"ai"`
|
config.RateLimitConfig
|
||||||
RateLimit RateLimitConfig `mapstructure:"rate_limiting"`
|
UserAuth AuthConfig
|
||||||
|
CustomerAuth AuthConfig
|
||||||
|
Assets AssetsConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
type AuthConfig struct {
|
||||||
|
JWT JWTConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
type JWTConfig struct {
|
||||||
|
AccessSecret string `env:"USER_AUTH_ACCESS_SECRET"`
|
||||||
|
RefreshSecret string `env:"USER_AUTH_REFRESH_SECRET"`
|
||||||
|
AccessExpiresIn int `env:"USER_AUTH_ACCESS_EXPIRES_IN"`
|
||||||
|
RefreshExpiresIn int `env:"USER_AUTH_REFRESH_EXPIRES_IN"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type AssetsConfig struct {
|
||||||
|
FlagsPath string `env:"ASSETS_FLAGS_PATH"`
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -124,28 +140,21 @@ type Config struct {
|
|||||||
|
|
||||||
```go
|
```go
|
||||||
type Config struct {
|
type Config struct {
|
||||||
config.BaseConfig
|
config.DatabaseConfig
|
||||||
TED TEDConfig `mapstructure:"ted"`
|
config.LoggingConfig
|
||||||
|
TED ScraperConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
type TEDConfig struct {
|
type ScraperConfig struct {
|
||||||
BaseURL string `mapstructure:"base_url"`
|
BaseURL string `env:"TED_BASE_URL"`
|
||||||
MaxRetries int `mapstructure:"max_retries"`
|
Timeout time.Duration `env:"TED_TIMEOUT"`
|
||||||
MaxConcurrency int `mapstructure:"max_concurrency"`
|
MaxRetries int `env:"TED_MAX_RETRIES"`
|
||||||
DownloadDir string `mapstructure:"download_dir"`
|
RetryDelay time.Duration `env:"TED_RETRY_DELAY"`
|
||||||
CleanupAfter time.Duration `mapstructure:"cleanup_after"`
|
UserAgent string `env:"TED_USER_AGENT"`
|
||||||
}
|
MaxConcurrency int `env:"TED_MAX_CONCURRENCY"`
|
||||||
```
|
DownloadDir string `env:"TED_DOWNLOAD_DIR"`
|
||||||
|
CleanupAfter time.Duration `env:"TED_CLEANUP_AFTER"`
|
||||||
### Worker Command Config
|
ScrapingInterval time.Duration `env:"TED_SCRAPING_INTERVAL"`
|
||||||
|
|
||||||
```go
|
|
||||||
type Config struct {
|
|
||||||
config.BaseConfig
|
|
||||||
Queue QueueConfig `mapstructure:"queue"`
|
|
||||||
AI AIConfig `mapstructure:"ai"`
|
|
||||||
Worker WorkerConfig `mapstructure:"worker"`
|
|
||||||
Scraping ScrapingConfig `mapstructure:"scraping"`
|
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -157,58 +166,87 @@ The configuration loader uses Go generics to provide type-safe configuration loa
|
|||||||
|
|
||||||
### Environment Variable Support
|
### Environment Variable Support
|
||||||
|
|
||||||
The configuration system automatically supports environment variable overrides via Viper's `AutomaticEnv()` feature.
|
The configuration system loads all values from `.env` files using the `github.com/subosito/gotenv` library.
|
||||||
|
|
||||||
### YAML Support
|
### Reflection-Based Parsing
|
||||||
|
|
||||||
Configuration files use YAML format for easy readability and maintenance.
|
The system uses reflection to automatically parse environment variables into struct fields based on the `env` tags.
|
||||||
|
|
||||||
### Validation
|
### Supported Types
|
||||||
|
|
||||||
Each command can implement its own validation logic after loading the configuration.
|
The configuration system supports the following Go types:
|
||||||
|
- `string`
|
||||||
|
- `int`, `int8`, `int16`, `int32`, `int64`
|
||||||
|
- `bool`
|
||||||
|
- `float32`, `float64`
|
||||||
|
- `time.Duration` (parsed from strings like "30s", "1h", "5m")
|
||||||
|
|
||||||
## Best Practices
|
## Best Practices
|
||||||
|
|
||||||
1. **Keep BaseConfig minimal**: Only add fields to `BaseConfig` that are truly common across all commands.
|
1. **Use descriptive environment variable names**: Choose clear, descriptive names for your environment variables (e.g., `MONGODB_URI` instead of `DB_URI`).
|
||||||
|
|
||||||
2. **Use descriptive names**: Choose clear, descriptive names for your configuration structs and fields.
|
2. **Use consistent naming conventions**: Use uppercase with underscores for environment variable names (e.g., `SERVER_HOST`, `LOG_LEVEL`).
|
||||||
|
|
||||||
3. **Document your config**: Add comments to your configuration structs explaining what each field does.
|
3. **Group related variables**: Use prefixes to group related configuration (e.g., `MONGODB_*`, `REDIS_*`, `LOG_*`).
|
||||||
|
|
||||||
4. **Use appropriate types**: Use `time.Duration` for durations, proper numeric types for numbers, etc.
|
4. **Document your configuration**: Add comments in your `.env` files explaining what each variable does.
|
||||||
|
|
||||||
5. **Provide defaults**: Consider providing sensible defaults in your YAML files.
|
5. **Use appropriate types**: Use `time.Duration` for durations, proper numeric types for numbers, etc.
|
||||||
|
|
||||||
6. **Environment variables**: Use environment variables for sensitive data like API keys and passwords.
|
6. **Provide sensible defaults**: Consider providing sensible defaults in your code when environment variables are not set.
|
||||||
|
|
||||||
## Migration from Old System
|
## Migration from YAML System
|
||||||
|
|
||||||
If you're migrating from the old `infra.Config` system:
|
If you're migrating from the old YAML-based configuration system:
|
||||||
|
|
||||||
1. Remove imports of `tm/infra`
|
1. Remove `mapstructure` tags from your config structs
|
||||||
2. Add imports of `tm/pkg/config`
|
2. Add `env` tags to your config struct fields
|
||||||
3. Change your config struct to embed `config.BaseConfig`
|
3. Create `.env` files instead of `config.yaml` files
|
||||||
4. Update your `LoadConfig` call to use the new generic function
|
4. Update your `LoadConfig` calls to use the new system
|
||||||
5. Update function signatures to use the new config types
|
5. Update function signatures to use the new config types
|
||||||
|
|
||||||
Example migration:
|
Example migration:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
// Old way
|
// Old way (YAML)
|
||||||
func initConfig() infra.Config {
|
type Config struct {
|
||||||
config, err := infra.LoadConfig(".")
|
Server config.ServerConfig `mapstructure:"server"`
|
||||||
if err != nil {
|
Database config.DatabaseConfig `mapstructure:"database"`
|
||||||
panic(fmt.Sprintf("Failed to load config: %v", err))
|
|
||||||
}
|
|
||||||
return *config
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// New way
|
// New way (.env)
|
||||||
func initConfig() Config {
|
type Config struct {
|
||||||
conf, err := config.LoadConfig(".", &Config{})
|
Server config.ServerConfig
|
||||||
if err != nil {
|
Database config.DatabaseConfig
|
||||||
panic(fmt.Sprintf("Failed to load config: %v", err))
|
|
||||||
}
|
|
||||||
return *conf
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Old way (YAML loading)
|
||||||
|
conf, err := config.LoadConfig(".", &Config{})
|
||||||
|
|
||||||
|
// New way (.env loading)
|
||||||
|
conf, err := config.LoadConfig(".", &Config{})
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Environment Variable Naming Convention
|
||||||
|
|
||||||
|
The system follows these naming conventions:
|
||||||
|
|
||||||
|
- **Server**: `SERVER_*` (e.g., `SERVER_HOST`, `SERVER_PORT`)
|
||||||
|
- **Database**: `MONGODB_*` (e.g., `MONGODB_URI`, `MONGODB_NAME`)
|
||||||
|
- **Cache**: `REDIS_*` (e.g., `REDIS_HOST`, `REDIS_PORT`)
|
||||||
|
- **Logging**: `LOG_*` (e.g., `LOG_LEVEL`, `LOG_FORMAT`)
|
||||||
|
- **Rate Limiting**: `RATE_LIMIT_*` (e.g., `RATE_LIMIT_REQUESTS_PER_MINUTE`)
|
||||||
|
|
||||||
|
## Error Handling
|
||||||
|
|
||||||
|
The configuration system provides detailed error messages when:
|
||||||
|
- The `.env` file cannot be loaded
|
||||||
|
- Environment variables cannot be parsed into the expected types
|
||||||
|
- Required fields are missing (handled by your application logic)
|
||||||
|
|
||||||
|
## Security Considerations
|
||||||
|
|
||||||
|
1. **Never commit `.env` files**: Add `.env` to your `.gitignore` file
|
||||||
|
2. **Use environment-specific files**: Create different `.env` files for different environments (`.env.local`, `.env.production`, etc.)
|
||||||
|
3. **Validate sensitive values**: Always validate sensitive configuration values like API keys and database URIs
|
||||||
|
4. **Use secure defaults**: Provide secure default values for security-related configuration
|
||||||
|
|||||||
+177
-41
@@ -1,79 +1,215 @@
|
|||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/subosito/gotenv"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ServerConfig struct {
|
type ServerConfig struct {
|
||||||
Host string `mapstructure:"host"`
|
Host string `env:"SERVER_HOST"`
|
||||||
Port int `mapstructure:"port"`
|
Port int `env:"SERVER_PORT"`
|
||||||
Timeout time.Duration `mapstructure:"timeout"`
|
Timeout time.Duration `env:"SERVER_TIMEOUT"`
|
||||||
ReadTimeout time.Duration `mapstructure:"read_timeout"`
|
ReadTimeout time.Duration `env:"SERVER_READ_TIMEOUT"`
|
||||||
WriteTimeout time.Duration `mapstructure:"write_timeout"`
|
WriteTimeout time.Duration `env:"SERVER_WRITE_TIMEOUT"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type DatabaseConfig struct {
|
type DatabaseConfig struct {
|
||||||
MongoDB MongoConfig `mapstructure:"mongodb"`
|
MongoDB MongoConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
type MongoConfig struct {
|
type MongoConfig struct {
|
||||||
URI string `mapstructure:"uri"`
|
URI string `env:"MONGODB_URI"`
|
||||||
Name string `mapstructure:"name"`
|
Name string `env:"MONGODB_NAME"`
|
||||||
Timeout time.Duration `mapstructure:"timeout"`
|
Timeout time.Duration `env:"MONGODB_TIMEOUT"`
|
||||||
MaxPoolSize int `mapstructure:"max_pool_size"`
|
MaxPoolSize int `env:"MONGODB_MAX_POOL_SIZE"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type CacheConfig struct {
|
type CacheConfig struct {
|
||||||
Redis RedisConfig `mapstructure:"redis"`
|
Redis RedisConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
type RedisConfig struct {
|
type RedisConfig struct {
|
||||||
Host string `mapstructure:"host"`
|
Host string `env:"REDIS_HOST"`
|
||||||
Port int `mapstructure:"port"`
|
Port int `env:"REDIS_PORT"`
|
||||||
Password string `mapstructure:"password"`
|
Password string `env:"REDIS_PASSWORD"`
|
||||||
DB int `mapstructure:"db"`
|
DB int `env:"REDIS_DB"`
|
||||||
PoolSize int `mapstructure:"pool_size"`
|
PoolSize int `env:"REDIS_POOL_SIZE"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type LoggingConfig struct {
|
type LoggingConfig struct {
|
||||||
Level string `mapstructure:"level"`
|
Level string `env:"LOG_LEVEL"`
|
||||||
Format string `mapstructure:"format"`
|
Format string `env:"LOG_FORMAT"`
|
||||||
Output string `mapstructure:"output"`
|
Output string `env:"LOG_OUTPUT"`
|
||||||
File LogFileConfig `mapstructure:"file"`
|
File LogFileConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
type LogFileConfig struct {
|
type LogFileConfig struct {
|
||||||
Path string `mapstructure:"path"`
|
Path string `env:"LOG_FILE_PATH"`
|
||||||
MaxSize int `mapstructure:"max_size"` // MB
|
MaxSize int `env:"LOG_FILE_MAX_SIZE"`
|
||||||
MaxBackups int `mapstructure:"max_backups"`
|
MaxBackups int `env:"LOG_FILE_MAX_BACKUPS"`
|
||||||
MaxAge int `mapstructure:"max_age"` // days
|
MaxAge int `env:"LOG_FILE_MAX_AGE"`
|
||||||
Compress bool `mapstructure:"compress"`
|
Compress bool `env:"LOG_FILE_COMPRESS"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type AssetsConfig struct {
|
||||||
|
FlagsPath string `env:"ASSETS_FLAGS_PATH"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type RateLimitConfig struct {
|
type RateLimitConfig struct {
|
||||||
RequestsPerMinute int `mapstructure:"requests_per_minute"`
|
RequestsPerMinute int `env:"RATE_LIMIT_REQUESTS_PER_MINUTE"`
|
||||||
Burst int `mapstructure:"burst"`
|
Burst int `env:"RATE_LIMIT_BURST"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// LoadConfig loads configuration from file and environment variables
|
// LoadConfig loads configuration from .env file
|
||||||
func LoadConfig[T any](path string, config T) (T, error) {
|
func LoadConfig[T any](path string, config T) (T, error) {
|
||||||
viper.SetConfigName("config")
|
// Load .env file
|
||||||
viper.SetConfigType("yaml")
|
envPath := fmt.Sprintf("%s/.env", path)
|
||||||
viper.AddConfigPath(path)
|
if err := gotenv.Load(envPath); err != nil {
|
||||||
|
return config, fmt.Errorf("failed to load .env file from %s: %w", envPath, err)
|
||||||
// Enable reading from environment variables
|
|
||||||
viper.AutomaticEnv()
|
|
||||||
|
|
||||||
// Read configuration file
|
|
||||||
if err := viper.ReadInConfig(); err != nil {
|
|
||||||
return config, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := viper.Unmarshal(config); err != nil {
|
// Parse configuration using reflection
|
||||||
return config, err
|
if err := parseConfig(config); err != nil {
|
||||||
|
return config, fmt.Errorf("failed to parse configuration: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return config, nil
|
return config, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseConfig uses reflection to parse environment variables into the config struct
|
||||||
|
func parseConfig(config interface{}) error {
|
||||||
|
v := reflect.ValueOf(config)
|
||||||
|
if v.Kind() != reflect.Ptr {
|
||||||
|
return fmt.Errorf("config must be a pointer")
|
||||||
|
}
|
||||||
|
|
||||||
|
v = v.Elem()
|
||||||
|
if v.Kind() != reflect.Struct {
|
||||||
|
return fmt.Errorf("config must be a struct")
|
||||||
|
}
|
||||||
|
|
||||||
|
return parseStruct(v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseStruct recursively parses struct fields
|
||||||
|
func parseStruct(v reflect.Value) error {
|
||||||
|
t := v.Type()
|
||||||
|
|
||||||
|
for i := 0; i < v.NumField(); i++ {
|
||||||
|
field := v.Field(i)
|
||||||
|
fieldType := t.Field(i)
|
||||||
|
|
||||||
|
// Skip unexported fields
|
||||||
|
if !field.CanSet() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get env tag
|
||||||
|
envTag := fieldType.Tag.Get("env")
|
||||||
|
if envTag == "" {
|
||||||
|
// If no env tag, try to parse nested struct
|
||||||
|
if field.Kind() == reflect.Struct {
|
||||||
|
if err := parseStruct(field); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get environment variable value
|
||||||
|
envValue := os.Getenv(envTag)
|
||||||
|
if envValue == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse and set the field value
|
||||||
|
if err := setFieldValue(field, envValue); err != nil {
|
||||||
|
return fmt.Errorf("failed to set field %s: %w", fieldType.Name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// setFieldValue sets the field value from environment variable string
|
||||||
|
func setFieldValue(field reflect.Value, value string) error {
|
||||||
|
switch field.Kind() {
|
||||||
|
case reflect.String:
|
||||||
|
field.SetString(value)
|
||||||
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||||
|
if field.Type() == reflect.TypeOf(time.Duration(0)) {
|
||||||
|
// Handle time.Duration
|
||||||
|
duration, err := time.ParseDuration(value)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid duration format: %s", value)
|
||||||
|
}
|
||||||
|
field.Set(reflect.ValueOf(duration))
|
||||||
|
} else {
|
||||||
|
// Handle regular integers
|
||||||
|
intValue, err := strconv.ParseInt(value, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid integer format: %s", value)
|
||||||
|
}
|
||||||
|
field.SetInt(intValue)
|
||||||
|
}
|
||||||
|
case reflect.Bool:
|
||||||
|
boolValue, err := strconv.ParseBool(value)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid boolean format: %s", value)
|
||||||
|
}
|
||||||
|
field.SetBool(boolValue)
|
||||||
|
case reflect.Float32, reflect.Float64:
|
||||||
|
floatValue, err := strconv.ParseFloat(value, 64)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("invalid float format: %s", value)
|
||||||
|
}
|
||||||
|
field.SetFloat(floatValue)
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("unsupported field type: %s", field.Kind())
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to get environment variable with default value
|
||||||
|
func getEnv(key, defaultValue string) string {
|
||||||
|
if value := os.Getenv(key); value != "" {
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to get environment variable as int with default value
|
||||||
|
func getEnvAsInt(key string, defaultValue int) int {
|
||||||
|
if value := os.Getenv(key); value != "" {
|
||||||
|
if intValue, err := strconv.Atoi(value); err == nil {
|
||||||
|
return intValue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to get environment variable as duration with default value
|
||||||
|
func getEnvAsDuration(key string, defaultValue time.Duration) time.Duration {
|
||||||
|
if value := os.Getenv(key); value != "" {
|
||||||
|
if duration, err := time.ParseDuration(value); err == nil {
|
||||||
|
return duration
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper function to get environment variable as bool with default value
|
||||||
|
func getEnvAsBool(key string, defaultValue bool) bool {
|
||||||
|
if value := os.Getenv(key); value != "" {
|
||||||
|
if boolValue, err := strconv.ParseBool(value); err == nil {
|
||||||
|
return boolValue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return defaultValue
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user