mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor configuration management by migrating to a modular config system

Browse Source 
- Removed the old `infra.Config` structure and replaced it with a new modular configuration system in `pkg/config`.
- Introduced a `Config` struct in `cmd/web/config.go` to hold command-specific configurations.
- Updated configuration loading functions to utilize the new `LoadConfig` method with generics for type safety.
- Adjusted various initialization functions in `cmd/web/bootstrap.go` and `cmd/web/main.go` to reflect the new configuration structure.
- Cleaned up the `config.yaml` by removing obsolete fields and ensuring it aligns with the new configuration schema.
This commit is contained in:
n.nakhostin
2025-08-13 12:51:36 +03:30
parent e3a32e151f
commit b7fa012d13
7 changed files with 331 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/web/bootstrap.go
+9 -9
View File
@@ -4,8 +4,8 @@ import (
"fmt"
"net/http"
"time"
"tm/infra"
"tm/pkg/authorization"
"tm/pkg/config"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/redis"
@@ -16,17 +16,17 @@ import (
)
// Init Application Configuration
func initConfig() infra.Config {
config, err := infra.LoadConfig(".")
func initConfig() Config {
conf, err := config.LoadConfig(".", &Config{})
if err != nil {
panic(fmt.Sprintf("Failed to load config: %v", err))
}
return *config
return *conf
}
// Init Logger Service
func initLogger(conf infra.LoggingConfig) logger.Logger {
func initLogger(conf config.LoggingConfig) logger.Logger {
return logger.NewLogger(&logger.Config{
Level: conf.Level,
Format: conf.Format,
@@ -42,7 +42,7 @@ func initLogger(conf infra.LoggingConfig) logger.Logger {
}
// Init MongoDB Connection Manager
func initMongoDB(conf infra.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
func initMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
// Convert infra.MongoConfig to mongo.ConnectionConfig
connectionConfig := mongo.ConnectionConfig{
URI: conf.URI,
@@ -75,7 +75,7 @@ func initMongoDB(conf infra.MongoConfig, log logger.Logger) *mongo.ConnectionMan
}
// initHTTPServer initializes the Echo HTTP server with middleware
func initHTTPServer(conf infra.Config, log logger.Logger) *echo.Echo {
func initHTTPServer(conf Config, log logger.Logger) *echo.Echo {
e := echo.New()
// Configure Echo
@@ -129,7 +129,7 @@ func initHTTPServer(conf infra.Config, log logger.Logger) *echo.Echo {
}
// Init Redis Connection Manager
func initRedis(conf infra.RedisConfig, log logger.Logger) redis.Client {
func initRedis(conf config.RedisConfig, log logger.Logger) redis.Client {
connectionConfig := &redis.Config{
Host: conf.Host,
Port: conf.Port,
@@ -158,7 +158,7 @@ func initRedis(conf infra.RedisConfig, log logger.Logger) redis.Client {
}
// Init Authorization Service
func initAuthorizationService(conf infra.JWTConfig, redisClient redis.Client, log logger.Logger) authorization.AuthorizationService {
func initAuthorizationService(conf JWTConfig, redisClient redis.Client, log logger.Logger) authorization.AuthorizationService {
authConfig := &authorization.AuthorizationConfig{
AccessTokenSecret: conf.AccessSecret,
RefreshTokenSecret: conf.RefreshSecret,
cmd/web/config.go
+27
View File
@@ -0,0 +1,27 @@
package main
import (
"tm/pkg/config"
)
// Config holds configuration for the web command
type Config struct {
Server config.ServerConfig `mapstructure:"server"`
Database config.DatabaseConfig `mapstructure:"database"`
Cache config.CacheConfig `mapstructure:"cache"`
Logging config.LoggingConfig `mapstructure:"logging"`
RateLimit config.RateLimitConfig `mapstructure:"rate_limiting"`
UserAuth AuthConfig `mapstructure:"user_authorization"`
CustomerAuth AuthConfig `mapstructure:"customer_authorization"`
}
type AuthConfig struct {
JWT JWTConfig `mapstructure:"jwt"`
}
type JWTConfig struct {
AccessSecret string `mapstructure:"access_secret"`
RefreshSecret string `mapstructure:"refresh_secret"`
AccessExpiresIn int `mapstructure:"access_expires_in"`
RefreshExpiresIn int `mapstructure:"refresh_expires_in"`
}
cmd/web/config.yaml
-28
View File
@@ -20,22 +20,6 @@ cache:
db: 0
pool_size: 10
queue:
rabbitmq:
url: "amqp://guest:guest@localhost:5672/"
exchange: "tender_exchange"
queues:
scraping: "scraping_queue"
processing: "processing_queue"
notifications: "notifications_queue"
search:
elasticsearch:
urls: ["http://localhost:9200"]
username: ""
password: ""
index_prefix: "tender_"
user_authorization:
jwt:
access_secret: "your-super-secret-access-token-key-here-make-it-long-and-secure"
@@ -50,18 +34,6 @@ customer_authorization:
access_expires_in: 3600 # 1 hour in seconds
refresh_expires_in: 2592000 # 30 days in seconds
ai:
openai:
api_key: ""
model: "gpt-3.5-turbo"
max_tokens: 1000
scraping:
user_agent: "TenderBot/1.0"
timeout: 30s
max_retries: 3
delay_between_requests: 1s
logging:
level: "info"
format: "json"
cmd/web/main.go
+2 -2
View File
@@ -80,8 +80,8 @@ func main() {
}()
// Initialize authorization service
userAuthService := initAuthorizationService(conf.UserAuthorization.JWT, redisClient, logger)
customerAuthService := initAuthorizationService(conf.CustomerAuthorization.JWT, redisClient, logger)
userAuthService := initAuthorizationService(conf.UserAuth.JWT, redisClient, logger)
customerAuthService := initAuthorizationService(conf.CustomerAuth.JWT, redisClient, logger)
// Initialize repositories with MongoDB connection manager
userRepository := user.NewUserRepository(mongoManager, logger)
infra/config.go
-144
View File
@@ -1,144 +0,0 @@
package infra
import (
"time"
"github.com/spf13/viper"
)
// Config holds all configuration for the application
type Config struct {
Server ServerConfig `mapstructure:"server"`
Database DatabaseConfig `mapstructure:"database"`
Cache CacheConfig `mapstructure:"cache"`
Queue QueueConfig `mapstructure:"queue"`
Search SearchConfig `mapstructure:"search"`
UserAuthorization AuthConfig `mapstructure:"user_authorization"`
CustomerAuthorization AuthConfig `mapstructure:"customer_authorization"`
AI AIConfig `mapstructure:"ai"`
Scraping ScrapingConfig `mapstructure:"scraping"`
Logging LoggingConfig `mapstructure:"logging"`
RateLimit RateLimitConfig `mapstructure:"rate_limiting"`
}
type ServerConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
Timeout time.Duration `mapstructure:"timeout"`
ReadTimeout time.Duration `mapstructure:"read_timeout"`
WriteTimeout time.Duration `mapstructure:"write_timeout"`
}
type DatabaseConfig struct {
MongoDB MongoConfig `mapstructure:"mongodb"`
}
type MongoConfig struct {
URI string `mapstructure:"uri"`
Name string `mapstructure:"name"`
Timeout time.Duration `mapstructure:"timeout"`
MaxPoolSize int `mapstructure:"max_pool_size"`
}
type CacheConfig struct {
Redis RedisConfig `mapstructure:"redis"`
}
type RedisConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
Password string `mapstructure:"password"`
DB int `mapstructure:"db"`
PoolSize int `mapstructure:"pool_size"`
}
type QueueConfig struct {
RabbitMQ RabbitMQConfig `mapstructure:"rabbitmq"`
}
type RabbitMQConfig struct {
URL string `mapstructure:"url"`
Exchange string `mapstructure:"exchange"`
Queues map[string]string `mapstructure:"queues"`
}
type SearchConfig struct {
Elasticsearch ElasticsearchConfig `mapstructure:"elasticsearch"`
}
type ElasticsearchConfig struct {
URLs []string `mapstructure:"urls"`
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
IndexPrefix string `mapstructure:"index_prefix"`
}
type AuthConfig struct {
JWT JWTConfig `mapstructure:"jwt"`
}
type JWTConfig struct {
AccessSecret string `mapstructure:"access_secret"`
RefreshSecret string `mapstructure:"refresh_secret"`
AccessExpiresIn int `mapstructure:"access_expires_in"`
RefreshExpiresIn int `mapstructure:"refresh_expires_in"`
}
type AIConfig struct {
OpenAI OpenAIConfig `mapstructure:"openai"`
}
type OpenAIConfig struct {
APIKey string `mapstructure:"api_key"`
Model string `mapstructure:"model"`
MaxTokens int `mapstructure:"max_tokens"`
}
type ScrapingConfig struct {
UserAgent string `mapstructure:"user_agent"`
Timeout time.Duration `mapstructure:"timeout"`
MaxRetries int `mapstructure:"max_retries"`
DelayBetweenRequests time.Duration `mapstructure:"delay_between_requests"`
}
type LoggingConfig struct {
Level string `mapstructure:"level"`
Format string `mapstructure:"format"`
Output string `mapstructure:"output"`
File LogFileConfig `mapstructure:"file"`
}
type LogFileConfig struct {
Path string `mapstructure:"path"`
MaxSize int `mapstructure:"max_size"` // MB
MaxBackups int `mapstructure:"max_backups"`
MaxAge int `mapstructure:"max_age"` // days
Compress bool `mapstructure:"compress"`
}
type RateLimitConfig struct {
RequestsPerMinute int `mapstructure:"requests_per_minute"`
Burst int `mapstructure:"burst"`
}
// LoadConfig loads configuration from file and environment variables
func LoadConfig(path string) (*Config, error) {
viper.SetConfigName("config")
viper.SetConfigType("yaml")
viper.AddConfigPath(path)
// Enable reading from environment variables
viper.AutomaticEnv()
// Read configuration file
if err := viper.ReadInConfig(); err != nil {
return nil, err
}
var config Config
if err := viper.Unmarshal(&config); err != nil {
return nil, err
}
return &config, nil
}
pkg/config/README.md
+214
View File
@@ -0,0 +1,214 @@
# Modular Configuration System
This package provides a modular configuration system for the Tender Management backend. It allows each command (web, scraper, worker, etc.) to have its own specific configuration while sharing common base configurations.
## Architecture
### Base Configuration (`BaseConfig`)
The `BaseConfig` struct contains common configuration fields that are used across all commands:
- **Server**: HTTP server configuration (host, port, timeouts)
- **Database**: MongoDB connection configuration
- **Cache**: Redis connection configuration
- **Logging**: Logging system configuration
### Command-Specific Configuration
Each command can extend the `BaseConfig` by embedding it and adding its own specific configuration fields.
## Usage
### 1. Create Command-Specific Config
Create a config file for your command (e.g., `cmd/yourcommand/config.go`):
```go
package main
import (
"time"
"tm/pkg/config"
)
// Config holds configuration for your command
type Config struct {
config.BaseConfig
YourSpecific YourSpecificConfig `mapstructure:"your_specific"`
}
type YourSpecificConfig struct {
SomeField string `mapstructure:"some_field"`
SomeTimeout time.Duration `mapstructure:"some_timeout"`
SomeNumber int `mapstructure:"some_number"`
}
```
### 2. Load Configuration
In your command's `main.go` or bootstrap file:
```go
func initConfig() Config {
conf, err := config.LoadConfig(".", &Config{})
if err != nil {
panic(fmt.Sprintf("Failed to load config: %v", err))
}
return *conf
}
```
### 3. Create YAML Configuration File
Create a `config.yaml` file in your command directory:
```yaml
# Base configuration (common to all commands)
server:
host: "localhost"
port: 8080
timeout: 30s
read_timeout: 10s
write_timeout: 10s
database:
mongodb:
uri: "mongodb://localhost:27017"
name: "your_database"
timeout: 10s
max_pool_size: 100
cache:
redis:
host: "localhost"
port: 6379
password: ""
db: 0
pool_size: 10
logging:
level: "info"
format: "json"
output: "stdout"
file:
path: "/var/log/tm/your-command.log"
max_size: 100
max_backups: 5
max_age: 30
compress: true
# Your command-specific configuration
your_specific:
some_field: "value"
some_timeout: 30s
some_number: 42
```
## Examples
### Web Command Config
```go
type Config struct {
config.BaseConfig
Queue QueueConfig `mapstructure:"queue"`
UserAuthorization AuthConfig `mapstructure:"user_authorization"`
CustomerAuthorization AuthConfig `mapstructure:"customer_authorization"`
AI AIConfig `mapstructure:"ai"`
RateLimit RateLimitConfig `mapstructure:"rate_limiting"`
}
```
### Scraper Command Config
```go
type Config struct {
config.BaseConfig
TED TEDConfig `mapstructure:"ted"`
}
type TEDConfig struct {
BaseURL string `mapstructure:"base_url"`
MaxRetries int `mapstructure:"max_retries"`
MaxConcurrency int `mapstructure:"max_concurrency"`
DownloadDir string `mapstructure:"download_dir"`
CleanupAfter time.Duration `mapstructure:"cleanup_after"`
}
```
### Worker Command Config
```go
type Config struct {
config.BaseConfig
Queue QueueConfig `mapstructure:"queue"`
AI AIConfig `mapstructure:"ai"`
Worker WorkerConfig `mapstructure:"worker"`
Scraping ScrapingConfig `mapstructure:"scraping"`
}
```
## Features
### Type Safety
The configuration loader uses Go generics to provide type-safe configuration loading. The function signature ensures that you get back exactly the type you expect.
### Environment Variable Support
The configuration system automatically supports environment variable overrides via Viper's `AutomaticEnv()` feature.
### YAML Support
Configuration files use YAML format for easy readability and maintenance.
### Validation
Each command can implement its own validation logic after loading the configuration.
## Best Practices
1. **Keep BaseConfig minimal**: Only add fields to `BaseConfig` that are truly common across all commands.
2. **Use descriptive names**: Choose clear, descriptive names for your configuration structs and fields.
3. **Document your config**: Add comments to your configuration structs explaining what each field does.
4. **Use appropriate types**: Use `time.Duration` for durations, proper numeric types for numbers, etc.
5. **Provide defaults**: Consider providing sensible defaults in your YAML files.
6. **Environment variables**: Use environment variables for sensitive data like API keys and passwords.
## Migration from Old System
If you're migrating from the old `infra.Config` system:
1. Remove imports of `tm/infra`
2. Add imports of `tm/pkg/config`
3. Change your config struct to embed `config.BaseConfig`
4. Update your `LoadConfig` call to use the new generic function
5. Update function signatures to use the new config types
Example migration:
```go
// Old way
func initConfig() infra.Config {
config, err := infra.LoadConfig(".")
if err != nil {
panic(fmt.Sprintf("Failed to load config: %v", err))
}
return *config
}
// New way
func initConfig() Config {
conf, err := config.LoadConfig(".", &Config{})
if err != nil {
panic(fmt.Sprintf("Failed to load config: %v", err))
}
return *conf
}
```
pkg/config/config.go
+79
View File
@@ -0,0 +1,79 @@
package config
import (
"time"
"github.com/spf13/viper"
)
type ServerConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
Timeout time.Duration `mapstructure:"timeout"`
ReadTimeout time.Duration `mapstructure:"read_timeout"`
WriteTimeout time.Duration `mapstructure:"write_timeout"`
}
type DatabaseConfig struct {
MongoDB MongoConfig `mapstructure:"mongodb"`
}
type MongoConfig struct {
URI string `mapstructure:"uri"`
Name string `mapstructure:"name"`
Timeout time.Duration `mapstructure:"timeout"`
MaxPoolSize int `mapstructure:"max_pool_size"`
}
type CacheConfig struct {
Redis RedisConfig `mapstructure:"redis"`
}
type RedisConfig struct {
Host string `mapstructure:"host"`
Port int `mapstructure:"port"`
Password string `mapstructure:"password"`
DB int `mapstructure:"db"`
PoolSize int `mapstructure:"pool_size"`
}
type LoggingConfig struct {
Level string `mapstructure:"level"`
Format string `mapstructure:"format"`
Output string `mapstructure:"output"`
File LogFileConfig `mapstructure:"file"`
}
type LogFileConfig struct {
Path string `mapstructure:"path"`
MaxSize int `mapstructure:"max_size"` // MB
MaxBackups int `mapstructure:"max_backups"`
MaxAge int `mapstructure:"max_age"` // days
Compress bool `mapstructure:"compress"`
}
type RateLimitConfig struct {
RequestsPerMinute int `mapstructure:"requests_per_minute"`
Burst int `mapstructure:"burst"`
}
// LoadConfig loads configuration from file and environment variables
func LoadConfig[T any](path string, config T) (T, error) {
viper.SetConfigName("config")
viper.SetConfigType("yaml")
viper.AddConfigPath(path)
// Enable reading from environment variables
viper.AutomaticEnv()
// Read configuration file
if err := viper.ReadInConfig(); err != nil {
return config, err
}
if err := viper.Unmarshal(config); err != nil {
return config, err
}
return config, nil
}