From b7fa012d132585333be09b0563e80c4ec6141d41 Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Wed, 13 Aug 2025 12:51:36 +0330 Subject: [PATCH] Refactor configuration management by migrating to a modular config system - Removed the old `infra.Config` structure and replaced it with a new modular configuration system in `pkg/config`. - Introduced a `Config` struct in `cmd/web/config.go` to hold command-specific configurations. - Updated configuration loading functions to utilize the new `LoadConfig` method with generics for type safety. - Adjusted various initialization functions in `cmd/web/bootstrap.go` and `cmd/web/main.go` to reflect the new configuration structure. - Cleaned up the `config.yaml` by removing obsolete fields and ensuring it aligns with the new configuration schema. --- cmd/web/bootstrap.go | 18 ++-- cmd/web/config.go | 27 ++++++ cmd/web/config.yaml | 28 ------ cmd/web/main.go | 4 +- infra/config.go | 144 ----------------------------- pkg/config/README.md | 214 +++++++++++++++++++++++++++++++++++++++++++ pkg/config/config.go | 79 ++++++++++++++++ 7 files changed, 331 insertions(+), 183 deletions(-) create mode 100644 cmd/web/config.go delete mode 100644 infra/config.go create mode 100644 pkg/config/README.md create mode 100644 pkg/config/config.go diff --git a/cmd/web/bootstrap.go b/cmd/web/bootstrap.go index 2756073..7457359 100644 --- a/cmd/web/bootstrap.go +++ b/cmd/web/bootstrap.go @@ -4,8 +4,8 @@ import ( "fmt" "net/http" "time" - "tm/infra" "tm/pkg/authorization" + "tm/pkg/config" "tm/pkg/logger" "tm/pkg/mongo" "tm/pkg/redis" @@ -16,17 +16,17 @@ import ( ) // Init Application Configuration -func initConfig() infra.Config { - config, err := infra.LoadConfig(".") +func initConfig() Config { + conf, err := config.LoadConfig(".", &Config{}) if err != nil { panic(fmt.Sprintf("Failed to load config: %v", err)) } - return *config + return *conf } // Init Logger Service -func initLogger(conf infra.LoggingConfig) logger.Logger { +func initLogger(conf config.LoggingConfig) logger.Logger { return logger.NewLogger(&logger.Config{ Level: conf.Level, Format: conf.Format, @@ -42,7 +42,7 @@ func initLogger(conf infra.LoggingConfig) logger.Logger { } // Init MongoDB Connection Manager -func initMongoDB(conf infra.MongoConfig, log logger.Logger) *mongo.ConnectionManager { +func initMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager { // Convert infra.MongoConfig to mongo.ConnectionConfig connectionConfig := mongo.ConnectionConfig{ URI: conf.URI, @@ -75,7 +75,7 @@ func initMongoDB(conf infra.MongoConfig, log logger.Logger) *mongo.ConnectionMan } // initHTTPServer initializes the Echo HTTP server with middleware -func initHTTPServer(conf infra.Config, log logger.Logger) *echo.Echo { +func initHTTPServer(conf Config, log logger.Logger) *echo.Echo { e := echo.New() // Configure Echo @@ -129,7 +129,7 @@ func initHTTPServer(conf infra.Config, log logger.Logger) *echo.Echo { } // Init Redis Connection Manager -func initRedis(conf infra.RedisConfig, log logger.Logger) redis.Client { +func initRedis(conf config.RedisConfig, log logger.Logger) redis.Client { connectionConfig := &redis.Config{ Host: conf.Host, Port: conf.Port, @@ -158,7 +158,7 @@ func initRedis(conf infra.RedisConfig, log logger.Logger) redis.Client { } // Init Authorization Service -func initAuthorizationService(conf infra.JWTConfig, redisClient redis.Client, log logger.Logger) authorization.AuthorizationService { +func initAuthorizationService(conf JWTConfig, redisClient redis.Client, log logger.Logger) authorization.AuthorizationService { authConfig := &authorization.AuthorizationConfig{ AccessTokenSecret: conf.AccessSecret, RefreshTokenSecret: conf.RefreshSecret, diff --git a/cmd/web/config.go b/cmd/web/config.go new file mode 100644 index 0000000..6e9ac52 --- /dev/null +++ b/cmd/web/config.go @@ -0,0 +1,27 @@ +package main + +import ( + "tm/pkg/config" +) + +// Config holds configuration for the web command +type Config struct { + Server config.ServerConfig `mapstructure:"server"` + Database config.DatabaseConfig `mapstructure:"database"` + Cache config.CacheConfig `mapstructure:"cache"` + Logging config.LoggingConfig `mapstructure:"logging"` + RateLimit config.RateLimitConfig `mapstructure:"rate_limiting"` + UserAuth AuthConfig `mapstructure:"user_authorization"` + CustomerAuth AuthConfig `mapstructure:"customer_authorization"` +} + +type AuthConfig struct { + JWT JWTConfig `mapstructure:"jwt"` +} + +type JWTConfig struct { + AccessSecret string `mapstructure:"access_secret"` + RefreshSecret string `mapstructure:"refresh_secret"` + AccessExpiresIn int `mapstructure:"access_expires_in"` + RefreshExpiresIn int `mapstructure:"refresh_expires_in"` +} diff --git a/cmd/web/config.yaml b/cmd/web/config.yaml index 94646fa..ae435a3 100644 --- a/cmd/web/config.yaml +++ b/cmd/web/config.yaml @@ -20,22 +20,6 @@ cache: db: 0 pool_size: 10 -queue: - rabbitmq: - url: "amqp://guest:guest@localhost:5672/" - exchange: "tender_exchange" - queues: - scraping: "scraping_queue" - processing: "processing_queue" - notifications: "notifications_queue" - -search: - elasticsearch: - urls: ["http://localhost:9200"] - username: "" - password: "" - index_prefix: "tender_" - user_authorization: jwt: access_secret: "your-super-secret-access-token-key-here-make-it-long-and-secure" @@ -50,18 +34,6 @@ customer_authorization: access_expires_in: 3600 # 1 hour in seconds refresh_expires_in: 2592000 # 30 days in seconds -ai: - openai: - api_key: "" - model: "gpt-3.5-turbo" - max_tokens: 1000 - -scraping: - user_agent: "TenderBot/1.0" - timeout: 30s - max_retries: 3 - delay_between_requests: 1s - logging: level: "info" format: "json" diff --git a/cmd/web/main.go b/cmd/web/main.go index cf3a2b6..e091018 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -80,8 +80,8 @@ func main() { }() // Initialize authorization service - userAuthService := initAuthorizationService(conf.UserAuthorization.JWT, redisClient, logger) - customerAuthService := initAuthorizationService(conf.CustomerAuthorization.JWT, redisClient, logger) + userAuthService := initAuthorizationService(conf.UserAuth.JWT, redisClient, logger) + customerAuthService := initAuthorizationService(conf.CustomerAuth.JWT, redisClient, logger) // Initialize repositories with MongoDB connection manager userRepository := user.NewUserRepository(mongoManager, logger) diff --git a/infra/config.go b/infra/config.go deleted file mode 100644 index f8a0c93..0000000 --- a/infra/config.go +++ /dev/null @@ -1,144 +0,0 @@ -package infra - -import ( - "time" - - "github.com/spf13/viper" -) - -// Config holds all configuration for the application -type Config struct { - Server ServerConfig `mapstructure:"server"` - Database DatabaseConfig `mapstructure:"database"` - Cache CacheConfig `mapstructure:"cache"` - Queue QueueConfig `mapstructure:"queue"` - Search SearchConfig `mapstructure:"search"` - UserAuthorization AuthConfig `mapstructure:"user_authorization"` - CustomerAuthorization AuthConfig `mapstructure:"customer_authorization"` - AI AIConfig `mapstructure:"ai"` - Scraping ScrapingConfig `mapstructure:"scraping"` - Logging LoggingConfig `mapstructure:"logging"` - RateLimit RateLimitConfig `mapstructure:"rate_limiting"` -} - -type ServerConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - Timeout time.Duration `mapstructure:"timeout"` - ReadTimeout time.Duration `mapstructure:"read_timeout"` - WriteTimeout time.Duration `mapstructure:"write_timeout"` -} - -type DatabaseConfig struct { - MongoDB MongoConfig `mapstructure:"mongodb"` -} - -type MongoConfig struct { - URI string `mapstructure:"uri"` - Name string `mapstructure:"name"` - Timeout time.Duration `mapstructure:"timeout"` - MaxPoolSize int `mapstructure:"max_pool_size"` -} - -type CacheConfig struct { - Redis RedisConfig `mapstructure:"redis"` -} - -type RedisConfig struct { - Host string `mapstructure:"host"` - Port int `mapstructure:"port"` - Password string `mapstructure:"password"` - DB int `mapstructure:"db"` - PoolSize int `mapstructure:"pool_size"` -} - -type QueueConfig struct { - RabbitMQ RabbitMQConfig `mapstructure:"rabbitmq"` -} - -type RabbitMQConfig struct { - URL string `mapstructure:"url"` - Exchange string `mapstructure:"exchange"` - Queues map[string]string `mapstructure:"queues"` -} - -type SearchConfig struct { - Elasticsearch ElasticsearchConfig `mapstructure:"elasticsearch"` -} - -type ElasticsearchConfig struct { - URLs []string `mapstructure:"urls"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password"` - IndexPrefix string `mapstructure:"index_prefix"` -} - -type AuthConfig struct { - JWT JWTConfig `mapstructure:"jwt"` -} - -type JWTConfig struct { - AccessSecret string `mapstructure:"access_secret"` - RefreshSecret string `mapstructure:"refresh_secret"` - AccessExpiresIn int `mapstructure:"access_expires_in"` - RefreshExpiresIn int `mapstructure:"refresh_expires_in"` -} - -type AIConfig struct { - OpenAI OpenAIConfig `mapstructure:"openai"` -} - -type OpenAIConfig struct { - APIKey string `mapstructure:"api_key"` - Model string `mapstructure:"model"` - MaxTokens int `mapstructure:"max_tokens"` -} - -type ScrapingConfig struct { - UserAgent string `mapstructure:"user_agent"` - Timeout time.Duration `mapstructure:"timeout"` - MaxRetries int `mapstructure:"max_retries"` - DelayBetweenRequests time.Duration `mapstructure:"delay_between_requests"` -} - -type LoggingConfig struct { - Level string `mapstructure:"level"` - Format string `mapstructure:"format"` - Output string `mapstructure:"output"` - File LogFileConfig `mapstructure:"file"` -} - -type LogFileConfig struct { - Path string `mapstructure:"path"` - MaxSize int `mapstructure:"max_size"` // MB - MaxBackups int `mapstructure:"max_backups"` - MaxAge int `mapstructure:"max_age"` // days - Compress bool `mapstructure:"compress"` -} - -type RateLimitConfig struct { - RequestsPerMinute int `mapstructure:"requests_per_minute"` - Burst int `mapstructure:"burst"` -} - -// LoadConfig loads configuration from file and environment variables -func LoadConfig(path string) (*Config, error) { - viper.SetConfigName("config") - viper.SetConfigType("yaml") - viper.AddConfigPath(path) - - // Enable reading from environment variables - viper.AutomaticEnv() - - // Read configuration file - if err := viper.ReadInConfig(); err != nil { - return nil, err - } - - var config Config - if err := viper.Unmarshal(&config); err != nil { - return nil, err - } - - return &config, nil -} diff --git a/pkg/config/README.md b/pkg/config/README.md new file mode 100644 index 0000000..733fdf3 --- /dev/null +++ b/pkg/config/README.md @@ -0,0 +1,214 @@ +# Modular Configuration System + +This package provides a modular configuration system for the Tender Management backend. It allows each command (web, scraper, worker, etc.) to have its own specific configuration while sharing common base configurations. + +## Architecture + +### Base Configuration (`BaseConfig`) + +The `BaseConfig` struct contains common configuration fields that are used across all commands: + +- **Server**: HTTP server configuration (host, port, timeouts) +- **Database**: MongoDB connection configuration +- **Cache**: Redis connection configuration +- **Logging**: Logging system configuration + +### Command-Specific Configuration + +Each command can extend the `BaseConfig` by embedding it and adding its own specific configuration fields. + +## Usage + +### 1. Create Command-Specific Config + +Create a config file for your command (e.g., `cmd/yourcommand/config.go`): + +```go +package main + +import ( + "time" + "tm/pkg/config" +) + +// Config holds configuration for your command +type Config struct { + config.BaseConfig + YourSpecific YourSpecificConfig `mapstructure:"your_specific"` +} + +type YourSpecificConfig struct { + SomeField string `mapstructure:"some_field"` + SomeTimeout time.Duration `mapstructure:"some_timeout"` + SomeNumber int `mapstructure:"some_number"` +} +``` + +### 2. Load Configuration + +In your command's `main.go` or bootstrap file: + +```go +func initConfig() Config { + conf, err := config.LoadConfig(".", &Config{}) + if err != nil { + panic(fmt.Sprintf("Failed to load config: %v", err)) + } + + return *conf +} +``` + +### 3. Create YAML Configuration File + +Create a `config.yaml` file in your command directory: + +```yaml +# Base configuration (common to all commands) +server: + host: "localhost" + port: 8080 + timeout: 30s + read_timeout: 10s + write_timeout: 10s + +database: + mongodb: + uri: "mongodb://localhost:27017" + name: "your_database" + timeout: 10s + max_pool_size: 100 + +cache: + redis: + host: "localhost" + port: 6379 + password: "" + db: 0 + pool_size: 10 + +logging: + level: "info" + format: "json" + output: "stdout" + file: + path: "/var/log/tm/your-command.log" + max_size: 100 + max_backups: 5 + max_age: 30 + compress: true + +# Your command-specific configuration +your_specific: + some_field: "value" + some_timeout: 30s + some_number: 42 +``` + +## Examples + +### Web Command Config + +```go +type Config struct { + config.BaseConfig + Queue QueueConfig `mapstructure:"queue"` + UserAuthorization AuthConfig `mapstructure:"user_authorization"` + CustomerAuthorization AuthConfig `mapstructure:"customer_authorization"` + AI AIConfig `mapstructure:"ai"` + RateLimit RateLimitConfig `mapstructure:"rate_limiting"` +} +``` + +### Scraper Command Config + +```go +type Config struct { + config.BaseConfig + TED TEDConfig `mapstructure:"ted"` +} + +type TEDConfig struct { + BaseURL string `mapstructure:"base_url"` + MaxRetries int `mapstructure:"max_retries"` + MaxConcurrency int `mapstructure:"max_concurrency"` + DownloadDir string `mapstructure:"download_dir"` + CleanupAfter time.Duration `mapstructure:"cleanup_after"` +} +``` + +### Worker Command Config + +```go +type Config struct { + config.BaseConfig + Queue QueueConfig `mapstructure:"queue"` + AI AIConfig `mapstructure:"ai"` + Worker WorkerConfig `mapstructure:"worker"` + Scraping ScrapingConfig `mapstructure:"scraping"` +} +``` + +## Features + +### Type Safety + +The configuration loader uses Go generics to provide type-safe configuration loading. The function signature ensures that you get back exactly the type you expect. + +### Environment Variable Support + +The configuration system automatically supports environment variable overrides via Viper's `AutomaticEnv()` feature. + +### YAML Support + +Configuration files use YAML format for easy readability and maintenance. + +### Validation + +Each command can implement its own validation logic after loading the configuration. + +## Best Practices + +1. **Keep BaseConfig minimal**: Only add fields to `BaseConfig` that are truly common across all commands. + +2. **Use descriptive names**: Choose clear, descriptive names for your configuration structs and fields. + +3. **Document your config**: Add comments to your configuration structs explaining what each field does. + +4. **Use appropriate types**: Use `time.Duration` for durations, proper numeric types for numbers, etc. + +5. **Provide defaults**: Consider providing sensible defaults in your YAML files. + +6. **Environment variables**: Use environment variables for sensitive data like API keys and passwords. + +## Migration from Old System + +If you're migrating from the old `infra.Config` system: + +1. Remove imports of `tm/infra` +2. Add imports of `tm/pkg/config` +3. Change your config struct to embed `config.BaseConfig` +4. Update your `LoadConfig` call to use the new generic function +5. Update function signatures to use the new config types + +Example migration: + +```go +// Old way +func initConfig() infra.Config { + config, err := infra.LoadConfig(".") + if err != nil { + panic(fmt.Sprintf("Failed to load config: %v", err)) + } + return *config +} + +// New way +func initConfig() Config { + conf, err := config.LoadConfig(".", &Config{}) + if err != nil { + panic(fmt.Sprintf("Failed to load config: %v", err)) + } + return *conf +} +``` \ No newline at end of file diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..248cd08 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,79 @@ +package config + +import ( + "time" + + "github.com/spf13/viper" +) + +type ServerConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Timeout time.Duration `mapstructure:"timeout"` + ReadTimeout time.Duration `mapstructure:"read_timeout"` + WriteTimeout time.Duration `mapstructure:"write_timeout"` +} + +type DatabaseConfig struct { + MongoDB MongoConfig `mapstructure:"mongodb"` +} + +type MongoConfig struct { + URI string `mapstructure:"uri"` + Name string `mapstructure:"name"` + Timeout time.Duration `mapstructure:"timeout"` + MaxPoolSize int `mapstructure:"max_pool_size"` +} + +type CacheConfig struct { + Redis RedisConfig `mapstructure:"redis"` +} + +type RedisConfig struct { + Host string `mapstructure:"host"` + Port int `mapstructure:"port"` + Password string `mapstructure:"password"` + DB int `mapstructure:"db"` + PoolSize int `mapstructure:"pool_size"` +} + +type LoggingConfig struct { + Level string `mapstructure:"level"` + Format string `mapstructure:"format"` + Output string `mapstructure:"output"` + File LogFileConfig `mapstructure:"file"` +} + +type LogFileConfig struct { + Path string `mapstructure:"path"` + MaxSize int `mapstructure:"max_size"` // MB + MaxBackups int `mapstructure:"max_backups"` + MaxAge int `mapstructure:"max_age"` // days + Compress bool `mapstructure:"compress"` +} + +type RateLimitConfig struct { + RequestsPerMinute int `mapstructure:"requests_per_minute"` + Burst int `mapstructure:"burst"` +} + +// LoadConfig loads configuration from file and environment variables +func LoadConfig[T any](path string, config T) (T, error) { + viper.SetConfigName("config") + viper.SetConfigType("yaml") + viper.AddConfigPath(path) + + // Enable reading from environment variables + viper.AutomaticEnv() + + // Read configuration file + if err := viper.ReadInConfig(); err != nil { + return config, err + } + + if err := viper.Unmarshal(config); err != nil { + return config, err + } + + return config, nil +}