n.nakhostin
45d2e84fd7
- Added a new Dockerfile for the scraper application located at cmd/scraper/Dockerfile to streamline the build process. - Updated the CI configuration in .drone.yml to correctly build the scraper application with the new Dockerfile. - Refactored the main application entry point to utilize a new bootstrap package for configuration, logging, and MongoDB initialization. - Enhanced the scraper's configuration management by defining a dedicated Config struct for better organization and maintainability.
51 lines
1.5 KiB
Go
51 lines
1.5 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"time"
|
|
"tm/pkg/config"
|
|
)
|
|
|
|
// Config defines the scraper application configuration
|
|
type Config struct {
|
|
Database config.DatabaseConfig `mapstructure:"database"`
|
|
Logging config.LoggingConfig `mapstructure:"logging"`
|
|
TED ScraperConfig `mapstructure:"ted"`
|
|
}
|
|
|
|
type ScraperConfig struct {
|
|
BaseURL string `mapstructure:"base_url"`
|
|
Timeout time.Duration `mapstructure:"timeout"`
|
|
MaxRetries int `mapstructure:"max_retries"`
|
|
RetryDelay time.Duration `mapstructure:"retry_delay"`
|
|
UserAgent string `mapstructure:"user_agent"`
|
|
MaxConcurrency int `mapstructure:"max_concurrency"`
|
|
DownloadDir string `mapstructure:"download_dir"`
|
|
CleanupAfter time.Duration `mapstructure:"cleanup_after"`
|
|
ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
|
|
}
|
|
|
|
// LoadConfig loads the configuration from file
|
|
func LoadConfig(path string) (*Config, error) {
|
|
cfg := &Config{
|
|
// Set default values
|
|
TED: ScraperConfig{
|
|
BaseURL: "https://ted.europa.eu/packages/daily",
|
|
Timeout: 30 * time.Second,
|
|
MaxRetries: 3,
|
|
RetryDelay: 5 * time.Second,
|
|
UserAgent: "TenderManagement-TED-Scraper/1.0",
|
|
MaxConcurrency: 3,
|
|
DownloadDir: "/tmp/ted_downloads",
|
|
CleanupAfter: 24 * time.Hour,
|
|
ScrapingInterval: 12 * time.Hour,
|
|
},
|
|
}
|
|
|
|
result, err := config.LoadConfig(path, cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return result, nil
|
|
}
|