Implement TED Scraper and Configuration Management
- Introduced a new TED scraper in `cmd/scraper` to handle downloading and parsing TED XML files. - Added configuration management for the scraper, including a new `Config` struct and YAML configuration file. - Created necessary handler, service, and repository layers for tender management, adhering to Clean Architecture principles. - Implemented comprehensive logging and error handling throughout the scraper functionality. - Updated API routes to include tender management operations, enhancing the overall system capabilities.
This commit is contained in:
n.nakhostin
@@ -0,0 +1,140 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
"tm/internal/tender"
|
||||
"tm/pkg/config"
|
||||
"tm/pkg/logger"
|
||||
"tm/pkg/mongo"
|
||||
"tm/ted"
|
||||
)
|
||||
|
||||
// Init Application Configuration
|
||||
func initConfig() *Config {
|
||||
conf, err := config.LoadConfig(".", &Config{})
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("Failed to load config: %v", err))
|
||||
}
|
||||
|
||||
return conf
|
||||
}
|
||||
|
||||
// Init Logger Service
|
||||
func initLogger(conf config.LoggingConfig) logger.Logger {
|
||||
return logger.NewLogger(&logger.Config{
|
||||
Level: conf.Level,
|
||||
Format: conf.Format,
|
||||
Output: conf.Output,
|
||||
File: logger.FileConfig{
|
||||
Path: conf.File.Path,
|
||||
MaxSize: conf.File.MaxSize,
|
||||
MaxBackups: conf.File.MaxBackups,
|
||||
MaxAge: conf.File.MaxAge,
|
||||
Compress: conf.File.Compress,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Init MongoDB Connection Manager
|
||||
func initMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
|
||||
// Convert infra.MongoConfig to mongo.ConnectionConfig
|
||||
connectionConfig := mongo.ConnectionConfig{
|
||||
URI: conf.URI,
|
||||
Database: conf.Name,
|
||||
MaxPoolSize: uint64(conf.MaxPoolSize),
|
||||
MinPoolSize: 5, // Default minimum pool size
|
||||
MaxConnIdleTime: 30 * time.Minute, // Default idle time
|
||||
ConnectTimeout: conf.Timeout,
|
||||
ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
|
||||
}
|
||||
|
||||
// Create MongoDB connection manager
|
||||
connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
|
||||
if err != nil {
|
||||
log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"uri": conf.URI,
|
||||
"db": conf.Name,
|
||||
})
|
||||
panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
|
||||
}
|
||||
|
||||
log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
|
||||
"database": conf.Name,
|
||||
"uri": conf.URI,
|
||||
"pool_size": conf.MaxPoolSize,
|
||||
})
|
||||
|
||||
return connectionManager
|
||||
}
|
||||
|
||||
// init TED scraper
|
||||
func initTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger) {
|
||||
// Initialize tender repository
|
||||
tenderRepo := tender.NewTenderRepository(mongoManager, appLogger)
|
||||
|
||||
// Initialize TED scraper
|
||||
tedScraper := ted.NewTEDScraper(
|
||||
&ted.Config{
|
||||
BaseURL: config.TED.BaseURL,
|
||||
Timeout: config.TED.Timeout,
|
||||
MaxRetries: config.TED.MaxRetries,
|
||||
RetryDelay: config.TED.RetryDelay,
|
||||
UserAgent: config.TED.UserAgent,
|
||||
MaxConcurrency: config.TED.MaxConcurrency,
|
||||
DownloadDir: config.TED.DownloadDir,
|
||||
CleanupAfter: config.TED.CleanupAfter,
|
||||
ScrapingInterval: config.TED.ScrapingInterval,
|
||||
},
|
||||
appLogger,
|
||||
mongoManager,
|
||||
tenderRepo,
|
||||
)
|
||||
|
||||
// Create context with cancellation
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Set up signal handling for graceful shutdown
|
||||
signalChan := make(chan os.Signal, 1)
|
||||
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
// Start the scraper in a goroutine
|
||||
scraperDone := make(chan error, 1)
|
||||
go func() {
|
||||
appLogger.Info("Starting TED scraper", map[string]interface{}{
|
||||
"interval": config.TED.ScrapingInterval.String(),
|
||||
})
|
||||
scraperDone <- tedScraper.RunPeriodicScraping(ctx)
|
||||
}()
|
||||
|
||||
// Wait for shutdown signal or scraper completion
|
||||
select {
|
||||
case <-signalChan:
|
||||
appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{})
|
||||
cancel()
|
||||
|
||||
// Wait for scraper to finish gracefully with timeout
|
||||
shutdownTimeout := time.NewTimer(30 * time.Second)
|
||||
select {
|
||||
case <-scraperDone:
|
||||
appLogger.Info("Scraper stopped gracefully", map[string]interface{}{})
|
||||
case <-shutdownTimeout.C:
|
||||
appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{})
|
||||
}
|
||||
|
||||
case err := <-scraperDone:
|
||||
if err != nil {
|
||||
appLogger.Error("Scraper finished with error", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
os.Exit(1)
|
||||
}
|
||||
appLogger.Info("Scraper finished successfully", map[string]interface{}{})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
"tm/pkg/config"
|
||||
)
|
||||
|
||||
// Config defines the scraper application configuration
|
||||
type Config struct {
|
||||
Database config.DatabaseConfig `mapstructure:"database"`
|
||||
Logging config.LoggingConfig `mapstructure:"logging"`
|
||||
TED ScraperConfig `mapstructure:"ted"`
|
||||
}
|
||||
|
||||
type ScraperConfig struct {
|
||||
BaseURL string `mapstructure:"base_url"`
|
||||
Timeout time.Duration `mapstructure:"timeout"`
|
||||
MaxRetries int `mapstructure:"max_retries"`
|
||||
RetryDelay time.Duration `mapstructure:"retry_delay"`
|
||||
UserAgent string `mapstructure:"user_agent"`
|
||||
MaxConcurrency int `mapstructure:"max_concurrency"`
|
||||
DownloadDir string `mapstructure:"download_dir"`
|
||||
CleanupAfter time.Duration `mapstructure:"cleanup_after"`
|
||||
ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
|
||||
}
|
||||
|
||||
// LoadConfig loads the configuration from file
|
||||
func LoadConfig(path string) (*Config, error) {
|
||||
cfg := &Config{
|
||||
// Set default values
|
||||
TED: ScraperConfig{
|
||||
BaseURL: "https://ted.europa.eu/packages/daily",
|
||||
Timeout: 30 * time.Second,
|
||||
MaxRetries: 3,
|
||||
RetryDelay: 5 * time.Second,
|
||||
UserAgent: "TenderManagement-TED-Scraper/1.0",
|
||||
MaxConcurrency: 3,
|
||||
DownloadDir: "/tmp/ted_downloads",
|
||||
CleanupAfter: 24 * time.Hour,
|
||||
ScrapingInterval: 12 * time.Hour,
|
||||
},
|
||||
}
|
||||
|
||||
result, err := config.LoadConfig(path, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
database:
|
||||
mongodb:
|
||||
uri: "mongodb://localhost:27017"
|
||||
name: "tender_management"
|
||||
timeout: 10s
|
||||
max_pool_size: 50
|
||||
|
||||
logging:
|
||||
level: "info"
|
||||
format: "json"
|
||||
output: "file" # stdout, stderr, file
|
||||
file:
|
||||
path: "./logs/scraper.log"
|
||||
max_size: 100 # Max size in MB before rotation
|
||||
max_backups: 5 # Number of backup files to keep
|
||||
max_age: 30 # Max age in days to keep log files
|
||||
compress: true # Compress rotated files
|
||||
|
||||
ted:
|
||||
base_url: "https://ted.europa.eu/packages/daily"
|
||||
timeout: 300s # Increased to 5 minutes for large files
|
||||
max_retries: 5
|
||||
retry_delay: 10s
|
||||
user_agent: "TenderManagement-TED-Scraper/1.0"
|
||||
max_concurrency: 2 # Reduced to avoid overwhelming the server
|
||||
download_dir: "./tmp/ted_downloads"
|
||||
cleanup_after: 24h
|
||||
scraping_interval: 12h # Run scraping every 12 hours
|
||||
@@ -0,0 +1,37 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Load configuration
|
||||
config, err := LoadConfig(".")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load configuration: %v", err)
|
||||
}
|
||||
|
||||
// Initialize logger
|
||||
appLogger := initLogger(config.Logging)
|
||||
appLogger.Info("Starting TED scraper application", map[string]interface{}{
|
||||
"version": "1.0.0",
|
||||
})
|
||||
|
||||
// Initialize MongoDB connection
|
||||
mongoManager := initMongoDB(config.Database.MongoDB, appLogger)
|
||||
defer func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := mongoManager.Close(ctx); err != nil {
|
||||
appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}()
|
||||
|
||||
initTEDScraper(*config, mongoManager, appLogger)
|
||||
|
||||
appLogger.Info("TED scraper application stopped", map[string]interface{}{})
|
||||
}
|
||||
Reference in New Issue
Block a user