Add TED Scraper Configuration and Cron Scheduler Implementation
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
This commit is contained in:
n.nakhostin
+167
-123
@@ -1,140 +1,184 @@
|
||||
package bootstrap
|
||||
|
||||
// import (
|
||||
// "context"
|
||||
// "fmt"
|
||||
// "os"
|
||||
// "os/signal"
|
||||
// "syscall"
|
||||
// "time"
|
||||
// "tm/internal/tender"
|
||||
// "tm/pkg/config"
|
||||
// "tm/pkg/logger"
|
||||
// "tm/pkg/mongo"
|
||||
// "tm/ted"
|
||||
// )
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
"tm/internal/tender"
|
||||
"tm/pkg/config"
|
||||
"tm/pkg/logger"
|
||||
"tm/pkg/mongo"
|
||||
"tm/pkg/notification"
|
||||
"tm/pkg/schedule"
|
||||
"tm/ted"
|
||||
)
|
||||
|
||||
// // Init Application Configuration
|
||||
// func InitConfig() (*Config, error) {
|
||||
// conf, err := config.LoadConfig(".", &Config{})
|
||||
// if err != nil {
|
||||
// return nil, fmt.Errorf("failed to load config: %v", err)
|
||||
// }
|
||||
// Init Application Configuration
|
||||
func InitConfig() (*Config, error) {
|
||||
conf, err := config.LoadConfig(".", &Config{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load config: %v", err)
|
||||
}
|
||||
|
||||
// return conf, nil
|
||||
// }
|
||||
return conf, nil
|
||||
}
|
||||
|
||||
// // Init Logger Service
|
||||
// func InitLogger(conf config.LoggingConfig) logger.Logger {
|
||||
// return logger.NewLogger(&logger.Config{
|
||||
// Level: conf.Level,
|
||||
// Format: conf.Format,
|
||||
// Output: conf.Output,
|
||||
// File: logger.FileConfig{
|
||||
// Path: conf.File.Path,
|
||||
// MaxSize: conf.File.MaxSize,
|
||||
// MaxBackups: conf.File.MaxBackups,
|
||||
// MaxAge: conf.File.MaxAge,
|
||||
// Compress: conf.File.Compress,
|
||||
// },
|
||||
// })
|
||||
// }
|
||||
// Init Logger Service
|
||||
func InitLogger(conf config.LoggingConfig) logger.Logger {
|
||||
return logger.NewLogger(&logger.Config{
|
||||
Level: conf.Level,
|
||||
Format: conf.Format,
|
||||
Output: conf.Output,
|
||||
File: logger.FileConfig{
|
||||
Path: conf.File.Path,
|
||||
MaxSize: conf.File.MaxSize,
|
||||
MaxBackups: conf.File.MaxBackups,
|
||||
MaxAge: conf.File.MaxAge,
|
||||
Compress: conf.File.Compress,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// // Init MongoDB Connection Manager
|
||||
// func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
|
||||
// // Convert infra.MongoConfig to mongo.ConnectionConfig
|
||||
// connectionConfig := mongo.ConnectionConfig{
|
||||
// URI: conf.URI,
|
||||
// Database: conf.Name,
|
||||
// MaxPoolSize: uint64(conf.MaxPoolSize),
|
||||
// MinPoolSize: 5, // Default minimum pool size
|
||||
// MaxConnIdleTime: 30 * time.Minute, // Default idle time
|
||||
// ConnectTimeout: conf.Timeout,
|
||||
// ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
|
||||
// }
|
||||
// Init MongoDB Connection Manager
|
||||
func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
|
||||
// Convert infra.MongoConfig to mongo.ConnectionConfig
|
||||
connectionConfig := mongo.ConnectionConfig{
|
||||
URI: conf.URI,
|
||||
Database: conf.Name,
|
||||
MaxPoolSize: uint64(conf.MaxPoolSize),
|
||||
MinPoolSize: 5, // Default minimum pool size
|
||||
MaxConnIdleTime: 30 * time.Minute, // Default idle time
|
||||
ConnectTimeout: conf.Timeout,
|
||||
ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
|
||||
}
|
||||
|
||||
// // Create MongoDB connection manager
|
||||
// connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
|
||||
// if err != nil {
|
||||
// log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
|
||||
// "error": err.Error(),
|
||||
// "uri": conf.URI,
|
||||
// "db": conf.Name,
|
||||
// })
|
||||
// panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
|
||||
// }
|
||||
// Create MongoDB connection manager
|
||||
connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
|
||||
if err != nil {
|
||||
log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"uri": conf.URI,
|
||||
"db": conf.Name,
|
||||
})
|
||||
panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
|
||||
}
|
||||
|
||||
// log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
|
||||
// "database": conf.Name,
|
||||
// "uri": conf.URI,
|
||||
// "pool_size": conf.MaxPoolSize,
|
||||
// })
|
||||
log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
|
||||
"database": conf.Name,
|
||||
"uri": conf.URI,
|
||||
"pool_size": conf.MaxPoolSize,
|
||||
})
|
||||
|
||||
// return connectionManager
|
||||
// }
|
||||
return connectionManager
|
||||
}
|
||||
|
||||
// // init TED scraper
|
||||
// func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger) {
|
||||
// // Initialize tender repository
|
||||
// tenderRepo := tender.NewRepository(mongoManager, appLogger)
|
||||
// init TED scraper
|
||||
func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) {
|
||||
// Initialize tender repository
|
||||
tenderRepo := tender.NewRepository(mongoManager, appLogger)
|
||||
|
||||
// // Initialize TED scraper
|
||||
// tedScraper := ted.NewTEDScraper(
|
||||
// &ted.Config{
|
||||
// BaseURL: config.TED.BaseURL,
|
||||
// Timeout: config.TED.Timeout,
|
||||
// MaxRetries: config.TED.MaxRetries,
|
||||
// RetryDelay: config.TED.RetryDelay,
|
||||
// UserAgent: config.TED.UserAgent,
|
||||
// MaxConcurrency: config.TED.MaxConcurrency,
|
||||
// DownloadDir: config.TED.DownloadDir,
|
||||
// CleanupAfter: config.TED.CleanupAfter,
|
||||
// ScrapingInterval: config.TED.ScrapingInterval,
|
||||
// },
|
||||
// appLogger,
|
||||
// mongoManager,
|
||||
// tenderRepo,
|
||||
// )
|
||||
// Initialize TED scraper
|
||||
tedScraper := ted.NewTEDScraper(
|
||||
&ted.Config{
|
||||
BaseURL: config.TED.BaseURL,
|
||||
Timeout: config.TED.Timeout,
|
||||
MaxRetries: config.TED.MaxRetries,
|
||||
RetryDelay: config.TED.RetryDelay,
|
||||
UserAgent: config.TED.UserAgent,
|
||||
MaxConcurrency: config.TED.MaxConcurrency,
|
||||
DownloadDir: config.TED.DownloadDir,
|
||||
CleanupAfter: config.TED.CleanupAfter,
|
||||
ScrapingInterval: config.TED.ScrapingInterval,
|
||||
},
|
||||
appLogger,
|
||||
mongoManager,
|
||||
tenderRepo,
|
||||
notify,
|
||||
)
|
||||
|
||||
// // Create context with cancellation
|
||||
// ctx, cancel := context.WithCancel(context.Background())
|
||||
// defer cancel()
|
||||
// start TED scraper job
|
||||
schedule.NewCronScheduler(appLogger, mongoManager, tenderRepo).AddJob(schedule.Job{
|
||||
Name: "TED Scraper Job",
|
||||
Func: func() {
|
||||
_ = tedScraper.Run(context.Background())
|
||||
},
|
||||
Expr: config.TED.ScrapingInterval,
|
||||
})
|
||||
|
||||
// // Set up signal handling for graceful shutdown
|
||||
// signalChan := make(chan os.Signal, 1)
|
||||
// signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
// Create context with cancellation
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// // Start the scraper in a goroutine
|
||||
// scraperDone := make(chan error, 1)
|
||||
// go func() {
|
||||
// appLogger.Info("Starting TED scraper", map[string]interface{}{
|
||||
// "interval": config.TED.ScrapingInterval.String(),
|
||||
// })
|
||||
// scraperDone <- tedScraper.RunPeriodicScraping(ctx)
|
||||
// }()
|
||||
// Set up signal handling for graceful shutdown
|
||||
signalChan := make(chan os.Signal, 1)
|
||||
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
// // Wait for shutdown signal or scraper completion
|
||||
// select {
|
||||
// case <-signalChan:
|
||||
// appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{})
|
||||
// cancel()
|
||||
// Start the scraper in a goroutine
|
||||
scraperDone := make(chan error, 1)
|
||||
go func() {
|
||||
appLogger.Info("Starting TED scraper", map[string]interface{}{
|
||||
"interval": config.TED.ScrapingInterval,
|
||||
})
|
||||
scraperDone <- tedScraper.Run(ctx)
|
||||
}()
|
||||
|
||||
// // Wait for scraper to finish gracefully with timeout
|
||||
// shutdownTimeout := time.NewTimer(30 * time.Second)
|
||||
// select {
|
||||
// case <-scraperDone:
|
||||
// appLogger.Info("Scraper stopped gracefully", map[string]interface{}{})
|
||||
// case <-shutdownTimeout.C:
|
||||
// appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{})
|
||||
// }
|
||||
// Wait for shutdown signal or scraper completion
|
||||
select {
|
||||
case <-signalChan:
|
||||
appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{})
|
||||
cancel()
|
||||
|
||||
// case err := <-scraperDone:
|
||||
// if err != nil {
|
||||
// appLogger.Error("Scraper finished with error", map[string]interface{}{
|
||||
// "error": err.Error(),
|
||||
// })
|
||||
// os.Exit(1)
|
||||
// }
|
||||
// appLogger.Info("Scraper finished successfully", map[string]interface{}{})
|
||||
// }
|
||||
// }
|
||||
// Wait for scraper to finish gracefully with timeout
|
||||
shutdownTimeout := time.NewTimer(30 * time.Second)
|
||||
select {
|
||||
case <-scraperDone:
|
||||
appLogger.Info("Scraper stopped gracefully", map[string]interface{}{})
|
||||
case <-shutdownTimeout.C:
|
||||
appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{})
|
||||
}
|
||||
|
||||
case err := <-scraperDone:
|
||||
|
||||
if err != nil {
|
||||
appLogger.Error("Scraper finished with error", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
os.Exit(1)
|
||||
}
|
||||
appLogger.Info("Scraper finished successfully", map[string]interface{}{})
|
||||
}
|
||||
}
|
||||
|
||||
// Init Notification Service
|
||||
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
|
||||
notificationConfig := ¬ification.Config{
|
||||
BaseURL: conf.BaseURL,
|
||||
Timeout: conf.Timeout,
|
||||
RetryAttempts: conf.RetryAttempts,
|
||||
RetryDelay: conf.RetryDelay,
|
||||
EnableLogging: conf.EnableLogging,
|
||||
UserAgent: conf.UserAgent,
|
||||
}
|
||||
|
||||
notificationSDK, err := notification.New(notificationConfig, log)
|
||||
if err != nil {
|
||||
log.Error("Failed to initialize Notification SDK", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
fmt.Printf("Failed to initialize Notification SDK: %v\n", err.Error())
|
||||
}
|
||||
|
||||
log.Info("Notification SDK initialized successfully", map[string]interface{}{
|
||||
"base_url": conf.BaseURL,
|
||||
"timeout": conf.Timeout,
|
||||
"retry_attempts": conf.RetryAttempts,
|
||||
"retry_delay": conf.RetryDelay,
|
||||
"enable_logging": conf.EnableLogging,
|
||||
"user_agent": conf.UserAgent,
|
||||
})
|
||||
|
||||
return *notificationSDK
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user