n.nakhostin
05c7eae8a2
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
55 lines
1.4 KiB
Go
55 lines
1.4 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
"tm/cmd/scraper/bootstrap"
|
|
)
|
|
|
|
func main() {
|
|
// Load configuration
|
|
config, err := bootstrap.InitConfig()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
// Initialize logger
|
|
appLogger := bootstrap.InitLogger(config.Logging)
|
|
appLogger.Info("Starting TED scraper application", map[string]interface{}{
|
|
"version": "1.0.0",
|
|
})
|
|
|
|
// Initialize notification service
|
|
|
|
// Initialize MongoDB connection
|
|
mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
|
|
defer func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := mongoManager.Close(ctx); err != nil {
|
|
appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
}
|
|
}()
|
|
|
|
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
|
|
bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService)
|
|
|
|
// Set up signal handling for graceful shutdown
|
|
signalChan := make(chan os.Signal, 1)
|
|
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
|
|
|
// Wait for shutdown signal
|
|
<-signalChan
|
|
appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{})
|
|
|
|
// Stop the scheduler
|
|
// scheduler.Stop()
|
|
|
|
appLogger.Info("TED scraper application stopped", map[string]interface{}{})
|
|
}
|