diff --git a/cmd/scraper/bootstrap/bootstrap.go b/cmd/scraper/bootstrap/bootstrap.go index 1af1ed0..1d57a97 100644 --- a/cmd/scraper/bootstrap/bootstrap.go +++ b/cmd/scraper/bootstrap/bootstrap.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "os" - "os/signal" - "syscall" "time" "tm/internal/notice" "tm/pkg/config" @@ -76,7 +74,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa } // init TED scraper -func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) { +func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) *schedule.CronScheduler { // Initialize tender repository noticeRepo := notice.NewRepository(mongoManager, appLogger) @@ -100,57 +98,39 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog notify, ) - // start TED scraper job - schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ + // start TED scraper job with error handling + scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo) + err := scheduler.AddJob(schedule.Job{ Name: "TED Scraper Job", Func: func() { - _ = tedScraper.Run(context.Background(), nil, nil) + ctx := context.Background() + err := tedScraper.Run(ctx, nil, nil) + if err != nil { + appLogger.Error("Scheduled scraper run failed", map[string]interface{}{ + "error": err.Error(), + }) + // Don't exit - continue running for next scheduled execution + } else { + appLogger.Info("Scheduled scraper run completed successfully", map[string]interface{}{}) + } }, Expr: config.TED.ScrapingInterval, }) - - // Create context with cancellation - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // Set up signal handling for graceful shutdown - signalChan := make(chan os.Signal, 1) - signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) - - // Start the scraper in a goroutine - scraperDone := make(chan error, 1) - go func() { - appLogger.Info("Starting TED scraper", map[string]interface{}{ - "interval": config.TED.ScrapingInterval, + if err != nil { + appLogger.Error("Failed to schedule TED scraper job", map[string]interface{}{ + "error": err.Error(), }) - scraperDone <- tedScraper.Run(ctx, nil, nil) - }() - - // Wait for shutdown signal or scraper completion - select { - case <-signalChan: - appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{}) - cancel() - - // Wait for scraper to finish gracefully with timeout - shutdownTimeout := time.NewTimer(30 * time.Second) - select { - case <-scraperDone: - appLogger.Info("Scraper stopped gracefully", map[string]interface{}{}) - case <-shutdownTimeout.C: - appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{}) - } - - case err := <-scraperDone: - - if err != nil { - appLogger.Error("Scraper finished with error", map[string]interface{}{ - "error": err.Error(), - }) - os.Exit(1) - } - appLogger.Info("Scraper finished successfully", map[string]interface{}{}) + return nil } + + // Start the scheduler + scheduler.Start() + + appLogger.Info("TED scraper scheduled and started", map[string]interface{}{ + "interval": config.TED.ScrapingInterval, + }) + + return scheduler } // Init Notification Service diff --git a/cmd/scraper/main.go b/cmd/scraper/main.go index 3242069..940f540 100644 --- a/cmd/scraper/main.go +++ b/cmd/scraper/main.go @@ -73,7 +73,11 @@ func main() { } // Initialize TED scraper for scheduled mode - bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService) + scheduler := bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService) + if scheduler == nil { + appLogger.Error("Failed to initialize TED scraper scheduler", map[string]interface{}{}) + os.Exit(1) + } // Set up signal handling for graceful shutdown signalChan := make(chan os.Signal, 1) @@ -83,5 +87,10 @@ func main() { <-signalChan appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{}) + // Stop scheduler gracefully + if scheduler != nil { + scheduler.Stop() + } + appLogger.Info("TED scraper application stopped", map[string]interface{}{}) }