Add TED Scraper Configuration and Cron Scheduler Implementation

- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
2025-09-30 16:03:53 +03:30
parent 0c50935c42
commit 05c7eae8a2
36 changed files with 3427 additions and 311 deletions
@@ -1,31 +1,54 @@
 package main

+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+	"tm/cmd/scraper/bootstrap"
+)
+
 func main() {
-	// // Load configuration
-	// config, err := bootstrap.InitConfig()
-	// if err != nil {
-	// 	panic(err)
-	// }
+	// Load configuration
+	config, err := bootstrap.InitConfig()
+	if err != nil {
+		panic(err)
+	}

-	// // Initialize logger
-	// appLogger := bootstrap.InitLogger(config.Logging)
-	// appLogger.Info("Starting TED scraper application", map[string]interface{}{
-	// 	"version": "1.0.0",
-	// })
+	// Initialize logger
+	appLogger := bootstrap.InitLogger(config.Logging)
+	appLogger.Info("Starting TED scraper application", map[string]interface{}{
+		"version": "1.0.0",
+	})

-	// // Initialize MongoDB connection
-	// mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
-	// defer func() {
-	// 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	// 	defer cancel()
-	// 	if err := mongoManager.Close(ctx); err != nil {
-	// 		appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
-	// 			"error": err.Error(),
-	// 		})
-	// 	}
-	// }()
+	// Initialize notification service

-	// bootstrap.InitTEDScraper(*config, mongoManager, appLogger)
+	// Initialize MongoDB connection
+	mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
+	defer func() {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		if err := mongoManager.Close(ctx); err != nil {
+			appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
+				"error": err.Error(),
+			})
+		}
+	}()

-	// appLogger.Info("TED scraper application stopped", map[string]interface{}{})
+	notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
+	bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService)
+
+	// Set up signal handling for graceful shutdown
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
+
+	// Wait for shutdown signal
+	<-signalChan
+	appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{})
+
+	// Stop the scheduler
+	// scheduler.Stop()
+
+	appLogger.Info("TED scraper application stopped", map[string]interface{}{})
 }