Implement Scraper Dockerfile and Refactor CI Configuration

- Added a new Dockerfile for the scraper application located at cmd/scraper/Dockerfile to streamline the build process. - Updated the CI configuration in .drone.yml to correctly build the scraper application with the new Dockerfile. - Refactored the main application entry point to utilize a new bootstrap package for configuration, logging, and MongoDB initialization. - Enhanced the scraper's configuration management by defining a dedicated Config struct for better organization and maintainability.
2025-08-31 14:23:01 +03:30
parent 511155e0a7
commit 45d2e84fd7
5 changed files with 48 additions and 26 deletions
@@ -0,0 +1,140 @@
+package bootstrap
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+	"tm/internal/tender"
+	"tm/pkg/config"
+	"tm/pkg/logger"
+	"tm/pkg/mongo"
+	"tm/ted"
+)
+
+// Init Application Configuration
+func InitConfig() (*Config, error) {
+	conf, err := config.LoadConfig(".", &Config{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to load config: %v", err)
+	}
+
+	return conf, nil
+}
+
+// Init Logger Service
+func InitLogger(conf config.LoggingConfig) logger.Logger {
+	return logger.NewLogger(&logger.Config{
+		Level:  conf.Level,
+		Format: conf.Format,
+		Output: conf.Output,
+		File: logger.FileConfig{
+			Path:       conf.File.Path,
+			MaxSize:    conf.File.MaxSize,
+			MaxBackups: conf.File.MaxBackups,
+			MaxAge:     conf.File.MaxAge,
+			Compress:   conf.File.Compress,
+		},
+	})
+}
+
+// Init MongoDB Connection Manager
+func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
+	// Convert infra.MongoConfig to mongo.ConnectionConfig
+	connectionConfig := mongo.ConnectionConfig{
+		URI:                    conf.URI,
+		Database:               conf.Name,
+		MaxPoolSize:            uint64(conf.MaxPoolSize),
+		MinPoolSize:            5,                // Default minimum pool size
+		MaxConnIdleTime:        30 * time.Minute, // Default idle time
+		ConnectTimeout:         conf.Timeout,
+		ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
+	}
+
+	// Create MongoDB connection manager
+	connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
+	if err != nil {
+		log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
+			"error": err.Error(),
+			"uri":   conf.URI,
+			"db":    conf.Name,
+		})
+		panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
+	}
+
+	log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
+		"database":  conf.Name,
+		"uri":       conf.URI,
+		"pool_size": conf.MaxPoolSize,
+	})
+
+	return connectionManager
+}
+
+// init TED scraper
+func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger) {
+	// Initialize tender repository
+	tenderRepo := tender.NewTenderRepository(mongoManager, appLogger)
+
+	// Initialize TED scraper
+	tedScraper := ted.NewTEDScraper(
+		&ted.Config{
+			BaseURL:          config.TED.BaseURL,
+			Timeout:          config.TED.Timeout,
+			MaxRetries:       config.TED.MaxRetries,
+			RetryDelay:       config.TED.RetryDelay,
+			UserAgent:        config.TED.UserAgent,
+			MaxConcurrency:   config.TED.MaxConcurrency,
+			DownloadDir:      config.TED.DownloadDir,
+			CleanupAfter:     config.TED.CleanupAfter,
+			ScrapingInterval: config.TED.ScrapingInterval,
+		},
+		appLogger,
+		mongoManager,
+		tenderRepo,
+	)
+
+	// Create context with cancellation
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	// Set up signal handling for graceful shutdown
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
+
+	// Start the scraper in a goroutine
+	scraperDone := make(chan error, 1)
+	go func() {
+		appLogger.Info("Starting TED scraper", map[string]interface{}{
+			"interval": config.TED.ScrapingInterval.String(),
+		})
+		scraperDone <- tedScraper.RunPeriodicScraping(ctx)
+	}()
+
+	// Wait for shutdown signal or scraper completion
+	select {
+	case <-signalChan:
+		appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{})
+		cancel()
+
+		// Wait for scraper to finish gracefully with timeout
+		shutdownTimeout := time.NewTimer(30 * time.Second)
+		select {
+		case <-scraperDone:
+			appLogger.Info("Scraper stopped gracefully", map[string]interface{}{})
+		case <-shutdownTimeout.C:
+			appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{})
+		}
+
+	case err := <-scraperDone:
+		if err != nil {
+			appLogger.Error("Scraper finished with error", map[string]interface{}{
+				"error": err.Error(),
+			})
+			os.Exit(1)
+		}
+		appLogger.Info("Scraper finished successfully", map[string]interface{}{})
+	}
+}