tm_back/cmd/scraper/bootstrap/bootstrap.go

package bootstrap

import (
	"context"
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"time"
	"tm/internal/notice"
	"tm/pkg/config"
	"tm/pkg/logger"
	"tm/pkg/mongo"
	"tm/pkg/notification"
	"tm/pkg/schedule"
	"tm/ted"
)

// Init Application Configuration
func InitConfig() (*Config, error) {
	conf, err := config.LoadConfig(".", &Config{})
	if err != nil {
		return nil, fmt.Errorf("failed to load config: %v", err)
	}

	return conf, nil
}

// Init Logger Service
func InitLogger(conf config.LoggingConfig) logger.Logger {
	return logger.NewLogger(&logger.Config{
		Level:  conf.Level,
		Format: conf.Format,
		Output: conf.Output,
		File: logger.FileConfig{
			Path:       conf.File.Path,
			MaxSize:    conf.File.MaxSize,
			MaxBackups: conf.File.MaxBackups,
			MaxAge:     conf.File.MaxAge,
			Compress:   conf.File.Compress,
		},
	})
}

// Init MongoDB Connection Manager
func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
	// Convert infra.MongoConfig to mongo.ConnectionConfig
	connectionConfig := mongo.ConnectionConfig{
		URI:                    conf.URI,
		Database:               conf.Name,
		MaxPoolSize:            uint64(conf.MaxPoolSize),
		MinPoolSize:            5,                // Default minimum pool size
		MaxConnIdleTime:        30 * time.Minute, // Default idle time
		ConnectTimeout:         conf.Timeout,
		ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
	}

	// Create MongoDB connection manager
	connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
	if err != nil {
		log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
			"error": err.Error(),
			"uri":   conf.URI,
			"db":    conf.Name,
		})
		panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
	}

	log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
		"database":  conf.Name,
		"uri":       conf.URI,
		"pool_size": conf.MaxPoolSize,
	})

	return connectionManager
}

// init TED scraper
func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) {
	// Initialize tender repository
	noticeRepo := notice.NewRepository(mongoManager, appLogger)

	// Initialize TED scraper
	tedScraper := ted.NewTEDScraper(
		&ted.Config{
			BaseURL:          config.TED.BaseURL,
			Timeout:          config.TED.Timeout,
			MaxRetries:       config.TED.MaxRetries,
			RetryDelay:       config.TED.RetryDelay,
			UserAgent:        config.TED.UserAgent,
			MaxConcurrency:   config.TED.MaxConcurrency,
			DownloadDir:      config.TED.DownloadDir,
			CleanupAfter:     config.TED.CleanupAfter,
			ScrapingInterval: config.TED.ScrapingInterval,
			AlertMail:        config.AlertMail,
		},
		appLogger,
		mongoManager,
		noticeRepo,
		notify,
	)

	// start TED scraper job
	schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
		Name: "TED Scraper Job",
		Func: func() {
			_ = tedScraper.Run(context.Background(), nil, nil)
		},
		Expr: config.TED.ScrapingInterval,
	})

	// Create context with cancellation
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// Set up signal handling for graceful shutdown
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	// Start the scraper in a goroutine
	scraperDone := make(chan error, 1)
	go func() {
		appLogger.Info("Starting TED scraper", map[string]interface{}{
			"interval": config.TED.ScrapingInterval,
		})
		scraperDone <- tedScraper.Run(ctx, nil, nil)
	}()

	// Wait for shutdown signal or scraper completion
	select {
	case <-signalChan:
		appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{})
		cancel()

		// Wait for scraper to finish gracefully with timeout
		shutdownTimeout := time.NewTimer(30 * time.Second)
		select {
		case <-scraperDone:
			appLogger.Info("Scraper stopped gracefully", map[string]interface{}{})
		case <-shutdownTimeout.C:
			appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{})
		}

	case err := <-scraperDone:

		if err != nil {
			appLogger.Error("Scraper finished with error", map[string]interface{}{
				"error": err.Error(),
			})
			os.Exit(1)
		}
		appLogger.Info("Scraper finished successfully", map[string]interface{}{})
	}
}

// Init Notification Service
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
	notificationConfig := &notification.Config{
		BaseURL:       conf.BaseURL,
		Timeout:       conf.Timeout,
		RetryAttempts: conf.RetryAttempts,
		RetryDelay:    conf.RetryDelay,
		EnableLogging: conf.EnableLogging,
		UserAgent:     conf.UserAgent,
	}

	notificationSDK, err := notification.New(notificationConfig, log)
	if err != nil {
		log.Error("Failed to initialize Notification SDK", map[string]interface{}{
			"error": err.Error(),
		})
		fmt.Printf("Failed to initialize Notification SDK: %v\n", err.Error())
	}

	log.Info("Notification SDK initialized successfully", map[string]interface{}{
		"base_url":       conf.BaseURL,
		"timeout":        conf.Timeout,
		"retry_attempts": conf.RetryAttempts,
		"retry_delay":    conf.RetryDelay,
		"enable_logging": conf.EnableLogging,
		"user_agent":     conf.UserAgent,
	})

	return *notificationSDK
}

// RunOneTimeScraping runs the TED scraper once for a specific date range
func RunOneTimeScraping(
	config Config,
	mongoManager *mongo.ConnectionManager,
	appLogger logger.Logger,
	notificationService notification.SDK,
	from, to *time.Time,
) {
	// Initialize notice repository
	noticeRepo := notice.NewRepository(mongoManager, appLogger)

	// Convert scraper config to TED config
	tedConfig := &ted.Config{
		BaseURL:          config.TED.BaseURL,
		UserAgent:        config.TED.UserAgent,
		DownloadDir:      config.TED.DownloadDir,
		MaxRetries:       config.TED.MaxRetries,
		MaxConcurrency:   config.TED.MaxConcurrency,
		Timeout:          config.TED.Timeout,
		RetryDelay:       config.TED.RetryDelay,
		CleanupAfter:     config.TED.CleanupAfter,
		ScrapingInterval: config.TED.ScrapingInterval,
		AlertMail:        config.AlertMail,
	}

	// Initialize TED scraper
	tedScraper := ted.NewTEDScraper(
		tedConfig,
		appLogger,
		mongoManager,
		noticeRepo,
		notificationService,
	)

	appLogger.Info("Starting one-time TED scraping", map[string]interface{}{
		"from_date": from.Format("02/01/2006"),
		"to_date":   to.Format("02/01/2006"),
	})

	// Create context with timeout for the entire operation
	ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) // 24 hour timeout
	defer cancel()

	// Run the scraper with date range
	err := tedScraper.Run(ctx, from, to)
	if err != nil {
		appLogger.Error("One-time scraping failed", map[string]interface{}{
			"error":     err.Error(),
			"from_date": from.Format("02/01/2006"),
			"to_date":   to.Format("02/01/2006"),
		})
		os.Exit(1)
	}

	appLogger.Info("One-time TED scraping completed successfully", map[string]interface{}{
		"from_date": from.Format("02/01/2006"),
		"to_date":   to.Format("02/01/2006"),
	})
}