tm_back/cmd/scraper/bootstrap/bootstrap.go

package bootstrap

import (
	"context"
	"fmt"
	"os"
	"sync"
	"time"
	"tm/internal/notice"
	"tm/pkg/config"
	"tm/pkg/logger"
	"tm/pkg/mongo"
	"tm/pkg/notification"
	"tm/pkg/schedule"
	"tm/ted"
)

// tedScraperRunMu ensures only one TED scraper run executes at a time (startup catch-up and cron).
var tedScraperRunMu sync.Mutex

// Init Application Configuration
func InitConfig() (*Config, error) {
	conf, err := config.LoadConfig(".", &Config{})
	if err != nil {
		return nil, fmt.Errorf("failed to load config: %v", err)
	}

	return conf, nil
}

// Init Logger Service
func InitLogger(conf config.LoggingConfig) logger.Logger {
	return logger.NewLogger(&logger.Config{
		Level:  conf.Level,
		Format: conf.Format,
		Output: conf.Output,
		File: logger.FileConfig{
			Path:       conf.File.Path,
			MaxSize:    conf.File.MaxSize,
			MaxBackups: conf.File.MaxBackups,
			MaxAge:     conf.File.MaxAge,
			Compress:   conf.File.Compress,
		},
	})
}

// Init MongoDB Connection Manager
func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
	// Convert infra.MongoConfig to mongo.ConnectionConfig
	connectionConfig := mongo.ConnectionConfig{
		URI:                    conf.URI,
		Database:               conf.Name,
		MaxPoolSize:            uint64(conf.MaxPoolSize),
		MinPoolSize:            5,                // Default minimum pool size
		MaxConnIdleTime:        30 * time.Minute, // Default idle time
		ConnectTimeout:         conf.Timeout,
		ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
	}

	// Create MongoDB connection manager
	connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
	if err != nil {
		log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
			"error": err.Error(),
			"uri":   conf.URI,
			"db":    conf.Name,
		})
		panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
	}

	log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
		"database":  conf.Name,
		"uri":       conf.URI,
		"pool_size": conf.MaxPoolSize,
	})

	return connectionManager
}

// init TED scraper
func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) *schedule.CronScheduler {
	// Initialize tender repository
	noticeRepo := notice.NewRepository(mongoManager, appLogger)

	// Initialize TED scraper
	tedScraper := ted.NewTEDScraper(
		&ted.Config{
			BaseURL:          config.TED.BaseURL,
			Timeout:          config.TED.Timeout,
			MaxRetries:       config.TED.MaxRetries,
			RetryDelay:       config.TED.RetryDelay,
			UserAgent:        config.TED.UserAgent,
			MaxConcurrency:   config.TED.MaxConcurrency,
			DownloadDir:      config.TED.DownloadDir,
			CleanupAfter:     config.TED.CleanupAfter,
			ScrapingInterval: config.TED.ScrapingInterval,
			AlertMail:        config.AlertMail,
			MaxNoticesPerDay: config.TED.MaxNoticesPerDay,
		},
		appLogger,
		mongoManager,
		noticeRepo,
		notify,
	)

	// start TED scraper job with error handling
	scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
	dailyJobTracker := schedule.NewDailyJobTracker(mongoManager, appLogger)

	runDailyScrape := func(trigger string) {
		tedScraperRunMu.Lock()
		defer tedScraperRunMu.Unlock()

		ctx := context.Background()
		today := time.Now().Local()

		if trigger == "startup" {
			completed, checkErr := dailyJobTracker.IsCompleted(ctx, schedule.TEDScraperJobName, today)
			if checkErr != nil {
				appLogger.Error("Failed to check daily scrape completion status", map[string]interface{}{
					"error": checkErr.Error(),
					"date":  today.Format("02/01/2006"),
				})
			} else if completed {
				appLogger.Info("Startup catch-up skipped: today's TED scrape already completed", map[string]interface{}{
					"date": today.Format("02/01/2006"),
				})
				return
			}
			appLogger.Info("Running startup catch-up for today's TED scrape", map[string]interface{}{
				"date": today.Format("02/01/2006"),
			})
		} else {
			appLogger.Info("Running scheduled TED scrape", map[string]interface{}{
				"date": today.Format("02/01/2006"),
			})
		}

		err := tedScraper.Run(ctx, nil, nil)
		if err != nil {
			appLogger.Error("TED scraper run failed", map[string]interface{}{
				"trigger": trigger,
				"date":    today.Format("02/01/2006"),
				"error":   err.Error(),
			})
			return
		}

		if markErr := dailyJobTracker.MarkCompleted(ctx, schedule.TEDScraperJobName, today); markErr != nil {
			appLogger.Error("Failed to mark daily scrape as completed", map[string]interface{}{
				"trigger": trigger,
				"date":    today.Format("02/01/2006"),
				"error":   markErr.Error(),
			})
		}

		appLogger.Info("TED scraper run completed successfully", map[string]interface{}{
			"trigger": trigger,
			"date":    today.Format("02/01/2006"),
		})
	}

	err := scheduler.AddJob(schedule.Job{
		Name: "TED Scraper Job",
		Func: func() { runDailyScrape("scheduled") },
		Expr: config.TED.ScrapingInterval,
	})
	if err != nil {
		appLogger.Error("Failed to schedule TED scraper job", map[string]interface{}{
			"error": err.Error(),
		})
		return nil
	}

	// After a server restart, resume today's scrape if it was interrupted or never ran.
	go func() {
		runDailyScrape("startup")
	}()

	// Start the scheduler
	scheduler.Start()

	appLogger.Info("TED scraper scheduled and started", map[string]interface{}{
		"interval": config.TED.ScrapingInterval,
	})

	return scheduler
}

// Init Notification Service
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
	notificationConfig := &notification.Config{
		BaseURL:       conf.BaseURL,
		Timeout:       conf.Timeout,
		RetryAttempts: conf.RetryAttempts,
		RetryDelay:    conf.RetryDelay,
		EnableLogging: conf.EnableLogging,
		UserAgent:     conf.UserAgent,
	}

	notificationSDK, err := notification.New(notificationConfig, log)
	if err != nil {
		log.Error("Failed to initialize Notification SDK", map[string]interface{}{
			"error": err.Error(),
		})
		fmt.Printf("Failed to initialize Notification SDK: %v\n", err.Error())
	}

	log.Info("Notification SDK initialized successfully", map[string]interface{}{
		"base_url":       conf.BaseURL,
		"timeout":        conf.Timeout,
		"retry_attempts": conf.RetryAttempts,
		"retry_delay":    conf.RetryDelay,
		"enable_logging": conf.EnableLogging,
		"user_agent":     conf.UserAgent,
	})

	return *notificationSDK
}

// RunOneTimeScraping runs the TED scraper once for a specific date range
func RunOneTimeScraping(
	config Config,
	mongoManager *mongo.ConnectionManager,
	appLogger logger.Logger,
	notificationService notification.SDK,
	from, to *time.Time,
) {
	// Initialize notice repository
	noticeRepo := notice.NewRepository(mongoManager, appLogger)

	// Convert scraper config to TED config
	tedConfig := &ted.Config{
		BaseURL:          config.TED.BaseURL,
		UserAgent:        config.TED.UserAgent,
		DownloadDir:      config.TED.DownloadDir,
		MaxRetries:       config.TED.MaxRetries,
		MaxConcurrency:   config.TED.MaxConcurrency,
		Timeout:          config.TED.Timeout,
		RetryDelay:       config.TED.RetryDelay,
		CleanupAfter:     config.TED.CleanupAfter,
		ScrapingInterval: config.TED.ScrapingInterval,
		AlertMail:        config.AlertMail,
		MaxNoticesPerDay: config.TED.MaxNoticesPerDay,
	}

	// Initialize TED scraper
	tedScraper := ted.NewTEDScraper(
		tedConfig,
		appLogger,
		mongoManager,
		noticeRepo,
		notificationService,
	)

	appLogger.Info("Starting one-time TED scraping", map[string]interface{}{
		"from_date": from.Format("02/01/2006"),
		"to_date":   to.Format("02/01/2006"),
	})

	// Create context with timeout for the entire operation
	ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) // 24 hour timeout
	defer cancel()

	// Run the scraper with date range
	err := tedScraper.Run(ctx, from, to)
	if err != nil {
		appLogger.Error("One-time scraping failed", map[string]interface{}{
			"error":     err.Error(),
			"from_date": from.Format("02/01/2006"),
			"to_date":   to.Format("02/01/2006"),
		})
		os.Exit(1)
	}

	appLogger.Info("One-time TED scraping completed successfully", map[string]interface{}{
		"from_date": from.Format("02/01/2006"),
		"to_date":   to.Format("02/01/2006"),
	})
}