package bootstrap import ( "fmt" "time" "tm/cmd/worker/workers" "tm/internal/notice" "tm/internal/tender" "tm/pkg/config" "tm/pkg/glm" "tm/pkg/logger" "tm/pkg/minio" "tm/pkg/mongo" "tm/pkg/notification" "tm/pkg/ollama" "tm/pkg/queue" "tm/pkg/schedule" "tm/pkg/scraper" ) // Init Application Configuration func InitConfig() (*Config, error) { conf, err := config.LoadConfig(".", &Config{}) if err != nil { return nil, fmt.Errorf("failed to load config: %v", err) } return conf, nil } // Init Logger Service func InitLogger(conf config.LoggingConfig) logger.Logger { return logger.NewLogger(&logger.Config{ Level: conf.Level, Format: conf.Format, Output: conf.Output, File: logger.FileConfig{ Path: conf.File.Path, MaxSize: conf.File.MaxSize, MaxBackups: conf.File.MaxBackups, MaxAge: conf.File.MaxAge, Compress: conf.File.Compress, }, }) } // Init MongoDB Connection Manager func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager { // Convert infra.MongoConfig to mongo.ConnectionConfig connectionConfig := mongo.ConnectionConfig{ URI: conf.URI, Database: conf.Name, MaxPoolSize: uint64(conf.MaxPoolSize), MinPoolSize: 5, // Default minimum pool size MaxConnIdleTime: 30 * time.Minute, // Default idle time ConnectTimeout: conf.Timeout, ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout } // Create MongoDB connection manager connectionManager, err := mongo.NewConnectionManager(connectionConfig, log) if err != nil { log.Error("Failed to initialize MongoDB connection", map[string]interface{}{ "error": err.Error(), "uri": conf.URI, "db": conf.Name, }) panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err)) } log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{ "database": conf.Name, "uri": conf.URI, "pool_size": conf.MaxPoolSize, }) return connectionManager } // Init Worker func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, scraperService scraper.SDK, minioService *minio.Service) { // Debug: Log worker config appLogger.Info("Worker configuration", map[string]interface{}{ "worker_interval": config.Worker.Interval, "queue_enabled": config.Queue.Enabled, "queue_mode": config.Queue.Mode, }) // Initialize repositories noticeRepo := notice.NewRepository(mongoManager, appLogger) tenderRepo := tender.NewRepository(mongoManager, appLogger) // Initialize Notice Worker appLogger.Info("Starting notice worker", map[string]interface{}{ "processing_limit": config.Worker.NoticeProcessingLimit, }) noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, ¬ify, noticeRepo, tenderRepo, glmService, scraperService, config.Worker.NoticeProcessingLimit) noticeWorker.Run() // Initialize Queue-based scraping if enabled var queueService queue.Queue if config.Queue.Enabled { var err error queueConfig := queue.QueueConfig{ RedisURL: config.Queue.RedisURL, MaxRetries: config.Queue.MaxRetries, RetryBackoff: config.Queue.RetryBackoff, JobTimeout: config.Queue.JobTimeout, PoolSize: config.Queue.PoolSize, IsDLQEnabled: config.Queue.IsDLQEnabled, MaxQueueLength: 0, // Unlimited } queueService, err = queue.NewRedisQueue(queueConfig, appLogger) if err != nil { appLogger.Error("Failed to initialize queue service", map[string]interface{}{ "error": err.Error(), }) appLogger.Warn("Queue-based scraping is disabled, falling back to scheduled scraping", map[string]interface{}{}) config.Queue.Enabled = false } else { appLogger.Info("Queue service initialized successfully", map[string]interface{}{ "mode": config.Queue.Mode, }) } } // Start Queue Consumer Worker (24/7 continuous processing) if config.Queue.Enabled && (config.Queue.Mode == "consumer" || config.Queue.Mode == "both") { appLogger.Info("Starting queue consumer worker", map[string]interface{}{ "concurrency": config.Queue.Concurrency, }) consumerWorker := workers.NewDocumentScraperConsumerWorker( mongoManager, appLogger, tenderRepo, queueService, scraperService, config.Queue.Concurrency, ) go consumerWorker.Run() } else if !config.Queue.Enabled { appLogger.Info("Queue consumer worker disabled, using scheduled scraping instead", map[string]interface{}{}) } // Start Queue Producer Worker (enqueues unscraped tenders periodically) if config.Queue.Enabled && (config.Queue.Mode == "producer" || config.Queue.Mode == "both") { appLogger.Info("Scheduling queue producer worker", map[string]interface{}{ "batch_size": config.Queue.ProducerBatch, "interval": config.Queue.ProducerInterval, }) schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ Name: "Queue Producer Worker Job", Func: func() { producer := workers.NewQueueProducerWorker( mongoManager, appLogger, tenderRepo, queueService, config.Queue.ProducerBatch, ) producer.Run() }, Expr: config.Queue.ProducerInterval, }) } // Fallback: Keep the old scheduled scraper for backward compatibility if !config.Queue.Enabled { appLogger.Info("Scheduling fallback document scraper worker", map[string]interface{}{}) schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ Name: "Document Scraper Worker Job (Fallback)", Func: func() { worker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService) worker.Run() }, Expr: "0 10 * * * *", // Run at 10 AM every day }) } // Initialize Document Summarization Worker (only if MinIO is available) if minioService != nil { appLogger.Info("Starting document summarization worker", map[string]interface{}{}) summarizerWorker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, minioService, glmService) summarizerWorker.Run() // Schedule Document Summarization Worker job schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ Name: "Document Summarization Worker Job", Func: func() { worker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, minioService, glmService) worker.Run() }, Expr: "0 11 * * * *", // Run at 11 AM every day (1 hour after document scraping) }) } else { appLogger.Warn("MinIO service not available, document summarization will be skipped", map[string]interface{}{}) } // Schedule Notice Worker job workerInterval := config.Worker.Interval if workerInterval == "" { workerInterval = "* 10 * * * *" // Default: every hour at minute 10 appLogger.Warn("WORKER_INTERVAL not set, using default schedule", map[string]interface{}{ "interval": workerInterval, }) } schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ Name: "Notice Worker Job", Func: func() { worker := workers.NewNoticeWorker(mongoManager, appLogger, ¬ify, noticeRepo, tenderRepo, glmService, scraperService, config.Worker.NoticeProcessingLimit) worker.Run() }, Expr: workerInterval, }) } // Init Notification Service func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK { notificationConfig := ¬ification.Config{ BaseURL: conf.BaseURL, Timeout: conf.Timeout, RetryAttempts: conf.RetryAttempts, RetryDelay: conf.RetryDelay, EnableLogging: conf.EnableLogging, UserAgent: conf.UserAgent, } notificationSDK, err := notification.New(notificationConfig, log) if err != nil { log.Error("Failed to initialize Notification SDK", map[string]interface{}{ "error": err.Error(), }) fmt.Printf("Failed to initialize Notification SDK: %v\n", err.Error()) } log.Info("Notification SDK initialized successfully", map[string]interface{}{ "base_url": conf.BaseURL, "timeout": conf.Timeout, "retry_attempts": conf.RetryAttempts, "retry_delay": conf.RetryDelay, "enable_logging": conf.EnableLogging, "user_agent": conf.UserAgent, }) return *notificationSDK } // Init Ollama Service func InitOllamaService(conf config.OllamaConfig, log logger.Logger) *ollama.SDK { ollamaSDK, err := ollama.New(&ollama.Config{ BaseURL: conf.BaseURL, Timeout: conf.Timeout, RetryAttempts: conf.RetryAttempts, RetryDelay: conf.RetryDelay, EnableLogging: conf.EnableLogging, UserAgent: conf.UserAgent, DefaultModel: conf.DefaultModel, DefaultTemperature: conf.DefaultTemperature, DefaultTopP: conf.DefaultTopP, DefaultMaxTokens: conf.DefaultMaxTokens, EnableStreaming: conf.EnableStreaming, }, log) if err != nil { log.Error("Failed to initialize Ollama SDK", map[string]interface{}{ "error": err.Error(), }) } return ollamaSDK } // Init GLM Service func InitGLMService(conf GLMConfig, log logger.Logger) *glm.SDK { glmSDK, err := glm.New( &glm.Config{ BaseURL: conf.BaseURL, APIKey: conf.APIKey, Timeout: conf.Timeout, RetryAttempts: conf.RetryAttempts, RetryDelay: conf.RetryDelay, EnableLogging: conf.EnableLogging, UserAgent: conf.UserAgent, DefaultModel: conf.DefaultModel, DefaultMaxTokens: conf.DefaultMaxTokens, DefaultTopP: conf.DefaultTopP, DefaultTemperature: conf.DefaultTemperature, DefaultFrequencyPenalty: conf.DefaultFrequencyPenalty, DefaultPresencePenalty: conf.DefaultPresencePenalty, EnableStreaming: conf.EnableStreaming, EnableThinking: conf.EnableThinking, TranslationOptions: glm.TranslationOption{ Model: conf.TranslateOptions.Model, MaxTokens: conf.TranslateOptions.MaxTokens, Temperature: conf.TranslateOptions.Temperature, FrequencyPenalty: conf.TranslateOptions.FrequencyPenalty, PresencePenalty: conf.TranslateOptions.PresencePenalty, TopP: conf.TranslateOptions.TopP, EnableThinking: conf.TranslateOptions.EnableThinking, }, }, log) if err != nil { log.Error("Failed to initialize GLM SDK", map[string]interface{}{ "error": err.Error(), }) log.Warn("GLM service will be disabled, translation features will not be available", map[string]interface{}{}) return nil } log.Info("GLM SDK initialized successfully", map[string]interface{}{ "base_url": conf.BaseURL, "timeout": conf.Timeout, }) return glmSDK } // Init MinIO Service func InitMinIOService(conf config.MinIOConfig, log logger.Logger) *minio.Service { // Convert bootstrap config to minio config minioConfig := minio.Config{ Endpoint: conf.Endpoint, AccessKeyID: conf.AccessKeyID, SecretAccessKey: conf.SecretAccessKey, UseSSL: conf.UseSSL, Region: conf.Region, DefaultBucket: conf.DefaultBucket, HierarchicalBucket: conf.HierarchicalBucket, } // Validate config if err := minioConfig.Validate(); err != nil { log.Warn("Invalid MinIO configuration, MinIO features will be disabled", map[string]interface{}{ "error": err.Error(), }) return nil // Return nil instead of panicking } // Create connection manager connManager, err := minio.NewConnectionManager(minioConfig, log) if err != nil { log.Error("Failed to create MinIO connection manager", map[string]interface{}{ "error": err.Error(), }) panic(fmt.Sprintf("Failed to create MinIO connection manager: %v", err)) } // Connect to MinIO if err := connManager.Connect(); err != nil { log.Warn("Failed to connect to MinIO, MinIO features will be disabled", map[string]interface{}{ "endpoint": minioConfig.Endpoint, "error": err.Error(), }) return nil // Return nil instead of panicking } // Create service service := minio.NewService(connManager, minioConfig, log) log.Info("MinIO service initialized successfully", map[string]interface{}{ "endpoint": conf.Endpoint, "default_bucket": conf.DefaultBucket, "hierarchical_bucket": conf.HierarchicalBucket, }) return service } func InitScraperService(config Config, logger logger.Logger) scraper.SDK { return scraper.NewClient( config.Scraper.BaseURL, config.Scraper.Timeout, logger, ) }