diff --git a/cmd/scraper/bootstrap/bootstrap.go b/cmd/scraper/bootstrap/bootstrap.go index 9cbdf52..1f866c4 100644 --- a/cmd/scraper/bootstrap/bootstrap.go +++ b/cmd/scraper/bootstrap/bootstrap.go @@ -92,6 +92,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog DownloadDir: config.TED.DownloadDir, CleanupAfter: config.TED.CleanupAfter, ScrapingInterval: config.TED.ScrapingInterval, + AlertMail: config.AlertMail, }, appLogger, mongoManager, diff --git a/cmd/scraper/bootstrap/config.go b/cmd/scraper/bootstrap/config.go index 7766ae2..28fc584 100644 --- a/cmd/scraper/bootstrap/config.go +++ b/cmd/scraper/bootstrap/config.go @@ -11,6 +11,7 @@ type Config struct { Logging config.LoggingConfig TED ScraperConfig Notification config.NotificationConfig + AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"` } type ScraperConfig struct { diff --git a/cmd/worker/bootstrap/bootstrap.go b/cmd/worker/bootstrap/bootstrap.go index 96fbea5..844ac09 100644 --- a/cmd/worker/bootstrap/bootstrap.go +++ b/cmd/worker/bootstrap/bootstrap.go @@ -79,6 +79,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger noticeRepo := notice.NewRepository(mongoManager, appLogger) tenderRepo := tender.NewRepository(mongoManager, appLogger) + worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK) + worker.Run() + // start Worker job schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ Name: "Worker Job", diff --git a/cmd/worker/bootstrap/config.go b/cmd/worker/bootstrap/config.go index 7d219ff..8c3485c 100644 --- a/cmd/worker/bootstrap/config.go +++ b/cmd/worker/bootstrap/config.go @@ -11,6 +11,7 @@ type Config struct { Notification config.NotificationConfig Ollama config.OllamaConfig Worker WorkerConfig + AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"` } type WorkerConfig struct { diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 8fdb9ee..e01131e 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -39,6 +39,13 @@ func main() { // Initialize ollama service ollamaSDK := bootstrap.InitOllamaService(config.Ollama, appLogger) + aiModels, err := ollamaSDK.ListModels(context.Background()) + if err != nil { + appLogger.Error("Failed to list ollama models", map[string]interface{}{ + "error": err.Error(), + }) + } + appLogger.Info("Ollama models", map[string]interface{}{"models": aiModels}) // Initialize Worker bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, ollamaSDK) diff --git a/ted/scraper.go b/ted/scraper.go index c37bd6e..3f28d15 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -27,6 +27,7 @@ type Config struct { RetryDelay time.Duration `mapstructure:"retry_delay"` CleanupAfter time.Duration `mapstructure:"cleanup_after"` ScrapingInterval string `mapstructure:"scraping_interval"` + AlertMail string `mapstructure:"ALERT_MAIL"` } // TEDScraper handles downloading and parsing TED XML files @@ -251,26 +252,29 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o } } - go s.notify.SendNotification( - context.Background(), - ¬ification.NotificationRequest{ - EventType: notification.EventTypeEmail, - Title: "TED scraper completed", - Message: GenerateFileProcessingEmailTemplate( - &FileProcessingEmailTemplateData{ - OJS: ojs, - ProcessedCount: result.ProcessedCount, - SuccessCount: result.SuccessCount, - Errors: result.Errors, - ProcessedAt: result.ProcessedAt, - }), - Priority: "important", - Methods: notification.NotificationMethods{ - Email: "nakhostin.nima1998@gmail.com", - }, - UserID: "ted-scraper", - Type: "alert", - }) + if s.config.AlertMail != "" { + + go s.notify.SendNotification( + context.Background(), + ¬ification.NotificationRequest{ + EventType: notification.EventTypeEmail, + Title: "TED scraper completed", + Message: GenerateFileProcessingEmailTemplate( + &FileProcessingEmailTemplateData{ + OJS: ojs, + ProcessedCount: result.ProcessedCount, + SuccessCount: result.SuccessCount, + Errors: result.Errors, + ProcessedAt: result.ProcessedAt, + }), + Priority: "important", + Methods: notification.NotificationMethods{ + Email: s.config.AlertMail, + }, + UserID: "ted-scraper", + Type: "alert", + }) + } return result, nil }