Add AlertMail Configuration and Enhance TED Scraper Notification Logic

- Introduced AlertMail configuration in both scraper and worker bootstrap files, allowing for customizable email notifications.
- Updated the TED scraper to utilize the AlertMail configuration for sending completion notifications, improving flexibility in notification management.
- Enhanced error logging in the worker's main function to capture issues when listing Ollama models, ensuring better visibility into potential failures.
- Refactored notification sending logic to check for a valid AlertMail before dispatching emails, ensuring notifications are only sent when configured.
- Improved overall structure and readability of the bootstrap configuration files, aligning with best practices for maintainability.

cmd/scraper/bootstrap/bootstrap.go

@@ -92,6 +92,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
 			DownloadDir:      config.TED.DownloadDir,
 			CleanupAfter:     config.TED.CleanupAfter,
 			ScrapingInterval: config.TED.ScrapingInterval,
+			AlertMail:        config.AlertMail,
 		},
 		appLogger,
 		mongoManager,

cmd/scraper/bootstrap/config.go

@@ -11,6 +11,7 @@ type Config struct {
 	Logging      config.LoggingConfig
 	TED          ScraperConfig
 	Notification config.NotificationConfig
+	AlertMail    string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
 }
 
 type ScraperConfig struct {

cmd/worker/bootstrap/bootstrap.go

@@ -79,6 +79,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 	noticeRepo := notice.NewRepository(mongoManager, appLogger)
 	tenderRepo := tender.NewRepository(mongoManager, appLogger)
 
+	worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK)
+	worker.Run()
+
 	// start Worker job
 	schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
 		Name: "Worker Job",

cmd/worker/bootstrap/config.go

@@ -11,6 +11,7 @@ type Config struct {
 	Notification config.NotificationConfig
 	Ollama       config.OllamaConfig
 	Worker       WorkerConfig
+	AlertMail    string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
 }
 
 type WorkerConfig struct {

cmd/worker/main.go

@@ -39,6 +39,13 @@ func main() {
 
 	// Initialize ollama service
 	ollamaSDK := bootstrap.InitOllamaService(config.Ollama, appLogger)
+	aiModels, err := ollamaSDK.ListModels(context.Background())
+	if err != nil {
+		appLogger.Error("Failed to list ollama models", map[string]interface{}{
+			"error": err.Error(),
+		})
+	}
+	appLogger.Info("Ollama models", map[string]interface{}{"models": aiModels})
 
 	// Initialize Worker
 	bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, ollamaSDK)

ted/scraper.go

@@ -27,6 +27,7 @@ type Config struct {
 	RetryDelay       time.Duration `mapstructure:"retry_delay"`
 	CleanupAfter     time.Duration `mapstructure:"cleanup_after"`
 	ScrapingInterval string        `mapstructure:"scraping_interval"`
+	AlertMail        string        `mapstructure:"ALERT_MAIL"`
 }
 
 // TEDScraper handles downloading and parsing TED XML files
@@ -251,26 +252,29 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
 		}
 	}
 
-	go s.notify.SendNotification(
+	if s.config.AlertMail != "" {
-		context.Background(),
+
-		&notification.NotificationRequest{
+		go s.notify.SendNotification(
-			EventType: notification.EventTypeEmail,
+			context.Background(),
-			Title:     "TED scraper completed",
+			&notification.NotificationRequest{
-			Message: GenerateFileProcessingEmailTemplate(
+				EventType: notification.EventTypeEmail,
-				&FileProcessingEmailTemplateData{
+				Title:     "TED scraper completed",
-					OJS:            ojs,
+				Message: GenerateFileProcessingEmailTemplate(
-					ProcessedCount: result.ProcessedCount,
+					&FileProcessingEmailTemplateData{
-					SuccessCount:   result.SuccessCount,
+						OJS:            ojs,
-					Errors:         result.Errors,
+						ProcessedCount: result.ProcessedCount,
-					ProcessedAt:    result.ProcessedAt,
+						SuccessCount:   result.SuccessCount,
-				}),
+						Errors:         result.Errors,
-			Priority: "important",
+						ProcessedAt:    result.ProcessedAt,
-			Methods: notification.NotificationMethods{
+					}),
-				Email: "nakhostin.nima1998@gmail.com",
+				Priority: "important",
-			},
+				Methods: notification.NotificationMethods{
-			UserID: "ted-scraper",
+					Email: s.config.AlertMail,
-			Type:   "alert",
+				},
-		})
+				UserID: "ted-scraper",
+				Type:   "alert",
+			})
+	}
 
 	return result, nil
 }