Implemented the integration with scraper python server, tenders summarise, and some fixes.

2025-12-27 12:47:08 +03:30
parent e4da3b5bb2
commit f6bdcc6d7c
18 changed files with 1054 additions and 45 deletions
@@ -9,10 +9,12 @@ import (
 	"tm/pkg/config"
 	"tm/pkg/glm"
 	"tm/pkg/logger"
+	"tm/pkg/minio"
 	"tm/pkg/mongo"
 	"tm/pkg/notification"
 	"tm/pkg/ollama"
 	"tm/pkg/schedule"
+	"tm/pkg/scraper"
 )

 // Init Application Configuration
@@ -75,24 +77,61 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
 }

 // Init Worker
-func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK) {
+func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, scraperService scraper.SDK, minioService *minio.Service) {
 	// Initialize repositories
 	noticeRepo := notice.NewRepository(mongoManager, appLogger)
 	tenderRepo := tender.NewRepository(mongoManager, appLogger)

-	worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService)
-	worker.Run()
+	// Initialize Notice Worker
+	appLogger.Info("Starting notice worker", map[string]interface{}{})
+	noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, scraperService)
+	noticeWorker.Run()

-	// start Worker job
+	// Initialize Document Scraper Worker
+	// appLogger.Info("Initializing document scraper worker", map[string]interface{}{})
+	// scraperWorker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
+	// appLogger.Info("Starting document scraper worker", map[string]interface{}{})
+	// scraperWorker.Run()
+
+	// Initialize Document Summarization Worker (only if MinIO is available)
+	if minioService != nil {
+		appLogger.Info("Starting document summarization worker", map[string]interface{}{})
+		summarizerWorker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, minioService, glmService)
+		summarizerWorker.Run()
+
+		// Schedule Document Summarization Worker job
+		schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
+			Name: "Document Summarization Worker Job",
+			Func: func() {
+				worker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, minioService, glmService)
+				worker.Run()
+			},
+			Expr: "0 11 * * * *", // Run at 11 AM every day (1 hour after document scraping)
+		})
+	} else {
+		appLogger.Warn("MinIO service not available, document summarization will be skipped", map[string]interface{}{})
+	}
+
+	// Schedule Notice Worker job
 	schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
-		Name: "Worker Job",
+		Name: "Notice Worker Job",
 		Func: func() {
-			worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService)
+			worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, scraperService)
 			worker.Run()
 		},
 		Expr: config.Worker.Interval,
 	})

+	// Schedule Document Scraper Worker job (runs after notice worker)
+	schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
+		Name: "Document Scraper Worker Job",
+		Func: func() {
+			worker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
+			worker.Run()
+		},
+		Expr: "0 10 * * * *", // Run at 10 AM every day (1 hour after notice worker)
+	})
+
 }

 // Init Notification Service
@@ -194,3 +233,62 @@ func InitGLMService(conf GLMConfig, log logger.Logger) *glm.SDK {

 	return glmSDK
 }
+
+// Init MinIO Service
+func InitMinIOService(conf config.MinIOConfig, log logger.Logger) *minio.Service {
+	// Convert bootstrap config to minio config
+	minioConfig := minio.Config{
+		Endpoint:           conf.Endpoint,
+		AccessKeyID:        conf.AccessKeyID,
+		SecretAccessKey:    conf.SecretAccessKey,
+		UseSSL:             conf.UseSSL,
+		Region:             conf.Region,
+		DefaultBucket:      conf.DefaultBucket,
+		HierarchicalBucket: conf.HierarchicalBucket,
+	}
+
+	// Validate config
+	if err := minioConfig.Validate(); err != nil {
+		log.Warn("Invalid MinIO configuration, MinIO features will be disabled", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil // Return nil instead of panicking
+	}
+
+	// Create connection manager
+	connManager, err := minio.NewConnectionManager(minioConfig, log)
+	if err != nil {
+		log.Error("Failed to create MinIO connection manager", map[string]interface{}{
+			"error": err.Error(),
+		})
+		panic(fmt.Sprintf("Failed to create MinIO connection manager: %v", err))
+	}
+
+	// Connect to MinIO
+	if err := connManager.Connect(); err != nil {
+		log.Warn("Failed to connect to MinIO, MinIO features will be disabled", map[string]interface{}{
+			"endpoint": minioConfig.Endpoint,
+			"error":    err.Error(),
+		})
+		return nil // Return nil instead of panicking
+	}
+
+	// Create service
+	service := minio.NewService(connManager, minioConfig, log)
+
+	log.Info("MinIO service initialized successfully", map[string]interface{}{
+		"endpoint":            conf.Endpoint,
+		"default_bucket":      conf.DefaultBucket,
+		"hierarchical_bucket": conf.HierarchicalBucket,
+	})
+
+	return service
+}
+
+func InitScraperService(config Config, logger logger.Logger) scraper.SDK {
+	return scraper.NewClient(
+		config.Scraper.BaseURL,
+		config.Scraper.Timeout,
+		logger,
+	)
+}