mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Removed old document scraper service

Browse Source
This commit is contained in:
m.nazemi
2026-05-03 13:46:38 +03:30
parent 0bd5e41b70
commit 446c11dfbf
11 changed files with 9 additions and 1371 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/bootstrap/bootstrap.go
+3 -43
View File
@@ -15,7 +15,6 @@ import (
"tm/pkg/ollama"
"tm/pkg/queue"
"tm/pkg/schedule"
"tm/pkg/scraper"
)
// Init Application Configuration
@@ -78,7 +77,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
}
// Init Worker
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, scraperService scraper.SDK, minioService *minio.Service) {
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, minioService *minio.Service) {
// Debug: Log worker config
appLogger.Info("Worker configuration", map[string]interface{}{
"worker_interval": config.Worker.Interval,
@@ -93,7 +92,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
appLogger.Info("Starting notice worker", map[string]interface{}{
"processing_limit": config.Worker.NoticeProcessingLimit,
})
noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, scraperService, config.Worker.NoticeProcessingLimit)
noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit)
noticeWorker.Run()
// Initialize Queue-based scraping if enabled
@@ -124,24 +123,6 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
}
}
// Start Queue Consumer Worker (24/7 continuous processing)
if config.Queue.Enabled && (config.Queue.Mode == "consumer" || config.Queue.Mode == "both") {
appLogger.Info("Starting queue consumer worker", map[string]interface{}{
"concurrency": config.Queue.Concurrency,
})
consumerWorker := workers.NewDocumentScraperConsumerWorker(
mongoManager,
appLogger,
tenderRepo,
queueService,
scraperService,
config.Queue.Concurrency,
)
go consumerWorker.Run()
} else if !config.Queue.Enabled {
appLogger.Info("Queue consumer worker disabled, using scheduled scraping instead", map[string]interface{}{})
}
// Start Queue Producer Worker (enqueues unscraped tenders periodically)
if config.Queue.Enabled && (config.Queue.Mode == "producer" || config.Queue.Mode == "both") {
appLogger.Info("Scheduling queue producer worker", map[string]interface{}{
@@ -165,19 +146,6 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
})
}
// Fallback: Keep the old scheduled scraper for backward compatibility
if !config.Queue.Enabled {
appLogger.Info("Scheduling fallback document scraper worker", map[string]interface{}{})
schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
Name: "Document Scraper Worker Job (Fallback)",
Func: func() {
worker := workers.NewDocumentScraperWorker(mongoManager, appLogger, tenderRepo, scraperService)
worker.Run()
},
Expr: "0 10 * * * *", // Run at 10 AM every day
})
}
// Initialize Document Summarization Worker (only if MinIO is available)
if minioService != nil {
appLogger.Info("Starting document summarization worker", map[string]interface{}{})
@@ -209,7 +177,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
Name: "Notice Worker Job",
Func: func() {
worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, scraperService, config.Worker.NoticeProcessingLimit)
worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit)
worker.Run()
},
Expr: workerInterval,
@@ -367,11 +335,3 @@ func InitMinIOService(conf config.MinIOConfig, log logger.Logger) *minio.Service
return service
}
func InitScraperService(config Config, logger logger.Logger) scraper.SDK {
return scraper.NewClient(
config.Scraper.BaseURL,
config.Scraper.Timeout,
logger,
)
}