Enhance TED scraper and worker initialization with startup catch-up logic
continuous-integration/drone/push Build is passing
continuous-integration/drone/push Build is passing
- Introduced a mutex to ensure only one TED scraper run executes at a time, preventing concurrent executions during startup and scheduled runs. - Implemented a mechanism to check if today's TED scrape has already been completed during startup, logging appropriate messages for both completed and new runs. - Added startup catch-up logic for tender translations and unprocessed notices, ensuring that any missed tasks are executed without blocking the application startup. This update improves the reliability and efficiency of the TED scraper and worker processes, ensuring that all necessary tasks are completed after a server restart.
This commit is contained in:
@@ -158,6 +158,23 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
"interval": config.Worker.TranslationInterval,
|
||||
"batch_size": config.Worker.TranslationBatchSize,
|
||||
})
|
||||
|
||||
go func() {
|
||||
appLogger.Info("Running startup catch-up for tender translations", map[string]interface{}{
|
||||
"target_languages": targetLanguages,
|
||||
})
|
||||
worker := workers.NewTranslationWorker(
|
||||
mongoManager,
|
||||
appLogger,
|
||||
tenderRepo,
|
||||
aiClient,
|
||||
aiStorage,
|
||||
translationSuccessCounter,
|
||||
targetLanguages,
|
||||
config.Worker.TranslationBatchSize,
|
||||
)
|
||||
worker.Run()
|
||||
}()
|
||||
} else {
|
||||
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
|
||||
}
|
||||
@@ -182,8 +199,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
"interval": notificationInterval,
|
||||
})
|
||||
|
||||
// Kick off one notice-processing pass without blocking startup (cron continues on schedule)
|
||||
// After a server restart, process any unprocessed notices (including today's) without blocking startup.
|
||||
go func() {
|
||||
appLogger.Info("Running startup catch-up for unprocessed notices", map[string]interface{}{})
|
||||
w := workers.NewNoticeWorker(
|
||||
mongoManager,
|
||||
appLogger,
|
||||
|
||||
Reference in New Issue
Block a user