Refactor AI summarizer client initialization to include MongoDB manager and add statistics endpoint
- Updated `InitAISummarizerClient` to accept `mongoManager` for tracking translation success. - Introduced new `Statistics` endpoint in the dashboard to fetch scraping and translation statistics. - Enhanced `TranslationWorker` to utilize the new success counter for tracking successful translations. - Added necessary data structures and query forms for statistics reporting. This refactor improves the tracking of AI translation success and provides new insights through the dashboard statistics.
This commit is contained in:
@@ -13,13 +13,14 @@ import (
|
||||
// TranslationWorker backfills missing tender translations using the AI pipeline
|
||||
// and on-demand translate API. Results are persisted by the AI service in MinIO only.
|
||||
type TranslationWorker struct {
|
||||
Mongo *mongo.ConnectionManager
|
||||
Logger logger.Logger
|
||||
TenderRepo tender.TenderRepository
|
||||
AIClient *ai_summarizer.Client
|
||||
AIStorage *ai_summarizer.StorageClient
|
||||
TargetLanguages []string
|
||||
BatchSize int
|
||||
Mongo *mongo.ConnectionManager
|
||||
Logger logger.Logger
|
||||
TenderRepo tender.TenderRepository
|
||||
AIClient *ai_summarizer.Client
|
||||
AIStorage *ai_summarizer.StorageClient
|
||||
TranslationSuccessCounter *mongo.Counter
|
||||
TargetLanguages []string
|
||||
BatchSize int
|
||||
}
|
||||
|
||||
// NewTranslationWorker creates a translation worker for the given target languages.
|
||||
@@ -29,6 +30,7 @@ func NewTranslationWorker(
|
||||
tenderRepo tender.TenderRepository,
|
||||
aiClient *ai_summarizer.Client,
|
||||
aiStorage *ai_summarizer.StorageClient,
|
||||
translationSuccessCounter *mongo.Counter,
|
||||
targetLanguages []string,
|
||||
batchSize int,
|
||||
) *TranslationWorker {
|
||||
@@ -53,13 +55,14 @@ func NewTranslationWorker(
|
||||
}
|
||||
|
||||
return &TranslationWorker{
|
||||
Mongo: mongo,
|
||||
Logger: logger,
|
||||
TenderRepo: tenderRepo,
|
||||
AIClient: aiClient,
|
||||
AIStorage: aiStorage,
|
||||
TargetLanguages: langs,
|
||||
BatchSize: batchSize,
|
||||
Mongo: mongo,
|
||||
Logger: logger,
|
||||
TenderRepo: tenderRepo,
|
||||
AIClient: aiClient,
|
||||
AIStorage: aiStorage,
|
||||
TranslationSuccessCounter: translationSuccessCounter,
|
||||
TargetLanguages: langs,
|
||||
BatchSize: batchSize,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -77,6 +80,8 @@ func (w *TranslationWorker) Run() {
|
||||
"batch_size": w.BatchSize,
|
||||
})
|
||||
|
||||
startCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
|
||||
|
||||
if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
|
||||
w.Logger.Warn("Failed to trigger AI pipeline translate (continuing with per-tender sync)", map[string]interface{}{
|
||||
"target_languages": w.TargetLanguages,
|
||||
@@ -88,11 +93,36 @@ func (w *TranslationWorker) Run() {
|
||||
w.runForLanguage(language)
|
||||
}
|
||||
|
||||
endCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
|
||||
totalCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKey)
|
||||
runCount := endCount - startCount
|
||||
if runCount < 0 {
|
||||
runCount = 0
|
||||
}
|
||||
|
||||
w.Logger.Info("Translation worker completed", map[string]interface{}{
|
||||
"target_languages": w.TargetLanguages,
|
||||
"target_languages": w.TargetLanguages,
|
||||
"successful_ai_translation_requests_run": runCount,
|
||||
"successful_ai_translation_requests_daily_job": endCount,
|
||||
"successful_ai_translation_requests_total": totalCount,
|
||||
})
|
||||
}
|
||||
|
||||
func (w *TranslationWorker) getSuccessfulTranslationCount(key string) int64 {
|
||||
if w.TranslationSuccessCounter == nil {
|
||||
return 0
|
||||
}
|
||||
count, err := w.TranslationSuccessCounter.Get(context.Background(), key)
|
||||
if err != nil {
|
||||
w.Logger.Warn("Failed to read AI translation success counter", map[string]interface{}{
|
||||
"counter_key": key,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return 0
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (w *TranslationWorker) runForLanguage(language string) {
|
||||
skip := 0
|
||||
for {
|
||||
@@ -210,6 +240,7 @@ func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string
|
||||
Title: t.Title,
|
||||
Description: t.Description,
|
||||
Language: language,
|
||||
RequestSource: ai_summarizer.TranslationRequestSourceDailyJob,
|
||||
})
|
||||
if apiErr != nil {
|
||||
return "", "", "api", apiErr
|
||||
|
||||
Reference in New Issue
Block a user