mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor AI summarizer client initialization to include MongoDB manager and add statistics endpoint

Browse Source 
- Updated `InitAISummarizerClient` to accept `mongoManager` for tracking translation success.
- Introduced new `Statistics` endpoint in the dashboard to fetch scraping and translation statistics.
- Enhanced `TranslationWorker` to utilize the new success counter for tracking successful translations.
- Added necessary data structures and query forms for statistics reporting.

This refactor improves the tracking of AI translation success and provides new insights through the dashboard statistics.
This commit is contained in:
Mazyar
2026-06-06 21:20:53 +03:30
parent 5af3bfaa1a
commit 68b170126d
17 changed files with 507 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/web/bootstrap/bootstrap.go
+13 -11
View File
@@ -290,23 +290,25 @@ func InitFileStore(mongoManager *mongo.ConnectionManager, log logger.Logger) (*f
}
// InitAISummarizerClient initializes the AI summarizer HTTP client for on-demand summarization
func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.Client {
func InitAISummarizerClient(conf AISummarizerConfig, mongoManager *mongo.ConnectionManager, log logger.Logger) *ai_summarizer.Client {
if conf.APIBaseURL == "" {
log.Warn("AI Summarizer API base URL not configured, on-demand summarization will be unavailable", map[string]interface{}{})
return nil
}
translationCounter := mongo.NewCounter(mongoManager)
cfg := &ai_summarizer.Config{
APIBaseURL: conf.APIBaseURL,
APITimeout: conf.APITimeout,
APIRetryCount: conf.APIRetryCount,
APIRetryDelay: conf.APIRetryDelay,
MinioEndpoint: conf.MinioEndpoint,
MinioAccessKey: conf.MinioAccessKey,
MinioSecretKey: conf.MinioSecretKey,
MinioUseSSL: conf.MinioUseSSL,
MinioRegion: conf.MinioRegion,
MinioBucket: conf.MinioBucket,
APIBaseURL: conf.APIBaseURL,
APITimeout: conf.APITimeout,
APIRetryCount: conf.APIRetryCount,
APIRetryDelay: conf.APIRetryDelay,
MinioEndpoint: conf.MinioEndpoint,
MinioAccessKey: conf.MinioAccessKey,
MinioSecretKey: conf.MinioSecretKey,
MinioUseSSL: conf.MinioUseSSL,
MinioRegion: conf.MinioRegion,
MinioBucket: conf.MinioBucket,
OnSuccessfulTranslation: mongo.AITranslationSuccessCallback(translationCounter),
}
client, err := ai_summarizer.NewClient(cfg, log)
cmd/web/main.go
+1 -1
View File
@@ -168,7 +168,7 @@ func main() {
var aiSummarizerClient tender.AISummarizerClient
var aiSummarizerStorage tender.AISummarizerStorage
if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, logger); c != nil {
if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, mongoManager, logger); c != nil {
aiSummarizerClient = c
}
if s := bootstrap.InitAISummarizerStorage(conf.AISummarizer, logger); s != nil {
cmd/web/router/routes.go
+1
View File
@@ -222,6 +222,7 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
dashboardGP.GET("/notice-types", dashboardHandler.NoticeTypes)
dashboardGP.GET("/closing-soon", dashboardHandler.ClosingSoon)
dashboardGP.GET("/recent", dashboardHandler.Recent)
dashboardGP.GET("/statistics", dashboardHandler.Statistics)
}
}
cmd/worker/bootstrap/bootstrap.go
+9 -5
View File
@@ -196,6 +196,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
// Initialize translation backfill worker for configured languages
if aiClient != nil {
targetLanguages := parseTranslationLanguages(config.AISummarizer)
translationSuccessCounter := mongo.NewCounter(mongoManager)
scheduler.AddJob(schedule.Job{
Name: "Tender Translation Worker Job",
Func: func() {
@@ -205,6 +206,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
tenderRepo,
aiClient,
aiStorage,
translationSuccessCounter,
targetLanguages,
config.Worker.TranslationBatchSize,
)
@@ -247,17 +249,19 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
}
// InitAISummarizerClient initializes the AI summarizer HTTP client used by worker jobs.
func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.Client {
func InitAISummarizerClient(conf AISummarizerConfig, mongoManager *mongo.ConnectionManager, log logger.Logger) *ai_summarizer.Client {
if conf.APIBaseURL == "" {
log.Warn("AI Summarizer API base URL not configured, translation worker will be unavailable", map[string]interface{}{})
return nil
}
translationCounter := mongo.NewCounter(mongoManager)
cfg := &ai_summarizer.Config{
APIBaseURL: conf.APIBaseURL,
APITimeout: conf.APITimeout,
APIRetryCount: conf.APIRetryCount,
APIRetryDelay: conf.APIRetryDelay,
APIBaseURL: conf.APIBaseURL,
APITimeout: conf.APITimeout,
APIRetryCount: conf.APIRetryCount,
APIRetryDelay: conf.APIRetryDelay,
OnSuccessfulTranslation: mongo.AITranslationSuccessCallback(translationCounter),
}
client, err := ai_summarizer.NewClient(cfg, log)
cmd/worker/main.go
+1 -1
View File
@@ -44,7 +44,7 @@ func main() {
}
// Initialize AI summarizer client (translation + document summarization workers)
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, appLogger)
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, mongoManager, appLogger)
aiSummarizerStorage := bootstrap.InitAISummarizerStorage(config.AISummarizer, appLogger)
// Initialize Worker
cmd/worker/workers/translation.go
+46 -15
View File
@@ -13,13 +13,14 @@ import (
// TranslationWorker backfills missing tender translations using the AI pipeline
// and on-demand translate API. Results are persisted by the AI service in MinIO only.
type TranslationWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tender.TenderRepository
AIClient *ai_summarizer.Client
AIStorage *ai_summarizer.StorageClient
TargetLanguages []string
BatchSize int
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tender.TenderRepository
AIClient *ai_summarizer.Client
AIStorage *ai_summarizer.StorageClient
TranslationSuccessCounter *mongo.Counter
TargetLanguages []string
BatchSize int
}
// NewTranslationWorker creates a translation worker for the given target languages.
@@ -29,6 +30,7 @@ func NewTranslationWorker(
tenderRepo tender.TenderRepository,
aiClient *ai_summarizer.Client,
aiStorage *ai_summarizer.StorageClient,
translationSuccessCounter *mongo.Counter,
targetLanguages []string,
batchSize int,
) *TranslationWorker {
@@ -53,13 +55,14 @@ func NewTranslationWorker(
}
return &TranslationWorker{
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
AIClient: aiClient,
AIStorage: aiStorage,
TargetLanguages: langs,
BatchSize: batchSize,
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
AIClient: aiClient,
AIStorage: aiStorage,
TranslationSuccessCounter: translationSuccessCounter,
TargetLanguages: langs,
BatchSize: batchSize,
}
}
@@ -77,6 +80,8 @@ func (w *TranslationWorker) Run() {
"batch_size": w.BatchSize,
})
startCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
w.Logger.Warn("Failed to trigger AI pipeline translate (continuing with per-tender sync)", map[string]interface{}{
"target_languages": w.TargetLanguages,
@@ -88,11 +93,36 @@ func (w *TranslationWorker) Run() {
w.runForLanguage(language)
}
endCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
totalCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKey)
runCount := endCount - startCount
if runCount < 0 {
runCount = 0
}
w.Logger.Info("Translation worker completed", map[string]interface{}{
"target_languages": w.TargetLanguages,
"target_languages": w.TargetLanguages,
"successful_ai_translation_requests_run": runCount,
"successful_ai_translation_requests_daily_job": endCount,
"successful_ai_translation_requests_total": totalCount,
})
}
func (w *TranslationWorker) getSuccessfulTranslationCount(key string) int64 {
if w.TranslationSuccessCounter == nil {
return 0
}
count, err := w.TranslationSuccessCounter.Get(context.Background(), key)
if err != nil {
w.Logger.Warn("Failed to read AI translation success counter", map[string]interface{}{
"counter_key": key,
"error": err.Error(),
})
return 0
}
return count
}
func (w *TranslationWorker) runForLanguage(language string) {
skip := 0
for {
@@ -210,6 +240,7 @@ func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string
Title: t.Title,
Description: t.Description,
Language: language,
RequestSource: ai_summarizer.TranslationRequestSourceDailyJob,
})
if apiErr != nil {
return "", "", "api", apiErr