Refactor AI summarizer client initialization to include MongoDB manager and add statistics endpoint

- Updated `InitAISummarizerClient` to accept `mongoManager` for tracking translation success.
- Introduced new `Statistics` endpoint in the dashboard to fetch scraping and translation statistics.
- Enhanced `TranslationWorker` to utilize the new success counter for tracking successful translations.
- Added necessary data structures and query forms for statistics reporting.

This refactor improves the tracking of AI translation success and provides new insights through the dashboard statistics.
This commit is contained in:
Mazyar
2026-06-06 21:20:53 +03:30
parent 5af3bfaa1a
commit 68b170126d
17 changed files with 507 additions and 44 deletions
+13 -11
View File
@@ -290,23 +290,25 @@ func InitFileStore(mongoManager *mongo.ConnectionManager, log logger.Logger) (*f
} }
// InitAISummarizerClient initializes the AI summarizer HTTP client for on-demand summarization // InitAISummarizerClient initializes the AI summarizer HTTP client for on-demand summarization
func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.Client { func InitAISummarizerClient(conf AISummarizerConfig, mongoManager *mongo.ConnectionManager, log logger.Logger) *ai_summarizer.Client {
if conf.APIBaseURL == "" { if conf.APIBaseURL == "" {
log.Warn("AI Summarizer API base URL not configured, on-demand summarization will be unavailable", map[string]interface{}{}) log.Warn("AI Summarizer API base URL not configured, on-demand summarization will be unavailable", map[string]interface{}{})
return nil return nil
} }
translationCounter := mongo.NewCounter(mongoManager)
cfg := &ai_summarizer.Config{ cfg := &ai_summarizer.Config{
APIBaseURL: conf.APIBaseURL, APIBaseURL: conf.APIBaseURL,
APITimeout: conf.APITimeout, APITimeout: conf.APITimeout,
APIRetryCount: conf.APIRetryCount, APIRetryCount: conf.APIRetryCount,
APIRetryDelay: conf.APIRetryDelay, APIRetryDelay: conf.APIRetryDelay,
MinioEndpoint: conf.MinioEndpoint, MinioEndpoint: conf.MinioEndpoint,
MinioAccessKey: conf.MinioAccessKey, MinioAccessKey: conf.MinioAccessKey,
MinioSecretKey: conf.MinioSecretKey, MinioSecretKey: conf.MinioSecretKey,
MinioUseSSL: conf.MinioUseSSL, MinioUseSSL: conf.MinioUseSSL,
MinioRegion: conf.MinioRegion, MinioRegion: conf.MinioRegion,
MinioBucket: conf.MinioBucket, MinioBucket: conf.MinioBucket,
OnSuccessfulTranslation: mongo.AITranslationSuccessCallback(translationCounter),
} }
client, err := ai_summarizer.NewClient(cfg, log) client, err := ai_summarizer.NewClient(cfg, log)
+1 -1
View File
@@ -168,7 +168,7 @@ func main() {
var aiSummarizerClient tender.AISummarizerClient var aiSummarizerClient tender.AISummarizerClient
var aiSummarizerStorage tender.AISummarizerStorage var aiSummarizerStorage tender.AISummarizerStorage
if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, logger); c != nil { if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, mongoManager, logger); c != nil {
aiSummarizerClient = c aiSummarizerClient = c
} }
if s := bootstrap.InitAISummarizerStorage(conf.AISummarizer, logger); s != nil { if s := bootstrap.InitAISummarizerStorage(conf.AISummarizer, logger); s != nil {
+1
View File
@@ -222,6 +222,7 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
dashboardGP.GET("/notice-types", dashboardHandler.NoticeTypes) dashboardGP.GET("/notice-types", dashboardHandler.NoticeTypes)
dashboardGP.GET("/closing-soon", dashboardHandler.ClosingSoon) dashboardGP.GET("/closing-soon", dashboardHandler.ClosingSoon)
dashboardGP.GET("/recent", dashboardHandler.Recent) dashboardGP.GET("/recent", dashboardHandler.Recent)
dashboardGP.GET("/statistics", dashboardHandler.Statistics)
} }
} }
+9 -5
View File
@@ -196,6 +196,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
// Initialize translation backfill worker for configured languages // Initialize translation backfill worker for configured languages
if aiClient != nil { if aiClient != nil {
targetLanguages := parseTranslationLanguages(config.AISummarizer) targetLanguages := parseTranslationLanguages(config.AISummarizer)
translationSuccessCounter := mongo.NewCounter(mongoManager)
scheduler.AddJob(schedule.Job{ scheduler.AddJob(schedule.Job{
Name: "Tender Translation Worker Job", Name: "Tender Translation Worker Job",
Func: func() { Func: func() {
@@ -205,6 +206,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
tenderRepo, tenderRepo,
aiClient, aiClient,
aiStorage, aiStorage,
translationSuccessCounter,
targetLanguages, targetLanguages,
config.Worker.TranslationBatchSize, config.Worker.TranslationBatchSize,
) )
@@ -247,17 +249,19 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
} }
// InitAISummarizerClient initializes the AI summarizer HTTP client used by worker jobs. // InitAISummarizerClient initializes the AI summarizer HTTP client used by worker jobs.
func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.Client { func InitAISummarizerClient(conf AISummarizerConfig, mongoManager *mongo.ConnectionManager, log logger.Logger) *ai_summarizer.Client {
if conf.APIBaseURL == "" { if conf.APIBaseURL == "" {
log.Warn("AI Summarizer API base URL not configured, translation worker will be unavailable", map[string]interface{}{}) log.Warn("AI Summarizer API base URL not configured, translation worker will be unavailable", map[string]interface{}{})
return nil return nil
} }
translationCounter := mongo.NewCounter(mongoManager)
cfg := &ai_summarizer.Config{ cfg := &ai_summarizer.Config{
APIBaseURL: conf.APIBaseURL, APIBaseURL: conf.APIBaseURL,
APITimeout: conf.APITimeout, APITimeout: conf.APITimeout,
APIRetryCount: conf.APIRetryCount, APIRetryCount: conf.APIRetryCount,
APIRetryDelay: conf.APIRetryDelay, APIRetryDelay: conf.APIRetryDelay,
OnSuccessfulTranslation: mongo.AITranslationSuccessCallback(translationCounter),
} }
client, err := ai_summarizer.NewClient(cfg, log) client, err := ai_summarizer.NewClient(cfg, log)
+1 -1
View File
@@ -44,7 +44,7 @@ func main() {
} }
// Initialize AI summarizer client (translation + document summarization workers) // Initialize AI summarizer client (translation + document summarization workers)
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, appLogger) aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, mongoManager, appLogger)
aiSummarizerStorage := bootstrap.InitAISummarizerStorage(config.AISummarizer, appLogger) aiSummarizerStorage := bootstrap.InitAISummarizerStorage(config.AISummarizer, appLogger)
// Initialize Worker // Initialize Worker
+46 -15
View File
@@ -13,13 +13,14 @@ import (
// TranslationWorker backfills missing tender translations using the AI pipeline // TranslationWorker backfills missing tender translations using the AI pipeline
// and on-demand translate API. Results are persisted by the AI service in MinIO only. // and on-demand translate API. Results are persisted by the AI service in MinIO only.
type TranslationWorker struct { type TranslationWorker struct {
Mongo *mongo.ConnectionManager Mongo *mongo.ConnectionManager
Logger logger.Logger Logger logger.Logger
TenderRepo tender.TenderRepository TenderRepo tender.TenderRepository
AIClient *ai_summarizer.Client AIClient *ai_summarizer.Client
AIStorage *ai_summarizer.StorageClient AIStorage *ai_summarizer.StorageClient
TargetLanguages []string TranslationSuccessCounter *mongo.Counter
BatchSize int TargetLanguages []string
BatchSize int
} }
// NewTranslationWorker creates a translation worker for the given target languages. // NewTranslationWorker creates a translation worker for the given target languages.
@@ -29,6 +30,7 @@ func NewTranslationWorker(
tenderRepo tender.TenderRepository, tenderRepo tender.TenderRepository,
aiClient *ai_summarizer.Client, aiClient *ai_summarizer.Client,
aiStorage *ai_summarizer.StorageClient, aiStorage *ai_summarizer.StorageClient,
translationSuccessCounter *mongo.Counter,
targetLanguages []string, targetLanguages []string,
batchSize int, batchSize int,
) *TranslationWorker { ) *TranslationWorker {
@@ -53,13 +55,14 @@ func NewTranslationWorker(
} }
return &TranslationWorker{ return &TranslationWorker{
Mongo: mongo, Mongo: mongo,
Logger: logger, Logger: logger,
TenderRepo: tenderRepo, TenderRepo: tenderRepo,
AIClient: aiClient, AIClient: aiClient,
AIStorage: aiStorage, AIStorage: aiStorage,
TargetLanguages: langs, TranslationSuccessCounter: translationSuccessCounter,
BatchSize: batchSize, TargetLanguages: langs,
BatchSize: batchSize,
} }
} }
@@ -77,6 +80,8 @@ func (w *TranslationWorker) Run() {
"batch_size": w.BatchSize, "batch_size": w.BatchSize,
}) })
startCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil { if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
w.Logger.Warn("Failed to trigger AI pipeline translate (continuing with per-tender sync)", map[string]interface{}{ w.Logger.Warn("Failed to trigger AI pipeline translate (continuing with per-tender sync)", map[string]interface{}{
"target_languages": w.TargetLanguages, "target_languages": w.TargetLanguages,
@@ -88,11 +93,36 @@ func (w *TranslationWorker) Run() {
w.runForLanguage(language) w.runForLanguage(language)
} }
endCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
totalCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKey)
runCount := endCount - startCount
if runCount < 0 {
runCount = 0
}
w.Logger.Info("Translation worker completed", map[string]interface{}{ w.Logger.Info("Translation worker completed", map[string]interface{}{
"target_languages": w.TargetLanguages, "target_languages": w.TargetLanguages,
"successful_ai_translation_requests_run": runCount,
"successful_ai_translation_requests_daily_job": endCount,
"successful_ai_translation_requests_total": totalCount,
}) })
} }
func (w *TranslationWorker) getSuccessfulTranslationCount(key string) int64 {
if w.TranslationSuccessCounter == nil {
return 0
}
count, err := w.TranslationSuccessCounter.Get(context.Background(), key)
if err != nil {
w.Logger.Warn("Failed to read AI translation success counter", map[string]interface{}{
"counter_key": key,
"error": err.Error(),
})
return 0
}
return count
}
func (w *TranslationWorker) runForLanguage(language string) { func (w *TranslationWorker) runForLanguage(language string) {
skip := 0 skip := 0
for { for {
@@ -210,6 +240,7 @@ func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string
Title: t.Title, Title: t.Title,
Description: t.Description, Description: t.Description,
Language: language, Language: language,
RequestSource: ai_summarizer.TranslationRequestSourceDailyJob,
}) })
if apiErr != nil { if apiErr != nil {
return "", "", "api", apiErr return "", "", "api", apiErr
+21
View File
@@ -85,3 +85,24 @@ type RecentItem struct {
CreatedAt int64 `json:"created_at"` CreatedAt int64 `json:"created_at"`
PublicationDate int64 `json:"publication_date"` PublicationDate int64 `json:"publication_date"`
} }
// StatisticsReportResponse powers the statistics and charts report.
type StatisticsReportResponse struct {
Days int `json:"days"`
GeneratedAt int64 `json:"generated_at"`
Daily StatisticsDailySeries `json:"daily"`
Totals StatisticsLifetimeTotals `json:"totals"`
}
// StatisticsDailySeries contains per-day chart series.
type StatisticsDailySeries struct {
ScrapedTED []TrendPoint `json:"scraped_ted"`
ScrapedDocuments []TrendPoint `json:"scraped_documents"`
TranslatedNotices []TrendPoint `json:"translated_notices"`
}
// StatisticsLifetimeTotals contains all-time totals.
type StatisticsLifetimeTotals struct {
ScrapedDocuments int64 `json:"scraped_documents"`
TranslatedTenders int64 `json:"translated_tenders"`
}
+5
View File
@@ -27,3 +27,8 @@ type RecentQuery struct {
Limit int `query:"limit"` Limit int `query:"limit"`
Cursor string `query:"cursor"` Cursor string `query:"cursor"`
} }
// StatisticsQuery binds query params for GET /dashboard/statistics.
type StatisticsQuery struct {
Days int `query:"days"`
}
+24
View File
@@ -161,6 +161,30 @@ func (h *Handler) Recent(c echo.Context) error {
return response.Success(c, out, "Recent tenders retrieved successfully") return response.Success(c, out, "Recent tenders retrieved successfully")
} }
// Statistics returns scraping and translation statistics for charts.
// @Summary Dashboard statistics report
// @Description Daily and lifetime statistics for TED scraping, document scraping, and AI translation
// @Tags Admin-Dashboard
// @Produce json
// @Param days query int false "Days back for daily series (default 14, max 90)"
// @Success 200 {object} response.APIResponse{data=StatisticsReportResponse}
// @Failure 500 {object} response.APIResponse
// @Router /admin/v1/dashboard/statistics [get]
// @Security BearerAuth
func (h *Handler) Statistics(c echo.Context) error {
query := StatisticsQuery{
Days: parseIntQuery(c, "days", 0),
}
out, err := h.service.Statistics(c.Request().Context(), query)
if err != nil {
return response.InternalServerError(c, "Failed to load dashboard statistics")
}
setPrivateCache(c, 60)
return response.Success(c, out, "Dashboard statistics retrieved successfully")
}
func parseIntQuery(c echo.Context, name string, fallback int) int { func parseIntQuery(c echo.Context, name string, fallback int) int {
raw := c.QueryParam(name) raw := c.QueryParam(name)
if raw == "" { if raw == "" {
+9 -4
View File
@@ -23,18 +23,23 @@ type Repository interface {
NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error) NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
ClosingSoon(ctx context.Context, limit int, windowSec int64) ([]ClosingSoonItem, error) ClosingSoon(ctx context.Context, limit int, windowSec int64) ([]ClosingSoonItem, error)
Recent(ctx context.Context, limit int, cursor string) ([]RecentItem, *string, error) Recent(ctx context.Context, limit int, cursor string) ([]RecentItem, *string, error)
Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error)
} }
type repository struct { type repository struct {
ormRepo orm.Repository[tender.Tender] ormRepo orm.Repository[tender.Tender]
logger logger.Logger mongoManager *orm.ConnectionManager
metricsCounter *orm.Counter
logger logger.Logger
} }
// NewRepository creates a dashboard repository backed by the tenders collection. // NewRepository creates a dashboard repository backed by the tenders collection.
func NewRepository(mongoManager *orm.ConnectionManager, log logger.Logger) Repository { func NewRepository(mongoManager *orm.ConnectionManager, log logger.Logger) Repository {
return &repository{ return &repository{
ormRepo: orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log), ormRepo: orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log),
logger: log, mongoManager: mongoManager,
metricsCounter: orm.NewCounter(mongoManager),
logger: log,
} }
} }
+20
View File
@@ -25,6 +25,7 @@ type Service interface {
NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error) NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
ClosingSoon(ctx context.Context, query ClosingSoonQuery) (*ClosingSoonResponse, error) ClosingSoon(ctx context.Context, query ClosingSoonQuery) (*ClosingSoonResponse, error)
Recent(ctx context.Context, query RecentQuery) (*RecentResponse, error) Recent(ctx context.Context, query RecentQuery) (*RecentResponse, error)
Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error)
} }
type service struct { type service struct {
@@ -157,6 +158,25 @@ func (s *service) Recent(ctx context.Context, query RecentQuery) (*RecentRespons
}, nil }, nil
} }
func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error) {
days := trendDays(query.Days)
startUnix := trendStartUnix(days)
s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
"days": days,
})
out, err := s.repo.Statistics(ctx, days, startUnix)
if err != nil {
s.logger.Error("Failed to fetch dashboard statistics report", map[string]interface{}{
"error": err.Error(),
})
return nil, fmt.Errorf("dashboard statistics: %w", err)
}
return out, nil
}
func closingWindowHours(hours int) int { func closingWindowHours(hours int) int {
if hours <= 0 { if hours <= 0 {
return defaultClosingWindowHours return defaultClosingWindowHours
+189
View File
@@ -0,0 +1,189 @@
package dashboard
import (
"context"
"fmt"
"time"
"tm/internal/notice"
orm "tm/pkg/mongo"
"go.mongodb.org/mongo-driver/v2/bson"
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
)
const noticesCollectionName = "notices"
func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
now := time.Now().UTC()
endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
startDay := endDay.AddDate(0, 0, -(days - 1))
scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{
"source": notice.TenderSourceTEDScraper,
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
}, "processing_metadata.scraped_at")
if err != nil {
return nil, fmt.Errorf("scraped ted per day: %w", err)
}
scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix)
if err != nil {
return nil, fmt.Errorf("scraped documents per day: %w", err)
}
translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay)
if err != nil {
return nil, fmt.Errorf("translated notices per day: %w", err)
}
totalDocuments, err := r.totalScrapedDocuments(ctx)
if err != nil {
return nil, fmt.Errorf("total scraped documents: %w", err)
}
totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey)
if err != nil {
return nil, fmt.Errorf("total translated tenders: %w", err)
}
return &StatisticsReportResponse{
Days: days,
GeneratedAt: now.Unix(),
Daily: StatisticsDailySeries{
ScrapedTED: fillTrendSeries(days, scrapedTED),
ScrapedDocuments: fillTrendSeries(days, scrapedDocuments),
TranslatedNotices: fillTrendSeries(days, translatedNotices),
},
Totals: StatisticsLifetimeTotals{
ScrapedDocuments: totalDocuments,
TranslatedTenders: totalTranslated,
},
}, nil
}
func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) {
pipeline := mongodriver.Pipeline{
{{Key: "$match", Value: match}},
{{Key: "$addFields", Value: bson.M{
"metric_ts": normalizeTimestampExpr(timestampField),
}}},
{{Key: "$group", Value: bson.M{
"_id": bson.M{
"$dateToString": bson.M{
"format": "%Y-%m-%d",
"date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
"timezone": "UTC",
},
},
"count": bson.M{"$sum": 1},
}}},
}
cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline)
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
return decodeDailyCounts(ctx, cursor)
}
func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) {
pipeline := mongodriver.Pipeline{
{{Key: "$match", Value: bson.M{
"processing_metadata.documents_scraped": true,
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
}}},
{{Key: "$unwind", Value: "$scraped_documents"}},
{{Key: "$addFields", Value: bson.M{
"metric_ts": bson.M{
"$cond": bson.A{
bson.M{"$gt": bson.A{"$scraped_documents.scraped_at", 0}},
"$scraped_documents.scraped_at",
"$processing_metadata.documents_scraped_at",
},
},
}}},
{{Key: "$match", Value: bson.M{
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
}}},
{{Key: "$addFields", Value: bson.M{
"metric_ts": normalizeTimestampExpr("metric_ts"),
}}},
{{Key: "$group", Value: bson.M{
"_id": bson.M{
"$dateToString": bson.M{
"format": "%Y-%m-%d",
"date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
"timezone": "UTC",
},
},
"count": bson.M{"$sum": 1},
}}},
}
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
if err != nil {
return nil, err
}
defer cursor.Close(ctx)
return decodeDailyCounts(ctx, cursor)
}
func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
}
func (r *repository) totalScrapedDocuments(ctx context.Context) (int64, error) {
pipeline := mongodriver.Pipeline{
{{Key: "$match", Value: bson.M{
"processing_metadata.documents_scraped": true,
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
}}},
{{Key: "$project", Value: bson.M{
"doc_count": bson.M{"$size": "$scraped_documents"},
}}},
{{Key: "$group", Value: bson.M{
"_id": nil,
"total": bson.M{"$sum": "$doc_count"},
}}},
}
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
if err != nil {
return 0, err
}
defer cursor.Close(ctx)
if !cursor.Next(ctx) {
return 0, nil
}
var row bson.M
if err := cursor.Decode(&row); err != nil {
return 0, err
}
return aggregateCount(row, "total"), nil
}
func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) {
counts := make(map[string]int64)
for cursor.Next(ctx) {
var row bson.M
if err := cursor.Decode(&row); err != nil {
return nil, err
}
date, ok := row["_id"].(string)
if !ok || date == "" {
continue
}
counts[date] = aggregateCount(row, "count")
}
if err := cursor.Err(); err != nil {
return nil, err
}
return counts, nil
}
+14 -5
View File
@@ -1476,7 +1476,7 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
return nil, fmt.Errorf("tender has no contract folder ID") return nil, fmt.Errorf("tender has no contract folder ID")
} }
title, description, err := s.resolveTranslation(ctx, t, targetLanguage) title, description, fromAPI, err := s.resolveTranslation(ctx, t, targetLanguage, ai_summarizer.TranslationRequestSourceManualTrigger)
if err != nil { if err != nil {
s.logger.Error("AI translation failed", map[string]interface{}{ s.logger.Error("AI translation failed", map[string]interface{}{
"tender_id": id, "tender_id": id,
@@ -1487,6 +1487,14 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
return nil, fmt.Errorf("AI translation request failed: %w", err) return nil, fmt.Errorf("AI translation request failed: %w", err)
} }
if fromAPI {
s.logger.Info("Manual AI translation request succeeded", map[string]interface{}{
"tender_id": id,
"notice_id": t.NoticePublicationID,
"language": targetLanguage,
})
}
return s.buildAITranslateResponse(t, targetLanguage, title, description), nil return s.buildAITranslateResponse(t, targetLanguage, title, description), nil
} }
@@ -1500,11 +1508,11 @@ func (s *tenderService) buildAITranslateResponse(t *Tender, language, title, des
} }
} }
func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, language string) (string, string, error) { func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, language, requestSource string) (string, string, bool, error) {
if s.aiSummarizerStorage != nil { if s.aiSummarizerStorage != nil {
stored, err := s.aiSummarizerStorage.GetTranslationFromStorage(ctx, t.ContractFolderID, t.NoticePublicationID, language) stored, err := s.aiSummarizerStorage.GetTranslationFromStorage(ctx, t.ContractFolderID, t.NoticePublicationID, language)
if err == nil { if err == nil {
return stored.Title, stored.Description, nil return stored.Title, stored.Description, false, nil
} }
if !errors.Is(err, ai_summarizer.ErrTranslationNotReady) && if !errors.Is(err, ai_summarizer.ErrTranslationNotReady) &&
!errors.Is(err, ai_summarizer.ErrObjectNotFound) && !errors.Is(err, ai_summarizer.ErrObjectNotFound) &&
@@ -1523,11 +1531,12 @@ func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, langu
Title: t.Title, Title: t.Title,
Description: t.Description, Description: t.Description,
Language: language, Language: language,
RequestSource: requestSource,
}) })
if err != nil { if err != nil {
return "", "", err return "", "", false, err
} }
return resp.TranslatedTitle, resp.TranslatedDescription, nil return resp.TranslatedTitle, resp.TranslatedDescription, true, nil
} }
func (s *tenderService) pickResponseLanguage(language *string) string { func (s *tenderService) pickResponseLanguage(language *string) string {
+13 -2
View File
@@ -107,7 +107,7 @@ func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody Translate
} }
} }
resp, err := c.doTranslatePost(ctx, url, jsonBody) resp, err := c.doTranslatePost(ctx, url, jsonBody, reqBody.RequestSource)
if err != nil { if err != nil {
lastErr = err lastErr = err
c.logger.Error("AI translate request failed", map[string]interface{}{ c.logger.Error("AI translate request failed", map[string]interface{}{
@@ -216,7 +216,7 @@ func (c *Client) doPost(ctx context.Context, url string, jsonBody []byte) (*Summ
} }
// doTranslatePost performs a single translation POST request and parses the response. // doTranslatePost performs a single translation POST request and parses the response.
func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byte) (*TranslateResponse, error) { func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byte, requestSource string) (*TranslateResponse, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody)) req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
if err != nil { if err != nil {
return nil, fmt.Errorf("ai_summarizer: failed to create translation request: %w", err) return nil, fmt.Errorf("ai_summarizer: failed to create translation request: %w", err)
@@ -253,6 +253,17 @@ func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byt
"language": result.Language, "language": result.Language,
}) })
if c.config.OnSuccessfulTranslation != nil {
if err := c.config.OnSuccessfulTranslation(ctx, requestSource); err != nil {
c.logger.Warn("Failed to increment AI translation success counter", map[string]interface{}{
"notice_publication_id": result.NoticePublicationID,
"language": result.Language,
"request_source": requestSource,
"error": err.Error(),
})
}
}
return result, nil return result, nil
} }
+5
View File
@@ -1,6 +1,7 @@
package ai_summarizer package ai_summarizer
import ( import (
"context"
"errors" "errors"
"fmt" "fmt"
"time" "time"
@@ -15,6 +16,10 @@ type Config struct {
APIRetryCount int // Number of retry attempts for failed API requests APIRetryCount int // Number of retry attempts for failed API requests
APIRetryDelay time.Duration // Delay between retry attempts APIRetryDelay time.Duration // Delay between retry attempts
// OnSuccessfulTranslation is called after a successful POST /ai/translate request.
// source identifies the caller (daily_job or manual_trigger).
OnSuccessfulTranslation func(ctx context.Context, source string) error
// MinIO storage settings // MinIO storage settings
MinioEndpoint string // MinIO endpoint, e.g. "ai-host:9000" MinioEndpoint string // MinIO endpoint, e.g. "ai-host:9000"
MinioAccessKey string // MinIO access key ID MinioAccessKey string // MinIO access key ID
+6
View File
@@ -109,8 +109,14 @@ type TranslateRequest struct {
Title string `json:"title"` Title string `json:"title"`
Description string `json:"description"` Description string `json:"description"`
Language string `json:"language"` Language string `json:"language"`
RequestSource string `json:"-"` // internal: daily_job | manual_trigger
} }
const (
TranslationRequestSourceDailyJob = "daily_job"
TranslationRequestSourceManualTrigger = "manual_trigger"
)
// TranslateResponse represents the translation payload from POST /ai/translate. // TranslateResponse represents the translation payload from POST /ai/translate.
type TranslateResponse struct { type TranslateResponse struct {
ContractFolderID string `json:"contract_folder_id"` ContractFolderID string `json:"contract_folder_id"`
+130
View File
@@ -0,0 +1,130 @@
package mongo
import (
"context"
"errors"
"fmt"
"time"
"go.mongodb.org/mongo-driver/v2/bson"
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
"go.mongodb.org/mongo-driver/v2/mongo/options"
)
const (
metricsCountersCollection = "metrics_counters"
AITranslationSuccessCounterKey = "ai_translation_successful_requests"
AITranslationSuccessCounterKeyDailyJob = "ai_translation_successful_requests_daily_job"
AITranslationSuccessCounterKeyManualTrigger = "ai_translation_successful_requests_manual_trigger"
)
// Counter provides atomic increment/read operations for named metrics stored in MongoDB.
type Counter struct {
mongo *ConnectionManager
}
// NewCounter creates a counter backed by the metrics_counters collection.
func NewCounter(mongo *ConnectionManager) *Counter {
return &Counter{mongo: mongo}
}
// Increment atomically increments the counter identified by key and returns the new value.
func (c *Counter) Increment(ctx context.Context, key string) (int64, error) {
if key == "" {
return 0, fmt.Errorf("mongo counter: key is required")
}
var res struct {
Count int64 `bson:"count"`
}
err := c.mongo.GetCollection(metricsCountersCollection).FindOneAndUpdate(
ctx,
bson.M{"_id": key},
bson.M{
"$inc": bson.M{"count": 1},
"$set": bson.M{"updated_at": time.Now().Unix()},
"$setOnInsert": bson.M{"_id": key},
},
options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After),
).Decode(&res)
if err != nil {
return 0, fmt.Errorf("mongo counter increment %q: %w", key, err)
}
return res.Count, nil
}
// Get returns the current value of the counter identified by key (0 when missing).
func (c *Counter) Get(ctx context.Context, key string) (int64, error) {
if key == "" {
return 0, fmt.Errorf("mongo counter: key is required")
}
var doc struct {
Count int64 `bson:"count"`
}
err := c.mongo.GetCollection(metricsCountersCollection).FindOne(ctx, bson.M{"_id": key}).Decode(&doc)
if err != nil {
if errors.Is(err, mongodriver.ErrNoDocuments) {
return 0, nil
}
return 0, fmt.Errorf("mongo counter get %q: %w", key, err)
}
return doc.Count, nil
}
// AITranslationSuccessCallback returns a callback that increments successful AI translation counters.
func AITranslationSuccessCallback(counter *Counter) func(ctx context.Context, source string) error {
return func(ctx context.Context, source string) error {
if _, err := counter.Increment(ctx, AITranslationSuccessCounterKey); err != nil {
return err
}
if _, err := counter.Increment(ctx, AITranslationSuccessDailyCounterKey(time.Now().UTC())); err != nil {
return err
}
if key := AITranslationSuccessCounterKeyForSource(source); key != "" {
_, err := counter.Increment(ctx, key)
return err
}
return nil
}
}
// AITranslationSuccessDailyCounterKey returns the metrics key for successful translations on a given UTC day.
func AITranslationSuccessDailyCounterKey(day time.Time) string {
return AITranslationSuccessCounterKey + "_day:" + day.UTC().Format("2006-01-02")
}
// GetDailyCounts returns counter values for each UTC day from start through end inclusive.
func (c *Counter) GetDailyCounts(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
startDay = time.Date(startDay.Year(), startDay.Month(), startDay.Day(), 0, 0, 0, 0, time.UTC)
endDay = time.Date(endDay.Year(), endDay.Month(), endDay.Day(), 0, 0, 0, 0, time.UTC)
if endDay.Before(startDay) {
return map[string]int64{}, nil
}
counts := make(map[string]int64)
for day := startDay; !day.After(endDay); day = day.AddDate(0, 0, 1) {
date := day.Format("2006-01-02")
value, err := c.Get(ctx, AITranslationSuccessDailyCounterKey(day))
if err != nil {
return nil, err
}
counts[date] = value
}
return counts, nil
}
// AITranslationSuccessCounterKeyForSource maps a request source to its metrics counter key.
func AITranslationSuccessCounterKeyForSource(source string) string {
switch source {
case "daily_job":
return AITranslationSuccessCounterKeyDailyJob
case "manual_trigger":
return AITranslationSuccessCounterKeyManualTrigger
default:
return ""
}
}