Refactor AI summarizer client initialization to include MongoDB manager and add statistics endpoint
- Updated `InitAISummarizerClient` to accept `mongoManager` for tracking translation success. - Introduced new `Statistics` endpoint in the dashboard to fetch scraping and translation statistics. - Enhanced `TranslationWorker` to utilize the new success counter for tracking successful translations. - Added necessary data structures and query forms for statistics reporting. This refactor improves the tracking of AI translation success and provides new insights through the dashboard statistics.
This commit is contained in:
@@ -290,12 +290,13 @@ func InitFileStore(mongoManager *mongo.ConnectionManager, log logger.Logger) (*f
|
||||
}
|
||||
|
||||
// InitAISummarizerClient initializes the AI summarizer HTTP client for on-demand summarization
|
||||
func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.Client {
|
||||
func InitAISummarizerClient(conf AISummarizerConfig, mongoManager *mongo.ConnectionManager, log logger.Logger) *ai_summarizer.Client {
|
||||
if conf.APIBaseURL == "" {
|
||||
log.Warn("AI Summarizer API base URL not configured, on-demand summarization will be unavailable", map[string]interface{}{})
|
||||
return nil
|
||||
}
|
||||
|
||||
translationCounter := mongo.NewCounter(mongoManager)
|
||||
cfg := &ai_summarizer.Config{
|
||||
APIBaseURL: conf.APIBaseURL,
|
||||
APITimeout: conf.APITimeout,
|
||||
@@ -307,6 +308,7 @@ func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summ
|
||||
MinioUseSSL: conf.MinioUseSSL,
|
||||
MinioRegion: conf.MinioRegion,
|
||||
MinioBucket: conf.MinioBucket,
|
||||
OnSuccessfulTranslation: mongo.AITranslationSuccessCallback(translationCounter),
|
||||
}
|
||||
|
||||
client, err := ai_summarizer.NewClient(cfg, log)
|
||||
|
||||
+1
-1
@@ -168,7 +168,7 @@ func main() {
|
||||
var aiSummarizerClient tender.AISummarizerClient
|
||||
var aiSummarizerStorage tender.AISummarizerStorage
|
||||
|
||||
if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, logger); c != nil {
|
||||
if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, mongoManager, logger); c != nil {
|
||||
aiSummarizerClient = c
|
||||
}
|
||||
if s := bootstrap.InitAISummarizerStorage(conf.AISummarizer, logger); s != nil {
|
||||
|
||||
@@ -222,6 +222,7 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
|
||||
dashboardGP.GET("/notice-types", dashboardHandler.NoticeTypes)
|
||||
dashboardGP.GET("/closing-soon", dashboardHandler.ClosingSoon)
|
||||
dashboardGP.GET("/recent", dashboardHandler.Recent)
|
||||
dashboardGP.GET("/statistics", dashboardHandler.Statistics)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -196,6 +196,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
// Initialize translation backfill worker for configured languages
|
||||
if aiClient != nil {
|
||||
targetLanguages := parseTranslationLanguages(config.AISummarizer)
|
||||
translationSuccessCounter := mongo.NewCounter(mongoManager)
|
||||
scheduler.AddJob(schedule.Job{
|
||||
Name: "Tender Translation Worker Job",
|
||||
Func: func() {
|
||||
@@ -205,6 +206,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
tenderRepo,
|
||||
aiClient,
|
||||
aiStorage,
|
||||
translationSuccessCounter,
|
||||
targetLanguages,
|
||||
config.Worker.TranslationBatchSize,
|
||||
)
|
||||
@@ -247,17 +249,19 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
}
|
||||
|
||||
// InitAISummarizerClient initializes the AI summarizer HTTP client used by worker jobs.
|
||||
func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.Client {
|
||||
func InitAISummarizerClient(conf AISummarizerConfig, mongoManager *mongo.ConnectionManager, log logger.Logger) *ai_summarizer.Client {
|
||||
if conf.APIBaseURL == "" {
|
||||
log.Warn("AI Summarizer API base URL not configured, translation worker will be unavailable", map[string]interface{}{})
|
||||
return nil
|
||||
}
|
||||
|
||||
translationCounter := mongo.NewCounter(mongoManager)
|
||||
cfg := &ai_summarizer.Config{
|
||||
APIBaseURL: conf.APIBaseURL,
|
||||
APITimeout: conf.APITimeout,
|
||||
APIRetryCount: conf.APIRetryCount,
|
||||
APIRetryDelay: conf.APIRetryDelay,
|
||||
OnSuccessfulTranslation: mongo.AITranslationSuccessCallback(translationCounter),
|
||||
}
|
||||
|
||||
client, err := ai_summarizer.NewClient(cfg, log)
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ func main() {
|
||||
}
|
||||
|
||||
// Initialize AI summarizer client (translation + document summarization workers)
|
||||
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, appLogger)
|
||||
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, mongoManager, appLogger)
|
||||
aiSummarizerStorage := bootstrap.InitAISummarizerStorage(config.AISummarizer, appLogger)
|
||||
|
||||
// Initialize Worker
|
||||
|
||||
@@ -18,6 +18,7 @@ type TranslationWorker struct {
|
||||
TenderRepo tender.TenderRepository
|
||||
AIClient *ai_summarizer.Client
|
||||
AIStorage *ai_summarizer.StorageClient
|
||||
TranslationSuccessCounter *mongo.Counter
|
||||
TargetLanguages []string
|
||||
BatchSize int
|
||||
}
|
||||
@@ -29,6 +30,7 @@ func NewTranslationWorker(
|
||||
tenderRepo tender.TenderRepository,
|
||||
aiClient *ai_summarizer.Client,
|
||||
aiStorage *ai_summarizer.StorageClient,
|
||||
translationSuccessCounter *mongo.Counter,
|
||||
targetLanguages []string,
|
||||
batchSize int,
|
||||
) *TranslationWorker {
|
||||
@@ -58,6 +60,7 @@ func NewTranslationWorker(
|
||||
TenderRepo: tenderRepo,
|
||||
AIClient: aiClient,
|
||||
AIStorage: aiStorage,
|
||||
TranslationSuccessCounter: translationSuccessCounter,
|
||||
TargetLanguages: langs,
|
||||
BatchSize: batchSize,
|
||||
}
|
||||
@@ -77,6 +80,8 @@ func (w *TranslationWorker) Run() {
|
||||
"batch_size": w.BatchSize,
|
||||
})
|
||||
|
||||
startCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
|
||||
|
||||
if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
|
||||
w.Logger.Warn("Failed to trigger AI pipeline translate (continuing with per-tender sync)", map[string]interface{}{
|
||||
"target_languages": w.TargetLanguages,
|
||||
@@ -88,11 +93,36 @@ func (w *TranslationWorker) Run() {
|
||||
w.runForLanguage(language)
|
||||
}
|
||||
|
||||
endCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
|
||||
totalCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKey)
|
||||
runCount := endCount - startCount
|
||||
if runCount < 0 {
|
||||
runCount = 0
|
||||
}
|
||||
|
||||
w.Logger.Info("Translation worker completed", map[string]interface{}{
|
||||
"target_languages": w.TargetLanguages,
|
||||
"successful_ai_translation_requests_run": runCount,
|
||||
"successful_ai_translation_requests_daily_job": endCount,
|
||||
"successful_ai_translation_requests_total": totalCount,
|
||||
})
|
||||
}
|
||||
|
||||
func (w *TranslationWorker) getSuccessfulTranslationCount(key string) int64 {
|
||||
if w.TranslationSuccessCounter == nil {
|
||||
return 0
|
||||
}
|
||||
count, err := w.TranslationSuccessCounter.Get(context.Background(), key)
|
||||
if err != nil {
|
||||
w.Logger.Warn("Failed to read AI translation success counter", map[string]interface{}{
|
||||
"counter_key": key,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return 0
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
func (w *TranslationWorker) runForLanguage(language string) {
|
||||
skip := 0
|
||||
for {
|
||||
@@ -210,6 +240,7 @@ func (w *TranslationWorker) resolveTranslation(t *tender.Tender, language string
|
||||
Title: t.Title,
|
||||
Description: t.Description,
|
||||
Language: language,
|
||||
RequestSource: ai_summarizer.TranslationRequestSourceDailyJob,
|
||||
})
|
||||
if apiErr != nil {
|
||||
return "", "", "api", apiErr
|
||||
|
||||
@@ -85,3 +85,24 @@ type RecentItem struct {
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
PublicationDate int64 `json:"publication_date"`
|
||||
}
|
||||
|
||||
// StatisticsReportResponse powers the statistics and charts report.
|
||||
type StatisticsReportResponse struct {
|
||||
Days int `json:"days"`
|
||||
GeneratedAt int64 `json:"generated_at"`
|
||||
Daily StatisticsDailySeries `json:"daily"`
|
||||
Totals StatisticsLifetimeTotals `json:"totals"`
|
||||
}
|
||||
|
||||
// StatisticsDailySeries contains per-day chart series.
|
||||
type StatisticsDailySeries struct {
|
||||
ScrapedTED []TrendPoint `json:"scraped_ted"`
|
||||
ScrapedDocuments []TrendPoint `json:"scraped_documents"`
|
||||
TranslatedNotices []TrendPoint `json:"translated_notices"`
|
||||
}
|
||||
|
||||
// StatisticsLifetimeTotals contains all-time totals.
|
||||
type StatisticsLifetimeTotals struct {
|
||||
ScrapedDocuments int64 `json:"scraped_documents"`
|
||||
TranslatedTenders int64 `json:"translated_tenders"`
|
||||
}
|
||||
|
||||
@@ -27,3 +27,8 @@ type RecentQuery struct {
|
||||
Limit int `query:"limit"`
|
||||
Cursor string `query:"cursor"`
|
||||
}
|
||||
|
||||
// StatisticsQuery binds query params for GET /dashboard/statistics.
|
||||
type StatisticsQuery struct {
|
||||
Days int `query:"days"`
|
||||
}
|
||||
|
||||
@@ -161,6 +161,30 @@ func (h *Handler) Recent(c echo.Context) error {
|
||||
return response.Success(c, out, "Recent tenders retrieved successfully")
|
||||
}
|
||||
|
||||
// Statistics returns scraping and translation statistics for charts.
|
||||
// @Summary Dashboard statistics report
|
||||
// @Description Daily and lifetime statistics for TED scraping, document scraping, and AI translation
|
||||
// @Tags Admin-Dashboard
|
||||
// @Produce json
|
||||
// @Param days query int false "Days back for daily series (default 14, max 90)"
|
||||
// @Success 200 {object} response.APIResponse{data=StatisticsReportResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Router /admin/v1/dashboard/statistics [get]
|
||||
// @Security BearerAuth
|
||||
func (h *Handler) Statistics(c echo.Context) error {
|
||||
query := StatisticsQuery{
|
||||
Days: parseIntQuery(c, "days", 0),
|
||||
}
|
||||
|
||||
out, err := h.service.Statistics(c.Request().Context(), query)
|
||||
if err != nil {
|
||||
return response.InternalServerError(c, "Failed to load dashboard statistics")
|
||||
}
|
||||
|
||||
setPrivateCache(c, 60)
|
||||
return response.Success(c, out, "Dashboard statistics retrieved successfully")
|
||||
}
|
||||
|
||||
func parseIntQuery(c echo.Context, name string, fallback int) int {
|
||||
raw := c.QueryParam(name)
|
||||
if raw == "" {
|
||||
|
||||
@@ -23,10 +23,13 @@ type Repository interface {
|
||||
NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
|
||||
ClosingSoon(ctx context.Context, limit int, windowSec int64) ([]ClosingSoonItem, error)
|
||||
Recent(ctx context.Context, limit int, cursor string) ([]RecentItem, *string, error)
|
||||
Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error)
|
||||
}
|
||||
|
||||
type repository struct {
|
||||
ormRepo orm.Repository[tender.Tender]
|
||||
mongoManager *orm.ConnectionManager
|
||||
metricsCounter *orm.Counter
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
@@ -34,6 +37,8 @@ type repository struct {
|
||||
func NewRepository(mongoManager *orm.ConnectionManager, log logger.Logger) Repository {
|
||||
return &repository{
|
||||
ormRepo: orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log),
|
||||
mongoManager: mongoManager,
|
||||
metricsCounter: orm.NewCounter(mongoManager),
|
||||
logger: log,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ type Service interface {
|
||||
NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
|
||||
ClosingSoon(ctx context.Context, query ClosingSoonQuery) (*ClosingSoonResponse, error)
|
||||
Recent(ctx context.Context, query RecentQuery) (*RecentResponse, error)
|
||||
Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error)
|
||||
}
|
||||
|
||||
type service struct {
|
||||
@@ -157,6 +158,25 @@ func (s *service) Recent(ctx context.Context, query RecentQuery) (*RecentRespons
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error) {
|
||||
days := trendDays(query.Days)
|
||||
startUnix := trendStartUnix(days)
|
||||
|
||||
s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
|
||||
"days": days,
|
||||
})
|
||||
|
||||
out, err := s.repo.Statistics(ctx, days, startUnix)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to fetch dashboard statistics report", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("dashboard statistics: %w", err)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func closingWindowHours(hours int) int {
|
||||
if hours <= 0 {
|
||||
return defaultClosingWindowHours
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"tm/internal/notice"
|
||||
orm "tm/pkg/mongo"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
|
||||
)
|
||||
|
||||
const noticesCollectionName = "notices"
|
||||
|
||||
func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
|
||||
now := time.Now().UTC()
|
||||
endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
|
||||
startDay := endDay.AddDate(0, 0, -(days - 1))
|
||||
|
||||
scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{
|
||||
"source": notice.TenderSourceTEDScraper,
|
||||
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}, "processing_metadata.scraped_at")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped ted per day: %w", err)
|
||||
}
|
||||
|
||||
scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped documents per day: %w", err)
|
||||
}
|
||||
|
||||
translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("translated notices per day: %w", err)
|
||||
}
|
||||
|
||||
totalDocuments, err := r.totalScrapedDocuments(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total scraped documents: %w", err)
|
||||
}
|
||||
|
||||
totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total translated tenders: %w", err)
|
||||
}
|
||||
|
||||
return &StatisticsReportResponse{
|
||||
Days: days,
|
||||
GeneratedAt: now.Unix(),
|
||||
Daily: StatisticsDailySeries{
|
||||
ScrapedTED: fillTrendSeries(days, scrapedTED),
|
||||
ScrapedDocuments: fillTrendSeries(days, scrapedDocuments),
|
||||
TranslatedNotices: fillTrendSeries(days, translatedNotices),
|
||||
},
|
||||
Totals: StatisticsLifetimeTotals{
|
||||
ScrapedDocuments: totalDocuments,
|
||||
TranslatedTenders: totalTranslated,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) {
|
||||
pipeline := mongodriver.Pipeline{
|
||||
{{Key: "$match", Value: match}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"metric_ts": normalizeTimestampExpr(timestampField),
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": bson.M{
|
||||
"$dateToString": bson.M{
|
||||
"format": "%Y-%m-%d",
|
||||
"date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
|
||||
"timezone": "UTC",
|
||||
},
|
||||
},
|
||||
"count": bson.M{"$sum": 1},
|
||||
}}},
|
||||
}
|
||||
|
||||
cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
return decodeDailyCounts(ctx, cursor)
|
||||
}
|
||||
|
||||
func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) {
|
||||
pipeline := mongodriver.Pipeline{
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"processing_metadata.documents_scraped": true,
|
||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
||||
}}},
|
||||
{{Key: "$unwind", Value: "$scraped_documents"}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"metric_ts": bson.M{
|
||||
"$cond": bson.A{
|
||||
bson.M{"$gt": bson.A{"$scraped_documents.scraped_at", 0}},
|
||||
"$scraped_documents.scraped_at",
|
||||
"$processing_metadata.documents_scraped_at",
|
||||
},
|
||||
},
|
||||
}}},
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"metric_ts": normalizeTimestampExpr("metric_ts"),
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": bson.M{
|
||||
"$dateToString": bson.M{
|
||||
"format": "%Y-%m-%d",
|
||||
"date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
|
||||
"timezone": "UTC",
|
||||
},
|
||||
},
|
||||
"count": bson.M{"$sum": 1},
|
||||
}}},
|
||||
}
|
||||
|
||||
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
return decodeDailyCounts(ctx, cursor)
|
||||
}
|
||||
|
||||
func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
|
||||
return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
|
||||
}
|
||||
|
||||
func (r *repository) totalScrapedDocuments(ctx context.Context) (int64, error) {
|
||||
pipeline := mongodriver.Pipeline{
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"processing_metadata.documents_scraped": true,
|
||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
||||
}}},
|
||||
{{Key: "$project", Value: bson.M{
|
||||
"doc_count": bson.M{"$size": "$scraped_documents"},
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": nil,
|
||||
"total": bson.M{"$sum": "$doc_count"},
|
||||
}}},
|
||||
}
|
||||
|
||||
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
if !cursor.Next(ctx) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var row bson.M
|
||||
if err := cursor.Decode(&row); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return aggregateCount(row, "total"), nil
|
||||
}
|
||||
|
||||
func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) {
|
||||
counts := make(map[string]int64)
|
||||
for cursor.Next(ctx) {
|
||||
var row bson.M
|
||||
if err := cursor.Decode(&row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
date, ok := row["_id"].(string)
|
||||
if !ok || date == "" {
|
||||
continue
|
||||
}
|
||||
counts[date] = aggregateCount(row, "count")
|
||||
}
|
||||
if err := cursor.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
@@ -1476,7 +1476,7 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
|
||||
return nil, fmt.Errorf("tender has no contract folder ID")
|
||||
}
|
||||
|
||||
title, description, err := s.resolveTranslation(ctx, t, targetLanguage)
|
||||
title, description, fromAPI, err := s.resolveTranslation(ctx, t, targetLanguage, ai_summarizer.TranslationRequestSourceManualTrigger)
|
||||
if err != nil {
|
||||
s.logger.Error("AI translation failed", map[string]interface{}{
|
||||
"tender_id": id,
|
||||
@@ -1487,6 +1487,14 @@ func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language str
|
||||
return nil, fmt.Errorf("AI translation request failed: %w", err)
|
||||
}
|
||||
|
||||
if fromAPI {
|
||||
s.logger.Info("Manual AI translation request succeeded", map[string]interface{}{
|
||||
"tender_id": id,
|
||||
"notice_id": t.NoticePublicationID,
|
||||
"language": targetLanguage,
|
||||
})
|
||||
}
|
||||
|
||||
return s.buildAITranslateResponse(t, targetLanguage, title, description), nil
|
||||
}
|
||||
|
||||
@@ -1500,11 +1508,11 @@ func (s *tenderService) buildAITranslateResponse(t *Tender, language, title, des
|
||||
}
|
||||
}
|
||||
|
||||
func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, language string) (string, string, error) {
|
||||
func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, language, requestSource string) (string, string, bool, error) {
|
||||
if s.aiSummarizerStorage != nil {
|
||||
stored, err := s.aiSummarizerStorage.GetTranslationFromStorage(ctx, t.ContractFolderID, t.NoticePublicationID, language)
|
||||
if err == nil {
|
||||
return stored.Title, stored.Description, nil
|
||||
return stored.Title, stored.Description, false, nil
|
||||
}
|
||||
if !errors.Is(err, ai_summarizer.ErrTranslationNotReady) &&
|
||||
!errors.Is(err, ai_summarizer.ErrObjectNotFound) &&
|
||||
@@ -1523,11 +1531,12 @@ func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, langu
|
||||
Title: t.Title,
|
||||
Description: t.Description,
|
||||
Language: language,
|
||||
RequestSource: requestSource,
|
||||
})
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
return "", "", false, err
|
||||
}
|
||||
return resp.TranslatedTitle, resp.TranslatedDescription, nil
|
||||
return resp.TranslatedTitle, resp.TranslatedDescription, true, nil
|
||||
}
|
||||
|
||||
func (s *tenderService) pickResponseLanguage(language *string) string {
|
||||
|
||||
@@ -107,7 +107,7 @@ func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody Translate
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := c.doTranslatePost(ctx, url, jsonBody)
|
||||
resp, err := c.doTranslatePost(ctx, url, jsonBody, reqBody.RequestSource)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
c.logger.Error("AI translate request failed", map[string]interface{}{
|
||||
@@ -216,7 +216,7 @@ func (c *Client) doPost(ctx context.Context, url string, jsonBody []byte) (*Summ
|
||||
}
|
||||
|
||||
// doTranslatePost performs a single translation POST request and parses the response.
|
||||
func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byte) (*TranslateResponse, error) {
|
||||
func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byte, requestSource string) (*TranslateResponse, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to create translation request: %w", err)
|
||||
@@ -253,6 +253,17 @@ func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byt
|
||||
"language": result.Language,
|
||||
})
|
||||
|
||||
if c.config.OnSuccessfulTranslation != nil {
|
||||
if err := c.config.OnSuccessfulTranslation(ctx, requestSource); err != nil {
|
||||
c.logger.Warn("Failed to increment AI translation success counter", map[string]interface{}{
|
||||
"notice_publication_id": result.NoticePublicationID,
|
||||
"language": result.Language,
|
||||
"request_source": requestSource,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package ai_summarizer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
@@ -15,6 +16,10 @@ type Config struct {
|
||||
APIRetryCount int // Number of retry attempts for failed API requests
|
||||
APIRetryDelay time.Duration // Delay between retry attempts
|
||||
|
||||
// OnSuccessfulTranslation is called after a successful POST /ai/translate request.
|
||||
// source identifies the caller (daily_job or manual_trigger).
|
||||
OnSuccessfulTranslation func(ctx context.Context, source string) error
|
||||
|
||||
// MinIO storage settings
|
||||
MinioEndpoint string // MinIO endpoint, e.g. "ai-host:9000"
|
||||
MinioAccessKey string // MinIO access key ID
|
||||
|
||||
@@ -109,8 +109,14 @@ type TranslateRequest struct {
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Language string `json:"language"`
|
||||
RequestSource string `json:"-"` // internal: daily_job | manual_trigger
|
||||
}
|
||||
|
||||
const (
|
||||
TranslationRequestSourceDailyJob = "daily_job"
|
||||
TranslationRequestSourceManualTrigger = "manual_trigger"
|
||||
)
|
||||
|
||||
// TranslateResponse represents the translation payload from POST /ai/translate.
|
||||
type TranslateResponse struct {
|
||||
ContractFolderID string `json:"contract_folder_id"`
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
package mongo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo/options"
|
||||
)
|
||||
|
||||
const (
|
||||
metricsCountersCollection = "metrics_counters"
|
||||
AITranslationSuccessCounterKey = "ai_translation_successful_requests"
|
||||
AITranslationSuccessCounterKeyDailyJob = "ai_translation_successful_requests_daily_job"
|
||||
AITranslationSuccessCounterKeyManualTrigger = "ai_translation_successful_requests_manual_trigger"
|
||||
)
|
||||
|
||||
// Counter provides atomic increment/read operations for named metrics stored in MongoDB.
|
||||
type Counter struct {
|
||||
mongo *ConnectionManager
|
||||
}
|
||||
|
||||
// NewCounter creates a counter backed by the metrics_counters collection.
|
||||
func NewCounter(mongo *ConnectionManager) *Counter {
|
||||
return &Counter{mongo: mongo}
|
||||
}
|
||||
|
||||
// Increment atomically increments the counter identified by key and returns the new value.
|
||||
func (c *Counter) Increment(ctx context.Context, key string) (int64, error) {
|
||||
if key == "" {
|
||||
return 0, fmt.Errorf("mongo counter: key is required")
|
||||
}
|
||||
|
||||
var res struct {
|
||||
Count int64 `bson:"count"`
|
||||
}
|
||||
err := c.mongo.GetCollection(metricsCountersCollection).FindOneAndUpdate(
|
||||
ctx,
|
||||
bson.M{"_id": key},
|
||||
bson.M{
|
||||
"$inc": bson.M{"count": 1},
|
||||
"$set": bson.M{"updated_at": time.Now().Unix()},
|
||||
"$setOnInsert": bson.M{"_id": key},
|
||||
},
|
||||
options.FindOneAndUpdate().SetUpsert(true).SetReturnDocument(options.After),
|
||||
).Decode(&res)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("mongo counter increment %q: %w", key, err)
|
||||
}
|
||||
|
||||
return res.Count, nil
|
||||
}
|
||||
|
||||
// Get returns the current value of the counter identified by key (0 when missing).
|
||||
func (c *Counter) Get(ctx context.Context, key string) (int64, error) {
|
||||
if key == "" {
|
||||
return 0, fmt.Errorf("mongo counter: key is required")
|
||||
}
|
||||
|
||||
var doc struct {
|
||||
Count int64 `bson:"count"`
|
||||
}
|
||||
err := c.mongo.GetCollection(metricsCountersCollection).FindOne(ctx, bson.M{"_id": key}).Decode(&doc)
|
||||
if err != nil {
|
||||
if errors.Is(err, mongodriver.ErrNoDocuments) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, fmt.Errorf("mongo counter get %q: %w", key, err)
|
||||
}
|
||||
|
||||
return doc.Count, nil
|
||||
}
|
||||
|
||||
// AITranslationSuccessCallback returns a callback that increments successful AI translation counters.
|
||||
func AITranslationSuccessCallback(counter *Counter) func(ctx context.Context, source string) error {
|
||||
return func(ctx context.Context, source string) error {
|
||||
if _, err := counter.Increment(ctx, AITranslationSuccessCounterKey); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := counter.Increment(ctx, AITranslationSuccessDailyCounterKey(time.Now().UTC())); err != nil {
|
||||
return err
|
||||
}
|
||||
if key := AITranslationSuccessCounterKeyForSource(source); key != "" {
|
||||
_, err := counter.Increment(ctx, key)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// AITranslationSuccessDailyCounterKey returns the metrics key for successful translations on a given UTC day.
|
||||
func AITranslationSuccessDailyCounterKey(day time.Time) string {
|
||||
return AITranslationSuccessCounterKey + "_day:" + day.UTC().Format("2006-01-02")
|
||||
}
|
||||
|
||||
// GetDailyCounts returns counter values for each UTC day from start through end inclusive.
|
||||
func (c *Counter) GetDailyCounts(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
|
||||
startDay = time.Date(startDay.Year(), startDay.Month(), startDay.Day(), 0, 0, 0, 0, time.UTC)
|
||||
endDay = time.Date(endDay.Year(), endDay.Month(), endDay.Day(), 0, 0, 0, 0, time.UTC)
|
||||
if endDay.Before(startDay) {
|
||||
return map[string]int64{}, nil
|
||||
}
|
||||
|
||||
counts := make(map[string]int64)
|
||||
for day := startDay; !day.After(endDay); day = day.AddDate(0, 0, 1) {
|
||||
date := day.Format("2006-01-02")
|
||||
value, err := c.Get(ctx, AITranslationSuccessDailyCounterKey(day))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
counts[date] = value
|
||||
}
|
||||
|
||||
return counts, nil
|
||||
}
|
||||
|
||||
// AITranslationSuccessCounterKeyForSource maps a request source to its metrics counter key.
|
||||
func AITranslationSuccessCounterKeyForSource(source string) string {
|
||||
switch source {
|
||||
case "daily_job":
|
||||
return AITranslationSuccessCounterKeyDailyJob
|
||||
case "manual_trigger":
|
||||
return AITranslationSuccessCounterKeyManualTrigger
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user