Refactor AI summarizer client initialization to include MongoDB manager and add statistics endpoint
- Updated `InitAISummarizerClient` to accept `mongoManager` for tracking translation success. - Introduced new `Statistics` endpoint in the dashboard to fetch scraping and translation statistics. - Enhanced `TranslationWorker` to utilize the new success counter for tracking successful translations. - Added necessary data structures and query forms for statistics reporting. This refactor improves the tracking of AI translation success and provides new insights through the dashboard statistics.
This commit is contained in:
Mazyar
@@ -0,0 +1,189 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"tm/internal/notice"
|
||||
orm "tm/pkg/mongo"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
|
||||
)
|
||||
|
||||
const noticesCollectionName = "notices"
|
||||
|
||||
func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
|
||||
now := time.Now().UTC()
|
||||
endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
|
||||
startDay := endDay.AddDate(0, 0, -(days - 1))
|
||||
|
||||
scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{
|
||||
"source": notice.TenderSourceTEDScraper,
|
||||
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}, "processing_metadata.scraped_at")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped ted per day: %w", err)
|
||||
}
|
||||
|
||||
scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped documents per day: %w", err)
|
||||
}
|
||||
|
||||
translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("translated notices per day: %w", err)
|
||||
}
|
||||
|
||||
totalDocuments, err := r.totalScrapedDocuments(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total scraped documents: %w", err)
|
||||
}
|
||||
|
||||
totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total translated tenders: %w", err)
|
||||
}
|
||||
|
||||
return &StatisticsReportResponse{
|
||||
Days: days,
|
||||
GeneratedAt: now.Unix(),
|
||||
Daily: StatisticsDailySeries{
|
||||
ScrapedTED: fillTrendSeries(days, scrapedTED),
|
||||
ScrapedDocuments: fillTrendSeries(days, scrapedDocuments),
|
||||
TranslatedNotices: fillTrendSeries(days, translatedNotices),
|
||||
},
|
||||
Totals: StatisticsLifetimeTotals{
|
||||
ScrapedDocuments: totalDocuments,
|
||||
TranslatedTenders: totalTranslated,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) {
|
||||
pipeline := mongodriver.Pipeline{
|
||||
{{Key: "$match", Value: match}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"metric_ts": normalizeTimestampExpr(timestampField),
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": bson.M{
|
||||
"$dateToString": bson.M{
|
||||
"format": "%Y-%m-%d",
|
||||
"date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
|
||||
"timezone": "UTC",
|
||||
},
|
||||
},
|
||||
"count": bson.M{"$sum": 1},
|
||||
}}},
|
||||
}
|
||||
|
||||
cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
return decodeDailyCounts(ctx, cursor)
|
||||
}
|
||||
|
||||
func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) {
|
||||
pipeline := mongodriver.Pipeline{
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"processing_metadata.documents_scraped": true,
|
||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
||||
}}},
|
||||
{{Key: "$unwind", Value: "$scraped_documents"}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"metric_ts": bson.M{
|
||||
"$cond": bson.A{
|
||||
bson.M{"$gt": bson.A{"$scraped_documents.scraped_at", 0}},
|
||||
"$scraped_documents.scraped_at",
|
||||
"$processing_metadata.documents_scraped_at",
|
||||
},
|
||||
},
|
||||
}}},
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"metric_ts": normalizeTimestampExpr("metric_ts"),
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": bson.M{
|
||||
"$dateToString": bson.M{
|
||||
"format": "%Y-%m-%d",
|
||||
"date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
|
||||
"timezone": "UTC",
|
||||
},
|
||||
},
|
||||
"count": bson.M{"$sum": 1},
|
||||
}}},
|
||||
}
|
||||
|
||||
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
return decodeDailyCounts(ctx, cursor)
|
||||
}
|
||||
|
||||
func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
|
||||
return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
|
||||
}
|
||||
|
||||
func (r *repository) totalScrapedDocuments(ctx context.Context) (int64, error) {
|
||||
pipeline := mongodriver.Pipeline{
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"processing_metadata.documents_scraped": true,
|
||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
||||
}}},
|
||||
{{Key: "$project", Value: bson.M{
|
||||
"doc_count": bson.M{"$size": "$scraped_documents"},
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": nil,
|
||||
"total": bson.M{"$sum": "$doc_count"},
|
||||
}}},
|
||||
}
|
||||
|
||||
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
if !cursor.Next(ctx) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var row bson.M
|
||||
if err := cursor.Decode(&row); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return aggregateCount(row, "total"), nil
|
||||
}
|
||||
|
||||
func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) {
|
||||
counts := make(map[string]int64)
|
||||
for cursor.Next(ctx) {
|
||||
var row bson.M
|
||||
if err := cursor.Decode(&row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
date, ok := row["_id"].(string)
|
||||
if !ok || date == "" {
|
||||
continue
|
||||
}
|
||||
counts[date] = aggregateCount(row, "count")
|
||||
}
|
||||
if err := cursor.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
Reference in New Issue
Block a user