Refactor AI summarizer client initialization to include MongoDB manager and add statistics endpoint
- Updated `InitAISummarizerClient` to accept `mongoManager` for tracking translation success. - Introduced new `Statistics` endpoint in the dashboard to fetch scraping and translation statistics. - Enhanced `TranslationWorker` to utilize the new success counter for tracking successful translations. - Added necessary data structures and query forms for statistics reporting. This refactor improves the tracking of AI translation success and provides new insights through the dashboard statistics.
This commit is contained in:
Mazyar
@@ -85,3 +85,24 @@ type RecentItem struct {
|
||||
CreatedAt int64 `json:"created_at"`
|
||||
PublicationDate int64 `json:"publication_date"`
|
||||
}
|
||||
|
||||
// StatisticsReportResponse powers the statistics and charts report.
|
||||
type StatisticsReportResponse struct {
|
||||
Days int `json:"days"`
|
||||
GeneratedAt int64 `json:"generated_at"`
|
||||
Daily StatisticsDailySeries `json:"daily"`
|
||||
Totals StatisticsLifetimeTotals `json:"totals"`
|
||||
}
|
||||
|
||||
// StatisticsDailySeries contains per-day chart series.
|
||||
type StatisticsDailySeries struct {
|
||||
ScrapedTED []TrendPoint `json:"scraped_ted"`
|
||||
ScrapedDocuments []TrendPoint `json:"scraped_documents"`
|
||||
TranslatedNotices []TrendPoint `json:"translated_notices"`
|
||||
}
|
||||
|
||||
// StatisticsLifetimeTotals contains all-time totals.
|
||||
type StatisticsLifetimeTotals struct {
|
||||
ScrapedDocuments int64 `json:"scraped_documents"`
|
||||
TranslatedTenders int64 `json:"translated_tenders"`
|
||||
}
|
||||
|
||||
@@ -27,3 +27,8 @@ type RecentQuery struct {
|
||||
Limit int `query:"limit"`
|
||||
Cursor string `query:"cursor"`
|
||||
}
|
||||
|
||||
// StatisticsQuery binds query params for GET /dashboard/statistics.
|
||||
type StatisticsQuery struct {
|
||||
Days int `query:"days"`
|
||||
}
|
||||
|
||||
@@ -161,6 +161,30 @@ func (h *Handler) Recent(c echo.Context) error {
|
||||
return response.Success(c, out, "Recent tenders retrieved successfully")
|
||||
}
|
||||
|
||||
// Statistics returns scraping and translation statistics for charts.
|
||||
// @Summary Dashboard statistics report
|
||||
// @Description Daily and lifetime statistics for TED scraping, document scraping, and AI translation
|
||||
// @Tags Admin-Dashboard
|
||||
// @Produce json
|
||||
// @Param days query int false "Days back for daily series (default 14, max 90)"
|
||||
// @Success 200 {object} response.APIResponse{data=StatisticsReportResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Router /admin/v1/dashboard/statistics [get]
|
||||
// @Security BearerAuth
|
||||
func (h *Handler) Statistics(c echo.Context) error {
|
||||
query := StatisticsQuery{
|
||||
Days: parseIntQuery(c, "days", 0),
|
||||
}
|
||||
|
||||
out, err := h.service.Statistics(c.Request().Context(), query)
|
||||
if err != nil {
|
||||
return response.InternalServerError(c, "Failed to load dashboard statistics")
|
||||
}
|
||||
|
||||
setPrivateCache(c, 60)
|
||||
return response.Success(c, out, "Dashboard statistics retrieved successfully")
|
||||
}
|
||||
|
||||
func parseIntQuery(c echo.Context, name string, fallback int) int {
|
||||
raw := c.QueryParam(name)
|
||||
if raw == "" {
|
||||
|
||||
@@ -23,18 +23,23 @@ type Repository interface {
|
||||
NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
|
||||
ClosingSoon(ctx context.Context, limit int, windowSec int64) ([]ClosingSoonItem, error)
|
||||
Recent(ctx context.Context, limit int, cursor string) ([]RecentItem, *string, error)
|
||||
Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error)
|
||||
}
|
||||
|
||||
type repository struct {
|
||||
ormRepo orm.Repository[tender.Tender]
|
||||
logger logger.Logger
|
||||
ormRepo orm.Repository[tender.Tender]
|
||||
mongoManager *orm.ConnectionManager
|
||||
metricsCounter *orm.Counter
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
// NewRepository creates a dashboard repository backed by the tenders collection.
|
||||
func NewRepository(mongoManager *orm.ConnectionManager, log logger.Logger) Repository {
|
||||
return &repository{
|
||||
ormRepo: orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log),
|
||||
logger: log,
|
||||
ormRepo: orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log),
|
||||
mongoManager: mongoManager,
|
||||
metricsCounter: orm.NewCounter(mongoManager),
|
||||
logger: log,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ type Service interface {
|
||||
NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
|
||||
ClosingSoon(ctx context.Context, query ClosingSoonQuery) (*ClosingSoonResponse, error)
|
||||
Recent(ctx context.Context, query RecentQuery) (*RecentResponse, error)
|
||||
Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error)
|
||||
}
|
||||
|
||||
type service struct {
|
||||
@@ -157,6 +158,25 @@ func (s *service) Recent(ctx context.Context, query RecentQuery) (*RecentRespons
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error) {
|
||||
days := trendDays(query.Days)
|
||||
startUnix := trendStartUnix(days)
|
||||
|
||||
s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
|
||||
"days": days,
|
||||
})
|
||||
|
||||
out, err := s.repo.Statistics(ctx, days, startUnix)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to fetch dashboard statistics report", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("dashboard statistics: %w", err)
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func closingWindowHours(hours int) int {
|
||||
if hours <= 0 {
|
||||
return defaultClosingWindowHours
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package dashboard
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"tm/internal/notice"
|
||||
orm "tm/pkg/mongo"
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
|
||||
)
|
||||
|
||||
const noticesCollectionName = "notices"
|
||||
|
||||
func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
|
||||
now := time.Now().UTC()
|
||||
endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
|
||||
startDay := endDay.AddDate(0, 0, -(days - 1))
|
||||
|
||||
scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{
|
||||
"source": notice.TenderSourceTEDScraper,
|
||||
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}, "processing_metadata.scraped_at")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped ted per day: %w", err)
|
||||
}
|
||||
|
||||
scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped documents per day: %w", err)
|
||||
}
|
||||
|
||||
translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("translated notices per day: %w", err)
|
||||
}
|
||||
|
||||
totalDocuments, err := r.totalScrapedDocuments(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total scraped documents: %w", err)
|
||||
}
|
||||
|
||||
totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total translated tenders: %w", err)
|
||||
}
|
||||
|
||||
return &StatisticsReportResponse{
|
||||
Days: days,
|
||||
GeneratedAt: now.Unix(),
|
||||
Daily: StatisticsDailySeries{
|
||||
ScrapedTED: fillTrendSeries(days, scrapedTED),
|
||||
ScrapedDocuments: fillTrendSeries(days, scrapedDocuments),
|
||||
TranslatedNotices: fillTrendSeries(days, translatedNotices),
|
||||
},
|
||||
Totals: StatisticsLifetimeTotals{
|
||||
ScrapedDocuments: totalDocuments,
|
||||
TranslatedTenders: totalTranslated,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) {
|
||||
pipeline := mongodriver.Pipeline{
|
||||
{{Key: "$match", Value: match}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"metric_ts": normalizeTimestampExpr(timestampField),
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": bson.M{
|
||||
"$dateToString": bson.M{
|
||||
"format": "%Y-%m-%d",
|
||||
"date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
|
||||
"timezone": "UTC",
|
||||
},
|
||||
},
|
||||
"count": bson.M{"$sum": 1},
|
||||
}}},
|
||||
}
|
||||
|
||||
cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
return decodeDailyCounts(ctx, cursor)
|
||||
}
|
||||
|
||||
func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) {
|
||||
pipeline := mongodriver.Pipeline{
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"processing_metadata.documents_scraped": true,
|
||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
||||
}}},
|
||||
{{Key: "$unwind", Value: "$scraped_documents"}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"metric_ts": bson.M{
|
||||
"$cond": bson.A{
|
||||
bson.M{"$gt": bson.A{"$scraped_documents.scraped_at", 0}},
|
||||
"$scraped_documents.scraped_at",
|
||||
"$processing_metadata.documents_scraped_at",
|
||||
},
|
||||
},
|
||||
}}},
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"metric_ts": normalizeTimestampExpr("metric_ts"),
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": bson.M{
|
||||
"$dateToString": bson.M{
|
||||
"format": "%Y-%m-%d",
|
||||
"date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
|
||||
"timezone": "UTC",
|
||||
},
|
||||
},
|
||||
"count": bson.M{"$sum": 1},
|
||||
}}},
|
||||
}
|
||||
|
||||
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
return decodeDailyCounts(ctx, cursor)
|
||||
}
|
||||
|
||||
func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
|
||||
return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
|
||||
}
|
||||
|
||||
func (r *repository) totalScrapedDocuments(ctx context.Context) (int64, error) {
|
||||
pipeline := mongodriver.Pipeline{
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"processing_metadata.documents_scraped": true,
|
||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
||||
}}},
|
||||
{{Key: "$project", Value: bson.M{
|
||||
"doc_count": bson.M{"$size": "$scraped_documents"},
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": nil,
|
||||
"total": bson.M{"$sum": "$doc_count"},
|
||||
}}},
|
||||
}
|
||||
|
||||
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
if !cursor.Next(ctx) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
var row bson.M
|
||||
if err := cursor.Decode(&row); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return aggregateCount(row, "total"), nil
|
||||
}
|
||||
|
||||
func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) {
|
||||
counts := make(map[string]int64)
|
||||
for cursor.Next(ctx) {
|
||||
var row bson.M
|
||||
if err := cursor.Decode(&row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
date, ok := row["_id"].(string)
|
||||
if !ok || date == "" {
|
||||
continue
|
||||
}
|
||||
counts[date] = aggregateCount(row, "count")
|
||||
}
|
||||
if err := cursor.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return counts, nil
|
||||
}
|
||||
Reference in New Issue
Block a user