Enhance dashboard repository and service with caching and aggregation improvements
- Introduced caching mechanisms for summary and statistics in the dashboard service, improving performance by reducing redundant data retrieval. - Refactored the Summary method to utilize MongoDB aggregation for more efficient data processing and retrieval. - Added synchronization features using singleflight to prevent duplicate processing of requests for cached data. - Updated the repository to include a cachedScrapedTendersScope method, enhancing the efficiency of scraped document statistics retrieval. This update significantly optimizes the dashboard's performance and data handling capabilities, ensuring faster response times and reduced load on the database.
This commit is contained in:
Mazyar
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"tm/internal/tender"
|
||||
"tm/pkg/ai_summarizer"
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
// ProcedureDocumentsLister lists contract folders that have scraped documents in MinIO.
|
||||
@@ -34,11 +36,16 @@ type Repository interface {
|
||||
}
|
||||
|
||||
type repository struct {
|
||||
ormRepo orm.Repository[tender.Tender]
|
||||
mongoManager *orm.ConnectionManager
|
||||
metricsCounter *orm.Counter
|
||||
procedureLister ProcedureDocumentsLister
|
||||
logger logger.Logger
|
||||
ormRepo orm.Repository[tender.Tender]
|
||||
mongoManager *orm.ConnectionManager
|
||||
metricsCounter *orm.Counter
|
||||
procedureLister ProcedureDocumentsLister
|
||||
logger logger.Logger
|
||||
scrapedScopeMu sync.Mutex
|
||||
scrapedScope scrapedTendersScope
|
||||
scrapedScopeExpiry time.Time
|
||||
scrapedScopeCached bool
|
||||
scrapedScopeGroup singleflight.Group
|
||||
}
|
||||
|
||||
// NewRepository creates a dashboard repository backed by the tenders collection.
|
||||
@@ -60,135 +67,114 @@ func (r *repository) Summary(ctx context.Context, closingWindowSec int64) (*Summ
|
||||
now := time.Now().Unix()
|
||||
windowEnd := now + closingWindowSec
|
||||
|
||||
total, err := r.ormRepo.Count(ctx, bson.M{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("count total tenders: %w", err)
|
||||
pipeline := mongo.Pipeline{
|
||||
{{Key: "$facet", Value: bson.M{
|
||||
"total": bson.A{
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"active": bson.A{
|
||||
bson.M{"$match": bson.M{"status": tender.TenderStatusActive}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"awarded": bson.A{
|
||||
bson.M{"$match": bson.M{"status": tender.TenderStatusAwarded}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"cancelled": bson.A{
|
||||
bson.M{"$match": bson.M{"status": tender.TenderStatusCancelled}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"expired": bson.A{
|
||||
bson.M{"$addFields": bson.M{"expiry_deadline": expiryDeadlineExpr()}},
|
||||
bson.M{"$match": bson.M{
|
||||
"status": bson.M{"$nin": []tender.TenderStatus{
|
||||
tender.TenderStatusAwarded,
|
||||
tender.TenderStatusCancelled,
|
||||
}},
|
||||
"$or": []bson.M{
|
||||
{"status": tender.TenderStatusExpired},
|
||||
{"expiry_deadline": bson.M{"$gt": 0, "$lte": now}},
|
||||
},
|
||||
}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"closing_soon": bson.A{
|
||||
bson.M{"$addFields": bson.M{"effective_deadline": effectiveDeadlineExpr()}},
|
||||
bson.M{"$match": bson.M{
|
||||
"effective_deadline": bson.M{"$gt": now, "$lte": windowEnd},
|
||||
}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"currency_value": bson.A{
|
||||
bson.M{"$match": bson.M{
|
||||
"estimated_value": bson.M{"$gt": 0},
|
||||
"currency": bson.M{"$type": "string", "$ne": ""},
|
||||
}},
|
||||
bson.M{"$group": bson.M{
|
||||
"_id": "$currency",
|
||||
"count": bson.M{"$sum": 1},
|
||||
"total": bson.M{"$sum": "$estimated_value"},
|
||||
}},
|
||||
bson.M{"$sort": bson.M{"count": -1}},
|
||||
bson.M{"$limit": 1},
|
||||
},
|
||||
}}},
|
||||
}
|
||||
|
||||
active, err := r.ormRepo.Count(ctx, bson.M{"status": tender.TenderStatusActive})
|
||||
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("count active tenders: %w", err)
|
||||
return nil, fmt.Errorf("dashboard summary aggregation: %w", err)
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return &SummaryResponse{
|
||||
ValueCurrency: defaultValueCurrency,
|
||||
GeneratedAt: now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
awarded, err := r.ormRepo.Count(ctx, bson.M{"status": tender.TenderStatusAwarded})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("count awarded tenders: %w", err)
|
||||
}
|
||||
|
||||
expired, err := r.countExpired(ctx, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cancelled, err := r.ormRepo.Count(ctx, bson.M{"status": tender.TenderStatusCancelled})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("count cancelled tenders: %w", err)
|
||||
}
|
||||
|
||||
closingSoon, err := r.countClosingSoon(ctx, now, windowEnd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
valueCurrency, totalValue, err := r.dominantCurrencyValue(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
facet := results[0]
|
||||
valueCurrency, totalValue := facetCurrencyValue(facet)
|
||||
|
||||
return &SummaryResponse{
|
||||
TotalTenders: total,
|
||||
ActiveTenders: active,
|
||||
AwardedTenders: awarded,
|
||||
ExpiredTenders: expired,
|
||||
CancelledTenders: cancelled,
|
||||
ClosingSoon: closingSoon,
|
||||
TotalTenders: facetCount(facet, "total"),
|
||||
ActiveTenders: facetCount(facet, "active"),
|
||||
AwardedTenders: facetCount(facet, "awarded"),
|
||||
ExpiredTenders: facetCount(facet, "expired"),
|
||||
CancelledTenders: facetCount(facet, "cancelled"),
|
||||
ClosingSoon: facetCount(facet, "closing_soon"),
|
||||
TotalEstimatedValue: totalValue,
|
||||
ValueCurrency: valueCurrency,
|
||||
GeneratedAt: now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *repository) countExpired(ctx context.Context, now int64) (int64, error) {
|
||||
pipeline := mongo.Pipeline{
|
||||
{{Key: "$addFields", Value: bson.M{"expiry_deadline": expiryDeadlineExpr()}}},
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"status": bson.M{"$nin": []tender.TenderStatus{
|
||||
tender.TenderStatusAwarded,
|
||||
tender.TenderStatusCancelled,
|
||||
}},
|
||||
"$or": []bson.M{
|
||||
{"status": tender.TenderStatusExpired},
|
||||
{
|
||||
"expiry_deadline": bson.M{"$gt": 0, "$lte": now},
|
||||
},
|
||||
},
|
||||
}}},
|
||||
{{Key: "$count", Value: "count"}},
|
||||
func facetCount(facet bson.M, key string) int64 {
|
||||
rows, ok := facet[key].(bson.A)
|
||||
if !ok || len(rows) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count expired tenders: %w", err)
|
||||
row, ok := rows[0].(bson.M)
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return aggregateCount(results[0], "count"), nil
|
||||
return aggregateCount(row, "count")
|
||||
}
|
||||
|
||||
func (r *repository) countClosingSoon(ctx context.Context, now, windowEnd int64) (int64, error) {
|
||||
pipeline := mongo.Pipeline{
|
||||
{{Key: "$addFields", Value: bson.M{"effective_deadline": effectiveDeadlineExpr()}}},
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"effective_deadline": bson.M{"$gt": now, "$lte": windowEnd},
|
||||
}}},
|
||||
{{Key: "$count", Value: "count"}},
|
||||
func facetCurrencyValue(facet bson.M) (string, float64) {
|
||||
rows, ok := facet["currency_value"].(bson.A)
|
||||
if !ok || len(rows) == 0 {
|
||||
return defaultValueCurrency, 0
|
||||
}
|
||||
|
||||
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count closing soon: %w", err)
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
return aggregateCount(results[0], "count"), nil
|
||||
}
|
||||
|
||||
func (r *repository) dominantCurrencyValue(ctx context.Context) (string, float64, error) {
|
||||
pipeline := mongo.Pipeline{
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"estimated_value": bson.M{"$gt": 0},
|
||||
"currency": bson.M{"$type": "string", "$ne": ""},
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": "$currency",
|
||||
"count": bson.M{"$sum": 1},
|
||||
"total": bson.M{"$sum": "$estimated_value"},
|
||||
}}},
|
||||
{{Key: "$sort", Value: bson.M{"count": -1}}},
|
||||
{{Key: "$limit", Value: 1}},
|
||||
}
|
||||
|
||||
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return defaultValueCurrency, 0, fmt.Errorf("dominant currency value: %w", err)
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
return defaultValueCurrency, 0, nil
|
||||
row, ok := rows[0].(bson.M)
|
||||
if !ok {
|
||||
return defaultValueCurrency, 0
|
||||
}
|
||||
|
||||
currency := defaultValueCurrency
|
||||
if c, ok := results[0]["_id"].(string); ok && c != "" {
|
||||
if c, ok := row["_id"].(string); ok && c != "" {
|
||||
currency = strings.ToUpper(c)
|
||||
}
|
||||
|
||||
return currency, toFloat64(results[0]["total"]), nil
|
||||
return currency, toFloat64(row["total"])
|
||||
}
|
||||
|
||||
func (r *repository) Trend(ctx context.Context, days int, metric string, startUnix int64) (map[string]int64, error) {
|
||||
|
||||
Reference in New Issue
Block a user