Enhance dashboard summary and statistics with new fields and caching improvements
continuous-integration/drone/push Build is passing
continuous-integration/drone/push Build is passing
- Added new field `Days` and `ScrapedTED` to `SummaryResponse` for tracking daily TED scrape counts. - Updated `SummaryQuery` to include `Days` parameter for querying scraped TED data. - Modified `Summary` handler to accept and process the new `Days` parameter. - Refactored `Summary` method in the repository to support the new `Days` parameter and improved aggregation logic. - Enhanced caching logic in the service layer to utilize a composite cache key based on `closingWindowSec` and `days`, improving cache management and retrieval efficiency. This update improves the dashboard's functionality by providing more detailed insights into TED scraping activities and optimizing the caching strategy for better performance.
This commit is contained in:
Mazyar
@@ -31,7 +31,7 @@ const defaultValueCurrency = "EUR"
|
||||
|
||||
// Repository defines dashboard data access.
|
||||
type Repository interface {
|
||||
Summary(ctx context.Context, closingWindowSec int64) (*SummaryResponse, error)
|
||||
Summary(ctx context.Context, closingWindowSec int64, days int) (*SummaryResponse, error)
|
||||
Trend(ctx context.Context, days int, metric string, startUnix int64) (map[string]int64, error)
|
||||
Countries(ctx context.Context, limit int) (*CountriesResponse, error)
|
||||
NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
|
||||
@@ -68,91 +68,188 @@ func tenderCollectionName() string {
|
||||
return "tenders"
|
||||
}
|
||||
|
||||
func (r *repository) Summary(ctx context.Context, closingWindowSec int64) (*SummaryResponse, error) {
|
||||
func (r *repository) Summary(ctx context.Context, closingWindowSec int64, days int) (*SummaryResponse, error) {
|
||||
now := time.Now().Unix()
|
||||
windowEnd := now + closingWindowSec
|
||||
|
||||
nowUTC := time.Now().UTC()
|
||||
endDay := time.Date(nowUTC.Year(), nowUTC.Month(), nowUTC.Day(), 0, 0, 0, 0, time.UTC)
|
||||
startDay := endDay.AddDate(0, 0, -(days - 1))
|
||||
|
||||
var (
|
||||
statusCounts summaryStatusCounts
|
||||
valueCurrency string
|
||||
totalValue float64
|
||||
scrapedTED map[string]int64
|
||||
wg sync.WaitGroup
|
||||
mu sync.Mutex
|
||||
firstErr error
|
||||
)
|
||||
|
||||
recordErr := func(section string, err error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("%s: %w", section, err)
|
||||
}
|
||||
}
|
||||
|
||||
wg.Add(3)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
counts, err := r.summaryStatusCounts(ctx, now, windowEnd)
|
||||
if err != nil {
|
||||
recordErr("status_counts", err)
|
||||
return
|
||||
}
|
||||
statusCounts = counts
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
currency, value, err := r.summaryCurrencyValue(ctx)
|
||||
if err != nil {
|
||||
recordErr("currency_value", err)
|
||||
return
|
||||
}
|
||||
valueCurrency, totalValue = currency, value
|
||||
}()
|
||||
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
qctx, cancel := context.WithTimeout(ctx, statisticsMongoQueryTimeout)
|
||||
defer cancel()
|
||||
|
||||
counts, err := r.metricsCounter.GetTEDNoticeScrapedDailyCounts(qctx, startDay, endDay)
|
||||
if err != nil {
|
||||
if r.logger != nil {
|
||||
r.logger.Warn("Dashboard summary scraped TED daily counts failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
scrapedTED = map[string]int64{}
|
||||
return
|
||||
}
|
||||
scrapedTED = counts
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
if firstErr != nil {
|
||||
return nil, fmt.Errorf("dashboard summary aggregation: %w", firstErr)
|
||||
}
|
||||
|
||||
return &SummaryResponse{
|
||||
TotalTenders: statusCounts.total,
|
||||
ActiveTenders: statusCounts.active,
|
||||
AwardedTenders: statusCounts.awarded,
|
||||
ExpiredTenders: statusCounts.expired,
|
||||
CancelledTenders: statusCounts.cancelled,
|
||||
ClosingSoon: statusCounts.closingSoon,
|
||||
TotalEstimatedValue: totalValue,
|
||||
ValueCurrency: valueCurrency,
|
||||
Days: days,
|
||||
ScrapedTED: fillTrendSeries(days, scrapedTED),
|
||||
GeneratedAt: now,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type summaryStatusCounts struct {
|
||||
total int64
|
||||
active int64
|
||||
awarded int64
|
||||
expired int64
|
||||
cancelled int64
|
||||
closingSoon int64
|
||||
}
|
||||
|
||||
func (r *repository) summaryStatusCounts(ctx context.Context, now, windowEnd int64) (summaryStatusCounts, error) {
|
||||
pipeline := mongo.Pipeline{
|
||||
{{Key: "$facet", Value: bson.M{
|
||||
"total": bson.A{
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"active": bson.A{
|
||||
bson.M{"$match": bson.M{"status": tender.TenderStatusActive}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"awarded": bson.A{
|
||||
bson.M{"$match": bson.M{"status": tender.TenderStatusAwarded}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"cancelled": bson.A{
|
||||
bson.M{"$match": bson.M{"status": tender.TenderStatusCancelled}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"expired": bson.A{
|
||||
bson.M{"$addFields": bson.M{"expiry_deadline": expiryDeadlineExpr()}},
|
||||
bson.M{"$match": bson.M{
|
||||
"status": bson.M{"$nin": []tender.TenderStatus{
|
||||
tender.TenderStatusAwarded,
|
||||
tender.TenderStatusCancelled,
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
"expiry_deadline": expiryDeadlineExpr(),
|
||||
"effective_deadline": effectiveDeadlineExpr(),
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": nil,
|
||||
"total": bson.M{"$sum": 1},
|
||||
"active": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||
bson.M{"$eq": bson.A{"$status", tender.TenderStatusActive}}, 1, 0,
|
||||
}}},
|
||||
"awarded": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||
bson.M{"$eq": bson.A{"$status", tender.TenderStatusAwarded}}, 1, 0,
|
||||
}}},
|
||||
"cancelled": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||
bson.M{"$eq": bson.A{"$status", tender.TenderStatusCancelled}}, 1, 0,
|
||||
}}},
|
||||
"expired": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||
bson.M{"$and": bson.A{
|
||||
bson.M{"$not": bson.M{"$in": bson.A{
|
||||
"$status",
|
||||
bson.A{tender.TenderStatusAwarded, tender.TenderStatusCancelled},
|
||||
}}},
|
||||
bson.M{"$or": bson.A{
|
||||
bson.M{"$eq": bson.A{"$status", tender.TenderStatusExpired}},
|
||||
bson.M{"$and": bson.A{
|
||||
bson.M{"$gt": bson.A{"$expiry_deadline", 0}},
|
||||
bson.M{"$lte": bson.A{"$expiry_deadline", now}},
|
||||
}},
|
||||
}},
|
||||
"$or": []bson.M{
|
||||
{"status": tender.TenderStatusExpired},
|
||||
{"expiry_deadline": bson.M{"$gt": 0, "$lte": now}},
|
||||
},
|
||||
}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"closing_soon": bson.A{
|
||||
bson.M{"$addFields": bson.M{"effective_deadline": effectiveDeadlineExpr()}},
|
||||
bson.M{"$match": bson.M{
|
||||
"effective_deadline": bson.M{"$gt": now, "$lte": windowEnd},
|
||||
}},
|
||||
bson.M{"$count": "count"},
|
||||
},
|
||||
"currency_value": bson.A{
|
||||
bson.M{"$match": bson.M{
|
||||
"estimated_value": bson.M{"$gt": 0},
|
||||
"currency": bson.M{"$type": "string", "$ne": ""},
|
||||
}},
|
||||
bson.M{"$group": bson.M{
|
||||
"_id": "$currency",
|
||||
"count": bson.M{"$sum": 1},
|
||||
"total": bson.M{"$sum": "$estimated_value"},
|
||||
}},
|
||||
bson.M{"$sort": bson.M{"count": -1}},
|
||||
bson.M{"$limit": 1},
|
||||
},
|
||||
}}, 1, 0,
|
||||
}}},
|
||||
"closing_soon": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||
bson.M{"$and": bson.A{
|
||||
bson.M{"$gt": bson.A{"$effective_deadline", now}},
|
||||
bson.M{"$lte": bson.A{"$effective_deadline", windowEnd}},
|
||||
}}, 1, 0,
|
||||
}}},
|
||||
}}},
|
||||
}
|
||||
|
||||
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dashboard summary aggregation: %w", err)
|
||||
return summaryStatusCounts{}, err
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return &SummaryResponse{
|
||||
ValueCurrency: defaultValueCurrency,
|
||||
GeneratedAt: now,
|
||||
}, nil
|
||||
return summaryStatusCounts{}, nil
|
||||
}
|
||||
|
||||
facet := results[0]
|
||||
valueCurrency, totalValue := facetCurrencyValue(facet)
|
||||
|
||||
return &SummaryResponse{
|
||||
TotalTenders: facetCount(facet, "total"),
|
||||
ActiveTenders: facetCount(facet, "active"),
|
||||
AwardedTenders: facetCount(facet, "awarded"),
|
||||
ExpiredTenders: facetCount(facet, "expired"),
|
||||
CancelledTenders: facetCount(facet, "cancelled"),
|
||||
ClosingSoon: facetCount(facet, "closing_soon"),
|
||||
TotalEstimatedValue: totalValue,
|
||||
ValueCurrency: valueCurrency,
|
||||
GeneratedAt: now,
|
||||
row := results[0]
|
||||
return summaryStatusCounts{
|
||||
total: aggregateCount(row, "total"),
|
||||
active: aggregateCount(row, "active"),
|
||||
awarded: aggregateCount(row, "awarded"),
|
||||
expired: aggregateCount(row, "expired"),
|
||||
cancelled: aggregateCount(row, "cancelled"),
|
||||
closingSoon: aggregateCount(row, "closing_soon"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (r *repository) summaryCurrencyValue(ctx context.Context) (string, float64, error) {
|
||||
pipeline := mongo.Pipeline{
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"estimated_value": bson.M{"$gt": 0},
|
||||
"currency": bson.M{"$type": "string", "$ne": ""},
|
||||
}}},
|
||||
{{Key: "$group", Value: bson.M{
|
||||
"_id": "$currency",
|
||||
"count": bson.M{"$sum": 1},
|
||||
"total": bson.M{"$sum": "$estimated_value"},
|
||||
}}},
|
||||
{{Key: "$sort", Value: bson.M{"count": -1}}},
|
||||
{{Key: "$limit", Value: 1}},
|
||||
}
|
||||
|
||||
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
||||
if err != nil {
|
||||
return defaultValueCurrency, 0, err
|
||||
}
|
||||
if len(results) == 0 {
|
||||
return defaultValueCurrency, 0, nil
|
||||
}
|
||||
|
||||
currency, total := facetCurrencyValue(bson.M{"currency_value": bson.A{results[0]}})
|
||||
return currency, total, nil
|
||||
}
|
||||
|
||||
func facetCount(facet bson.M, key string) int64 {
|
||||
rows := facetRows(facet, key)
|
||||
if len(rows) == 0 {
|
||||
|
||||
Reference in New Issue
Block a user