Update MinIO bucket configuration and enhance dashboard service caching
continuous-integration/drone/push Build is passing
continuous-integration/drone/push Build is passing
- Changed the default MinIO bucket name from "opplens-documents" to "opplens" across multiple configuration files. - Introduced caching for dashboard statistics in the service layer to improve performance and reduce redundant data fetching. - Implemented a mutex for thread-safe access to cached statistics, ensuring data integrity during concurrent requests. This update streamlines the configuration for the AI summarizer and optimizes the dashboard service, enhancing overall system efficiency.
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
const noticesCollectionName = "notices"
|
||||
@@ -19,32 +20,66 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
|
||||
endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
|
||||
startDay := endDay.AddDate(0, 0, -(days - 1))
|
||||
|
||||
scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{
|
||||
"source": notice.TenderSourceTEDScraper,
|
||||
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}, "processing_metadata.scraped_at")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped ted per day: %w", err)
|
||||
}
|
||||
var (
|
||||
scrapedTED map[string]int64
|
||||
scrapedDocuments map[string]int64
|
||||
translatedNotices map[string]int64
|
||||
totalDocuments int64
|
||||
totalTranslated int64
|
||||
)
|
||||
|
||||
scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped documents per day: %w", err)
|
||||
}
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
|
||||
translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("translated notices per day: %w", err)
|
||||
}
|
||||
g.Go(func() error {
|
||||
counts, err := r.dailyCountByTimestamp(gctx, noticesCollectionName, bson.M{
|
||||
"source": notice.TenderSourceTEDScraper,
|
||||
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}, "processing_metadata.scraped_at")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scraped ted per day: %w", err)
|
||||
}
|
||||
scrapedTED = counts
|
||||
return nil
|
||||
})
|
||||
|
||||
totalDocuments, err := r.totalScrapedDocuments(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total scraped documents: %w", err)
|
||||
}
|
||||
g.Go(func() error {
|
||||
counts, err := r.scrapedDocumentsPerDay(gctx, startUnix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scraped documents per day: %w", err)
|
||||
}
|
||||
scrapedDocuments = counts
|
||||
return nil
|
||||
})
|
||||
|
||||
totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total translated tenders: %w", err)
|
||||
g.Go(func() error {
|
||||
counts, err := r.translatedNoticesPerDay(gctx, startDay, endDay)
|
||||
if err != nil {
|
||||
return fmt.Errorf("translated notices per day: %w", err)
|
||||
}
|
||||
translatedNotices = counts
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
total, err := r.totalScrapedDocuments(gctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("total scraped documents: %w", err)
|
||||
}
|
||||
totalDocuments = total
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
total, err := r.metricsCounter.Get(gctx, orm.AITranslationSuccessCounterKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("total translated tenders: %w", err)
|
||||
}
|
||||
totalTranslated = total
|
||||
return nil
|
||||
})
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &StatisticsReportResponse{
|
||||
@@ -94,6 +129,10 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"processing_metadata.documents_scraped": true,
|
||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
||||
"$or": []bson.M{
|
||||
{"processing_metadata.documents_scraped_at": bson.M{"$gte": startUnix}},
|
||||
{"scraped_documents.scraped_at": bson.M{"$gte": startUnix}},
|
||||
},
|
||||
}}},
|
||||
{{Key: "$unwind", Value: "$scraped_documents"}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
|
||||
Reference in New Issue
Block a user