mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor dashboard repository to integrate ProcedureDocumentsLister

Browse Source 
- Introduced the ProcedureDocumentsLister interface to list contract folders with scraped documents, enhancing the accuracy of document-scrape statistics.
- Updated the dashboard repository to accept ProcedureDocumentsLister as a dependency, allowing for improved data retrieval.
- Implemented tests for the new functionality, ensuring proper handling of scraped document folder IDs and error propagation.

This update enhances the dashboard's capability to manage and report on scraped documents, improving overall system efficiency and data integrity.
This commit is contained in:
Mazyar
2026-06-17 14:17:41 +03:30
parent 7120a954b6
commit 9676f99304
4 changed files with 172 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/web/main.go
+3 -1
View File
@@ -171,6 +171,7 @@ func main() {
// Initialize AI Summarizer service
var aiSummarizerClient tender.AISummarizerClient
var aiSummarizerStorage tender.AISummarizerStorage
var procedureDocumentsLister dashboard.ProcedureDocumentsLister
var aiRecommendationClient company.AIRecommendationClient
var aiPipelineClient ai_pipeline.Client
@@ -181,6 +182,7 @@ func main() {
}
if s := bootstrap.InitAISummarizerStorage(conf.AISummarizer, logger); s != nil {
aiSummarizerStorage = s
procedureDocumentsLister = s
}
// Initialize authorization service
@@ -230,7 +232,7 @@ func main() {
cmsService := cms.NewService(cmsRepository, logger)
kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, goRulesClient, conf.GoRules.RuleID, logger)
documentScraperService := document_scraper.NewService(tenderRepository, logger)
dashboardRepository := dashboard.NewRepository(mongoManager, logger)
dashboardRepository := dashboard.NewRepository(mongoManager, logger, procedureDocumentsLister)
dashboardService := dashboard.NewService(dashboardRepository, logger)
aiPipelineService := ai_pipeline.NewService(aiPipelineClient, logger, tenderService)
logger.Info("Services initialized successfully", map[string]interface{}{
internal/dashboard/repository.go
+10 -1
View File
@@ -6,6 +6,7 @@ import (
"strings"
"time"
"tm/internal/tender"
"tm/pkg/ai_summarizer"
"tm/pkg/logger"
orm "tm/pkg/mongo"
@@ -13,6 +14,12 @@ import (
"go.mongodb.org/mongo-driver/v2/mongo"
)
// ProcedureDocumentsLister lists contract folders that have scraped documents in MinIO.
// This is the source of truth for document-scrape statistics, matching tender panel search.
type ProcedureDocumentsLister interface {
ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error)
}
const defaultValueCurrency = "EUR"
// Repository defines dashboard data access.
@@ -30,15 +37,17 @@ type repository struct {
ormRepo orm.Repository[tender.Tender]
mongoManager *orm.ConnectionManager
metricsCounter *orm.Counter
procedureLister ProcedureDocumentsLister
logger logger.Logger
}
// NewRepository creates a dashboard repository backed by the tenders collection.
func NewRepository(mongoManager *orm.ConnectionManager, log logger.Logger) Repository {
func NewRepository(mongoManager *orm.ConnectionManager, log logger.Logger, procedureLister ProcedureDocumentsLister) Repository {
return &repository{
ormRepo: orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log),
mongoManager: mongoManager,
metricsCounter: orm.NewCounter(mongoManager),
procedureLister: procedureLister,
logger: log,
}
}
internal/dashboard/statistics_repository.go
+62 -40
View File
@@ -3,6 +3,7 @@ package dashboard
import (
"context"
"fmt"
"strings"
"time"
"tm/internal/notice"
@@ -63,9 +64,9 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
})
g.Go(func() error {
total, err := r.totalScrapedDocuments(gctx)
total, err := r.totalScrapedTenders(gctx)
if err != nil {
return fmt.Errorf("total scraped documents: %w", err)
return fmt.Errorf("total scraped tenders: %w", err)
}
totalDocuments = total
return nil
@@ -127,23 +128,18 @@ func (r *repository) dailyCountByTimestamp(ctx context.Context, collection strin
}
func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) {
match, err := r.scrapedTendersMatchFilter(ctx)
if err != nil {
return nil, err
}
if len(match) == 0 {
return map[string]int64{}, nil
}
pipeline := mongodriver.Pipeline{
{{Key: "$match", Value: bson.M{
"processing_metadata.documents_scraped": true,
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
}}},
{{Key: "$unwind", Value: "$scraped_documents"}},
{{Key: "$match", Value: match}},
{{Key: "$addFields", Value: bson.M{
"metric_ts": bson.M{
"$cond": bson.A{
bson.M{"$gt": bson.A{"$scraped_documents.scraped_at", 0}},
"$scraped_documents.scraped_at",
"$processing_metadata.documents_scraped_at",
},
},
}}},
{{Key: "$addFields", Value: bson.M{
"metric_ts": normalizeTimestampExpr("metric_ts"),
"metric_ts": normalizeTimestampExpr("processing_metadata.documents_scraped_at"),
}}},
{{Key: "$match", Value: bson.M{
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
@@ -173,37 +169,63 @@ func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endD
return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
}
func (r *repository) totalScrapedDocuments(ctx context.Context) (int64, error) {
pipeline := mongodriver.Pipeline{
{{Key: "$match", Value: bson.M{
"processing_metadata.documents_scraped": true,
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
}}},
{{Key: "$project", Value: bson.M{
"doc_count": bson.M{"$size": "$scraped_documents"},
}}},
{{Key: "$group", Value: bson.M{
"_id": nil,
"total": bson.M{"$sum": "$doc_count"},
}}},
}
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
func (r *repository) totalScrapedTenders(ctx context.Context) (int64, error) {
match, err := r.scrapedTendersMatchFilter(ctx)
if err != nil {
return 0, err
}
defer cursor.Close(ctx)
if !cursor.Next(ctx) {
if len(match) == 0 {
return 0, nil
}
var row bson.M
if err := cursor.Decode(&row); err != nil {
return 0, err
return r.ormRepo.Count(ctx, match)
}
return aggregateCount(row, "total"), nil
func (r *repository) scrapedTendersMatchFilter(ctx context.Context) (bson.M, error) {
folderIDs, err := r.scrapedDocumentsFolderIDs(ctx)
if err != nil {
return nil, err
}
if len(folderIDs) > 0 {
return bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}, nil
}
if r.procedureLister != nil {
return bson.M{}, nil
}
return bson.M{
"processing_metadata.documents_scraped": true,
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
}, nil
}
func (r *repository) scrapedDocumentsFolderIDs(ctx context.Context) ([]string, error) {
if r.procedureLister == nil {
return nil, nil
}
procedures, err := r.procedureLister.ListProceduresWithDocuments(ctx)
if err != nil {
return nil, err
}
folderIDs := make([]string, 0, len(procedures))
seen := make(map[string]struct{}, len(procedures))
for _, proc := range procedures {
if proc.DocumentCount <= 0 {
continue
}
id := strings.TrimSpace(proc.ContractFolderID)
if id == "" {
continue
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
folderIDs = append(folderIDs, id)
}
return folderIDs, nil
}
func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) {
internal/dashboard/statistics_repository_test.go
+74
View File
@@ -0,0 +1,74 @@
package dashboard
import (
"context"
"errors"
"testing"
"tm/pkg/ai_summarizer"
)
type mockProcedureDocumentsLister struct {
procedures []ai_summarizer.ProcedureDocumentsSummary
err error
}
func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
if m.err != nil {
return nil, m.err
}
return m.procedures, nil
}
func TestScrapedDocumentsFolderIDsUsesMinIOProcedures(t *testing.T) {
repo := &repository{
procedureLister: &mockProcedureDocumentsLister{
procedures: []ai_summarizer.ProcedureDocumentsSummary{
{ContractFolderID: "PROC-1", DocumentCount: 3},
{ContractFolderID: "PROC-2", DocumentCount: 0},
{ContractFolderID: " PROC-3 ", DocumentCount: 1},
{ContractFolderID: "PROC-1", DocumentCount: 2},
},
},
}
got, err := repo.scrapedDocumentsFolderIDs(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
want := []string{"PROC-1", "PROC-3"}
if len(got) != len(want) {
t.Fatalf("expected %d folder IDs, got %d: %v", len(want), len(got), got)
}
for i, id := range want {
if got[i] != id {
t.Fatalf("expected folder ID %q at index %d, got %q", id, i, got[i])
}
}
}
func TestScrapedDocumentsFolderIDsWithoutLister(t *testing.T) {
repo := &repository{}
got, err := repo.scrapedDocumentsFolderIDs(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != nil {
t.Fatalf("expected nil folder IDs without lister, got %v", got)
}
}
func TestScrapedDocumentsFolderIDsPropagatesError(t *testing.T) {
repo := &repository{
procedureLister: &mockProcedureDocumentsLister{
err: errors.New("minio unavailable"),
},
}
_, err := repo.scrapedDocumentsFolderIDs(context.Background())
if err == nil {
t.Fatal("expected error")
}
}