Add scrapedDocumentsScanner interface and enhance document scanning logic
continuous-integration/drone/push Build is passing
continuous-integration/drone/push Build is passing
- Introduced the `scrapedDocumentsScanner` interface to facilitate scanning of scraped documents from MinIO, returning both procedure summaries and daily document counts. - Updated the `ListProceduresWithDocuments` method to utilize the new scanning functionality, improving data retrieval efficiency. - Enhanced the `scrapedDocumentsPerDay` method to filter daily counts based on a specified start date, ensuring accurate reporting of document statistics. - Added unit tests for the new scanning logic and daily counts filtering, ensuring robust functionality and error handling. This update enhances the dashboard's document management capabilities, providing better insights into scraped documents and their daily counts.
This commit is contained in:
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"tm/pkg/ai_summarizer"
|
||||
|
||||
@@ -11,8 +12,9 @@ import (
|
||||
)
|
||||
|
||||
type mockProcedureDocumentsLister struct {
|
||||
procedures []ai_summarizer.ProcedureDocumentsSummary
|
||||
err error
|
||||
procedures []ai_summarizer.ProcedureDocumentsSummary
|
||||
dailyCounts map[string]int64
|
||||
err error
|
||||
}
|
||||
|
||||
func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
|
||||
@@ -22,6 +24,13 @@ func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Conte
|
||||
return m.procedures, nil
|
||||
}
|
||||
|
||||
func (m *mockProcedureDocumentsLister) ScanScrapedDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, map[string]int64, error) {
|
||||
if m.err != nil {
|
||||
return nil, nil, m.err
|
||||
}
|
||||
return m.procedures, m.dailyCounts, nil
|
||||
}
|
||||
|
||||
func TestScrapedDocumentsFolderIDsUsesMinIOProcedures(t *testing.T) {
|
||||
repo := &repository{
|
||||
procedureLister: &mockProcedureDocumentsLister{
|
||||
@@ -91,6 +100,9 @@ func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) {
|
||||
if scope.zero {
|
||||
t.Fatal("expected non-zero scope")
|
||||
}
|
||||
if !scope.fromMinIO {
|
||||
t.Fatal("expected MinIO-backed scope")
|
||||
}
|
||||
in, ok := scope.match["contract_folder_id"].(bson.M)
|
||||
if !ok {
|
||||
t.Fatalf("expected contract_folder_id filter, got %#v", scope.match)
|
||||
@@ -133,6 +145,52 @@ func TestResolveScrapedTendersScopeMongoFallbackWithoutLister(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFilterDailyCountsSince(t *testing.T) {
|
||||
counts := map[string]int64{
|
||||
"2026-06-01": 2,
|
||||
"2026-06-15": 5,
|
||||
"2026-06-28": 1,
|
||||
}
|
||||
|
||||
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
|
||||
got := filterDailyCountsSince(counts, startUnix)
|
||||
|
||||
want := map[string]int64{
|
||||
"2026-06-15": 5,
|
||||
"2026-06-28": 1,
|
||||
}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("expected %d dates, got %d: %v", len(want), len(got), got)
|
||||
}
|
||||
for date, count := range want {
|
||||
if got[date] != count {
|
||||
t.Fatalf("expected %d on %s, got %d", count, date, got[date])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScrapedDocumentsPerDayUsesMinIODailyCounts(t *testing.T) {
|
||||
repo := &repository{}
|
||||
scope := scrapedTendersScope{
|
||||
dailyCounts: map[string]int64{
|
||||
"2026-06-01": 2,
|
||||
"2026-06-28": 3,
|
||||
},
|
||||
}
|
||||
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
|
||||
|
||||
got, err := repo.scrapedDocumentsPerDay(context.Background(), startUnix, scope)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got["2026-06-01"] != 0 {
|
||||
t.Fatalf("expected old date filtered out, got %v", got)
|
||||
}
|
||||
if got["2026-06-28"] != 3 {
|
||||
t.Fatalf("expected 3 on 2026-06-28, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMongoScrapedTendersScopeMatchesStatisticsFilter(t *testing.T) {
|
||||
scope := mongoScrapedTendersScope()
|
||||
if scope.zero {
|
||||
|
||||
Reference in New Issue
Block a user