Add scrapedDocumentsScanner interface and enhance document scanning logic
continuous-integration/drone/push Build is passing

- Introduced the `scrapedDocumentsScanner` interface to facilitate scanning of scraped documents from MinIO, returning both procedure summaries and daily document counts.
- Updated the `ListProceduresWithDocuments` method to utilize the new scanning functionality, improving data retrieval efficiency.
- Enhanced the `scrapedDocumentsPerDay` method to filter daily counts based on a specified start date, ensuring accurate reporting of document statistics.
- Added unit tests for the new scanning logic and daily counts filtering, ensuring robust functionality and error handling.

This update enhances the dashboard's document management capabilities, providing better insights into scraped documents and their daily counts.
This commit is contained in:
Mazyar
2026-06-28 00:28:45 +03:30
parent 582f8b5c02
commit 39ac76e7b0
4 changed files with 160 additions and 23 deletions
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"testing"
"time"
"tm/pkg/ai_summarizer"
@@ -11,8 +12,9 @@ import (
)
type mockProcedureDocumentsLister struct {
procedures []ai_summarizer.ProcedureDocumentsSummary
err error
procedures []ai_summarizer.ProcedureDocumentsSummary
dailyCounts map[string]int64
err error
}
func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
@@ -22,6 +24,13 @@ func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Conte
return m.procedures, nil
}
func (m *mockProcedureDocumentsLister) ScanScrapedDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, map[string]int64, error) {
if m.err != nil {
return nil, nil, m.err
}
return m.procedures, m.dailyCounts, nil
}
func TestScrapedDocumentsFolderIDsUsesMinIOProcedures(t *testing.T) {
repo := &repository{
procedureLister: &mockProcedureDocumentsLister{
@@ -91,6 +100,9 @@ func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) {
if scope.zero {
t.Fatal("expected non-zero scope")
}
if !scope.fromMinIO {
t.Fatal("expected MinIO-backed scope")
}
in, ok := scope.match["contract_folder_id"].(bson.M)
if !ok {
t.Fatalf("expected contract_folder_id filter, got %#v", scope.match)
@@ -133,6 +145,52 @@ func TestResolveScrapedTendersScopeMongoFallbackWithoutLister(t *testing.T) {
}
}
func TestFilterDailyCountsSince(t *testing.T) {
counts := map[string]int64{
"2026-06-01": 2,
"2026-06-15": 5,
"2026-06-28": 1,
}
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
got := filterDailyCountsSince(counts, startUnix)
want := map[string]int64{
"2026-06-15": 5,
"2026-06-28": 1,
}
if len(got) != len(want) {
t.Fatalf("expected %d dates, got %d: %v", len(want), len(got), got)
}
for date, count := range want {
if got[date] != count {
t.Fatalf("expected %d on %s, got %d", count, date, got[date])
}
}
}
func TestScrapedDocumentsPerDayUsesMinIODailyCounts(t *testing.T) {
repo := &repository{}
scope := scrapedTendersScope{
dailyCounts: map[string]int64{
"2026-06-01": 2,
"2026-06-28": 3,
},
}
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
got, err := repo.scrapedDocumentsPerDay(context.Background(), startUnix, scope)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got["2026-06-01"] != 0 {
t.Fatalf("expected old date filtered out, got %v", got)
}
if got["2026-06-28"] != 3 {
t.Fatalf("expected 3 on 2026-06-28, got %v", got)
}
}
func TestMongoScrapedTendersScopeMatchesStatisticsFilter(t *testing.T) {
scope := mongoScrapedTendersScope()
if scope.zero {