39ac76e7b0
continuous-integration/drone/push Build is passing
- Introduced the `scrapedDocumentsScanner` interface to facilitate scanning of scraped documents from MinIO, returning both procedure summaries and daily document counts. - Updated the `ListProceduresWithDocuments` method to utilize the new scanning functionality, improving data retrieval efficiency. - Enhanced the `scrapedDocumentsPerDay` method to filter daily counts based on a specified start date, ensuring accurate reporting of document statistics. - Added unit tests for the new scanning logic and daily counts filtering, ensuring robust functionality and error handling. This update enhances the dashboard's document management capabilities, providing better insights into scraped documents and their daily counts.
203 lines
5.2 KiB
Go
203 lines
5.2 KiB
Go
package dashboard
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"tm/pkg/ai_summarizer"
|
|
|
|
"go.mongodb.org/mongo-driver/v2/bson"
|
|
)
|
|
|
|
type mockProcedureDocumentsLister struct {
|
|
procedures []ai_summarizer.ProcedureDocumentsSummary
|
|
dailyCounts map[string]int64
|
|
err error
|
|
}
|
|
|
|
func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
|
|
if m.err != nil {
|
|
return nil, m.err
|
|
}
|
|
return m.procedures, nil
|
|
}
|
|
|
|
func (m *mockProcedureDocumentsLister) ScanScrapedDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, map[string]int64, error) {
|
|
if m.err != nil {
|
|
return nil, nil, m.err
|
|
}
|
|
return m.procedures, m.dailyCounts, nil
|
|
}
|
|
|
|
func TestScrapedDocumentsFolderIDsUsesMinIOProcedures(t *testing.T) {
|
|
repo := &repository{
|
|
procedureLister: &mockProcedureDocumentsLister{
|
|
procedures: []ai_summarizer.ProcedureDocumentsSummary{
|
|
{ContractFolderID: "PROC-1", DocumentCount: 3},
|
|
{ContractFolderID: "PROC-2", DocumentCount: 0},
|
|
{ContractFolderID: " PROC-3 ", DocumentCount: 1},
|
|
{ContractFolderID: "PROC-1", DocumentCount: 2},
|
|
},
|
|
},
|
|
}
|
|
|
|
got, err := repo.scrapedDocumentsFolderIDs(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
|
|
want := []string{"PROC-1", "PROC-3"}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("expected %d folder IDs, got %d: %v", len(want), len(got), got)
|
|
}
|
|
for i, id := range want {
|
|
if got[i] != id {
|
|
t.Fatalf("expected folder ID %q at index %d, got %q", id, i, got[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestScrapedDocumentsFolderIDsWithoutLister(t *testing.T) {
|
|
repo := &repository{}
|
|
|
|
got, err := repo.scrapedDocumentsFolderIDs(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got != nil {
|
|
t.Fatalf("expected nil folder IDs without lister, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestScrapedDocumentsFolderIDsPropagatesError(t *testing.T) {
|
|
repo := &repository{
|
|
procedureLister: &mockProcedureDocumentsLister{
|
|
err: errors.New("minio unavailable"),
|
|
},
|
|
}
|
|
|
|
_, err := repo.scrapedDocumentsFolderIDs(context.Background())
|
|
if err == nil {
|
|
t.Fatal("expected error")
|
|
}
|
|
}
|
|
|
|
func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) {
|
|
repo := &repository{
|
|
procedureLister: &mockProcedureDocumentsLister{
|
|
procedures: []ai_summarizer.ProcedureDocumentsSummary{
|
|
{ContractFolderID: "PROC-1", DocumentCount: 2},
|
|
},
|
|
},
|
|
}
|
|
|
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if scope.zero {
|
|
t.Fatal("expected non-zero scope")
|
|
}
|
|
if !scope.fromMinIO {
|
|
t.Fatal("expected MinIO-backed scope")
|
|
}
|
|
in, ok := scope.match["contract_folder_id"].(bson.M)
|
|
if !ok {
|
|
t.Fatalf("expected contract_folder_id filter, got %#v", scope.match)
|
|
}
|
|
ids, ok := in["$in"].([]string)
|
|
if !ok || len(ids) != 1 || ids[0] != "PROC-1" {
|
|
t.Fatalf("unexpected folder IDs: %#v", in["$in"])
|
|
}
|
|
}
|
|
|
|
func TestResolveScrapedTendersScopeMongoFallbackWhenMinIOEmpty(t *testing.T) {
|
|
repo := &repository{
|
|
procedureLister: &mockProcedureDocumentsLister{},
|
|
}
|
|
|
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if scope.zero {
|
|
t.Fatal("expected mongo fallback scope, not zero scope")
|
|
}
|
|
if scope.match["processing_metadata.documents_scraped"] != true {
|
|
t.Fatalf("expected mongo fallback filter, got %#v", scope.match)
|
|
}
|
|
}
|
|
|
|
func TestResolveScrapedTendersScopeMongoFallbackWithoutLister(t *testing.T) {
|
|
repo := &repository{}
|
|
|
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if scope.zero {
|
|
t.Fatal("expected mongo fallback scope")
|
|
}
|
|
if scope.match["processing_metadata.documents_scraped"] != true {
|
|
t.Fatalf("expected mongo fallback filter, got %#v", scope.match)
|
|
}
|
|
}
|
|
|
|
func TestFilterDailyCountsSince(t *testing.T) {
|
|
counts := map[string]int64{
|
|
"2026-06-01": 2,
|
|
"2026-06-15": 5,
|
|
"2026-06-28": 1,
|
|
}
|
|
|
|
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
|
|
got := filterDailyCountsSince(counts, startUnix)
|
|
|
|
want := map[string]int64{
|
|
"2026-06-15": 5,
|
|
"2026-06-28": 1,
|
|
}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("expected %d dates, got %d: %v", len(want), len(got), got)
|
|
}
|
|
for date, count := range want {
|
|
if got[date] != count {
|
|
t.Fatalf("expected %d on %s, got %d", count, date, got[date])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestScrapedDocumentsPerDayUsesMinIODailyCounts(t *testing.T) {
|
|
repo := &repository{}
|
|
scope := scrapedTendersScope{
|
|
dailyCounts: map[string]int64{
|
|
"2026-06-01": 2,
|
|
"2026-06-28": 3,
|
|
},
|
|
}
|
|
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
|
|
|
|
got, err := repo.scrapedDocumentsPerDay(context.Background(), startUnix, scope)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got["2026-06-01"] != 0 {
|
|
t.Fatalf("expected old date filtered out, got %v", got)
|
|
}
|
|
if got["2026-06-28"] != 3 {
|
|
t.Fatalf("expected 3 on 2026-06-28, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestMongoScrapedTendersScopeMatchesStatisticsFilter(t *testing.T) {
|
|
scope := mongoScrapedTendersScope()
|
|
if scope.zero {
|
|
t.Fatal("expected non-zero mongo statistics scope")
|
|
}
|
|
if scope.match["processing_metadata.documents_scraped"] != true {
|
|
t.Fatalf("expected documents_scraped filter, got %#v", scope.match)
|
|
}
|
|
}
|