Mazyar
1df44ec8ca
continuous-integration/drone/push Build is passing
- Introduced a new `translatedNoticesScanner` interface for scanning MinIO for daily translated notice counts. - Enhanced the `repository` struct to include fields for managing translated notice scope and caching. - Implemented the `ScanTranslatedNotices` method in the `StorageClient` to retrieve daily counts and total from MinIO. - Updated the `Statistics` method in the repository to utilize the new translated notices scope, improving data accuracy. - Added unit tests for the translated notices functionality, ensuring robust validation of the new features. This update significantly enhances the dashboard's ability to handle translated notice statistics, improving overall data management and reporting capabilities.
311 lines
8.4 KiB
Go
311 lines
8.4 KiB
Go
package dashboard
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
|
|
"tm/pkg/ai_summarizer"
|
|
"tm/pkg/logger"
|
|
|
|
"go.mongodb.org/mongo-driver/v2/bson"
|
|
)
|
|
|
|
type noopTestLogger struct{}
|
|
|
|
func (noopTestLogger) Debug(string, map[string]interface{}) {}
|
|
func (noopTestLogger) Info(string, map[string]interface{}) {}
|
|
func (noopTestLogger) Warn(string, map[string]interface{}) {}
|
|
func (noopTestLogger) Error(string, map[string]interface{}) {}
|
|
func (noopTestLogger) Fatal(string, map[string]interface{}) {}
|
|
func (l noopTestLogger) WithFields(map[string]interface{}) logger.Logger {
|
|
return l
|
|
}
|
|
func (noopTestLogger) Sync() error { return nil }
|
|
|
|
type mockProcedureDocumentsLister struct {
|
|
procedures []ai_summarizer.ProcedureDocumentsSummary
|
|
dailyCounts map[string]int64
|
|
err error
|
|
|
|
translatedDailyCounts map[string]int64
|
|
translatedTotal int64
|
|
translatedErr error
|
|
}
|
|
|
|
func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
|
|
if m.err != nil {
|
|
return nil, m.err
|
|
}
|
|
return m.procedures, nil
|
|
}
|
|
|
|
func (m *mockProcedureDocumentsLister) ScanScrapedDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, map[string]int64, error) {
|
|
if m.err != nil {
|
|
return nil, nil, m.err
|
|
}
|
|
return m.procedures, m.dailyCounts, nil
|
|
}
|
|
|
|
func (m *mockProcedureDocumentsLister) ScanTranslatedNotices(context.Context) (map[string]int64, int64, error) {
|
|
if m.translatedErr != nil {
|
|
return nil, 0, m.translatedErr
|
|
}
|
|
return m.translatedDailyCounts, m.translatedTotal, nil
|
|
}
|
|
|
|
func TestScrapedDocumentsFolderIDsUsesMinIOProcedures(t *testing.T) {
|
|
repo := &repository{
|
|
procedureLister: &mockProcedureDocumentsLister{
|
|
procedures: []ai_summarizer.ProcedureDocumentsSummary{
|
|
{ContractFolderID: "PROC-1", DocumentCount: 3},
|
|
{ContractFolderID: "PROC-2", DocumentCount: 0},
|
|
{ContractFolderID: " PROC-3 ", DocumentCount: 1},
|
|
{ContractFolderID: "PROC-1", DocumentCount: 2},
|
|
},
|
|
},
|
|
}
|
|
|
|
got, err := repo.scrapedDocumentsFolderIDs(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
|
|
want := []string{"PROC-1", "PROC-3"}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("expected %d folder IDs, got %d: %v", len(want), len(got), got)
|
|
}
|
|
for i, id := range want {
|
|
if got[i] != id {
|
|
t.Fatalf("expected folder ID %q at index %d, got %q", id, i, got[i])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestScrapedDocumentsFolderIDsWithoutLister(t *testing.T) {
|
|
repo := &repository{}
|
|
|
|
got, err := repo.scrapedDocumentsFolderIDs(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got != nil {
|
|
t.Fatalf("expected nil folder IDs without lister, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestScrapedDocumentsFolderIDsPropagatesError(t *testing.T) {
|
|
repo := &repository{
|
|
procedureLister: &mockProcedureDocumentsLister{
|
|
err: errors.New("minio unavailable"),
|
|
},
|
|
}
|
|
|
|
_, err := repo.scrapedDocumentsFolderIDs(context.Background())
|
|
if err == nil {
|
|
t.Fatal("expected error")
|
|
}
|
|
}
|
|
|
|
func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) {
|
|
repo := &repository{
|
|
procedureLister: &mockProcedureDocumentsLister{
|
|
procedures: []ai_summarizer.ProcedureDocumentsSummary{
|
|
{ContractFolderID: "PROC-1", DocumentCount: 2},
|
|
},
|
|
dailyCounts: map[string]int64{"2026-06-28": 2},
|
|
},
|
|
}
|
|
|
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if scope.zero {
|
|
t.Fatal("expected non-zero scope")
|
|
}
|
|
if !scope.fromMinIO {
|
|
t.Fatal("expected MinIO-backed scope")
|
|
}
|
|
if scope.totalTenderCount != 1 {
|
|
t.Fatalf("expected total tender count 1, got %d", scope.totalTenderCount)
|
|
}
|
|
if scope.dailyCounts == nil {
|
|
t.Fatal("expected MinIO daily counts")
|
|
}
|
|
in, ok := scope.match["contract_folder_id"].(bson.M)
|
|
if !ok {
|
|
t.Fatalf("expected contract_folder_id filter, got %#v", scope.match)
|
|
}
|
|
ids, ok := in["$in"].([]string)
|
|
if !ok || len(ids) != 1 || ids[0] != "PROC-1" {
|
|
t.Fatalf("unexpected folder IDs: %#v", in["$in"])
|
|
}
|
|
}
|
|
|
|
func TestResolveScrapedTendersScopeKeepsMinIOCountsWhenTooManyFolders(t *testing.T) {
|
|
procedures := make([]ai_summarizer.ProcedureDocumentsSummary, maxScrapedFolderInClause+1)
|
|
for i := range procedures {
|
|
procedures[i] = ai_summarizer.ProcedureDocumentsSummary{
|
|
ContractFolderID: fmt.Sprintf("PROC-%d", i),
|
|
DocumentCount: 1,
|
|
}
|
|
}
|
|
|
|
repo := &repository{
|
|
logger: noopTestLogger{},
|
|
procedureLister: &mockProcedureDocumentsLister{
|
|
procedures: procedures,
|
|
dailyCounts: map[string]int64{"2026-06-28": 3},
|
|
},
|
|
}
|
|
|
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if scope.zero {
|
|
t.Fatal("expected non-zero scope")
|
|
}
|
|
if !scope.fromMinIO {
|
|
t.Fatal("expected MinIO-backed scope")
|
|
}
|
|
if scope.totalTenderCount != int64(len(procedures)) {
|
|
t.Fatalf("expected total tender count %d, got %d", len(procedures), scope.totalTenderCount)
|
|
}
|
|
if scope.dailyCounts == nil {
|
|
t.Fatal("expected MinIO daily counts to be preserved")
|
|
}
|
|
if scope.match != nil {
|
|
t.Fatalf("expected no Mongo $in filter for large MinIO scope, got %#v", scope.match)
|
|
}
|
|
}
|
|
|
|
func TestResolveScrapedTendersScopeMongoFallbackWhenMinIOEmpty(t *testing.T) {
|
|
repo := &repository{
|
|
procedureLister: &mockProcedureDocumentsLister{},
|
|
}
|
|
|
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if scope.zero {
|
|
t.Fatal("expected mongo fallback scope, not zero scope")
|
|
}
|
|
if scope.match["processing_metadata.documents_scraped"] != true {
|
|
t.Fatalf("expected mongo fallback filter, got %#v", scope.match)
|
|
}
|
|
}
|
|
|
|
func TestResolveScrapedTendersScopeMongoFallbackWithoutLister(t *testing.T) {
|
|
repo := &repository{}
|
|
|
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if scope.zero {
|
|
t.Fatal("expected mongo fallback scope")
|
|
}
|
|
if scope.match["processing_metadata.documents_scraped"] != true {
|
|
t.Fatalf("expected mongo fallback filter, got %#v", scope.match)
|
|
}
|
|
}
|
|
|
|
func TestFilterDailyCountsSince(t *testing.T) {
|
|
counts := map[string]int64{
|
|
"2026-06-01": 2,
|
|
"2026-06-15": 5,
|
|
"2026-06-28": 1,
|
|
}
|
|
|
|
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
|
|
got := filterDailyCountsSince(counts, startUnix)
|
|
|
|
want := map[string]int64{
|
|
"2026-06-15": 5,
|
|
"2026-06-28": 1,
|
|
}
|
|
if len(got) != len(want) {
|
|
t.Fatalf("expected %d dates, got %d: %v", len(want), len(got), got)
|
|
}
|
|
for date, count := range want {
|
|
if got[date] != count {
|
|
t.Fatalf("expected %d on %s, got %d", count, date, got[date])
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestScrapedDocumentsPerDayUsesMinIODailyCounts(t *testing.T) {
|
|
repo := &repository{}
|
|
scope := scrapedTendersScope{
|
|
dailyCounts: map[string]int64{
|
|
"2026-06-01": 2,
|
|
"2026-06-28": 3,
|
|
},
|
|
}
|
|
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
|
|
|
|
got, err := repo.scrapedDocumentsPerDay(context.Background(), startUnix, scope)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if got["2026-06-01"] != 0 {
|
|
t.Fatalf("expected old date filtered out, got %v", got)
|
|
}
|
|
if got["2026-06-28"] != 3 {
|
|
t.Fatalf("expected 3 on 2026-06-28, got %v", got)
|
|
}
|
|
}
|
|
|
|
func TestMongoScrapedTendersScopeMatchesStatisticsFilter(t *testing.T) {
|
|
scope := mongoScrapedTendersScope()
|
|
if scope.zero {
|
|
t.Fatal("expected non-zero mongo statistics scope")
|
|
}
|
|
if scope.match["processing_metadata.documents_scraped"] != true {
|
|
t.Fatalf("expected documents_scraped filter, got %#v", scope.match)
|
|
}
|
|
}
|
|
|
|
func TestResolveTranslatedNoticesScopeUsesMinIOCounts(t *testing.T) {
|
|
repo := &repository{
|
|
procedureLister: &mockProcedureDocumentsLister{
|
|
translatedDailyCounts: map[string]int64{
|
|
"2026-07-01": 4,
|
|
"2026-07-09": 2,
|
|
},
|
|
translatedTotal: 6,
|
|
},
|
|
}
|
|
|
|
scope, err := repo.resolveTranslatedNoticesScope(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if !scope.fromMinIO {
|
|
t.Fatal("expected MinIO-backed translation scope")
|
|
}
|
|
if scope.totalCount != 6 {
|
|
t.Fatalf("expected total 6, got %d", scope.totalCount)
|
|
}
|
|
if scope.dailyCounts["2026-07-09"] != 2 {
|
|
t.Fatalf("expected 2 on 2026-07-09, got %d", scope.dailyCounts["2026-07-09"])
|
|
}
|
|
}
|
|
|
|
func TestResolveTranslatedNoticesScopeWithoutLister(t *testing.T) {
|
|
repo := &repository{}
|
|
|
|
scope, err := repo.resolveTranslatedNoticesScope(context.Background())
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if scope.fromMinIO {
|
|
t.Fatal("expected metrics fallback without lister")
|
|
}
|
|
}
|