Merge pull request 'Enhance tender estimated value resolution and add unit tests' (#45) from est-value into develop

Reviewed-on: https://repo.ravanertebat.com/TM/tm_back/pulls/45

cmd/worker/workers/notice.go

@@ -476,6 +476,16 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 		awardedEntities = mergeAwardedEntitiesByLotID(t.AwardedEntities, awardedEntities)
 		title, description = mergeTitleAndDescriptionForMultiLot(t.Title, t.Description, title, description)
 
+		if estimatedVal == 0 && t.EstimatedValue > 0 {
+			estimatedVal = t.EstimatedValue
+		}
+		if awardedVal == 0 && t.AwardedValue > 0 {
+			awardedVal = t.AwardedValue
+		}
+		if currency == "" && t.Currency != "" {
+			currency = t.Currency
+		}
+
 		if strings.TrimSpace(previousNoticePublicationID) != "" && previousNoticePublicationID != n.NoticePublicationID {
 			if n.EstimatedValue > 0 {
 				estimatedVal = t.EstimatedValue + n.EstimatedValue
@@ -495,6 +505,15 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 		}
 	}
 
+	if estimatedVal == 0 {
+		if lotValue, lotCurrency := tender.AggregateProcurementLotEstimatedValue(procurementLots); lotValue > 0 {
+			estimatedVal = lotValue
+			if currency == "" {
+				currency = lotCurrency
+			}
+		}
+	}
+
 	// Title and description stay in the notice language; English translation is handled by the AI service (translation worker).
 	t.Title = title
 	t.Description = description

internal/dashboard/repository.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync"
 	"time"
 	"tm/internal/tender"
 	"tm/pkg/ai_summarizer"
@@ -12,6 +13,7 @@ import (
 
 	"go.mongodb.org/mongo-driver/v2/bson"
 	"go.mongodb.org/mongo-driver/v2/mongo"
+	"golang.org/x/sync/singleflight"
 )
 
 // ProcedureDocumentsLister lists contract folders that have scraped documents in MinIO.
@@ -34,11 +36,16 @@ type Repository interface {
 }
 
 type repository struct {
-	ormRepo         orm.Repository[tender.Tender]
+	ormRepo            orm.Repository[tender.Tender]
-	mongoManager    *orm.ConnectionManager
+	mongoManager       *orm.ConnectionManager
-	metricsCounter  *orm.Counter
+	metricsCounter     *orm.Counter
-	procedureLister ProcedureDocumentsLister
+	procedureLister    ProcedureDocumentsLister
-	logger          logger.Logger
+	logger             logger.Logger
+	scrapedScopeMu     sync.Mutex
+	scrapedScope         scrapedTendersScope
+	scrapedScopeExpiry   time.Time
+	scrapedScopeCached   bool
+	scrapedScopeGroup    singleflight.Group
 }
 
 // NewRepository creates a dashboard repository backed by the tenders collection.
@@ -60,135 +67,114 @@ func (r *repository) Summary(ctx context.Context, closingWindowSec int64) (*Summ
 	now := time.Now().Unix()
 	windowEnd := now + closingWindowSec
 
-	total, err := r.ormRepo.Count(ctx, bson.M{})
+	pipeline := mongo.Pipeline{
-	if err != nil {
+		{{Key: "$facet", Value: bson.M{
-		return nil, fmt.Errorf("count total tenders: %w", err)
+			"total": bson.A{
+				bson.M{"$count": "count"},
+			},
+			"active": bson.A{
+				bson.M{"$match": bson.M{"status": tender.TenderStatusActive}},
+				bson.M{"$count": "count"},
+			},
+			"awarded": bson.A{
+				bson.M{"$match": bson.M{"status": tender.TenderStatusAwarded}},
+				bson.M{"$count": "count"},
+			},
+			"cancelled": bson.A{
+				bson.M{"$match": bson.M{"status": tender.TenderStatusCancelled}},
+				bson.M{"$count": "count"},
+			},
+			"expired": bson.A{
+				bson.M{"$addFields": bson.M{"expiry_deadline": expiryDeadlineExpr()}},
+				bson.M{"$match": bson.M{
+					"status": bson.M{"$nin": []tender.TenderStatus{
+						tender.TenderStatusAwarded,
+						tender.TenderStatusCancelled,
+					}},
+					"$or": []bson.M{
+						{"status": tender.TenderStatusExpired},
+						{"expiry_deadline": bson.M{"$gt": 0, "$lte": now}},
+					},
+				}},
+				bson.M{"$count": "count"},
+			},
+			"closing_soon": bson.A{
+				bson.M{"$addFields": bson.M{"effective_deadline": effectiveDeadlineExpr()}},
+				bson.M{"$match": bson.M{
+					"effective_deadline": bson.M{"$gt": now, "$lte": windowEnd},
+				}},
+				bson.M{"$count": "count"},
+			},
+			"currency_value": bson.A{
+				bson.M{"$match": bson.M{
+					"estimated_value": bson.M{"$gt": 0},
+					"currency":        bson.M{"$type": "string", "$ne": ""},
+				}},
+				bson.M{"$group": bson.M{
+					"_id":   "$currency",
+					"count": bson.M{"$sum": 1},
+					"total": bson.M{"$sum": "$estimated_value"},
+				}},
+				bson.M{"$sort": bson.M{"count": -1}},
+				bson.M{"$limit": 1},
+			},
+		}}},
 	}
 
-	active, err := r.ormRepo.Count(ctx, bson.M{"status": tender.TenderStatusActive})
+	results, err := r.ormRepo.Aggregate(ctx, pipeline)
 	if err != nil {
-		return nil, fmt.Errorf("count active tenders: %w", err)
+		return nil, fmt.Errorf("dashboard summary aggregation: %w", err)
+	}
+	if len(results) == 0 {
+		return &SummaryResponse{
+			ValueCurrency: defaultValueCurrency,
+			GeneratedAt:   now,
+		}, nil
 	}
 
-	awarded, err := r.ormRepo.Count(ctx, bson.M{"status": tender.TenderStatusAwarded})
+	facet := results[0]
-	if err != nil {
+	valueCurrency, totalValue := facetCurrencyValue(facet)
-		return nil, fmt.Errorf("count awarded tenders: %w", err)
-	}
-
-	expired, err := r.countExpired(ctx, now)
-	if err != nil {
-		return nil, err
-	}
-
-	cancelled, err := r.ormRepo.Count(ctx, bson.M{"status": tender.TenderStatusCancelled})
-	if err != nil {
-		return nil, fmt.Errorf("count cancelled tenders: %w", err)
-	}
-
-	closingSoon, err := r.countClosingSoon(ctx, now, windowEnd)
-	if err != nil {
-		return nil, err
-	}
-
-	valueCurrency, totalValue, err := r.dominantCurrencyValue(ctx)
-	if err != nil {
-		return nil, err
-	}
 
 	return &SummaryResponse{
-		TotalTenders:        total,
+		TotalTenders:        facetCount(facet, "total"),
-		ActiveTenders:       active,
+		ActiveTenders:       facetCount(facet, "active"),
-		AwardedTenders:      awarded,
+		AwardedTenders:      facetCount(facet, "awarded"),
-		ExpiredTenders:      expired,
+		ExpiredTenders:      facetCount(facet, "expired"),
-		CancelledTenders:    cancelled,
+		CancelledTenders:    facetCount(facet, "cancelled"),
-		ClosingSoon:         closingSoon,
+		ClosingSoon:         facetCount(facet, "closing_soon"),
 		TotalEstimatedValue: totalValue,
 		ValueCurrency:       valueCurrency,
 		GeneratedAt:         now,
 	}, nil
 }
 
-func (r *repository) countExpired(ctx context.Context, now int64) (int64, error) {
+func facetCount(facet bson.M, key string) int64 {
-	pipeline := mongo.Pipeline{
+	rows, ok := facet[key].(bson.A)
-		{{Key: "$addFields", Value: bson.M{"expiry_deadline": expiryDeadlineExpr()}}},
+	if !ok || len(rows) == 0 {
-		{{Key: "$match", Value: bson.M{
+		return 0
-			"status": bson.M{"$nin": []tender.TenderStatus{
-				tender.TenderStatusAwarded,
-				tender.TenderStatusCancelled,
-			}},
-			"$or": []bson.M{
-				{"status": tender.TenderStatusExpired},
-				{
-					"expiry_deadline": bson.M{"$gt": 0, "$lte": now},
-				},
-			},
-		}}},
-		{{Key: "$count", Value: "count"}},
 	}
-
+	row, ok := rows[0].(bson.M)
-	results, err := r.ormRepo.Aggregate(ctx, pipeline)
+	if !ok {
-	if err != nil {
+		return 0
-		return 0, fmt.Errorf("count expired tenders: %w", err)
 	}
-
+	return aggregateCount(row, "count")
-	if len(results) == 0 {
-		return 0, nil
-	}
-
-	return aggregateCount(results[0], "count"), nil
 }
 
-func (r *repository) countClosingSoon(ctx context.Context, now, windowEnd int64) (int64, error) {
+func facetCurrencyValue(facet bson.M) (string, float64) {
-	pipeline := mongo.Pipeline{
+	rows, ok := facet["currency_value"].(bson.A)
-		{{Key: "$addFields", Value: bson.M{"effective_deadline": effectiveDeadlineExpr()}}},
+	if !ok || len(rows) == 0 {
-		{{Key: "$match", Value: bson.M{
+		return defaultValueCurrency, 0
-			"effective_deadline": bson.M{"$gt": now, "$lte": windowEnd},
-		}}},
-		{{Key: "$count", Value: "count"}},
 	}
-
+	row, ok := rows[0].(bson.M)
-	results, err := r.ormRepo.Aggregate(ctx, pipeline)
+	if !ok {
-	if err != nil {
+		return defaultValueCurrency, 0
-		return 0, fmt.Errorf("count closing soon: %w", err)
-	}
-
-	if len(results) == 0 {
-		return 0, nil
-	}
-
-	return aggregateCount(results[0], "count"), nil
-}
-
-func (r *repository) dominantCurrencyValue(ctx context.Context) (string, float64, error) {
-	pipeline := mongo.Pipeline{
-		{{Key: "$match", Value: bson.M{
-			"estimated_value": bson.M{"$gt": 0},
-			"currency":        bson.M{"$type": "string", "$ne": ""},
-		}}},
-		{{Key: "$group", Value: bson.M{
-			"_id":   "$currency",
-			"count": bson.M{"$sum": 1},
-			"total": bson.M{"$sum": "$estimated_value"},
-		}}},
-		{{Key: "$sort", Value: bson.M{"count": -1}}},
-		{{Key: "$limit", Value: 1}},
-	}
-
-	results, err := r.ormRepo.Aggregate(ctx, pipeline)
-	if err != nil {
-		return defaultValueCurrency, 0, fmt.Errorf("dominant currency value: %w", err)
-	}
-
-	if len(results) == 0 {
-		return defaultValueCurrency, 0, nil
 	}
 
 	currency := defaultValueCurrency
-	if c, ok := results[0]["_id"].(string); ok && c != "" {
+	if c, ok := row["_id"].(string); ok && c != "" {
 		currency = strings.ToUpper(c)
 	}
-
+	return currency, toFloat64(row["total"])
-	return currency, toFloat64(results[0]["total"]), nil
 }
 
 func (r *repository) Trend(ctx context.Context, days int, metric string, startUnix int64) (map[string]int64, error) {

internal/dashboard/service.go

@@ -3,10 +3,13 @@ package dashboard
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 	"tm/pkg/logger"
+
+	"golang.org/x/sync/singleflight"
 )
 
 const (
@@ -16,6 +19,7 @@ const (
 	defaultCountriesLimit     = 6
 	defaultListLimit          = 5
 	maxListLimit              = 20
+	summaryCacheTTL           = 60 * time.Second
 	statisticsCacheTTL        = 60 * time.Second
 )
 
@@ -30,16 +34,20 @@ type Service interface {
 	Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error)
 }
 
-type statisticsCacheEntry struct {
+type cacheEntry[T any] struct {
 	expiresAt time.Time
-	value     *StatisticsReportResponse
+	value     T
 }
 
 type service struct {
-	repo         Repository
+	repo            Repository
-	logger       logger.Logger
+	logger          logger.Logger
-	statisticsMu sync.Mutex
+	summaryMu       sync.Mutex
-	statistics   map[int]statisticsCacheEntry
+	summary         map[int64]cacheEntry[*SummaryResponse]
+	summaryGroup    singleflight.Group
+	statisticsMu    sync.Mutex
+	statistics      map[int]cacheEntry[*StatisticsReportResponse]
+	statisticsGroup singleflight.Group
 }
 
 // NewService creates a dashboard service.
@@ -47,7 +55,8 @@ func NewService(repo Repository, log logger.Logger) Service {
 	return &service{
 		repo:       repo,
 		logger:     log,
-		statistics: make(map[int]statisticsCacheEntry),
+		summary:    make(map[int64]cacheEntry[*SummaryResponse]),
+		statistics: make(map[int]cacheEntry[*StatisticsReportResponse]),
 	}
 }
 
@@ -55,19 +64,36 @@ func (s *service) Summary(ctx context.Context, query SummaryQuery) (*SummaryResp
 	windowHours := closingWindowHours(query.ClosingWindow)
 	windowSec := int64(windowHours) * 3600
 
-	s.logger.Info("Fetching dashboard summary", map[string]interface{}{
+	if cached, ok := s.cachedSummary(windowSec); ok {
-		"closing_window_hours": windowHours,
+		return cached, nil
-	})
-
-	out, err := s.repo.Summary(ctx, windowSec)
-	if err != nil {
-		s.logger.Error("Failed to fetch dashboard summary", map[string]interface{}{
-			"error": err.Error(),
-		})
-		return nil, fmt.Errorf("dashboard summary: %w", err)
 	}
 
-	return out, nil
+	cacheKey := strconv.FormatInt(windowSec, 10)
+	out, err, _ := s.summaryGroup.Do(cacheKey, func() (interface{}, error) {
+		if cached, ok := s.cachedSummary(windowSec); ok {
+			return cached, nil
+		}
+
+		s.logger.Info("Fetching dashboard summary", map[string]interface{}{
+			"closing_window_hours": windowHours,
+		})
+
+		result, err := s.repo.Summary(ctx, windowSec)
+		if err != nil {
+			s.logger.Error("Failed to fetch dashboard summary", map[string]interface{}{
+				"error": err.Error(),
+			})
+			return nil, fmt.Errorf("dashboard summary: %w", err)
+		}
+
+		s.storeSummary(windowSec, result)
+		return result, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return out.(*SummaryResponse), nil
 }
 
 func (s *service) Trend(ctx context.Context, query TrendQuery) (*TrendResponse, error) {
@@ -177,22 +203,61 @@ func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*Stati
 		return cached, nil
 	}
 
-	startUnix := trendStartUnix(days)
+	cacheKey := strconv.Itoa(days)
+	out, err, _ := s.statisticsGroup.Do(cacheKey, func() (interface{}, error) {
+		if cached, ok := s.cachedStatistics(days); ok {
+			return cached, nil
+		}
 
-	s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
+		startUnix := trendStartUnix(days)
-		"days": days,
-	})
 
-	out, err := s.repo.Statistics(ctx, days, startUnix)
+		s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
-	if err != nil {
+			"days": days,
-		s.logger.Error("Failed to fetch dashboard statistics report", map[string]interface{}{
-			"error": err.Error(),
 		})
-		return nil, fmt.Errorf("dashboard statistics: %w", err)
+
+		result, err := s.repo.Statistics(ctx, days, startUnix)
+		if err != nil {
+			s.logger.Error("Failed to fetch dashboard statistics report", map[string]interface{}{
+				"error": err.Error(),
+			})
+			return nil, fmt.Errorf("dashboard statistics: %w", err)
+		}
+
+		s.storeStatistics(days, result)
+		return result, nil
+	})
+	if err != nil {
+		return nil, err
 	}
 
-	s.storeStatistics(days, out)
+	return out.(*StatisticsReportResponse), nil
-	return out, nil
+}
+
+func (s *service) cachedSummary(windowSec int64) (*SummaryResponse, bool) {
+	now := time.Now()
+
+	s.summaryMu.Lock()
+	defer s.summaryMu.Unlock()
+
+	entry, ok := s.summary[windowSec]
+	if !ok || now.After(entry.expiresAt) {
+		if ok {
+			delete(s.summary, windowSec)
+		}
+		return nil, false
+	}
+
+	return entry.value, true
+}
+
+func (s *service) storeSummary(windowSec int64, value *SummaryResponse) {
+	s.summaryMu.Lock()
+	defer s.summaryMu.Unlock()
+
+	s.summary[windowSec] = cacheEntry[*SummaryResponse]{
+		expiresAt: time.Now().Add(summaryCacheTTL),
+		value:     value,
+	}
 }
 
 func (s *service) cachedStatistics(days int) (*StatisticsReportResponse, bool) {
@@ -216,7 +281,7 @@ func (s *service) storeStatistics(days int, value *StatisticsReportResponse) {
 	s.statisticsMu.Lock()
 	defer s.statisticsMu.Unlock()
 
-	s.statistics[days] = statisticsCacheEntry{
+	s.statistics[days] = cacheEntry[*StatisticsReportResponse]{
 		expiresAt: time.Now().Add(statisticsCacheTTL),
 		value:     value,
 	}

internal/dashboard/statistics_repository.go

@@ -16,6 +16,10 @@ import (
 
 const noticesCollectionName = "notices"
 
+// scrapedScopeCacheTTL avoids listing every MinIO object on each statistics request while
+// keeping counts aligned with the tender panel (MinIO is the source of truth for scraped docs).
+const scrapedScopeCacheTTL = 5 * time.Minute
+
 // scrapedTendersScope is the resolved Mongo filter for tenders with scraped documents.
 // zero is set when MinIO reports no procedure folders; callers must return zero without querying.
 type scrapedTendersScope struct {
@@ -28,7 +32,8 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
 	endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
 	startDay := endDay.AddDate(0, 0, -(days - 1))
 
-	scrapedScope, err := r.resolveScrapedTendersScope(ctx)
+	// Resolve scraped-document scope from MinIO (cached); falls back to Mongo flags on error.
+	scrapedScope, err := r.cachedScrapedTendersScope(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -187,6 +192,58 @@ func (r *repository) totalScrapedTenders(ctx context.Context, scope scrapedTende
 	return r.ormRepo.Count(ctx, scope.match)
 }
 
+func mongoScrapedTendersScope() scrapedTendersScope {
+	return scrapedTendersScope{
+		match: bson.M{
+			"processing_metadata.documents_scraped": true,
+			"scraped_documents":                   bson.M{"$exists": true, "$ne": bson.A{}},
+		},
+	}
+}
+
+func (r *repository) cachedScrapedTendersScope(ctx context.Context) (scrapedTendersScope, error) {
+	now := time.Now()
+
+	r.scrapedScopeMu.Lock()
+	if r.scrapedScopeCached && now.Before(r.scrapedScopeExpiry) {
+		scope := r.scrapedScope
+		r.scrapedScopeMu.Unlock()
+		return scope, nil
+	}
+	r.scrapedScopeMu.Unlock()
+
+	v, err, _ := r.scrapedScopeGroup.Do("scraped-scope", func() (interface{}, error) {
+		r.scrapedScopeMu.Lock()
+		if r.scrapedScopeCached && time.Now().Before(r.scrapedScopeExpiry) {
+			scope := r.scrapedScope
+			r.scrapedScopeMu.Unlock()
+			return scope, nil
+		}
+		r.scrapedScopeMu.Unlock()
+
+		scope, resolveErr := r.resolveScrapedTendersScope(ctx)
+		if resolveErr != nil {
+			r.logger.Warn("MinIO scraped scope unavailable, using Mongo fallback for dashboard statistics", map[string]interface{}{
+				"error": resolveErr.Error(),
+			})
+			return mongoScrapedTendersScope(), nil
+		}
+
+		r.scrapedScopeMu.Lock()
+		r.scrapedScope = scope
+		r.scrapedScopeExpiry = time.Now().Add(scrapedScopeCacheTTL)
+		r.scrapedScopeCached = true
+		r.scrapedScopeMu.Unlock()
+
+		return scope, nil
+	})
+	if err != nil {
+		return scrapedTendersScope{}, err
+	}
+
+	return v.(scrapedTendersScope), nil
+}
+
 func (r *repository) resolveScrapedTendersScope(ctx context.Context) (scrapedTendersScope, error) {
 	folderIDs, err := r.scrapedDocumentsFolderIDs(ctx)
 	if err != nil {

internal/dashboard/statistics_repository_test.go

@@ -129,3 +129,13 @@ func TestResolveScrapedTendersScopeMongoFallbackWithoutLister(t *testing.T) {
 		t.Fatalf("expected mongo fallback filter, got %#v", scope.match)
 	}
 }
+
+func TestMongoScrapedTendersScopeMatchesStatisticsFilter(t *testing.T) {
+	scope := mongoScrapedTendersScope()
+	if scope.zero {
+		t.Fatal("expected non-zero mongo statistics scope")
+	}
+	if scope.match["processing_metadata.documents_scraped"] != true {
+		t.Fatalf("expected documents_scraped filter, got %#v", scope.match)
+	}
+}

internal/tender/entity.go

@@ -305,20 +305,74 @@ func (t *Tender) GetMainOrganization() *Organization {
 
 // HasEstimatedValue returns true if the tender has an estimated value
 func (t *Tender) HasEstimatedValue() bool {
-	return t.EstimatedValue > 0
+	value, _ := t.ResolvedEstimatedValueAndCurrency()
+	return value > 0
+}
+
+// AggregateProcurementLotEstimatedValue sums lot-level estimated values when all non-empty
+// lot currencies match. Conflicting currencies are not aggregated.
+func AggregateProcurementLotEstimatedValue(lots []ProcurementLot) (value float64, currency string) {
+	var refCurrency string
+	for _, lot := range lots {
+		if lot.EstimatedValue <= 0 {
+			continue
+		}
+		c := strings.TrimSpace(lot.Currency)
+		if c == "" {
+			continue
+		}
+		if refCurrency == "" {
+			refCurrency = c
+			continue
+		}
+		if c != refCurrency {
+			return 0, ""
+		}
+	}
+
+	for _, lot := range lots {
+		if lot.EstimatedValue <= 0 {
+			continue
+		}
+		value += lot.EstimatedValue
+	}
+	if value <= 0 {
+		return 0, ""
+	}
+	return value, refCurrency
+}
+
+// ResolvedEstimatedValueAndCurrency returns the tender-level estimated value when set,
+// otherwise aggregates procurement lot values for API and display use.
+func (t *Tender) ResolvedEstimatedValueAndCurrency() (float64, string) {
+	if t == nil {
+		return 0, ""
+	}
+	if t.EstimatedValue > 0 {
+		return t.EstimatedValue, strings.TrimSpace(t.Currency)
+	}
+	value, currency := AggregateProcurementLotEstimatedValue(t.ProcurementLots)
+	if value > 0 {
+		if currency == "" {
+			currency = strings.TrimSpace(t.Currency)
+		}
+		return value, currency
+	}
+	return 0, strings.TrimSpace(t.Currency)
 }
 
 // GetFormattedEstimatedValue returns formatted estimated value with currency
 func (t *Tender) GetFormattedEstimatedValue() string {
-	if !t.HasEstimatedValue() {
+	value, currency := t.ResolvedEstimatedValueAndCurrency()
+	if value <= 0 {
 		return ""
 	}
 
-	if t.Currency != "" {
+	if currency != "" {
-		return fmt.Sprintf("%.2f %s", t.EstimatedValue, t.Currency)
+		return fmt.Sprintf("%.2f %s", value, currency)
 	}
 
-	return fmt.Sprintf("%.2f", t.EstimatedValue)
+	return fmt.Sprintf("%.2f", value)
 }
 
 // UpdateStatus updates the tender status and updated timestamp

internal/tender/entity_test.go

@@ -0,0 +1,74 @@
+package tender
+
+import "testing"
+
+func TestAggregateProcurementLotEstimatedValue(t *testing.T) {
+	value, currency := AggregateProcurementLotEstimatedValue([]ProcurementLot{
+		{EstimatedValue: 100000, Currency: "EUR"},
+		{EstimatedValue: 250000, Currency: "EUR"},
+	})
+	if value != 350000 {
+		t.Fatalf("expected total 350000, got %v", value)
+	}
+	if currency != "EUR" {
+		t.Fatalf("expected currency EUR, got %q", currency)
+	}
+}
+
+func TestAggregateProcurementLotEstimatedValueRejectsMixedCurrencies(t *testing.T) {
+	value, currency := AggregateProcurementLotEstimatedValue([]ProcurementLot{
+		{EstimatedValue: 100000, Currency: "EUR"},
+		{EstimatedValue: 250000, Currency: "USD"},
+	})
+	if value != 0 || currency != "" {
+		t.Fatalf("expected no aggregation for mixed currencies, got value=%v currency=%q", value, currency)
+	}
+}
+
+func TestAggregateProcurementLotEstimatedValueAllowsEmptyLotCurrency(t *testing.T) {
+	value, currency := AggregateProcurementLotEstimatedValue([]ProcurementLot{
+		{EstimatedValue: 100000, Currency: "EUR"},
+		{EstimatedValue: 250000},
+	})
+	if value != 350000 {
+		t.Fatalf("expected total 350000, got %v", value)
+	}
+	if currency != "EUR" {
+		t.Fatalf("expected currency EUR, got %q", currency)
+	}
+}
+
+func TestResolvedEstimatedValueAndCurrencyUsesLotsWhenTopLevelMissing(t *testing.T) {
+	tender := &Tender{
+		Currency: "EUR",
+		ProcurementLots: []ProcurementLot{
+			{LotID: "LOT-0001", EstimatedValue: 500000, Currency: "EUR"},
+		},
+	}
+
+	value, currency := tender.ResolvedEstimatedValueAndCurrency()
+	if value != 500000 {
+		t.Fatalf("expected 500000, got %v", value)
+	}
+	if currency != "EUR" {
+		t.Fatalf("expected EUR, got %q", currency)
+	}
+}
+
+func TestResolvedEstimatedValueAndCurrencyPrefersTopLevel(t *testing.T) {
+	tender := &Tender{
+		EstimatedValue: 900000,
+		Currency:       "USD",
+		ProcurementLots: []ProcurementLot{
+			{EstimatedValue: 100000, Currency: "EUR"},
+		},
+	}
+
+	value, currency := tender.ResolvedEstimatedValueAndCurrency()
+	if value != 900000 {
+		t.Fatalf("expected 900000, got %v", value)
+	}
+	if currency != "USD" {
+		t.Fatalf("expected USD, got %q", currency)
+	}
+}

internal/tender/form.go

@@ -277,6 +277,8 @@ func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse {
 		documentURL = strings.TrimSpace(t.TenderURL)
 	}
 
+	estimatedValue, currency := t.ResolvedEstimatedValueAndCurrency()
+
 	response := &TenderResponse{
 		ID:                          t.ID.Hex(),
 		NoticePublicationID:         t.NoticePublicationID,
@@ -307,8 +309,8 @@ func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse {
 		MainClassificationDescription: mainDesc,
 		MainClassificationDisplay:     classificationDisplay(mainCode, mainDesc),
 		AdditionalClassifications:     append([]string(nil), t.AdditionalClassifications...),
-		EstimatedValue:                t.EstimatedValue,
+		EstimatedValue:                estimatedValue,
-		Currency:                      strings.TrimSpace(t.Currency),
+		Currency:                      currency,
 		EstimatedValueVATBasis:        vatBasisUnspecified,
 
 		Duration:     t.Duration,

ted/aggregate.go

@@ -0,0 +1,41 @@
+package ted
+
+import "strings"
+
+type valuedAmount struct {
+	value    float64
+	currency string
+}
+
+// aggregateSameCurrencyAmounts sums values only when every non-empty currency matches.
+// Lots with an empty currency are included; conflicting currencies yield zero.
+func aggregateSameCurrencyAmounts(amounts []valuedAmount) (total float64, currency string) {
+	var refCurrency string
+	for _, item := range amounts {
+		if item.value <= 0 {
+			continue
+		}
+		c := strings.TrimSpace(item.currency)
+		if c == "" {
+			continue
+		}
+		if refCurrency == "" {
+			refCurrency = c
+			continue
+		}
+		if c != refCurrency {
+			return 0, ""
+		}
+	}
+
+	for _, item := range amounts {
+		if item.value <= 0 {
+			continue
+		}
+		total += item.value
+	}
+	if total <= 0 {
+		return 0, ""
+	}
+	return total, refCurrency
+}

ted/budget_test.go

@@ -0,0 +1,84 @@
+package ted
+
+import "testing"
+
+func TestCurrencyTextAmountAndCurrency(t *testing.T) {
+	t.Run("chardata", func(t *testing.T) {
+		amount, currency := (CurrencyText{CurrencyID: "EUR", Text: "1 234,56"}).AmountAndCurrency()
+		if amount != "1 234,56" {
+			t.Fatalf("unexpected amount %q", amount)
+		}
+		if currency != "EUR" {
+			t.Fatalf("unexpected currency %q", currency)
+		}
+	})
+
+	t.Run("value child", func(t *testing.T) {
+		amount, currency := (CurrencyText{CurrencyID: "EUR", Value: "500000"}).AmountAndCurrency()
+		if amount != "500000" {
+			t.Fatalf("unexpected amount %q", amount)
+		}
+		if currency != "EUR" {
+			t.Fatalf("unexpected currency %q", currency)
+		}
+	})
+}
+
+func TestContractNoticeGetBudgetSumsAllLots(t *testing.T) {
+	cn := ContractNotice{
+		ProcurementProjectLots: []ProcurementProjectLot{
+			{
+				ProcurementProject: &ProcurementProject{
+					RequestedTenderTotal: &RequestedTenderTotal{
+						EstimatedOverallContractAmount: CurrencyText{CurrencyID: "EUR", Text: "100000"},
+					},
+				},
+			},
+			{
+				ProcurementProject: &ProcurementProject{
+					RequestedTenderTotal: &RequestedTenderTotal{
+						EstimatedOverallContractAmount: CurrencyText{CurrencyID: "EUR", Text: "250000"},
+					},
+				},
+			},
+		},
+	}
+
+	amount, currency := cn.GetBudget()
+	value, ok := ParseEuropeanAmount(amount)
+	if !ok {
+		t.Fatalf("failed to parse budget amount %q", amount)
+	}
+	if value != 350000 {
+		t.Fatalf("expected 350000, got %v", value)
+	}
+	if currency != "EUR" {
+		t.Fatalf("expected EUR, got %q", currency)
+	}
+}
+
+func TestContractNoticeGetBudgetRejectsMixedCurrencies(t *testing.T) {
+	cn := ContractNotice{
+		ProcurementProjectLots: []ProcurementProjectLot{
+			{
+				ProcurementProject: &ProcurementProject{
+					RequestedTenderTotal: &RequestedTenderTotal{
+						EstimatedOverallContractAmount: CurrencyText{CurrencyID: "EUR", Text: "100000"},
+					},
+				},
+			},
+			{
+				ProcurementProject: &ProcurementProject{
+					RequestedTenderTotal: &RequestedTenderTotal{
+						EstimatedOverallContractAmount: CurrencyText{CurrencyID: "USD", Text: "250000"},
+					},
+				},
+			},
+		},
+	}
+
+	amount, currency := cn.GetBudget()
+	if amount != "" || currency != "" {
+		t.Fatalf("expected no budget for mixed currencies, got amount=%q currency=%q", amount, currency)
+	}
+}

ted/mapper.go

@@ -30,11 +30,11 @@ func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot
 		}
 	}
 	if pp.RequestedTenderTotal != nil {
-		a := pp.RequestedTenderTotal.EstimatedOverallContractAmount
+		amount, currencyID := pp.RequestedTenderTotal.AmountAndCurrency()
-		if v, ok := ParseEuropeanAmount(a.Text); ok {
+		if v, ok := ParseEuropeanAmount(amount); ok {
 			out.EstimatedValue = v
 		}
-		out.Currency = a.CurrencyID
+		out.Currency = currencyID
 	}
 	if pp.PlannedPeriod != nil && pp.PlannedPeriod.DurationMeasure != nil {
 		dm := pp.PlannedPeriod.DurationMeasure
@@ -167,6 +167,7 @@ func (s *TEDScraper) mapPriorInformationNoticeToTender(pin *PriorInformationNoti
 	for i := range pin.ProcurementProjectLot {
 		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&pin.ProcurementProjectLot[i]))
 	}
+	backfillNoticeEstimatedValueFromLots(t)
 
 	if duration, unit := pin.GetDuration(); duration != "" {
 		t.Duration = duration
@@ -351,6 +352,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 	for i := range cn.ProcurementProjectLots {
 		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&cn.ProcurementProjectLots[i]))
 	}
+	backfillNoticeEstimatedValueFromLots(t)
 
 	// Set duration information
 	if duration, unit := cn.GetDuration(); duration != "" {
@@ -506,6 +508,7 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
 	for i := range can.ProcurementProjectLot {
 		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&can.ProcurementProjectLot[i]))
 	}
+	backfillNoticeEstimatedValueFromLots(t)
 
 	// Set location information from main procurement project or first lot
 	var realizedAddr *Address
@@ -663,3 +666,27 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.
 
 	return org
 }
+
+func backfillNoticeEstimatedValueFromLots(t *notice.Notice) {
+	if t == nil || t.EstimatedValue > 0 {
+		return
+	}
+	amounts := make([]valuedAmount, 0, len(t.ProcurementLots))
+	for _, lot := range t.ProcurementLots {
+		if lot.EstimatedValue <= 0 {
+			continue
+		}
+		amounts = append(amounts, valuedAmount{
+			value:    lot.EstimatedValue,
+			currency: lot.Currency,
+		})
+	}
+	total, currency := aggregateSameCurrencyAmounts(amounts)
+	if total <= 0 {
+		return
+	}
+	t.EstimatedValue = total
+	if strings.TrimSpace(t.Currency) == "" {
+		t.Currency = currency
+	}
+}

ted/model.go

@@ -3,6 +3,7 @@ package ted
 import (
 	"encoding/xml"
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -337,15 +338,11 @@ func (pin PriorInformationNotice) GetTenderDeadline() string {
 
 // GetBudget returns the estimated contract amount and currency.
 func (pin PriorInformationNotice) GetBudget() (amount string, currency string) {
-	if pin.firstLot() != nil &&
+	if amount, currency = budgetFromProcurementLots(pin.ProcurementProjectLot); amount != "" {
-		pin.firstLot().ProcurementProject != nil &&
+		return amount, currency
-		pin.firstLot().ProcurementProject.RequestedTenderTotal != nil {
-		contractAmount := pin.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
-		return contractAmount.Text, contractAmount.CurrencyID
 	}
 	if pin.ProcurementProject != nil && pin.ProcurementProject.RequestedTenderTotal != nil {
-		a := pin.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
+		return pin.ProcurementProject.RequestedTenderTotal.AmountAndCurrency()
-		return a.Text, a.CurrencyID
 	}
 	return "", ""
 }
@@ -1551,12 +1548,34 @@ type ProcessJustification struct {
 type RequestedTenderTotal struct {
 	XMLName                        xml.Name     `xml:"RequestedTenderTotal"`
 	EstimatedOverallContractAmount CurrencyText `xml:"EstimatedOverallContractAmount"`
+	TotalAmount                    CurrencyText `xml:"TotalAmount,omitempty"`
+}
+
+// AmountAndCurrency returns the best-effort amount text and currency from RequestedTenderTotal.
+func (r *RequestedTenderTotal) AmountAndCurrency() (amount string, currency string) {
+	if r == nil {
+		return "", ""
+	}
+	if amount, currency = r.EstimatedOverallContractAmount.AmountAndCurrency(); amount != "" {
+		return amount, currency
+	}
+	return r.TotalAmount.AmountAndCurrency()
 }
 
 // CurrencyText represents text with currency attribute
 type CurrencyText struct {
 	CurrencyID string `xml:"currencyID,attr,omitempty"`
 	Text       string `xml:",chardata"`
+	Value      string `xml:"Value,omitempty"`
+}
+
+// AmountAndCurrency returns the amount text and currency id from a CurrencyText node.
+func (c CurrencyText) AmountAndCurrency() (amount string, currency string) {
+	amount = strings.TrimSpace(c.Text)
+	if amount == "" {
+		amount = strings.TrimSpace(c.Value)
+	}
+	return amount, strings.TrimSpace(c.CurrencyID)
 }
 
 // MainCommodityClassification contains main commodity classification
@@ -1972,21 +1991,42 @@ func (cn ContractNotice) GetTenderDeadline() string {
 }
 
 // GetBudget returns the estimated contract amount and currency.
-// Lot-level BT-27 is preferred; procedure-level BT-27 is used when lots omit RequestedTenderTotal.
+// Lot-level BT-27 values are summed when present; procedure-level BT-27 is used when lots omit RequestedTenderTotal.
 func (cn ContractNotice) GetBudget() (amount string, currency string) {
-	if cn.firstLot() != nil &&
+	if amount, currency = budgetFromProcurementLots(cn.ProcurementProjectLots); amount != "" {
-		cn.firstLot().ProcurementProject != nil &&
+		return amount, currency
-		cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
-		contractAmount := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
-		return contractAmount.Text, contractAmount.CurrencyID
 	}
 	if cn.ProcurementProject != nil && cn.ProcurementProject.RequestedTenderTotal != nil {
-		a := cn.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
+		return cn.ProcurementProject.RequestedTenderTotal.AmountAndCurrency()
-		return a.Text, a.CurrencyID
 	}
 	return "", ""
 }
 
+func budgetFromProcurementLots(lots []ProcurementProjectLot) (amount string, currency string) {
+	amounts := make([]valuedAmount, 0, len(lots))
+	for i := range lots {
+		pp := lots[i].ProcurementProject
+		if pp == nil || pp.RequestedTenderTotal == nil {
+			continue
+		}
+		lotAmount, lotCurrency := pp.RequestedTenderTotal.AmountAndCurrency()
+		if lotAmount == "" {
+			continue
+		}
+		value, ok := ParseEuropeanAmount(lotAmount)
+		if !ok || value <= 0 {
+			continue
+		}
+		amounts = append(amounts, valuedAmount{value: value, currency: lotCurrency})
+	}
+
+	total, currency := aggregateSameCurrencyAmounts(amounts)
+	if total <= 0 {
+		return "", ""
+	}
+	return strconv.FormatFloat(total, 'f', -1, 64), currency
+}
+
 // GetDuration returns the contract duration and unit
 func (cn ContractNotice) GetDuration() (duration string, unit string) {
 	if cn.firstLot() != nil &&
@@ -2223,7 +2263,7 @@ func (can ContractAwardNotice) GetMainClassificationDescription() string {
 func (can ContractAwardNotice) GetTotalAwardValue() (amount string, currency string) {
 	noticeResult := can.GetNoticeResult()
 	if noticeResult != nil && noticeResult.TotalAmount != nil {
-		return noticeResult.TotalAmount.Text, noticeResult.TotalAmount.CurrencyID
+		return noticeResult.TotalAmount.AmountAndCurrency()
 	}
 	return "", ""
 }