Merge pull request 'Enhance tender estimated value resolution and add unit tests' (#45) from est-value into develop

Reviewed-on: https://repo.ravanertebat.com/TM/tm_back/pulls/45

cmd/worker/workers/notice.go

@@ -476,6 +476,16 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 		awardedEntities = mergeAwardedEntitiesByLotID(t.AwardedEntities, awardedEntities)
 		title, description = mergeTitleAndDescriptionForMultiLot(t.Title, t.Description, title, description)
 
+		if estimatedVal == 0 && t.EstimatedValue > 0 {
+			estimatedVal = t.EstimatedValue
+		}
+		if awardedVal == 0 && t.AwardedValue > 0 {
+			awardedVal = t.AwardedValue
+		}
+		if currency == "" && t.Currency != "" {
+			currency = t.Currency
+		}
+
 		if strings.TrimSpace(previousNoticePublicationID) != "" && previousNoticePublicationID != n.NoticePublicationID {
 			if n.EstimatedValue > 0 {
 				estimatedVal = t.EstimatedValue + n.EstimatedValue
@@ -495,6 +505,15 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 		}
 	}
 
+	if estimatedVal == 0 {
+		if lotValue, lotCurrency := tender.AggregateProcurementLotEstimatedValue(procurementLots); lotValue > 0 {
+			estimatedVal = lotValue
+			if currency == "" {
+				currency = lotCurrency
+			}
+		}
+	}
+
 	// Title and description stay in the notice language; English translation is handled by the AI service (translation worker).
 	t.Title = title
 	t.Description = description

internal/tender/entity.go

@@ -305,20 +305,74 @@ func (t *Tender) GetMainOrganization() *Organization {
 
 // HasEstimatedValue returns true if the tender has an estimated value
 func (t *Tender) HasEstimatedValue() bool {
-	return t.EstimatedValue > 0
+	value, _ := t.ResolvedEstimatedValueAndCurrency()
+	return value > 0
+}
+
+// AggregateProcurementLotEstimatedValue sums lot-level estimated values when all non-empty
+// lot currencies match. Conflicting currencies are not aggregated.
+func AggregateProcurementLotEstimatedValue(lots []ProcurementLot) (value float64, currency string) {
+	var refCurrency string
+	for _, lot := range lots {
+		if lot.EstimatedValue <= 0 {
+			continue
+		}
+		c := strings.TrimSpace(lot.Currency)
+		if c == "" {
+			continue
+		}
+		if refCurrency == "" {
+			refCurrency = c
+			continue
+		}
+		if c != refCurrency {
+			return 0, ""
+		}
+	}
+
+	for _, lot := range lots {
+		if lot.EstimatedValue <= 0 {
+			continue
+		}
+		value += lot.EstimatedValue
+	}
+	if value <= 0 {
+		return 0, ""
+	}
+	return value, refCurrency
+}
+
+// ResolvedEstimatedValueAndCurrency returns the tender-level estimated value when set,
+// otherwise aggregates procurement lot values for API and display use.
+func (t *Tender) ResolvedEstimatedValueAndCurrency() (float64, string) {
+	if t == nil {
+		return 0, ""
+	}
+	if t.EstimatedValue > 0 {
+		return t.EstimatedValue, strings.TrimSpace(t.Currency)
+	}
+	value, currency := AggregateProcurementLotEstimatedValue(t.ProcurementLots)
+	if value > 0 {
+		if currency == "" {
+			currency = strings.TrimSpace(t.Currency)
+		}
+		return value, currency
+	}
+	return 0, strings.TrimSpace(t.Currency)
 }
 
 // GetFormattedEstimatedValue returns formatted estimated value with currency
 func (t *Tender) GetFormattedEstimatedValue() string {
-	if !t.HasEstimatedValue() {
+	value, currency := t.ResolvedEstimatedValueAndCurrency()
+	if value <= 0 {
 		return ""
 	}
 
-	if t.Currency != "" {
+	if currency != "" {
-		return fmt.Sprintf("%.2f %s", t.EstimatedValue, t.Currency)
+		return fmt.Sprintf("%.2f %s", value, currency)
 	}
 
-	return fmt.Sprintf("%.2f", t.EstimatedValue)
+	return fmt.Sprintf("%.2f", value)
 }
 
 // UpdateStatus updates the tender status and updated timestamp

internal/tender/entity_test.go

@@ -0,0 +1,74 @@
+package tender
+
+import "testing"
+
+func TestAggregateProcurementLotEstimatedValue(t *testing.T) {
+	value, currency := AggregateProcurementLotEstimatedValue([]ProcurementLot{
+		{EstimatedValue: 100000, Currency: "EUR"},
+		{EstimatedValue: 250000, Currency: "EUR"},
+	})
+	if value != 350000 {
+		t.Fatalf("expected total 350000, got %v", value)
+	}
+	if currency != "EUR" {
+		t.Fatalf("expected currency EUR, got %q", currency)
+	}
+}
+
+func TestAggregateProcurementLotEstimatedValueRejectsMixedCurrencies(t *testing.T) {
+	value, currency := AggregateProcurementLotEstimatedValue([]ProcurementLot{
+		{EstimatedValue: 100000, Currency: "EUR"},
+		{EstimatedValue: 250000, Currency: "USD"},
+	})
+	if value != 0 || currency != "" {
+		t.Fatalf("expected no aggregation for mixed currencies, got value=%v currency=%q", value, currency)
+	}
+}
+
+func TestAggregateProcurementLotEstimatedValueAllowsEmptyLotCurrency(t *testing.T) {
+	value, currency := AggregateProcurementLotEstimatedValue([]ProcurementLot{
+		{EstimatedValue: 100000, Currency: "EUR"},
+		{EstimatedValue: 250000},
+	})
+	if value != 350000 {
+		t.Fatalf("expected total 350000, got %v", value)
+	}
+	if currency != "EUR" {
+		t.Fatalf("expected currency EUR, got %q", currency)
+	}
+}
+
+func TestResolvedEstimatedValueAndCurrencyUsesLotsWhenTopLevelMissing(t *testing.T) {
+	tender := &Tender{
+		Currency: "EUR",
+		ProcurementLots: []ProcurementLot{
+			{LotID: "LOT-0001", EstimatedValue: 500000, Currency: "EUR"},
+		},
+	}
+
+	value, currency := tender.ResolvedEstimatedValueAndCurrency()
+	if value != 500000 {
+		t.Fatalf("expected 500000, got %v", value)
+	}
+	if currency != "EUR" {
+		t.Fatalf("expected EUR, got %q", currency)
+	}
+}
+
+func TestResolvedEstimatedValueAndCurrencyPrefersTopLevel(t *testing.T) {
+	tender := &Tender{
+		EstimatedValue: 900000,
+		Currency:       "USD",
+		ProcurementLots: []ProcurementLot{
+			{EstimatedValue: 100000, Currency: "EUR"},
+		},
+	}
+
+	value, currency := tender.ResolvedEstimatedValueAndCurrency()
+	if value != 900000 {
+		t.Fatalf("expected 900000, got %v", value)
+	}
+	if currency != "USD" {
+		t.Fatalf("expected USD, got %q", currency)
+	}
+}

internal/tender/form.go

@@ -277,6 +277,8 @@ func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse {
 		documentURL = strings.TrimSpace(t.TenderURL)
 	}
 
+	estimatedValue, currency := t.ResolvedEstimatedValueAndCurrency()
+
 	response := &TenderResponse{
 		ID:                          t.ID.Hex(),
 		NoticePublicationID:         t.NoticePublicationID,
@@ -307,8 +309,8 @@ func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse {
 		MainClassificationDescription: mainDesc,
 		MainClassificationDisplay:     classificationDisplay(mainCode, mainDesc),
 		AdditionalClassifications:     append([]string(nil), t.AdditionalClassifications...),
-		EstimatedValue:                t.EstimatedValue,
+		EstimatedValue:                estimatedValue,
-		Currency:                      strings.TrimSpace(t.Currency),
+		Currency:                      currency,
 		EstimatedValueVATBasis:        vatBasisUnspecified,
 
 		Duration:     t.Duration,

ted/aggregate.go

@@ -0,0 +1,41 @@
+package ted
+
+import "strings"
+
+type valuedAmount struct {
+	value    float64
+	currency string
+}
+
+// aggregateSameCurrencyAmounts sums values only when every non-empty currency matches.
+// Lots with an empty currency are included; conflicting currencies yield zero.
+func aggregateSameCurrencyAmounts(amounts []valuedAmount) (total float64, currency string) {
+	var refCurrency string
+	for _, item := range amounts {
+		if item.value <= 0 {
+			continue
+		}
+		c := strings.TrimSpace(item.currency)
+		if c == "" {
+			continue
+		}
+		if refCurrency == "" {
+			refCurrency = c
+			continue
+		}
+		if c != refCurrency {
+			return 0, ""
+		}
+	}
+
+	for _, item := range amounts {
+		if item.value <= 0 {
+			continue
+		}
+		total += item.value
+	}
+	if total <= 0 {
+		return 0, ""
+	}
+	return total, refCurrency
+}

ted/budget_test.go

@@ -0,0 +1,84 @@
+package ted
+
+import "testing"
+
+func TestCurrencyTextAmountAndCurrency(t *testing.T) {
+	t.Run("chardata", func(t *testing.T) {
+		amount, currency := (CurrencyText{CurrencyID: "EUR", Text: "1 234,56"}).AmountAndCurrency()
+		if amount != "1 234,56" {
+			t.Fatalf("unexpected amount %q", amount)
+		}
+		if currency != "EUR" {
+			t.Fatalf("unexpected currency %q", currency)
+		}
+	})
+
+	t.Run("value child", func(t *testing.T) {
+		amount, currency := (CurrencyText{CurrencyID: "EUR", Value: "500000"}).AmountAndCurrency()
+		if amount != "500000" {
+			t.Fatalf("unexpected amount %q", amount)
+		}
+		if currency != "EUR" {
+			t.Fatalf("unexpected currency %q", currency)
+		}
+	})
+}
+
+func TestContractNoticeGetBudgetSumsAllLots(t *testing.T) {
+	cn := ContractNotice{
+		ProcurementProjectLots: []ProcurementProjectLot{
+			{
+				ProcurementProject: &ProcurementProject{
+					RequestedTenderTotal: &RequestedTenderTotal{
+						EstimatedOverallContractAmount: CurrencyText{CurrencyID: "EUR", Text: "100000"},
+					},
+				},
+			},
+			{
+				ProcurementProject: &ProcurementProject{
+					RequestedTenderTotal: &RequestedTenderTotal{
+						EstimatedOverallContractAmount: CurrencyText{CurrencyID: "EUR", Text: "250000"},
+					},
+				},
+			},
+		},
+	}
+
+	amount, currency := cn.GetBudget()
+	value, ok := ParseEuropeanAmount(amount)
+	if !ok {
+		t.Fatalf("failed to parse budget amount %q", amount)
+	}
+	if value != 350000 {
+		t.Fatalf("expected 350000, got %v", value)
+	}
+	if currency != "EUR" {
+		t.Fatalf("expected EUR, got %q", currency)
+	}
+}
+
+func TestContractNoticeGetBudgetRejectsMixedCurrencies(t *testing.T) {
+	cn := ContractNotice{
+		ProcurementProjectLots: []ProcurementProjectLot{
+			{
+				ProcurementProject: &ProcurementProject{
+					RequestedTenderTotal: &RequestedTenderTotal{
+						EstimatedOverallContractAmount: CurrencyText{CurrencyID: "EUR", Text: "100000"},
+					},
+				},
+			},
+			{
+				ProcurementProject: &ProcurementProject{
+					RequestedTenderTotal: &RequestedTenderTotal{
+						EstimatedOverallContractAmount: CurrencyText{CurrencyID: "USD", Text: "250000"},
+					},
+				},
+			},
+		},
+	}
+
+	amount, currency := cn.GetBudget()
+	if amount != "" || currency != "" {
+		t.Fatalf("expected no budget for mixed currencies, got amount=%q currency=%q", amount, currency)
+	}
+}

ted/mapper.go

@@ -30,11 +30,11 @@ func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot
 		}
 	}
 	if pp.RequestedTenderTotal != nil {
-		a := pp.RequestedTenderTotal.EstimatedOverallContractAmount
+		amount, currencyID := pp.RequestedTenderTotal.AmountAndCurrency()
-		if v, ok := ParseEuropeanAmount(a.Text); ok {
+		if v, ok := ParseEuropeanAmount(amount); ok {
 			out.EstimatedValue = v
 		}
-		out.Currency = a.CurrencyID
+		out.Currency = currencyID
 	}
 	if pp.PlannedPeriod != nil && pp.PlannedPeriod.DurationMeasure != nil {
 		dm := pp.PlannedPeriod.DurationMeasure
@@ -167,6 +167,7 @@ func (s *TEDScraper) mapPriorInformationNoticeToTender(pin *PriorInformationNoti
 	for i := range pin.ProcurementProjectLot {
 		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&pin.ProcurementProjectLot[i]))
 	}
+	backfillNoticeEstimatedValueFromLots(t)
 
 	if duration, unit := pin.GetDuration(); duration != "" {
 		t.Duration = duration
@@ -351,6 +352,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 	for i := range cn.ProcurementProjectLots {
 		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&cn.ProcurementProjectLots[i]))
 	}
+	backfillNoticeEstimatedValueFromLots(t)
 
 	// Set duration information
 	if duration, unit := cn.GetDuration(); duration != "" {
@@ -506,6 +508,7 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
 	for i := range can.ProcurementProjectLot {
 		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&can.ProcurementProjectLot[i]))
 	}
+	backfillNoticeEstimatedValueFromLots(t)
 
 	// Set location information from main procurement project or first lot
 	var realizedAddr *Address
@@ -663,3 +666,27 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.
 
 	return org
 }
+
+func backfillNoticeEstimatedValueFromLots(t *notice.Notice) {
+	if t == nil || t.EstimatedValue > 0 {
+		return
+	}
+	amounts := make([]valuedAmount, 0, len(t.ProcurementLots))
+	for _, lot := range t.ProcurementLots {
+		if lot.EstimatedValue <= 0 {
+			continue
+		}
+		amounts = append(amounts, valuedAmount{
+			value:    lot.EstimatedValue,
+			currency: lot.Currency,
+		})
+	}
+	total, currency := aggregateSameCurrencyAmounts(amounts)
+	if total <= 0 {
+		return
+	}
+	t.EstimatedValue = total
+	if strings.TrimSpace(t.Currency) == "" {
+		t.Currency = currency
+	}
+}

ted/model.go

@@ -3,6 +3,7 @@ package ted
 import (
 	"encoding/xml"
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 )
@@ -337,15 +338,11 @@ func (pin PriorInformationNotice) GetTenderDeadline() string {
 
 // GetBudget returns the estimated contract amount and currency.
 func (pin PriorInformationNotice) GetBudget() (amount string, currency string) {
-	if pin.firstLot() != nil &&
+	if amount, currency = budgetFromProcurementLots(pin.ProcurementProjectLot); amount != "" {
-		pin.firstLot().ProcurementProject != nil &&
+		return amount, currency
-		pin.firstLot().ProcurementProject.RequestedTenderTotal != nil {
-		contractAmount := pin.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
-		return contractAmount.Text, contractAmount.CurrencyID
 	}
 	if pin.ProcurementProject != nil && pin.ProcurementProject.RequestedTenderTotal != nil {
-		a := pin.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
+		return pin.ProcurementProject.RequestedTenderTotal.AmountAndCurrency()
-		return a.Text, a.CurrencyID
 	}
 	return "", ""
 }
@@ -1551,12 +1548,34 @@ type ProcessJustification struct {
 type RequestedTenderTotal struct {
 	XMLName                        xml.Name     `xml:"RequestedTenderTotal"`
 	EstimatedOverallContractAmount CurrencyText `xml:"EstimatedOverallContractAmount"`
+	TotalAmount                    CurrencyText `xml:"TotalAmount,omitempty"`
+}
+
+// AmountAndCurrency returns the best-effort amount text and currency from RequestedTenderTotal.
+func (r *RequestedTenderTotal) AmountAndCurrency() (amount string, currency string) {
+	if r == nil {
+		return "", ""
+	}
+	if amount, currency = r.EstimatedOverallContractAmount.AmountAndCurrency(); amount != "" {
+		return amount, currency
+	}
+	return r.TotalAmount.AmountAndCurrency()
 }
 
 // CurrencyText represents text with currency attribute
 type CurrencyText struct {
 	CurrencyID string `xml:"currencyID,attr,omitempty"`
 	Text       string `xml:",chardata"`
+	Value      string `xml:"Value,omitempty"`
+}
+
+// AmountAndCurrency returns the amount text and currency id from a CurrencyText node.
+func (c CurrencyText) AmountAndCurrency() (amount string, currency string) {
+	amount = strings.TrimSpace(c.Text)
+	if amount == "" {
+		amount = strings.TrimSpace(c.Value)
+	}
+	return amount, strings.TrimSpace(c.CurrencyID)
 }
 
 // MainCommodityClassification contains main commodity classification
@@ -1972,21 +1991,42 @@ func (cn ContractNotice) GetTenderDeadline() string {
 }
 
 // GetBudget returns the estimated contract amount and currency.
-// Lot-level BT-27 is preferred; procedure-level BT-27 is used when lots omit RequestedTenderTotal.
+// Lot-level BT-27 values are summed when present; procedure-level BT-27 is used when lots omit RequestedTenderTotal.
 func (cn ContractNotice) GetBudget() (amount string, currency string) {
-	if cn.firstLot() != nil &&
+	if amount, currency = budgetFromProcurementLots(cn.ProcurementProjectLots); amount != "" {
-		cn.firstLot().ProcurementProject != nil &&
+		return amount, currency
-		cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
-		contractAmount := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
-		return contractAmount.Text, contractAmount.CurrencyID
 	}
 	if cn.ProcurementProject != nil && cn.ProcurementProject.RequestedTenderTotal != nil {
-		a := cn.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
+		return cn.ProcurementProject.RequestedTenderTotal.AmountAndCurrency()
-		return a.Text, a.CurrencyID
 	}
 	return "", ""
 }
 
+func budgetFromProcurementLots(lots []ProcurementProjectLot) (amount string, currency string) {
+	amounts := make([]valuedAmount, 0, len(lots))
+	for i := range lots {
+		pp := lots[i].ProcurementProject
+		if pp == nil || pp.RequestedTenderTotal == nil {
+			continue
+		}
+		lotAmount, lotCurrency := pp.RequestedTenderTotal.AmountAndCurrency()
+		if lotAmount == "" {
+			continue
+		}
+		value, ok := ParseEuropeanAmount(lotAmount)
+		if !ok || value <= 0 {
+			continue
+		}
+		amounts = append(amounts, valuedAmount{value: value, currency: lotCurrency})
+	}
+
+	total, currency := aggregateSameCurrencyAmounts(amounts)
+	if total <= 0 {
+		return "", ""
+	}
+	return strconv.FormatFloat(total, 'f', -1, 64), currency
+}
+
 // GetDuration returns the contract duration and unit
 func (cn ContractNotice) GetDuration() (duration string, unit string) {
 	if cn.firstLot() != nil &&
@@ -2223,7 +2263,7 @@ func (can ContractAwardNotice) GetMainClassificationDescription() string {
 func (can ContractAwardNotice) GetTotalAwardValue() (amount string, currency string) {
 	noticeResult := can.GetNoticeResult()
 	if noticeResult != nil && noticeResult.TotalAmount != nil {
-		return noticeResult.TotalAmount.Text, noticeResult.TotalAmount.CurrencyID
+		return noticeResult.TotalAmount.AmountAndCurrency()
 	}
 	return "", ""
 }