Added missing fields - scraper and worker fixes

ted/mapper.go

@@ -1,12 +1,63 @@
 package ted
 
 import (
-	"strconv"
 	"strings"
 	"time"
 	"tm/internal/notice"
 )
 
+func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot {
+	if lot == nil {
+		return notice.ProcurementLot{}
+	}
+	out := notice.ProcurementLot{LotID: lot.ID}
+	if lot.ProcurementProject == nil {
+		return out
+	}
+	pp := lot.ProcurementProject
+	out.Title = pp.Name
+	out.Description = pp.Description
+	out.MainNatureOfContract = pp.ProcurementTypeCode
+	if pp.MainCommodityClassification != nil {
+		out.MainClassification = strings.TrimSpace(pp.MainCommodityClassification.ItemClassificationCode)
+	}
+	for _, ac := range pp.AdditionalCommodityClassification {
+		c := strings.TrimSpace(ac.ItemClassificationCode)
+		if c != "" {
+			out.AdditionalClassifications = append(out.AdditionalClassifications, c)
+		}
+	}
+	if pp.RequestedTenderTotal != nil {
+		a := pp.RequestedTenderTotal.EstimatedOverallContractAmount
+		if v, ok := ParseEuropeanAmount(a.Text); ok {
+			out.EstimatedValue = v
+		}
+		out.Currency = a.CurrencyID
+	}
+	if pp.PlannedPeriod != nil && pp.PlannedPeriod.DurationMeasure != nil {
+		dm := pp.PlannedPeriod.DurationMeasure
+		txt := strings.TrimSpace(dm.Text)
+		unit := strings.TrimSpace(dm.UnitCode)
+		switch unit {
+		case "MONTH":
+			out.Duration = txt + " Months"
+		case "DAY":
+			out.Duration = txt + " Days"
+		case "WEEK":
+			out.Duration = txt + " Weeks"
+		case "YEAR":
+			out.Duration = txt + " Years"
+		default:
+			if unit != "" && txt != "" {
+				out.Duration = txt + " " + unit
+			} else {
+				out.Duration = txt
+			}
+		}
+	}
+	return out
+}
+
 // mapToTenderFromParsedDoc maps a parsed TED document to tender entity
 func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice {
 	switch parsedDoc.Type {
@@ -89,8 +140,13 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 	t := &notice.Notice{
 		ContractNoticeID:    cn.ID,
 		ContractFolderID:    cn.ContractFolderID,
-		NoticeTypeCode:      cn.GetNoticeSubType(),
+		NoticeTypeCode:      strings.TrimSpace(cn.GetNoticeType()),
+		FormType:            cn.GetNoticeFormType(),
 		NoticeSubTypeCode:   cn.GetNoticeSubType(),
+		NoticeIdentifier:    cn.ID,
+		NoticeVersion:       cn.VersionID,
+		NoticeDispatchAt:    ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime),
+		ESenderDispatchAt:   TransmissionDispatchUnix(cn.Extensions),
 		NoticeLanguageCode:  cn.NoticeLanguageCode,
 		IssueDate:           ParseDateToUnix(cn.IssueDate),
 		IssueTime:           ParseTimeToUnix(cn.IssueTime),
@@ -131,12 +187,20 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 
 	// Set estimated value and currency
 	if amount, currency := cn.GetBudget(); amount != "" {
-		if value, err := strconv.ParseFloat(amount, 64); err == nil {
+		if value, ok := ParseEuropeanAmount(amount); ok {
 			t.EstimatedValue = value
 			t.Currency = currency
 		}
 	}
 
+	if cn.ContractingParty != nil {
+		t.BuyerProfileURL = cn.ContractingParty.BuyerProfileURI
+	}
+
+	for i := range cn.ProcurementProjectLots {
+		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&cn.ProcurementProjectLots[i]))
+	}
+
 	// Set duration information
 	if duration, unit := cn.GetDuration(); duration != "" {
 		t.Duration = duration
@@ -159,8 +223,8 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 	}
 
 	// Set SubmissionURL from TenderingTerms
-	if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil {
-		if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
+	if lot := cn.firstLot(); lot != nil && lot.TenderingTerms != nil {
+		if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
 			if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
 				if strings.HasPrefix(endpointID, "http") {
 					t.SubmissionURL = endpointID
@@ -235,8 +299,13 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
 	t := &notice.Notice{
 		ContractNoticeID:    can.ID,
 		ContractFolderID:    can.ContractFolderID,
-		NoticeTypeCode:      can.NoticeTypeCode,
+		NoticeTypeCode:      strings.TrimSpace(can.GetNoticeType()),
+		FormType:            can.GetNoticeFormType(),
 		NoticeSubTypeCode:   can.GetNoticeSubType(),
+		NoticeIdentifier:    can.ID,
+		NoticeVersion:       can.VersionID,
+		NoticeDispatchAt:    ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime),
+		ESenderDispatchAt:   TransmissionDispatchUnix(can.Extensions),
 		NoticeLanguageCode:  can.NoticeLanguageCode,
 		IssueDate:           ParseDateToUnix(can.IssueDate),
 		IssueTime:           ParseTimeToUnix(can.IssueTime),
@@ -266,21 +335,75 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
 
 	// Set total award value
 	if amount, currency := can.GetTotalAwardValue(); amount != "" {
-		if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
+		if parsedAmount, ok := ParseEuropeanAmount(amount); ok {
 			t.EstimatedValue = parsedAmount
+			t.AwardedValue = parsedAmount
 		}
 		t.Currency = currency
 	}
 
-	// Set location information from main procurement project
+	if ad := can.GetAwardDate(); ad != "" {
+		t.AwardDate = ParseDateToUnix(ad)
+	}
+
+	if can.ContractingParty != nil {
+		t.BuyerProfileURL = can.ContractingParty.BuyerProfileURI
+	}
+
+	for i := range can.ProcurementProjectLot {
+		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&can.ProcurementProjectLot[i]))
+	}
+
+	// Set location information from main procurement project or first lot
+	var realizedAddr *Address
 	if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
-		if addr := can.ProcurementProject.RealizedLocation.Address; addr != nil {
-			t.CityName = addr.CityName
-			t.PostalCode = addr.PostalZone
-			t.RegionCode = addr.CountrySubentityCode
-			if addr.Country != nil {
-				t.CountryCode = addr.Country.IdentificationCode
+		realizedAddr = can.ProcurementProject.RealizedLocation.Address
+	} else if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil &&
+		lot.ProcurementProject.RealizedLocation != nil {
+		realizedAddr = lot.ProcurementProject.RealizedLocation.Address
+	}
+	if realizedAddr != nil {
+		t.CityName = realizedAddr.CityName
+		t.PostalCode = realizedAddr.PostalZone
+		t.RegionCode = realizedAddr.CountrySubentityCode
+		t.PlaceOfPerformance = realizedAddr.Region
+		if realizedAddr.Country != nil {
+			t.CountryCode = realizedAddr.Country.IdentificationCode
+		}
+	}
+
+	if wOrg := can.GetWinningTenderOrganization(); wOrg != nil {
+		t.WinningTenderer = s.mapOrganization(wOrg, "winner")
+	}
+
+	if nr := can.GetNoticeResult(); nr != nil {
+		for _, lr := range nr.LotResult {
+			winner := pickWinningLotTender(lr.LotTender)
+			if winner == nil || winner.TenderingParty == nil {
+				continue
 			}
+			org := can.GetOrganizationByID(winner.TenderingParty.ID)
+			aw := notice.Awarded{
+				LotID:          lr.ID,
+				OrganizationID: winner.TenderingParty.ID,
+				AwardDate:      t.AwardDate,
+			}
+			if org != nil && org.Company != nil {
+				if org.Company.PartyName != nil {
+					aw.Name = org.Company.PartyName.Name
+				}
+				if org.Company.PartyLegalEntity != nil {
+					aw.CompanyID = org.Company.PartyLegalEntity.CompanyID
+				}
+			}
+			if winner.LegalMonetaryTotal != nil {
+				pa := winner.LegalMonetaryTotal.PayableAmount
+				if v, ok := ParseEuropeanAmount(pa.Text); ok {
+					aw.Amount = v
+				}
+				aw.Currency = pa.CurrencyID
+			}
+			t.AwardedEntities = append(t.AwardedEntities, aw)
 		}
 	}
 
@@ -348,6 +471,7 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.
 			PostalZone:           company.PostalAddress.PostalZone,
 			CountrySubentityCode: company.PostalAddress.CountrySubentityCode,
 			Department:           company.PostalAddress.Department,
+			Region:               strings.TrimSpace(company.PostalAddress.CountrySubentity),
 		}
 		if company.PostalAddress.Country != nil {
 			org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode