Refactor TED Repository and Introduce TED Mapping Functionality

- Updated the tender repository to create a unique index on tender_id, enhancing database query performance. - Introduced new TED mapping functions to convert parsed TED documents into tender entities, improving integration with the tender management system. - Added error handling for contract notice processing in the TED scraper, ensuring robust logging and error management. - Removed deprecated eform structures to streamline the codebase and focus on essential components.
2025-10-04 12:34:37 +03:30
parent 366f7290a7
commit 0dd877a46d
32 changed files with 2386 additions and 1817 deletions
@@ -0,0 +1,367 @@
+package ted
+
+import (
+	"strconv"
+	"strings"
+	"time"
+	"tm/internal/tender"
+)
+
+// mapToTenderFromParsedDoc maps a parsed TED document to tender entity
+func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *tender.Tender {
+	switch parsedDoc.Type {
+	case DocumentTypeContractNotice:
+		if parsedDoc.ContractNotice == nil {
+			s.logger.Error("ContractNotice is nil", map[string]interface{}{
+				"document_type": string(parsedDoc.Type),
+			})
+			return nil
+		}
+		return s.mapToTender(parsedDoc.ContractNotice, sourceFileURL, sourceFileName)
+
+	case DocumentTypeContractAwardNotice:
+		if parsedDoc.ContractAwardNotice == nil {
+			s.logger.Error("ContractAwardNotice is nil", map[string]interface{}{
+				"document_type": string(parsedDoc.Type),
+			})
+			return nil
+		}
+		return s.mapContractAwardNoticeToTender(parsedDoc.ContractAwardNotice, sourceFileURL, sourceFileName)
+
+	case DocumentTypePriorInformationNotice:
+		if parsedDoc.PriorInformationNotice == nil {
+			s.logger.Error("PriorInformationNotice is nil", map[string]interface{}{
+				"document_type": string(parsedDoc.Type),
+			})
+			return nil
+		}
+		return s.mapGenericNoticeToTender(parsedDoc.PriorInformationNotice, sourceFileURL, sourceFileName)
+	default:
+		s.logger.Error("Unknown document type for mapping", map[string]interface{}{
+			"document_type": string(parsedDoc.Type),
+		})
+		return nil
+	}
+}
+
+// mapGenericNoticeToTender maps any TED document type to tender entity using the common interface
+func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *tender.Tender {
+	if doc == nil {
+		s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{})
+		return nil
+	}
+
+	now := time.Now().Unix()
+
+	t := &tender.Tender{
+		ContractNoticeID:    doc.GetID(),
+		ContractFolderID:    "", // Not available in generic interface
+		NoticeTypeCode:      doc.GetNoticeType(),
+		NoticeSubTypeCode:   "", // Not available in generic interface
+		NoticeLanguageCode:  doc.GetLanguage(),
+		IssueDate:           now, // TODO: Parse actual date from document
+		IssueTime:           now, // TODO: Parse actual time from document
+		GazetteID:           "",  // Not available in generic interface
+		NoticePublicationID: "",  // Not available in generic interface
+		PublicationDate:     now, // TODO: Parse actual publication date
+
+		// Basic tender information
+		Title:          "Generic Notice " + doc.GetNoticeType(),
+		Description:    "Processed from " + doc.GetNoticeType() + " document",
+		Status:         tender.TenderStatusActive,
+		Source:         tender.TenderSourceTEDScraper,
+		SourceFileURL:  sourceFileURL,
+		SourceFileName: sourceFileName,
+		ProcessingMetadata: tender.ProcessingMetadata{
+			ScrapedAt:         now,
+			ProcessedAt:       now,
+			ProcessingVersion: "1.0",
+		},
+	}
+
+	// Generate the tender ID after creating the tender
+	t.TenderID = GenerateTenderID(t)
+
+	return t
+}
+
+// mapToTender maps a TED ContractNotice to tender entity
+func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *tender.Tender {
+	now := time.Now().Unix()
+
+	t := &tender.Tender{
+		ContractNoticeID:    cn.ID,
+		ContractFolderID:    cn.ContractFolderID,
+		NoticeTypeCode:      cn.GetNoticeSubType(),
+		NoticeSubTypeCode:   cn.GetNoticeSubType(),
+		NoticeLanguageCode:  cn.NoticeLanguageCode,
+		IssueDate:           ParseDateToUnix(cn.IssueDate),
+		IssueTime:           ParseTimeToUnix(cn.IssueTime),
+		GazetteID:           cn.GetOJSID(),
+		Title:               cn.GetProcurementTitle(),
+		Description:         cn.GetProcurementDescription(),
+		ProcurementTypeCode: cn.GetContractNature(),
+		ProcedureCode:       cn.GetProcedureCode(),
+		MainClassification:  cn.GetMainClassification(),
+		PlaceOfPerformance:  cn.GetPlaceOfPerformance(),
+		DocumentURI:         cn.GetDocumentURI(),
+		Status:              tender.TenderStatusActive,
+		Source:              tender.TenderSourceTEDScraper,
+		SourceFileURL:       sourceFileURL,
+		SourceFileName:      sourceFileName,
+		ProcessingMetadata: tender.ProcessingMetadata{
+			ScrapedAt:         now,
+			ProcessedAt:       now,
+			ProcessingVersion: "1.0",
+		},
+	}
+
+	// Set publication information
+	if pubInfo := cn.GetPublicationInfo(); pubInfo != nil {
+		t.NoticePublicationID = pubInfo.NoticePublicationID
+		t.PublicationDate = ParseDateToUnix(pubInfo.PublicationDate)
+		t.TenderURL = GenerateTenderURL(pubInfo.NoticePublicationID)
+	}
+
+	// Set additional classifications
+	if cn.ProcurementProject != nil && cn.ProcurementProject.AdditionalCommodityClassification != nil {
+		for _, additionalClass := range cn.ProcurementProject.AdditionalCommodityClassification {
+			if additionalClass.ItemClassificationCode != "" {
+				t.AdditionalClassifications = append(t.AdditionalClassifications, additionalClass.ItemClassificationCode)
+			}
+		}
+	}
+
+	// Set estimated value and currency
+	if amount, currency := cn.GetBudget(); amount != "" {
+		if value, err := strconv.ParseFloat(amount, 64); err == nil {
+			t.EstimatedValue = value
+			t.Currency = currency
+		}
+	}
+
+	// Set duration information
+	if duration, unit := cn.GetDuration(); duration != "" {
+		t.Duration = duration
+		t.DurationUnit = unit
+	}
+
+	// Set tender deadline
+	if deadline := cn.GetTenderDeadline(); deadline != "" {
+		t.TenderDeadline = ParseDateToUnix(deadline)
+
+		if deadlineTime, err := ParseDate(deadline); err == nil {
+			applicationDeadline := CalculateApplicationDeadline(deadlineTime)
+			t.ApplicationDeadline = applicationDeadline.Unix()
+		}
+	}
+
+	if pubDate, err := ParseDate(cn.GetPublicationDate()); err == nil {
+		submissionDeadline := CalculateSubmissionDeadline(pubDate)
+		t.SubmissionDeadline = submissionDeadline.Unix()
+	}
+
+	// Set SubmissionURL from TenderingTerms
+	if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil {
+		if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
+			if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
+				if strings.HasPrefix(endpointID, "http") {
+					t.SubmissionURL = endpointID
+				} else {
+					t.SubmissionURL = "https://" + endpointID
+				}
+			}
+		}
+	}
+
+	// Set location information
+	if cn.ProcurementProject != nil && cn.ProcurementProject.RealizedLocation != nil {
+		if addr := cn.ProcurementProject.RealizedLocation.Address; addr != nil {
+			t.CityName = addr.CityName
+			t.PostalCode = addr.PostalZone
+			t.RegionCode = addr.CountrySubentityCode
+			if addr.Country != nil {
+				t.CountryCode = addr.Country.IdentificationCode
+			}
+		}
+	}
+
+	// Set buyer organization
+	buyerID := cn.GetBuyerID()
+	if buyerID != "" {
+		if buyerOrg := cn.GetOrganizationByID(buyerID); buyerOrg != nil {
+			t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
+		}
+	}
+
+	// Set review organization
+	reviewID := cn.GetReviewOrganizationID()
+	if reviewID != "" {
+		if reviewOrg := cn.GetOrganizationByID(reviewID); reviewOrg != nil {
+			t.ReviewOrganization = s.mapOrganization(reviewOrg, "review")
+		}
+	}
+
+	// Set all organizations
+	if orgs := cn.GetOrganizations(); orgs != nil {
+		for _, org := range orgs.Organization {
+			mappedOrg := s.mapOrganization(&org, "")
+			if mappedOrg != nil {
+				t.Organizations = append(t.Organizations, *mappedOrg)
+			}
+		}
+	}
+
+	// Set selection criteria
+	if criteria := cn.GetSelectionCriteria(); criteria != nil {
+		for _, criterion := range criteria {
+			t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{
+				TypeCode:    criterion.CriterionTypeCode,
+				Description: criterion.Description,
+				LanguageID:  criterion.LanguageID,
+			})
+		}
+	}
+
+	// Set official languages
+	if languages := cn.GetOfficialLanguages(); languages != nil {
+		t.OfficialLanguages = languages
+	}
+
+	// Generate unique tender ID
+	t.TenderID = GenerateTenderID(t)
+
+	return t
+}
+
+// mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity
+func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *tender.Tender {
+	now := time.Now().Unix()
+
+	t := &tender.Tender{
+		ContractNoticeID:    can.ID,
+		ContractFolderID:    can.ContractFolderID,
+		NoticeTypeCode:      can.NoticeTypeCode,
+		NoticeSubTypeCode:   can.GetNoticeSubType(),
+		NoticeLanguageCode:  can.NoticeLanguageCode,
+		IssueDate:           ParseDateToUnix(can.IssueDate),
+		IssueTime:           ParseTimeToUnix(can.IssueTime),
+		GazetteID:           can.GetOJSID(),
+		Title:               can.GetProcurementTitle(),
+		Description:         can.GetProcurementDescription(),
+		ProcurementTypeCode: can.GetContractNature(),
+		ProcedureCode:       can.GetProcedureCode(),
+		MainClassification:  can.GetMainClassification(),
+		Status:              tender.TenderStatusAwarded, // Award notices are for completed tenders
+		Source:              tender.TenderSourceTEDScraper,
+		SourceFileURL:       sourceFileURL,
+		SourceFileName:      sourceFileName,
+		ProcessingMetadata: tender.ProcessingMetadata{
+			ScrapedAt:         now,
+			ProcessedAt:       now,
+			ProcessingVersion: "1.0",
+		},
+	}
+
+	// Set publication information
+	if pubInfo := can.GetPublicationInfo(); pubInfo != nil {
+		t.NoticePublicationID = pubInfo.NoticePublicationID
+		t.PublicationDate = ParseDateToUnix(pubInfo.PublicationDate)
+		t.TenderURL = GenerateTenderURL(pubInfo.NoticePublicationID)
+	}
+
+	// Set total award value
+	if amount, currency := can.GetTotalAwardValue(); amount != "" {
+		if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
+			t.EstimatedValue = parsedAmount
+		}
+		t.Currency = currency
+	}
+
+	// Set location information from main procurement project
+	if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
+		if addr := can.ProcurementProject.RealizedLocation.Address; addr != nil {
+			t.CityName = addr.CityName
+			t.PostalCode = addr.PostalZone
+			t.RegionCode = addr.CountrySubentityCode
+			if addr.Country != nil {
+				t.CountryCode = addr.Country.IdentificationCode
+			}
+		}
+	}
+
+	// Set buyer organization
+	buyerID := can.GetBuyerID()
+	if buyerID != "" {
+		if buyerOrg := can.GetOrganizationByID(buyerID); buyerOrg != nil {
+			t.BuyerOrganization = s.mapOrganization(buyerOrg, "buyer")
+		}
+	}
+
+	// Set all organizations
+	if orgs := can.GetOrganizations(); orgs != nil {
+		for _, org := range orgs.Organization {
+			mappedOrg := s.mapOrganization(&org, "")
+			if mappedOrg != nil {
+				t.Organizations = append(t.Organizations, *mappedOrg)
+			}
+		}
+	}
+
+	// Generate unique tender ID
+	t.TenderID = GenerateTenderID(t)
+
+	return t
+}
+
+// mapOrganization maps a TED Organization to tender Organization
+func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.Organization {
+	if tedOrg == nil || tedOrg.Company == nil {
+		return nil
+	}
+
+	company := tedOrg.Company
+	org := &tender.Organization{
+		Role: role,
+	}
+
+	// Set organization ID and name
+	if company.PartyIdentification != nil {
+		org.ID = company.PartyIdentification.ID
+	}
+	if company.PartyName != nil {
+		org.Name = company.PartyName.Name
+	}
+
+	// Set company ID
+	if company.PartyLegalEntity != nil {
+		org.CompanyID = company.PartyLegalEntity.CompanyID
+	}
+
+	// Set website
+	org.WebsiteURI = company.WebsiteURI
+
+	// Set contact information
+	if company.Contact != nil {
+		org.ContactName = company.Contact.Name
+		org.ContactTelephone = company.Contact.Telephone
+		org.ContactEmail = company.Contact.ElectronicMail
+		org.ContactFax = company.Contact.Telefax
+	}
+
+	// Set address
+	if company.PostalAddress != nil {
+		org.Address = tender.Address{
+			StreetName:           company.PostalAddress.StreetName,
+			CityName:             company.PostalAddress.CityName,
+			PostalZone:           company.PostalAddress.PostalZone,
+			CountrySubentityCode: company.PostalAddress.CountrySubentityCode,
+			Department:           company.PostalAddress.Department,
+		}
+		if company.PostalAddress.Country != nil {
+			org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode
+		}
+	}
+
+	return org
+}