From f57e53bd9b3c883616366303460a18069cc554db Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Sun, 28 Sep 2025 10:27:20 +0330 Subject: [PATCH] Enhance Tender Entity and TED Parser with Status Management and Upsert Logic - Added new fields to the Tender entity for handling cancellation, award, and suspension details, improving the entity's capability to manage tender statuses. - Implemented a DetectNoticeStatus method in the TEDParser to identify the status of TED notices from XML data, enhancing the parser's functionality. - Updated the ParseXML method to include notice status detection, ensuring accurate status representation in parsed documents. - Introduced upsert logic in the TED scraper to handle tender records more effectively, allowing for updates or creation based on existing ContractIDs. - Enhanced logging and error handling throughout the scraping process to improve traceability and robustness. - Added a comprehensive usage example document to illustrate the new features and usage patterns for the TED XML parser and scraper. --- internal/tender/entity.go | 45 ++- ted/parser.go | 151 +++++++- ted/scraper.go | 742 ++++++++++++++++++++++++++++++++++++-- ted/ted.go | 492 ++++++++++++++++++++----- 4 files changed, 1306 insertions(+), 124 deletions(-) diff --git a/internal/tender/entity.go b/internal/tender/entity.go index e72488c..bd35eab 100644 --- a/internal/tender/entity.go +++ b/internal/tender/entity.go @@ -54,6 +54,17 @@ type Tender struct { ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} + // Status-specific fields + CancellationReason string `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"` + CancellationDate int64 `bson:"cancellation_date,omitempty" json:"cancellation_date,omitempty"` // Unix milliseconds + AwardDate int64 `bson:"award_date,omitempty" json:"award_date,omitempty"` // Unix milliseconds + AwardedValue float64 `bson:"awarded_value,omitempty" json:"awarded_value,omitempty"` + WinningTenderer *Organization `bson:"winning_tenderer,omitempty" json:"winning_tenderer,omitempty"` + ContractNumber string `bson:"contract_number,omitempty" json:"contract_number,omitempty"` + SuspensionReason string `bson:"suspension_reason,omitempty" json:"suspension_reason,omitempty"` + SuspensionDate int64 `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"` // Unix milliseconds + Modifications []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"` + AwardedEntities []AwardedEntity `bson:"awarded_entities,omitempty" json:"awarded_entities,omitempty"` } // Organization represents organization information @@ -98,6 +109,30 @@ type ProcessingMetadata struct { EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"` } +// TenderModification represents a modification made to a tender +type TenderModification struct { + ModificationDate int64 `bson:"modification_date" json:"modification_date"` // Unix milliseconds + ModificationReason string `bson:"modification_reason" json:"modification_reason"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"` +} + +// AwardedEntity represents an entity that has been awarded a contract +type AwardedEntity struct { + Name string `bson:"name" json:"name"` + Address string `bson:"address,omitempty" json:"address,omitempty"` + Country string `bson:"country,omitempty" json:"country,omitempty"` + Amount float64 `bson:"amount" json:"amount"` + Currency string `bson:"currency,omitempty" json:"currency,omitempty"` + Share float64 `bson:"share,omitempty" json:"share,omitempty"` // Percentage share for multiple winners + AwardDate int64 `bson:"award_date" json:"award_date"` // Unix milliseconds + ContractID string `bson:"contract_id,omitempty" json:"contract_id,omitempty"` // Contract reference ID + TenderID string `bson:"tender_id,omitempty" json:"tender_id,omitempty"` // Tender reference ID + LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` // Lot reference ID + CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"` // Company registration ID + OrganizationID string `bson:"organization_id,omitempty" json:"organization_id,omitempty"` // Organization reference ID +} + // TenderStatus represents the status of a tender type TenderStatus string @@ -107,6 +142,10 @@ const ( TenderStatusCancelled TenderStatus = "cancelled" TenderStatusAwarded TenderStatus = "awarded" TenderStatusDraft TenderStatus = "draft" + TenderStatusClosed TenderStatus = "closed" + TenderStatusModified TenderStatus = "modified" + TenderStatusSuspended TenderStatus = "suspended" + TenderStatusPublished TenderStatus = "published" ) // TenderSource represents the source of tender data @@ -153,7 +192,7 @@ func (t *Tender) IsExpired() bool { } // Check if deadline (in milliseconds) is in the past - now := time.Now().UnixMilli() + now := time.Now().Unix() return t.TenderDeadline < now } @@ -214,7 +253,7 @@ func (t *Tender) GetFormattedEstimatedValue() string { // UpdateStatus updates the tender status and updated timestamp func (t *Tender) UpdateStatus(status TenderStatus) { t.Status = status - t.UpdatedAt = time.Now().UnixMilli() + t.UpdatedAt = time.Now().Unix() } // GetDaysUntilDeadline returns the number of days until the tender deadline @@ -223,7 +262,7 @@ func (t *Tender) GetDaysUntilDeadline() int { return 0 } - now := time.Now().UnixMilli() + now := time.Now().Unix() if t.TenderDeadline <= now { return 0 // Already expired } diff --git a/ted/parser.go b/ted/parser.go index 45a37b7..5ec04ac 100644 --- a/ted/parser.go +++ b/ted/parser.go @@ -121,6 +121,141 @@ func (p *TEDParser) DetectDocumentType(xmlData []byte) (DocumentType, error) { return "", fmt.Errorf("unknown document type") } +// DetectNoticeStatus detects the status of a TED notice from XML data +func (p *TEDParser) DetectNoticeStatus(xmlData []byte) NoticeStatus { + // Convert to string for fast text search + xmlStr := string(xmlData) + + // Check for common status indicators in XML content + // Many TED XMLs contain status information in extensions or specific tags + + // Check for cancellation indicators + if strings.Contains(xmlStr, " 0 { job.Results.ErrorCount += len(fileResult.Errors) } @@ -272,6 +289,8 @@ func (s *TEDScraper) ScrapeLatest(ctx context.Context) error { s.logger.Info("TED XML scraping completed", map[string]interface{}{ "files_processed": job.Progress.ProcessedFiles, "tenders_created": job.Results.NewTenders, + "tenders_updated": job.Results.UpdatedTenders, + "total_tenders": job.Results.TotalTenders, "error_count": job.Results.ErrorCount, "success_rate": job.Results.SuccessRate, }) @@ -347,6 +366,8 @@ type FileProcessingResult struct { Number int `json:"number"` ProcessedCount int `json:"processed_count"` SuccessCount int `json:"success_count"` + NewTenders int `json:"new_tenders"` + UpdatedTenders int `json:"updated_tenders"` Errors []string `json:"errors"` ProcessedAt int64 `json:"processed_at"` } @@ -527,9 +548,17 @@ func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fil } // Parse and process XML content - if err := s.processXMLContent(ctx, content, fileName, fileInfo, result); err != nil { + processResult, err := s.processXMLContent(ctx, content, fileName, fileInfo) + if err != nil { errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err) result.Errors = append(result.Errors, errMsg) + } else { + result.SuccessCount++ + if processResult.IsNew { + result.NewTenders++ + } else { + result.UpdatedTenders++ + } } result.ProcessedCount++ @@ -556,8 +585,15 @@ func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fil return result, nil } +// XMLProcessResult represents the result of processing a single XML file +type XMLProcessResult struct { + IsNew bool `json:"is_new"` + TenderID string `json:"tender_id"` + ContractID string `json:"contract_id"` +} + // processXMLContent parses XML content and creates/updates tender -func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo, result *FileProcessingResult) error { +func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo) (*XMLProcessResult, error) { s.logger.Info("Starting XML processing", map[string]interface{}{ "file_name": fileName, "file_size": len(content), @@ -570,7 +606,7 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file "file_name": fileName, "error": err.Error(), }) - return fmt.Errorf("failed to parse XML: %w", err) + return nil, fmt.Errorf("failed to parse XML: %w", err) } // Get the underlying document for ID and validation @@ -612,33 +648,45 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file sourceURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number) tenderEntity := s.mapToTenderFromParsedDoc(parsedDoc, sourceURL, fileName) + // Handle status-specific information and award logic + s.applyStatusSpecificInformation(tenderEntity, parsedDoc) + s.applyAwardLogic(tenderEntity, parsedDoc) + s.logger.Info("Mapped to tender entity", map[string]interface{}{ "file_name": fileName, "tender_id": tenderEntity.TenderID, "contract_notice_id": tenderEntity.ContractNoticeID, "title": tenderEntity.Title, + "status": string(tenderEntity.Status), }) - // Try to find existing tender by contract notice ID + // Try to find existing tender by contract notice ID (ContractID) s.logger.Info("Checking for existing tender", map[string]interface{}{ "contract_notice_id": tenderEntity.ContractNoticeID, }) existingTender, err := s.findTenderByContractNoticeID(ctx, tenderEntity.ContractNoticeID) - if err != nil { + if err != nil && err.Error() != "document not found" { s.logger.Warn("Error checking for existing tender", map[string]interface{}{ "contract_notice_id": tenderEntity.ContractNoticeID, "error": err.Error(), }) } + var processResult *XMLProcessResult + if err == nil && existingTender != nil { // Update existing tender s.logger.Info("Found existing tender, updating", map[string]interface{}{ "existing_tender_id": existingTender.ID, "contract_notice_id": tenderEntity.ContractNoticeID, + "old_status": string(existingTender.Status), + "new_status": string(tenderEntity.Status), }) + // Merge updates with existing data + s.mergeExistingTenderData(existingTender, tenderEntity) + tenderEntity.ID = existingTender.ID tenderEntity.CreatedAt = existingTender.CreatedAt tenderEntity.UpdatedAt = time.Now().Unix() @@ -649,17 +697,24 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file "tender_id": tenderEntity.ID, "error": err.Error(), }) - return fmt.Errorf("failed to update existing tender: %w", err) + return nil, fmt.Errorf("failed to update existing tender: %w", err) } s.logger.Info("Successfully updated tender", map[string]interface{}{ "tender_id": tenderEntity.ID, }) + + processResult = &XMLProcessResult{ + IsNew: false, + TenderID: tenderEntity.TenderID, + ContractID: tenderEntity.ContractNoticeID, + } } else { // Create new tender s.logger.Info("Creating new tender", map[string]interface{}{ "contract_notice_id": tenderEntity.ContractNoticeID, "tender_id": tenderEntity.TenderID, + "status": string(tenderEntity.Status), }) tenderEntity.CreatedAt = time.Now().Unix() @@ -672,25 +727,30 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file "tender_id": tenderEntity.TenderID, "error": err.Error(), }) - return fmt.Errorf("failed to create tender: %w", err) + return nil, fmt.Errorf("failed to create tender: %w", err) } s.logger.Info("Successfully created new tender", map[string]interface{}{ "tender_id": tenderEntity.TenderID, "contract_notice_id": tenderEntity.ContractNoticeID, }) - } - result.SuccessCount++ + processResult = &XMLProcessResult{ + IsNew: true, + TenderID: tenderEntity.TenderID, + ContractID: tenderEntity.ContractNoticeID, + } + } s.logger.Debug("Successfully processed XML file", map[string]interface{}{ "file_name": fileName, "document_type": string(parsedDoc.Type), "document_id": documentID, "tender_id": tenderEntity.TenderID, + "is_new": processResult.IsNew, }) - return nil + return processResult, nil } // Helper methods for database operations @@ -940,8 +1000,8 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa NoticeTypeCode: cn.GetNoticeSubType(), NoticeSubTypeCode: cn.GetNoticeSubType(), NoticeLanguageCode: cn.NoticeLanguageCode, - IssueDate: s.parseDateToUnixMilli(cn.IssueDate), - IssueTime: s.parseTimeToUnixMilli(cn.IssueTime), + IssueDate: s.parseDateToUnix(cn.IssueDate), + IssueTime: s.parseTimeToUnix(cn.IssueTime), GazetteID: cn.GetOJSID(), Title: cn.GetProcurementTitle(), Description: cn.GetProcurementDescription(), @@ -964,7 +1024,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa // Set publication information if pubInfo := cn.GetPublicationInfo(); pubInfo != nil { t.NoticePublicationID = pubInfo.NoticePublicationID - t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate) + t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate) t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID) } @@ -993,17 +1053,17 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa // Set tender deadline if deadline := cn.GetTenderDeadline(); deadline != "" { - t.TenderDeadline = s.parseDateToUnixMilli(deadline) + t.TenderDeadline = s.parseDateToUnix(deadline) if deadlineTime, err := s.parseDate(deadline); err == nil { applicationDeadline := s.calculateApplicationDeadline(deadlineTime) - t.ApplicationDeadline = applicationDeadline.UnixMilli() + t.ApplicationDeadline = applicationDeadline.Unix() } } if pubDate, err := s.parseDate(cn.GetPublicationDate()); err == nil { submissionDeadline := s.calculateSubmissionDeadline(pubDate) - t.SubmissionDeadline = submissionDeadline.UnixMilli() + t.SubmissionDeadline = submissionDeadline.Unix() } // Set SubmissionURL from TenderingTerms @@ -1089,8 +1149,8 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so NoticeTypeCode: can.NoticeTypeCode, NoticeSubTypeCode: can.GetNoticeSubType(), NoticeLanguageCode: can.NoticeLanguageCode, - IssueDate: s.parseDateToUnixMilli(can.IssueDate), - IssueTime: s.parseTimeToUnixMilli(can.IssueTime), + IssueDate: s.parseDateToUnix(can.IssueDate), + IssueTime: s.parseTimeToUnix(can.IssueTime), GazetteID: can.GetOJSID(), Title: can.GetProcurementTitle(), Description: can.GetProcurementDescription(), @@ -1111,7 +1171,7 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so // Set publication information if pubInfo := can.GetPublicationInfo(); pubInfo != nil { t.NoticePublicationID = pubInfo.NoticePublicationID - t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate) + t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate) t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID) } @@ -1234,7 +1294,7 @@ func (s *TEDScraper) generateTenderID(t *tender.Tender) string { // parts = append(parts, buyerCode) // Date (YYYYMMDD format from issue date) - parts = append(parts, s.padOrTruncate(s.unixMilliToDateString(t.IssueDate), 4)) + parts = append(parts, s.padOrTruncate(s.UnixToDateString(t.IssueDate), 4)) // Location code (country code + region code, padded to 6 chars) locationCode := "EU" @@ -1310,21 +1370,21 @@ func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) { return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr) } -// parseDateToUnixMilli converts string date to Unix milliseconds (int64) -func (s *TEDScraper) parseDateToUnixMilli(dateStr string) int64 { +// parseDateToUnix converts string date to Unix milliseconds (int64) +func (s *TEDScraper) parseDateToUnix(dateStr string) int64 { if dateStr == "" { return 0 } if parsedTime, err := s.parseDate(dateStr); err == nil { - return parsedTime.UnixMilli() + return parsedTime.Unix() } return 0 } -// parseTimeToUnixMilli converts string time to Unix milliseconds (int64) -func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 { +// parseTimeToUnix converts string time to Unix milliseconds (int64) +func (s *TEDScraper) parseTimeToUnix(timeStr string) int64 { if timeStr == "" { return 0 } @@ -1337,19 +1397,19 @@ func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 { } if parsedTime, err := s.parseDate(timeStr); err == nil { - return parsedTime.UnixMilli() + return parsedTime.Unix() } return 0 } -// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation -func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string { - if unixMilli == 0 { +// UnixToDateString converts Unix milliseconds to date string for TenderID generation +func (s *TEDScraper) UnixToDateString(u int64) string { + if u == 0 { return "00000000" } - return time.UnixMilli(unixMilli).Format("20060102") + return time.Unix(u, 0).Format("20060102") } func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time { @@ -1395,3 +1455,613 @@ func (s *TEDScraper) adjustToWorkingDay(targetDate time.Time, forward bool) time return current } + +// mapTEDStatusToTenderStatus maps TED notice status to tender status +func (s *TEDScraper) mapTEDStatusToTenderStatus(tedStatus NoticeStatus) tender.TenderStatus { + switch tedStatus { + case NoticeStatusDraft: + return tender.TenderStatusDraft + case NoticeStatusPublished: + return tender.TenderStatusPublished + case NoticeStatusCancelled: + return tender.TenderStatusCancelled + case NoticeStatusAwarded: + return tender.TenderStatusAwarded + case NoticeStatusClosed: + return tender.TenderStatusClosed + case NoticeStatusModified: + return tender.TenderStatusModified + case NoticeStatusSuspended: + return tender.TenderStatusSuspended + default: + // Default to active for unknown statuses, but log a warning + s.logger.Warn("Unknown TED notice status, defaulting to active", map[string]interface{}{ + "ted_status": string(tedStatus), + }) + return tender.TenderStatusActive + } +} + +// applyStatusSpecificInformation applies status-specific information to the tender entity +func (s *TEDScraper) applyStatusSpecificInformation(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) { + if tenderEntity == nil || parsedDoc == nil { + return + } + + // Map TED status to tender status + tenderEntity.Status = s.mapTEDStatusToTenderStatus(parsedDoc.Status) + + // Get the document to access status-specific methods + document := parsedDoc.GetDocument() + if document == nil { + return + } + + // Apply status-specific information based on document type + switch parsedDoc.Type { + case DocumentTypeContractNotice: + if cn, ok := document.(*ContractNotice); ok { + s.applyContractNoticeStatusInfo(tenderEntity, cn, parsedDoc.Status) + } + case DocumentTypeContractAwardNotice: + if can, ok := document.(*ContractAwardNotice); ok { + s.applyContractAwardNoticeStatusInfo(tenderEntity, can, parsedDoc.Status) + } + default: + // For other document types, we rely on the parsed status + s.logger.Debug("Applied basic status information for document type", map[string]interface{}{ + "document_type": string(parsedDoc.Type), + "status": string(parsedDoc.Status), + }) + } +} + +// applyContractNoticeStatusInfo applies status-specific information from ContractNotice +func (s *TEDScraper) applyContractNoticeStatusInfo(tenderEntity *tender.Tender, cn *ContractNotice, status NoticeStatus) { + switch status { + case NoticeStatusCancelled: + if cancellation := cn.GetCancellationReason(); cancellation != nil { + tenderEntity.CancellationReason = cancellation.Description + if cancellation.ReasonCode != "" { + tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description) + } + // Note: TED XML doesn't typically have cancellation date, so we use current time + tenderEntity.CancellationDate = time.Now().Unix() + } + + case NoticeStatusAwarded: + if award := cn.GetAwardInformation(); award != nil { + if award.AwardDate != "" { + tenderEntity.AwardDate = s.parseDateToUnix(award.AwardDate) + } + if award.AwardedValue.Text != "" { + if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil { + tenderEntity.AwardedValue = value + } + } + tenderEntity.ContractNumber = award.ContractNumber + if award.WinningTenderer != nil && award.WinningTenderer.Company != nil { + tenderEntity.WinningTenderer = s.mapOrganization(&Organization{ + Company: award.WinningTenderer.Company, + }, "winner") + } + } + + case NoticeStatusSuspended: + if suspension := cn.GetSuspensionReason(); suspension != nil { + tenderEntity.SuspensionReason = suspension.Description + if suspension.ReasonCode != "" { + tenderEntity.SuspensionReason = fmt.Sprintf("%s: %s", suspension.ReasonCode, suspension.Description) + } + tenderEntity.SuspensionDate = time.Now().Unix() + } + + case NoticeStatusModified: + if modifications := cn.GetModifications(); modifications != nil { + for _, mod := range modifications { + tenderMod := tender.TenderModification{ + ModificationDate: s.parseDateToUnix(mod.ModificationDate), + ModificationReason: mod.ModificationReason, + Description: mod.Description, + LanguageID: mod.LanguageID, + } + tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod) + } + } + } +} + +// applyContractAwardNoticeStatusInfo applies status-specific information from ContractAwardNotice +func (s *TEDScraper) applyContractAwardNoticeStatusInfo(tenderEntity *tender.Tender, can *ContractAwardNotice, status NoticeStatus) { + // Contract award notices are typically awarded status + tenderEntity.Status = tender.TenderStatusAwarded + + // Set award date from tender result + if can.TenderResult != nil && can.TenderResult.AwardDate != "" { + tenderEntity.AwardDate = s.parseDateToUnix(can.TenderResult.AwardDate) + } + + // Get award information from extensions if available + if award := can.GetAwardInformation(); award != nil { + if award.AwardedValue.Text != "" { + if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil { + tenderEntity.AwardedValue = value + } + } + tenderEntity.ContractNumber = award.ContractNumber + } + + // Get total award value from notice result + if amount, currency := can.GetTotalAwardValue(); amount != "" { + if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil { + tenderEntity.AwardedValue = parsedAmount + tenderEntity.Currency = currency + } + } + + // Extract detailed awarded contractors information + s.extractAwardedEntities(tenderEntity, can) + + // Handle other statuses if applicable + switch status { + case NoticeStatusCancelled: + if cancellation := can.GetCancellationReason(); cancellation != nil { + tenderEntity.Status = tender.TenderStatusCancelled + tenderEntity.CancellationReason = cancellation.Description + if cancellation.ReasonCode != "" { + tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description) + } + tenderEntity.CancellationDate = time.Now().Unix() + } + + case NoticeStatusModified: + if modifications := can.GetModifications(); modifications != nil { + tenderEntity.Status = tender.TenderStatusModified + for _, mod := range modifications { + tenderMod := tender.TenderModification{ + ModificationDate: s.parseDateToUnix(mod.ModificationDate), + ModificationReason: mod.ModificationReason, + Description: mod.Description, + LanguageID: mod.LanguageID, + } + tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod) + } + } + } +} + +// extractAwardedEntities extracts detailed information about awarded entities from a contract award notice +func (s *TEDScraper) extractAwardedEntities(tenderEntity *tender.Tender, can *ContractAwardNotice) { + awardedContractors := can.GetAwardedContractors() + if len(awardedContractors) == 0 { + return + } + + s.logger.Info("Extracting awarded entities", map[string]interface{}{ + "contract_notice_id": can.ID, + "num_contractors": len(awardedContractors), + }) + + for _, contractor := range awardedContractors { + entity := tender.AwardedEntity{ + Name: contractor.Name, + Address: contractor.Address, + Country: contractor.Country, + Currency: contractor.Currency, + ContractID: contractor.ContractID, + TenderID: contractor.TenderID, + LotID: contractor.LotID, + CompanyID: contractor.CompanyID, + OrganizationID: contractor.OrganizationID, + } + + // Parse amount + if contractor.Amount != "" { + if amount, err := strconv.ParseFloat(contractor.Amount, 64); err == nil { + entity.Amount = amount + } else { + s.logger.Warn("Failed to parse award amount", map[string]interface{}{ + "amount_string": contractor.Amount, + "contractor": contractor.Name, + "error": err.Error(), + }) + } + } + + // Parse award date + if contractor.AwardDate != "" { + entity.AwardDate = s.parseDateToUnix(contractor.AwardDate) + } else if contractor.IssueDate != "" { + entity.AwardDate = s.parseDateToUnix(contractor.IssueDate) + } + + // Calculate share percentage if there are multiple contractors + totalAmount := tenderEntity.AwardedValue + if totalAmount > 0 && entity.Amount > 0 { + entity.Share = (entity.Amount / totalAmount) * 100 + } + + tenderEntity.AwardedEntities = append(tenderEntity.AwardedEntities, entity) + + s.logger.Debug("Added awarded entity", map[string]interface{}{ + "name": entity.Name, + "amount": entity.Amount, + "currency": entity.Currency, + "share": entity.Share, + "contract_id": entity.ContractID, + }) + } + + s.logger.Info("Successfully extracted awarded entities", map[string]interface{}{ + "total_entities": len(tenderEntity.AwardedEntities), + "total_value": tenderEntity.AwardedValue, + "currency": tenderEntity.Currency, + }) +} + +// ScrapeByDateRange scrapes TED XML files within a specified date range +func (s *TEDScraper) ScrapeByDateRange(ctx context.Context, config DateRangeConfig) error { + startTime := time.Now() + + s.logger.Info("Starting TED XML scraping by date range", map[string]interface{}{ + "start_date": config.StartDate.Format("2006-01-02"), + "end_date": config.EndDate.Format("2006-01-02"), + "mode": string(config.Mode), + "timestamp": startTime.Unix(), + }) + + // Validate date range + if config.EndDate.Before(config.StartDate) { + return fmt.Errorf("end date cannot be before start date") + } + + // Calculate the date range in days + daysInRange := int(config.EndDate.Sub(config.StartDate).Hours()/24) + 1 + if daysInRange > 365 { + return fmt.Errorf("date range too large: maximum 365 days allowed") + } + + // Create scraping job + job := &ScrapingJob{ + Type: fmt.Sprintf("ted_%s", config.Mode), + Status: "running", + StartedAt: startTime.Unix(), + Progress: JobProgress{}, + Results: JobResults{}, + Config: map[string]interface{}{ + "base_url": s.config.BaseURL, + "start_date": config.StartDate.Format("2006-01-02"), + "end_date": config.EndDate.Format("2006-01-02"), + "mode": string(config.Mode), + }, + } + + // Save job + if err := s.saveScrapingJob(ctx, job); err != nil { + s.logger.Error("Failed to save scraping job", map[string]interface{}{ + "error": err.Error(), + }) + } + + // Calculate files to download based on date range + filesToDownload := s.calculateFilesInDateRange(config.StartDate, config.EndDate) + if len(filesToDownload) == 0 { + s.logger.Info("No files to download in date range", map[string]interface{}{ + "start_date": config.StartDate.Format("2006-01-02"), + "end_date": config.EndDate.Format("2006-01-02"), + }) + + // Complete the job + s.completeJob(ctx, job, nil, nil) + return nil + } + + // Update job progress + job.Progress.TotalFiles = len(filesToDownload) + s.saveScrapingJob(ctx, job) + + var lastSuccessfulFile *FileInfo + + // Download and process files + for i, fileInfo := range filesToDownload { + job.Progress.CurrentFile = fmt.Sprintf("%d%05d", fileInfo.Year, fileInfo.Number) + s.saveScrapingJob(ctx, job) + + fileResult, err := s.downloadAndProcess(ctx, fileInfo) + if err != nil { + s.logger.Error("Failed to download and process file", map[string]interface{}{ + "year": fileInfo.Year, + "number": fileInfo.Number, + "error": err.Error(), + }) + job.Progress.FailedFiles++ + job.Results.ErrorCount++ + } else { + job.Progress.ProcessedFiles++ + job.Results.TotalTenders += fileResult.ProcessedCount + job.Results.NewTenders += fileResult.NewTenders + job.Results.UpdatedTenders += fileResult.UpdatedTenders + if len(fileResult.Errors) > 0 { + job.Results.ErrorCount += len(fileResult.Errors) + } + lastSuccessfulFile = &fileInfo + } + + // Update progress percentage + job.Progress.PercentComplete = float64(i+1) / float64(len(filesToDownload)) * 100 + s.saveScrapingJob(ctx, job) + + // Check context for cancellation + select { + case <-ctx.Done(): + s.completeJob(ctx, job, nil, lastSuccessfulFile) + return ctx.Err() + default: + // Continue processing + } + } + + // Complete the job + s.completeJob(ctx, job, nil, lastSuccessfulFile) + + s.logger.Info("TED XML scraping by date range completed", map[string]interface{}{ + "files_processed": job.Progress.ProcessedFiles, + "tenders_created": job.Results.NewTenders, + "tenders_updated": job.Results.UpdatedTenders, + "error_count": job.Results.ErrorCount, + "success_rate": job.Results.SuccessRate, + }) + + return nil +} + +// calculateFilesInDateRange calculates which files need to be downloaded for a date range +func (s *TEDScraper) calculateFilesInDateRange(startDate, endDate time.Time) []FileInfo { + var files []FileInfo + + // TED files are published roughly once per working day + // We estimate file numbers based on working days from start of year + + for year := startDate.Year(); year <= endDate.Year(); year++ { + yearStart := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC) + yearEnd := time.Date(year, 12, 31, 23, 59, 59, 0, time.UTC) + + // Determine the effective start and end dates for this year + effectiveStart := startDate + if yearStart.After(startDate) { + effectiveStart = yearStart + } + + effectiveEnd := endDate + if yearEnd.Before(endDate) { + effectiveEnd = yearEnd + } + + // Skip if no overlap with this year + if effectiveStart.After(effectiveEnd) { + continue + } + + // Calculate approximate file range for this year + startDayOfYear := effectiveStart.YearDay() + endDayOfYear := effectiveEnd.YearDay() + + // Estimate file numbers (roughly 0.7 files per day, accounting for weekends) + startFileNum := int(float64(startDayOfYear) * 0.7) + endFileNum := int(float64(endDayOfYear) * 0.7) + + // Ensure minimum range + if startFileNum < 1 { + startFileNum = 1 + } + if endFileNum < startFileNum { + endFileNum = startFileNum + } + + // Add some buffer + startFileNum = max(1, startFileNum-2) + endFileNum = min(366, endFileNum+2) // Max ~366 files per year + + s.logger.Info("Calculating file range for year", map[string]interface{}{ + "year": year, + "start_file_num": startFileNum, + "end_file_num": endFileNum, + "start_date": effectiveStart.Format("2006-01-02"), + "end_date": effectiveEnd.Format("2006-01-02"), + }) + + for number := startFileNum; number <= endFileNum; number++ { + fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number) + ojsID := fmt.Sprintf("S %03d / %d", number, year) + + files = append(files, FileInfo{ + Year: year, + Number: number, + URL: fileURL, + OJS: ojsID, + }) + } + } + + s.logger.Info("Calculated files for date range", map[string]interface{}{ + "total_files": len(files), + "start_date": startDate.Format("2006-01-02"), + "end_date": endDate.Format("2006-01-02"), + }) + + return files +} + +// Helper function for max +func max(a, b int) int { + if a > b { + return a + } + return b +} + +// Helper function for min +func min(a, b int) int { + if a < b { + return a + } + return b +} + +// applyAwardLogic applies award-specific logic based on document type and content +func (s *TEDScraper) applyAwardLogic(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) { + if tenderEntity == nil || parsedDoc == nil { + return + } + + // Only set award information if the tender has a confirmed winner + hasConfirmedWinner := s.hasConfirmedWinner(parsedDoc) + + s.logger.Debug("Applying award logic", map[string]interface{}{ + "document_type": string(parsedDoc.Type), + "status": string(parsedDoc.Status), + "has_confirmed_winner": hasConfirmedWinner, + }) + + if !hasConfirmedWinner { + // Clear any award date and winner information for tenders without confirmed winners + tenderEntity.AwardDate = 0 + tenderEntity.AwardedValue = 0 + tenderEntity.WinningTenderer = nil + tenderEntity.AwardedEntities = nil + tenderEntity.ContractNumber = "" + + s.logger.Debug("Cleared award information for tender without confirmed winner", map[string]interface{}{ + "contract_notice_id": tenderEntity.ContractNoticeID, + }) + return + } + + // Process award information for tenders with confirmed winners + s.logger.Info("Processing award information for tender with confirmed winner", map[string]interface{}{ + "contract_notice_id": tenderEntity.ContractNoticeID, + "document_type": string(parsedDoc.Type), + }) +} + +// hasConfirmedWinner checks if the tender has a confirmed winner based on the document content +func (s *TEDScraper) hasConfirmedWinner(parsedDoc *ParsedDocument) bool { + if parsedDoc == nil { + return false + } + + // Check for award-specific document types + switch parsedDoc.Type { + case DocumentTypeContractAwardNotice, DocumentTypeResultNotice: + // These document types typically indicate awarded tenders + if parsedDoc.Status == NoticeStatusAwarded { + return true + } + + // Check for specific award indicators in the document + document := parsedDoc.GetDocument() + if document != nil { + return s.checkForAwardIndicators(document, parsedDoc.Type) + } + + case DocumentTypeContractNotice: + // Contract notices may have award information if they're updated with results + if parsedDoc.Status == NoticeStatusAwarded { + if cn := parsedDoc.ContractNotice; cn != nil { + return s.checkForAwardIndicators(cn, parsedDoc.Type) + } + } + } + + return false +} + +// checkForAwardIndicators checks for specific award indicators in the document +func (s *TEDScraper) checkForAwardIndicators(document TEDDocument, docType DocumentType) bool { + switch docType { + case DocumentTypeContractAwardNotice: + if can, ok := document.(*ContractAwardNotice); ok { + // Check for tender result with award date + if can.TenderResult != nil && can.TenderResult.AwardDate != "" { + return true + } + + // Check for awarded contractors + awardedContractors := can.GetAwardedContractors() + if len(awardedContractors) > 0 { + return true + } + + // Check for award information in extensions + if award := can.GetAwardInformation(); award != nil && award.AwardDate != "" { + return true + } + } + + case DocumentTypeContractNotice: + if cn, ok := document.(*ContractNotice); ok { + // Check for award information + if award := cn.GetAwardInformation(); award != nil && award.AwardDate != "" { + return true + } + } + + case DocumentTypeResultNotice: + // Result notices typically contain award information + return true + } + + return false +} + +// mergeExistingTenderData merges new data with existing tender data, preserving important existing information +func (s *TEDScraper) mergeExistingTenderData(existing *tender.Tender, updated *tender.Tender) { + // Preserve creation timestamp and original metadata + updated.CreatedAt = existing.CreatedAt + + // Merge processing metadata + if existing.ProcessingMetadata.ScrapedAt != 0 { + // Keep original scraped time, update processed time + updated.ProcessingMetadata.ScrapedAt = existing.ProcessingMetadata.ScrapedAt + } + + // Preserve existing award information if the new document doesn't have confirmed winners + // but existing data does (to avoid losing award data on subsequent updates) + if existing.AwardDate != 0 && updated.AwardDate == 0 && len(existing.AwardedEntities) > 0 { + s.logger.Info("Preserving existing award information", map[string]interface{}{ + "contract_notice_id": existing.ContractNoticeID, + "existing_award_date": existing.AwardDate, + "awarded_entities_count": len(existing.AwardedEntities), + }) + + updated.AwardDate = existing.AwardDate + updated.AwardedValue = existing.AwardedValue + updated.WinningTenderer = existing.WinningTenderer + updated.AwardedEntities = existing.AwardedEntities + updated.ContractNumber = existing.ContractNumber + } + + // Merge modifications (append new modifications to existing ones) + if len(existing.Modifications) > 0 { + // Create a map to avoid duplicate modifications + modMap := make(map[string]tender.TenderModification) + + // Add existing modifications + for _, mod := range existing.Modifications { + key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason) + modMap[key] = mod + } + + // Add new modifications + for _, mod := range updated.Modifications { + key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason) + modMap[key] = mod + } + + // Convert back to slice + updated.Modifications = make([]tender.TenderModification, 0, len(modMap)) + for _, mod := range modMap { + updated.Modifications = append(updated.Modifications, mod) + } + } +} diff --git a/ted/ted.go b/ted/ted.go index 14a34f9..70e9232 100644 --- a/ted/ted.go +++ b/ted/ted.go @@ -3,6 +3,7 @@ package ted import ( "encoding/xml" "fmt" + "strings" "time" ) @@ -27,9 +28,24 @@ const ( DocumentTypeSubcontractNotice DocumentType = "SubcontractNotice" ) +// NoticeStatus represents the status of a TED notice +type NoticeStatus string + +// Notice status constants for all possible TED notice statuses +const ( + NoticeStatusDraft NoticeStatus = "Draft" + NoticeStatusPublished NoticeStatus = "Published" + NoticeStatusCancelled NoticeStatus = "Cancelled" + NoticeStatusAwarded NoticeStatus = "Awarded" + NoticeStatusClosed NoticeStatus = "Closed" + NoticeStatusModified NoticeStatus = "Modified" + NoticeStatusSuspended NoticeStatus = "Suspended" +) + // ParsedDocument wraps any TED document with type information type ParsedDocument struct { - Type DocumentType + Type DocumentType `json:"type"` + Status NoticeStatus `json:"status"` ContractNotice *ContractNotice `json:"contract_notice,omitempty"` ContractAwardNotice *ContractAwardNotice `json:"contract_award_notice,omitempty"` PriorInformationNotice *PriorInformationNotice `json:"prior_information_notice,omitempty"` @@ -78,6 +94,7 @@ type TEDDocument interface { GetID() string GetNoticeType() string GetLanguage() string + GetNoticeStatus() NoticeStatus Validate() error } @@ -123,6 +140,11 @@ func (pin PriorInformationNotice) GetLanguage() string { return pin.NoticeLanguageCode } +func (pin PriorInformationNotice) GetNoticeStatus() NoticeStatus { + // Prior information notices are typically published + return NoticeStatusPublished +} + func (pin PriorInformationNotice) Validate() error { if pin.ID == "" { return fmt.Errorf("prior information notice ID is required") @@ -163,9 +185,10 @@ type DesignContest struct { } // Interface methods for DesignContest -func (dc DesignContest) GetID() string { return dc.ID } -func (dc DesignContest) GetNoticeType() string { return dc.NoticeTypeCode } -func (dc DesignContest) GetLanguage() string { return dc.NoticeLanguageCode } +func (dc DesignContest) GetID() string { return dc.ID } +func (dc DesignContest) GetNoticeType() string { return dc.NoticeTypeCode } +func (dc DesignContest) GetLanguage() string { return dc.NoticeLanguageCode } +func (dc DesignContest) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished } func (dc DesignContest) Validate() error { if dc.ID == "" { return fmt.Errorf("design contest ID is required") @@ -205,9 +228,10 @@ type QualificationSystemNotice struct { } // Interface methods for QualificationSystemNotice -func (qsn QualificationSystemNotice) GetID() string { return qsn.ID } -func (qsn QualificationSystemNotice) GetNoticeType() string { return qsn.NoticeTypeCode } -func (qsn QualificationSystemNotice) GetLanguage() string { return qsn.NoticeLanguageCode } +func (qsn QualificationSystemNotice) GetID() string { return qsn.ID } +func (qsn QualificationSystemNotice) GetNoticeType() string { return qsn.NoticeTypeCode } +func (qsn QualificationSystemNotice) GetLanguage() string { return qsn.NoticeLanguageCode } +func (qsn QualificationSystemNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished } func (qsn QualificationSystemNotice) Validate() error { if qsn.ID == "" { return fmt.Errorf("qualification system notice ID is required") @@ -249,9 +273,10 @@ type ConcessionNotice struct { } // Interface methods for ConcessionNotice -func (cn ConcessionNotice) GetID() string { return cn.ID } -func (cn ConcessionNotice) GetNoticeType() string { return cn.NoticeTypeCode } -func (cn ConcessionNotice) GetLanguage() string { return cn.NoticeLanguageCode } +func (cn ConcessionNotice) GetID() string { return cn.ID } +func (cn ConcessionNotice) GetNoticeType() string { return cn.NoticeTypeCode } +func (cn ConcessionNotice) GetLanguage() string { return cn.NoticeLanguageCode } +func (cn ConcessionNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished } func (cn ConcessionNotice) Validate() error { if cn.ID == "" { return fmt.Errorf("concession notice ID is required") @@ -291,9 +316,10 @@ type PlanningNotice struct { } // Interface methods for PlanningNotice -func (pn PlanningNotice) GetID() string { return pn.ID } -func (pn PlanningNotice) GetNoticeType() string { return pn.NoticeTypeCode } -func (pn PlanningNotice) GetLanguage() string { return pn.NoticeLanguageCode } +func (pn PlanningNotice) GetID() string { return pn.ID } +func (pn PlanningNotice) GetNoticeType() string { return pn.NoticeTypeCode } +func (pn PlanningNotice) GetLanguage() string { return pn.NoticeLanguageCode } +func (pn PlanningNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished } func (pn PlanningNotice) Validate() error { if pn.ID == "" { return fmt.Errorf("planning notice ID is required") @@ -333,9 +359,10 @@ type CompetitionNotice struct { } // Interface methods for CompetitionNotice -func (cn CompetitionNotice) GetID() string { return cn.ID } -func (cn CompetitionNotice) GetNoticeType() string { return cn.NoticeTypeCode } -func (cn CompetitionNotice) GetLanguage() string { return cn.NoticeLanguageCode } +func (cn CompetitionNotice) GetID() string { return cn.ID } +func (cn CompetitionNotice) GetNoticeType() string { return cn.NoticeTypeCode } +func (cn CompetitionNotice) GetLanguage() string { return cn.NoticeLanguageCode } +func (cn CompetitionNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished } func (cn CompetitionNotice) Validate() error { if cn.ID == "" { return fmt.Errorf("competition notice ID is required") @@ -376,9 +403,10 @@ type ResultNotice struct { } // Interface methods for ResultNotice -func (rn ResultNotice) GetID() string { return rn.ID } -func (rn ResultNotice) GetNoticeType() string { return rn.NoticeTypeCode } -func (rn ResultNotice) GetLanguage() string { return rn.NoticeLanguageCode } +func (rn ResultNotice) GetID() string { return rn.ID } +func (rn ResultNotice) GetNoticeType() string { return rn.NoticeTypeCode } +func (rn ResultNotice) GetLanguage() string { return rn.NoticeLanguageCode } +func (rn ResultNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusAwarded } func (rn ResultNotice) Validate() error { if rn.ID == "" { return fmt.Errorf("result notice ID is required") @@ -420,9 +448,10 @@ type ChangeNotice struct { } // Interface methods for ChangeNotice -func (cn ChangeNotice) GetID() string { return cn.ID } -func (cn ChangeNotice) GetNoticeType() string { return cn.NoticeTypeCode } -func (cn ChangeNotice) GetLanguage() string { return cn.NoticeLanguageCode } +func (cn ChangeNotice) GetID() string { return cn.ID } +func (cn ChangeNotice) GetNoticeType() string { return cn.NoticeTypeCode } +func (cn ChangeNotice) GetLanguage() string { return cn.NoticeLanguageCode } +func (cn ChangeNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusModified } func (cn ChangeNotice) Validate() error { if cn.ID == "" { return fmt.Errorf("change notice ID is required") @@ -463,9 +492,10 @@ type SubcontractNotice struct { } // Interface methods for SubcontractNotice -func (sn SubcontractNotice) GetID() string { return sn.ID } -func (sn SubcontractNotice) GetNoticeType() string { return sn.NoticeTypeCode } -func (sn SubcontractNotice) GetLanguage() string { return sn.NoticeLanguageCode } +func (sn SubcontractNotice) GetID() string { return sn.ID } +func (sn SubcontractNotice) GetNoticeType() string { return sn.NoticeTypeCode } +func (sn SubcontractNotice) GetLanguage() string { return sn.NoticeLanguageCode } +func (sn SubcontractNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished } func (sn SubcontractNotice) Validate() error { if sn.ID == "" { return fmt.Errorf("subcontract notice ID is required") @@ -665,67 +695,123 @@ type SubcontractingTerm struct { SubcontractingDescription string `xml:"SubcontractingDescription,omitempty"` } +// NoticeStatusInformation contains status-specific fields +type NoticeStatusInformation struct { + Status NoticeStatus `xml:"NoticeStatus,omitempty"` + StatusCode string `xml:"StatusCode,omitempty"` + CancellationReason *CancellationReason `xml:"CancellationReason,omitempty"` + AwardInformation *AwardInformation `xml:"AwardInformation,omitempty"` + Modifications []NoticeModification `xml:"Modifications>Modification,omitempty"` + SuspensionReason *SuspensionReason `xml:"SuspensionReason,omitempty"` +} + +// CancellationReason contains cancellation information +type CancellationReason struct { + XMLName xml.Name `xml:"CancellationReason"` + ReasonCode string `xml:"ReasonCode"` + Description string `xml:"Description,omitempty"` + LanguageID string `xml:"languageID,attr,omitempty"` +} + +// AwardInformation contains award-specific information +type AwardInformation struct { + XMLName xml.Name `xml:"AwardInformation"` + AwardDate string `xml:"AwardDate"` + AwardedValue CurrencyText `xml:"AwardedValue,omitempty"` + WinningTenderer *WinningTenderer `xml:"WinningTenderer,omitempty"` + ContractNumber string `xml:"ContractNumber,omitempty"` +} + +// WinningTenderer contains information about the winning tenderer +type WinningTenderer struct { + XMLName xml.Name `xml:"WinningTenderer"` + NaturalPersonIndicator string `xml:"NaturalPersonIndicator,omitempty"` + Company *Company `xml:"Company,omitempty"` +} + +// NoticeModification contains modification information +type NoticeModification struct { + XMLName xml.Name `xml:"Modification"` + ModificationDate string `xml:"ModificationDate"` + ModificationReason string `xml:"ModificationReason"` + Description string `xml:"Description,omitempty"` + LanguageID string `xml:"languageID,attr,omitempty"` +} + +// SuspensionReason contains suspension information +type SuspensionReason struct { + XMLName xml.Name `xml:"SuspensionReason"` + ReasonCode string `xml:"ReasonCode"` + Description string `xml:"Description,omitempty"` + LanguageID string `xml:"languageID,attr,omitempty"` +} + // ContractNotice represents the main TED XML contract notice structure type ContractNotice struct { - XMLName xml.Name `xml:"ContractNotice"` - Xmlns string `xml:"xmlns,attr,omitempty"` - XmlnsCac string `xml:"xmlns:cac,attr,omitempty"` - XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"` - XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"` - XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"` - XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"` - XmlnsExt string `xml:"xmlns:ext,attr,omitempty"` - UBLVersionID string `xml:"UBLVersionID"` - CustomizationID string `xml:"CustomizationID"` - ID string `xml:"ID"` - ContractFolderID string `xml:"ContractFolderID"` - IssueDate string `xml:"IssueDate"` - IssueTime string `xml:"IssueTime"` - VersionID string `xml:"VersionID"` - RegulatoryDomain string `xml:"RegulatoryDomain"` - NoticeTypeCode string `xml:"NoticeTypeCode"` - NoticeLanguageCode string `xml:"NoticeLanguageCode"` - Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"` - ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"` - TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"` - TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"` - ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"` - ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"` + XMLName xml.Name `xml:"ContractNotice"` + Xmlns string `xml:"xmlns,attr,omitempty"` + XmlnsCac string `xml:"xmlns:cac,attr,omitempty"` + XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"` + XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"` + XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"` + XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"` + XmlnsExt string `xml:"xmlns:ext,attr,omitempty"` + UBLVersionID string `xml:"UBLVersionID"` + CustomizationID string `xml:"CustomizationID"` + ID string `xml:"ID"` + ContractFolderID string `xml:"ContractFolderID"` + IssueDate string `xml:"IssueDate"` + IssueTime string `xml:"IssueTime"` + VersionID string `xml:"VersionID"` + RegulatoryDomain string `xml:"RegulatoryDomain"` + NoticeTypeCode string `xml:"NoticeTypeCode"` + NoticeLanguageCode string `xml:"NoticeLanguageCode"` + StatusInformation *NoticeStatusInformation `xml:"StatusInformation,omitempty"` + Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"` + ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"` + TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"` + TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"` + ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"` + ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"` } // ContractAwardNotice represents the main TED XML contract award notice structure type ContractAwardNotice struct { - XMLName xml.Name `xml:"ContractAwardNotice"` - Xmlns string `xml:"xmlns,attr,omitempty"` - XmlnsCac string `xml:"xmlns:cac,attr,omitempty"` - XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"` - XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"` - XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"` - XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"` - XmlnsExt string `xml:"xmlns:ext,attr,omitempty"` - Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"` - UBLVersionID string `xml:"UBLVersionID"` - CustomizationID string `xml:"CustomizationID"` - ID string `xml:"ID"` - ContractFolderID string `xml:"ContractFolderID"` - IssueDate string `xml:"IssueDate"` - IssueTime string `xml:"IssueTime"` - VersionID string `xml:"VersionID"` - RegulatoryDomain string `xml:"RegulatoryDomain"` - NoticeTypeCode string `xml:"NoticeTypeCode"` - NoticeLanguageCode string `xml:"NoticeLanguageCode"` - ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"` - TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"` - ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"` - ProcurementProjectLot []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"` - TenderResult *TenderResult `xml:"TenderResult,omitempty"` + XMLName xml.Name `xml:"ContractAwardNotice"` + Xmlns string `xml:"xmlns,attr,omitempty"` + XmlnsCac string `xml:"xmlns:cac,attr,omitempty"` + XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"` + XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"` + XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"` + XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"` + XmlnsExt string `xml:"xmlns:ext,attr,omitempty"` + Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"` + UBLVersionID string `xml:"UBLVersionID"` + CustomizationID string `xml:"CustomizationID"` + ID string `xml:"ID"` + ContractFolderID string `xml:"ContractFolderID"` + IssueDate string `xml:"IssueDate"` + IssueTime string `xml:"IssueTime"` + VersionID string `xml:"VersionID"` + RegulatoryDomain string `xml:"RegulatoryDomain"` + NoticeTypeCode string `xml:"NoticeTypeCode"` + NoticeLanguageCode string `xml:"NoticeLanguageCode"` + StatusInformation *NoticeStatusInformation `xml:"StatusInformation,omitempty"` + ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"` + TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"` + ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"` + ProcurementProjectLot []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"` + TenderResult *TenderResult `xml:"TenderResult,omitempty"` } // NoticeResult contains notice result information from extensions type NoticeResult struct { - XMLName xml.Name `xml:"NoticeResult"` - TotalAmount *CurrencyText `xml:"TotalAmount,omitempty"` - LotResult []LotResult `xml:"LotResult,omitempty"` + XMLName xml.Name `xml:"NoticeResult"` + TotalAmount *CurrencyText `xml:"TotalAmount,omitempty"` + LotResult []LotResult `xml:"LotResult,omitempty"` + LotTender []LotTender `xml:"LotTender,omitempty"` + SettledContract []SettledContract `xml:"SettledContract,omitempty"` + TenderingParty []TenderingParty `xml:"TenderingParty,omitempty"` } // LotResult contains lot result information @@ -761,7 +847,14 @@ type LegalMonetaryTotal struct { // TenderingParty contains tendering party information type TenderingParty struct { - XMLName xml.Name `xml:"TenderingParty"` + XMLName xml.Name `xml:"TenderingParty"` + ID string `xml:"ID"` + Tenderer []Tenderer `xml:"Tenderer,omitempty"` +} + +// Tenderer contains tenderer information +type Tenderer struct { + XMLName xml.Name `xml:"Tenderer"` ID string `xml:"ID"` } @@ -786,13 +879,16 @@ type ReceivedSubmissionsStatistics struct { // SettledContract contains settled contract information type SettledContract struct { - XMLName xml.Name `xml:"SettledContract"` - ID string `xml:"ID"` - IssueDate string `xml:"IssueDate,omitempty"` - Title string `xml:"Title,omitempty"` - SignatoryParty *SignatoryParty `xml:"SignatoryParty,omitempty"` - ContractReference *ContractReference `xml:"ContractReference,omitempty"` - LotTender *LotTender `xml:"LotTender,omitempty"` + XMLName xml.Name `xml:"SettledContract"` + ID string `xml:"ID"` + IssueDate string `xml:"IssueDate,omitempty"` + AwardDate string `xml:"AwardDate,omitempty"` + Title string `xml:"Title,omitempty"` + ContractFrameworkIndicator string `xml:"ContractFrameworkIndicator,omitempty"` + SignatoryParty *SignatoryParty `xml:"SignatoryParty,omitempty"` + ContractReference *ContractReference `xml:"ContractReference,omitempty"` + LotTender *LotTender `xml:"LotTender,omitempty"` + NoticeDocumentReference *NoticeDocumentReference `xml:"NoticeDocumentReference,omitempty"` } // SignatoryParty contains signatory party information @@ -1439,6 +1535,62 @@ func (cn ContractNotice) GetLanguage() string { return cn.NoticeLanguageCode } +// GetNoticeStatus returns the notice status +func (cn ContractNotice) GetNoticeStatus() NoticeStatus { + if cn.StatusInformation != nil { + if cn.StatusInformation.Status != "" { + return cn.StatusInformation.Status + } + // Check for specific status indicators + if cn.StatusInformation.CancellationReason != nil { + return NoticeStatusCancelled + } + if cn.StatusInformation.AwardInformation != nil { + return NoticeStatusAwarded + } + if cn.StatusInformation.SuspensionReason != nil { + return NoticeStatusSuspended + } + if len(cn.StatusInformation.Modifications) > 0 { + return NoticeStatusModified + } + } + // Default to Published if no status information is available + return NoticeStatusPublished +} + +// GetCancellationReason returns the cancellation reason if notice is cancelled +func (cn ContractNotice) GetCancellationReason() *CancellationReason { + if cn.StatusInformation != nil { + return cn.StatusInformation.CancellationReason + } + return nil +} + +// GetAwardInformation returns the award information if notice is awarded +func (cn ContractNotice) GetAwardInformation() *AwardInformation { + if cn.StatusInformation != nil { + return cn.StatusInformation.AwardInformation + } + return nil +} + +// GetModifications returns the list of modifications if notice is modified +func (cn ContractNotice) GetModifications() []NoticeModification { + if cn.StatusInformation != nil { + return cn.StatusInformation.Modifications + } + return nil +} + +// GetSuspensionReason returns the suspension reason if notice is suspended +func (cn ContractNotice) GetSuspensionReason() *SuspensionReason { + if cn.StatusInformation != nil { + return cn.StatusInformation.SuspensionReason + } + return nil +} + // GetPublicationInfo returns publication information if available func (cn ContractNotice) GetPublicationInfo() *Publication { if cn.Extensions != nil && @@ -1759,6 +1911,47 @@ func (can ContractAwardNotice) GetLanguage() string { return can.NoticeLanguageCode } +// GetNoticeStatus returns the notice status +func (can ContractAwardNotice) GetNoticeStatus() NoticeStatus { + if can.StatusInformation != nil && can.StatusInformation.Status != "" { + return can.StatusInformation.Status + } + // Award notices are typically awarded status + return NoticeStatusAwarded +} + +// GetCancellationReason returns the cancellation reason if notice is cancelled +func (can ContractAwardNotice) GetCancellationReason() *CancellationReason { + if can.StatusInformation != nil { + return can.StatusInformation.CancellationReason + } + return nil +} + +// GetAwardInformation returns the award information if notice is awarded +func (can ContractAwardNotice) GetAwardInformation() *AwardInformation { + if can.StatusInformation != nil { + return can.StatusInformation.AwardInformation + } + return nil +} + +// GetModifications returns the list of modifications if notice is modified +func (can ContractAwardNotice) GetModifications() []NoticeModification { + if can.StatusInformation != nil { + return can.StatusInformation.Modifications + } + return nil +} + +// GetSuspensionReason returns the suspension reason if notice is suspended +func (can ContractAwardNotice) GetSuspensionReason() *SuspensionReason { + if can.StatusInformation != nil { + return can.StatusInformation.SuspensionReason + } + return nil +} + // GetPublicationInfo returns publication information if available func (can ContractAwardNotice) GetPublicationInfo() *Publication { if can.Extensions != nil && @@ -1915,3 +2108,134 @@ func (can ContractAwardNotice) GetOrganizationByID(orgID string) *Organization { } return nil } + +// GetAwardedContractors returns detailed information about awarded contractors +func (can ContractAwardNotice) GetAwardedContractors() []AwardedContractorInfo { + var contractors []AwardedContractorInfo + + noticeResult := can.GetNoticeResult() + if noticeResult == nil { + return contractors + } + + // Create a map of tender ID to tender info for quick lookup + tenderMap := make(map[string]*LotTender) + for _, lotTender := range noticeResult.LotTender { + tenderMap[lotTender.ID] = &lotTender + } + + // Create a map of tendering party ID to organization ID + partyToOrgMap := make(map[string]string) + for _, tenderingParty := range noticeResult.TenderingParty { + for _, tenderer := range tenderingParty.Tenderer { + partyToOrgMap[tenderingParty.ID] = tenderer.ID + } + } + + // Process settled contracts to extract award information + for _, contract := range noticeResult.SettledContract { + if contract.LotTender != nil { + // Get tender information + tender := tenderMap[contract.LotTender.ID] + if tender == nil { + continue + } + + // Get organization information + var org *Organization + if tender.TenderingParty != nil { + orgID := partyToOrgMap[tender.TenderingParty.ID] + if orgID != "" { + org = can.GetOrganizationByID(orgID) + } + } + + // Create awarded contractor info + contractor := AwardedContractorInfo{ + ContractID: contract.ID, + TenderID: contract.LotTender.ID, + AwardDate: contract.AwardDate, + IssueDate: contract.IssueDate, + } + + // Set contract reference if available + if contract.ContractReference != nil { + contractor.ContractReference = contract.ContractReference.ID + } + + // Set amount and currency from tender + if tender.LegalMonetaryTotal != nil { + contractor.Amount = tender.LegalMonetaryTotal.PayableAmount.Text + contractor.Currency = tender.LegalMonetaryTotal.PayableAmount.CurrencyID + } + + // Set organization information + if org != nil && org.Company != nil { + if org.Company.PartyName != nil { + contractor.Name = org.Company.PartyName.Name + } + if org.Company.PartyLegalEntity != nil { + contractor.CompanyID = org.Company.PartyLegalEntity.CompanyID + } + if org.Company.PartyIdentification != nil { + contractor.OrganizationID = org.Company.PartyIdentification.ID + } + + // Set address information + if org.Company.PostalAddress != nil { + addressParts := []string{} + if org.Company.PostalAddress.StreetName != "" { + addressParts = append(addressParts, org.Company.PostalAddress.StreetName) + } + if org.Company.PostalAddress.CityName != "" { + addressParts = append(addressParts, org.Company.PostalAddress.CityName) + } + if org.Company.PostalAddress.PostalZone != "" { + addressParts = append(addressParts, org.Company.PostalAddress.PostalZone) + } + contractor.Address = strings.Join(addressParts, ", ") + + if org.Company.PostalAddress.Country != nil { + contractor.Country = org.Company.PostalAddress.Country.IdentificationCode + } + } + + // Set contact information + if org.Company.Contact != nil { + contractor.ContactName = org.Company.Contact.Name + contractor.ContactEmail = org.Company.Contact.ElectronicMail + contractor.ContactPhone = org.Company.Contact.Telephone + } + } + + // Set lot information + if tender.TenderLot != nil { + contractor.LotID = tender.TenderLot.ID + } + + contractors = append(contractors, contractor) + } + } + + return contractors +} + +// AwardedContractorInfo contains detailed information about an awarded contractor +type AwardedContractorInfo struct { + ContractID string `json:"contract_id"` + TenderID string `json:"tender_id"` + LotID string `json:"lot_id,omitempty"` + Name string `json:"name"` + CompanyID string `json:"company_id,omitempty"` + OrganizationID string `json:"organization_id,omitempty"` + Address string `json:"address,omitempty"` + Country string `json:"country,omitempty"` + Amount string `json:"amount"` + Currency string `json:"currency,omitempty"` + AwardDate string `json:"award_date,omitempty"` + IssueDate string `json:"issue_date,omitempty"` + ContractReference string `json:"contract_reference,omitempty"` + ContactName string `json:"contact_name,omitempty"` + ContactEmail string `json:"contact_email,omitempty"` + ContactPhone string `json:"contact_phone,omitempty"` +}