Enhance Tender Entity and TED Parser with Status Management and Upsert Logic

- Added new fields to the Tender entity for handling cancellation, award, and suspension details, improving the entity's capability to manage tender statuses.
- Implemented a DetectNoticeStatus method in the TEDParser to identify the status of TED notices from XML data, enhancing the parser's functionality.
- Updated the ParseXML method to include notice status detection, ensuring accurate status representation in parsed documents.
- Introduced upsert logic in the TED scraper to handle tender records more effectively, allowing for updates or creation based on existing ContractIDs.
- Enhanced logging and error handling throughout the scraping process to improve traceability and robustness.
- Added a comprehensive usage example document to illustrate the new features and usage patterns for the TED XML parser and scraper.
This commit is contained in:
n.nakhostin
2025-09-28 10:27:20 +03:30
parent b40615ec99
commit f57e53bd9b
4 changed files with 1306 additions and 124 deletions
+150 -1
View File
@@ -121,6 +121,141 @@ func (p *TEDParser) DetectDocumentType(xmlData []byte) (DocumentType, error) {
return "", fmt.Errorf("unknown document type")
}
// DetectNoticeStatus detects the status of a TED notice from XML data
func (p *TEDParser) DetectNoticeStatus(xmlData []byte) NoticeStatus {
// Convert to string for fast text search
xmlStr := string(xmlData)
// Check for common status indicators in XML content
// Many TED XMLs contain status information in extensions or specific tags
// Check for cancellation indicators
if strings.Contains(xmlStr, "<CancellationReason") ||
strings.Contains(xmlStr, "cancelled") ||
strings.Contains(xmlStr, "canceled") ||
strings.Contains(xmlStr, "CANCELLED") ||
strings.Contains(xmlStr, "CANCELED") {
return NoticeStatusCancelled
}
// Check for award indicators
if strings.Contains(xmlStr, "<AwardDate") ||
strings.Contains(xmlStr, "<TenderResult") ||
strings.Contains(xmlStr, "<SettledContract") ||
strings.Contains(xmlStr, "<ContractAwardNotice") ||
strings.Contains(xmlStr, "awarded") ||
strings.Contains(xmlStr, "AWARDED") {
return NoticeStatusAwarded
}
// Check for modification indicators
if strings.Contains(xmlStr, "<ChangeNotice") ||
strings.Contains(xmlStr, "<ChangeReason") ||
strings.Contains(xmlStr, "<Modification") ||
strings.Contains(xmlStr, "modified") ||
strings.Contains(xmlStr, "MODIFIED") {
return NoticeStatusModified
}
// Check for suspension indicators
if strings.Contains(xmlStr, "<SuspensionReason") ||
strings.Contains(xmlStr, "suspended") ||
strings.Contains(xmlStr, "SUSPENDED") {
return NoticeStatusSuspended
}
// Check for closed indicators
if strings.Contains(xmlStr, "closed") ||
strings.Contains(xmlStr, "CLOSED") ||
strings.Contains(xmlStr, "<ClosedDate") {
return NoticeStatusClosed
}
// Check for draft indicators
if strings.Contains(xmlStr, "draft") ||
strings.Contains(xmlStr, "DRAFT") {
return NoticeStatusDraft
}
// Check for explicit status in XML using XML parsing
status := p.extractStatusFromXML(xmlData)
if status != "" {
switch strings.ToLower(status) {
case "draft":
return NoticeStatusDraft
case "published":
return NoticeStatusPublished
case "cancelled", "canceled":
return NoticeStatusCancelled
case "awarded":
return NoticeStatusAwarded
case "closed":
return NoticeStatusClosed
case "modified":
return NoticeStatusModified
case "suspended":
return NoticeStatusSuspended
}
}
// Default to Published for most TED notices
return NoticeStatusPublished
}
// extractStatusFromXML extracts status information using XML parsing
func (p *TEDParser) extractStatusFromXML(xmlData []byte) string {
decoder := xml.NewDecoder(bytes.NewReader(xmlData))
for {
token, err := decoder.Token()
if err != nil {
break
}
if startElement, ok := token.(xml.StartElement); ok {
// Look for common status element names
switch startElement.Name.Local {
case "NoticeStatus", "Status", "StatusCode", "DocumentStatus":
// Read the content of this element
if content, err := p.readElementContent(decoder); err == nil {
return strings.TrimSpace(content)
}
}
// Check attributes for status information
for _, attr := range startElement.Attr {
if strings.Contains(strings.ToLower(attr.Name.Local), "status") {
return strings.TrimSpace(attr.Value)
}
}
}
}
return ""
}
// readElementContent reads the text content of the current XML element
func (p *TEDParser) readElementContent(decoder *xml.Decoder) (string, error) {
var content strings.Builder
for {
token, err := decoder.Token()
if err != nil {
return "", err
}
switch t := token.(type) {
case xml.CharData:
content.Write(t)
case xml.EndElement:
return content.String(), nil
case xml.StartElement:
// Skip nested elements
decoder.Skip()
}
}
}
// ParseXML automatically detects and parses any supported TED XML document type
func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
docType, err := p.DetectDocumentType(xmlData)
@@ -128,7 +263,13 @@ func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
return nil, fmt.Errorf("failed to detect document type: %w", err)
}
parsedDoc := &ParsedDocument{Type: docType}
// Detect notice status
noticeStatus := p.DetectNoticeStatus(xmlData)
parsedDoc := &ParsedDocument{
Type: docType,
Status: noticeStatus,
}
switch docType {
case DocumentTypeContractNotice:
@@ -137,6 +278,10 @@ func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
return nil, fmt.Errorf("failed to parse contract notice: %w", err)
}
parsedDoc.ContractNotice = parsed
// Override status based on document content
if parsed != nil {
parsedDoc.Status = parsed.GetNoticeStatus()
}
case DocumentTypeContractAwardNotice:
parsed, err := p.contractAwardNoticeParser.ParseBytes(xmlData)
@@ -144,6 +289,10 @@ func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
return nil, fmt.Errorf("failed to parse contract award notice: %w", err)
}
parsedDoc.ContractAwardNotice = parsed
// Override status based on document content
if parsed != nil {
parsedDoc.Status = parsed.GetNoticeStatus()
}
case DocumentTypePriorInformationNotice:
parsed, err := p.priorInformationNoticeParser.ParseBytes(xmlData)
+706 -36
View File
@@ -64,11 +64,11 @@ type JobResults struct {
// ScrapingState represents the current state of scraping
type ScrapingState struct {
mongo.Model
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"`
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"`
IsRunning bool `bson:"is_running" json:"is_running"`
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"`
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"`
IsRunning bool `bson:"is_running" json:"is_running"`
CurrentJobID bson.ObjectID `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
}
@@ -85,6 +85,22 @@ type Config struct {
ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
}
// ScrapingMode represents the type of scraping operation
type ScrapingMode string
const (
ScrapingModeDaily ScrapingMode = "daily" // Daily bulk fetch mode
ScrapingModeManual ScrapingMode = "manual" // Manual date range mode
ScrapingModeRealtime ScrapingMode = "realtime" // Real-time processing mode
)
// DateRangeConfig holds configuration for manual date range scraping
type DateRangeConfig struct {
StartDate time.Time `json:"start_date"`
EndDate time.Time `json:"end_date"`
Mode ScrapingMode `json:"mode"`
}
// NewTEDScraper creates a new TED scraper instance
func NewTEDScraper(
config *Config,
@@ -245,7 +261,8 @@ func (s *TEDScraper) ScrapeLatest(ctx context.Context) error {
} else {
job.Progress.ProcessedFiles++
job.Results.TotalTenders += fileResult.ProcessedCount
job.Results.NewTenders += fileResult.SuccessCount
job.Results.NewTenders += fileResult.NewTenders
job.Results.UpdatedTenders += fileResult.UpdatedTenders
if len(fileResult.Errors) > 0 {
job.Results.ErrorCount += len(fileResult.Errors)
}
@@ -272,6 +289,8 @@ func (s *TEDScraper) ScrapeLatest(ctx context.Context) error {
s.logger.Info("TED XML scraping completed", map[string]interface{}{
"files_processed": job.Progress.ProcessedFiles,
"tenders_created": job.Results.NewTenders,
"tenders_updated": job.Results.UpdatedTenders,
"total_tenders": job.Results.TotalTenders,
"error_count": job.Results.ErrorCount,
"success_rate": job.Results.SuccessRate,
})
@@ -347,6 +366,8 @@ type FileProcessingResult struct {
Number int `json:"number"`
ProcessedCount int `json:"processed_count"`
SuccessCount int `json:"success_count"`
NewTenders int `json:"new_tenders"`
UpdatedTenders int `json:"updated_tenders"`
Errors []string `json:"errors"`
ProcessedAt int64 `json:"processed_at"`
}
@@ -527,9 +548,17 @@ func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fil
}
// Parse and process XML content
if err := s.processXMLContent(ctx, content, fileName, fileInfo, result); err != nil {
processResult, err := s.processXMLContent(ctx, content, fileName, fileInfo)
if err != nil {
errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err)
result.Errors = append(result.Errors, errMsg)
} else {
result.SuccessCount++
if processResult.IsNew {
result.NewTenders++
} else {
result.UpdatedTenders++
}
}
result.ProcessedCount++
@@ -556,8 +585,15 @@ func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fil
return result, nil
}
// XMLProcessResult represents the result of processing a single XML file
type XMLProcessResult struct {
IsNew bool `json:"is_new"`
TenderID string `json:"tender_id"`
ContractID string `json:"contract_id"`
}
// processXMLContent parses XML content and creates/updates tender
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo, result *FileProcessingResult) error {
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo) (*XMLProcessResult, error) {
s.logger.Info("Starting XML processing", map[string]interface{}{
"file_name": fileName,
"file_size": len(content),
@@ -570,7 +606,7 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
"file_name": fileName,
"error": err.Error(),
})
return fmt.Errorf("failed to parse XML: %w", err)
return nil, fmt.Errorf("failed to parse XML: %w", err)
}
// Get the underlying document for ID and validation
@@ -612,33 +648,45 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
sourceURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number)
tenderEntity := s.mapToTenderFromParsedDoc(parsedDoc, sourceURL, fileName)
// Handle status-specific information and award logic
s.applyStatusSpecificInformation(tenderEntity, parsedDoc)
s.applyAwardLogic(tenderEntity, parsedDoc)
s.logger.Info("Mapped to tender entity", map[string]interface{}{
"file_name": fileName,
"tender_id": tenderEntity.TenderID,
"contract_notice_id": tenderEntity.ContractNoticeID,
"title": tenderEntity.Title,
"status": string(tenderEntity.Status),
})
// Try to find existing tender by contract notice ID
// Try to find existing tender by contract notice ID (ContractID)
s.logger.Info("Checking for existing tender", map[string]interface{}{
"contract_notice_id": tenderEntity.ContractNoticeID,
})
existingTender, err := s.findTenderByContractNoticeID(ctx, tenderEntity.ContractNoticeID)
if err != nil {
if err != nil && err.Error() != "document not found" {
s.logger.Warn("Error checking for existing tender", map[string]interface{}{
"contract_notice_id": tenderEntity.ContractNoticeID,
"error": err.Error(),
})
}
var processResult *XMLProcessResult
if err == nil && existingTender != nil {
// Update existing tender
s.logger.Info("Found existing tender, updating", map[string]interface{}{
"existing_tender_id": existingTender.ID,
"contract_notice_id": tenderEntity.ContractNoticeID,
"old_status": string(existingTender.Status),
"new_status": string(tenderEntity.Status),
})
// Merge updates with existing data
s.mergeExistingTenderData(existingTender, tenderEntity)
tenderEntity.ID = existingTender.ID
tenderEntity.CreatedAt = existingTender.CreatedAt
tenderEntity.UpdatedAt = time.Now().Unix()
@@ -649,17 +697,24 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
"tender_id": tenderEntity.ID,
"error": err.Error(),
})
return fmt.Errorf("failed to update existing tender: %w", err)
return nil, fmt.Errorf("failed to update existing tender: %w", err)
}
s.logger.Info("Successfully updated tender", map[string]interface{}{
"tender_id": tenderEntity.ID,
})
processResult = &XMLProcessResult{
IsNew: false,
TenderID: tenderEntity.TenderID,
ContractID: tenderEntity.ContractNoticeID,
}
} else {
// Create new tender
s.logger.Info("Creating new tender", map[string]interface{}{
"contract_notice_id": tenderEntity.ContractNoticeID,
"tender_id": tenderEntity.TenderID,
"status": string(tenderEntity.Status),
})
tenderEntity.CreatedAt = time.Now().Unix()
@@ -672,25 +727,30 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
"tender_id": tenderEntity.TenderID,
"error": err.Error(),
})
return fmt.Errorf("failed to create tender: %w", err)
return nil, fmt.Errorf("failed to create tender: %w", err)
}
s.logger.Info("Successfully created new tender", map[string]interface{}{
"tender_id": tenderEntity.TenderID,
"contract_notice_id": tenderEntity.ContractNoticeID,
})
}
result.SuccessCount++
processResult = &XMLProcessResult{
IsNew: true,
TenderID: tenderEntity.TenderID,
ContractID: tenderEntity.ContractNoticeID,
}
}
s.logger.Debug("Successfully processed XML file", map[string]interface{}{
"file_name": fileName,
"document_type": string(parsedDoc.Type),
"document_id": documentID,
"tender_id": tenderEntity.TenderID,
"is_new": processResult.IsNew,
})
return nil
return processResult, nil
}
// Helper methods for database operations
@@ -940,8 +1000,8 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
NoticeTypeCode: cn.GetNoticeSubType(),
NoticeSubTypeCode: cn.GetNoticeSubType(),
NoticeLanguageCode: cn.NoticeLanguageCode,
IssueDate: s.parseDateToUnixMilli(cn.IssueDate),
IssueTime: s.parseTimeToUnixMilli(cn.IssueTime),
IssueDate: s.parseDateToUnix(cn.IssueDate),
IssueTime: s.parseTimeToUnix(cn.IssueTime),
GazetteID: cn.GetOJSID(),
Title: cn.GetProcurementTitle(),
Description: cn.GetProcurementDescription(),
@@ -964,7 +1024,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
// Set publication information
if pubInfo := cn.GetPublicationInfo(); pubInfo != nil {
t.NoticePublicationID = pubInfo.NoticePublicationID
t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate)
t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate)
t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
}
@@ -993,17 +1053,17 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
// Set tender deadline
if deadline := cn.GetTenderDeadline(); deadline != "" {
t.TenderDeadline = s.parseDateToUnixMilli(deadline)
t.TenderDeadline = s.parseDateToUnix(deadline)
if deadlineTime, err := s.parseDate(deadline); err == nil {
applicationDeadline := s.calculateApplicationDeadline(deadlineTime)
t.ApplicationDeadline = applicationDeadline.UnixMilli()
t.ApplicationDeadline = applicationDeadline.Unix()
}
}
if pubDate, err := s.parseDate(cn.GetPublicationDate()); err == nil {
submissionDeadline := s.calculateSubmissionDeadline(pubDate)
t.SubmissionDeadline = submissionDeadline.UnixMilli()
t.SubmissionDeadline = submissionDeadline.Unix()
}
// Set SubmissionURL from TenderingTerms
@@ -1089,8 +1149,8 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
NoticeTypeCode: can.NoticeTypeCode,
NoticeSubTypeCode: can.GetNoticeSubType(),
NoticeLanguageCode: can.NoticeLanguageCode,
IssueDate: s.parseDateToUnixMilli(can.IssueDate),
IssueTime: s.parseTimeToUnixMilli(can.IssueTime),
IssueDate: s.parseDateToUnix(can.IssueDate),
IssueTime: s.parseTimeToUnix(can.IssueTime),
GazetteID: can.GetOJSID(),
Title: can.GetProcurementTitle(),
Description: can.GetProcurementDescription(),
@@ -1111,7 +1171,7 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
// Set publication information
if pubInfo := can.GetPublicationInfo(); pubInfo != nil {
t.NoticePublicationID = pubInfo.NoticePublicationID
t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate)
t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate)
t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
}
@@ -1234,7 +1294,7 @@ func (s *TEDScraper) generateTenderID(t *tender.Tender) string {
// parts = append(parts, buyerCode)
// Date (YYYYMMDD format from issue date)
parts = append(parts, s.padOrTruncate(s.unixMilliToDateString(t.IssueDate), 4))
parts = append(parts, s.padOrTruncate(s.UnixToDateString(t.IssueDate), 4))
// Location code (country code + region code, padded to 6 chars)
locationCode := "EU"
@@ -1310,21 +1370,21 @@ func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) {
return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr)
}
// parseDateToUnixMilli converts string date to Unix milliseconds (int64)
func (s *TEDScraper) parseDateToUnixMilli(dateStr string) int64 {
// parseDateToUnix converts string date to Unix milliseconds (int64)
func (s *TEDScraper) parseDateToUnix(dateStr string) int64 {
if dateStr == "" {
return 0
}
if parsedTime, err := s.parseDate(dateStr); err == nil {
return parsedTime.UnixMilli()
return parsedTime.Unix()
}
return 0
}
// parseTimeToUnixMilli converts string time to Unix milliseconds (int64)
func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 {
// parseTimeToUnix converts string time to Unix milliseconds (int64)
func (s *TEDScraper) parseTimeToUnix(timeStr string) int64 {
if timeStr == "" {
return 0
}
@@ -1337,19 +1397,19 @@ func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 {
}
if parsedTime, err := s.parseDate(timeStr); err == nil {
return parsedTime.UnixMilli()
return parsedTime.Unix()
}
return 0
}
// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation
func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string {
if unixMilli == 0 {
// UnixToDateString converts Unix milliseconds to date string for TenderID generation
func (s *TEDScraper) UnixToDateString(u int64) string {
if u == 0 {
return "00000000"
}
return time.UnixMilli(unixMilli).Format("20060102")
return time.Unix(u, 0).Format("20060102")
}
func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time {
@@ -1395,3 +1455,613 @@ func (s *TEDScraper) adjustToWorkingDay(targetDate time.Time, forward bool) time
return current
}
// mapTEDStatusToTenderStatus maps TED notice status to tender status
func (s *TEDScraper) mapTEDStatusToTenderStatus(tedStatus NoticeStatus) tender.TenderStatus {
switch tedStatus {
case NoticeStatusDraft:
return tender.TenderStatusDraft
case NoticeStatusPublished:
return tender.TenderStatusPublished
case NoticeStatusCancelled:
return tender.TenderStatusCancelled
case NoticeStatusAwarded:
return tender.TenderStatusAwarded
case NoticeStatusClosed:
return tender.TenderStatusClosed
case NoticeStatusModified:
return tender.TenderStatusModified
case NoticeStatusSuspended:
return tender.TenderStatusSuspended
default:
// Default to active for unknown statuses, but log a warning
s.logger.Warn("Unknown TED notice status, defaulting to active", map[string]interface{}{
"ted_status": string(tedStatus),
})
return tender.TenderStatusActive
}
}
// applyStatusSpecificInformation applies status-specific information to the tender entity
func (s *TEDScraper) applyStatusSpecificInformation(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) {
if tenderEntity == nil || parsedDoc == nil {
return
}
// Map TED status to tender status
tenderEntity.Status = s.mapTEDStatusToTenderStatus(parsedDoc.Status)
// Get the document to access status-specific methods
document := parsedDoc.GetDocument()
if document == nil {
return
}
// Apply status-specific information based on document type
switch parsedDoc.Type {
case DocumentTypeContractNotice:
if cn, ok := document.(*ContractNotice); ok {
s.applyContractNoticeStatusInfo(tenderEntity, cn, parsedDoc.Status)
}
case DocumentTypeContractAwardNotice:
if can, ok := document.(*ContractAwardNotice); ok {
s.applyContractAwardNoticeStatusInfo(tenderEntity, can, parsedDoc.Status)
}
default:
// For other document types, we rely on the parsed status
s.logger.Debug("Applied basic status information for document type", map[string]interface{}{
"document_type": string(parsedDoc.Type),
"status": string(parsedDoc.Status),
})
}
}
// applyContractNoticeStatusInfo applies status-specific information from ContractNotice
func (s *TEDScraper) applyContractNoticeStatusInfo(tenderEntity *tender.Tender, cn *ContractNotice, status NoticeStatus) {
switch status {
case NoticeStatusCancelled:
if cancellation := cn.GetCancellationReason(); cancellation != nil {
tenderEntity.CancellationReason = cancellation.Description
if cancellation.ReasonCode != "" {
tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description)
}
// Note: TED XML doesn't typically have cancellation date, so we use current time
tenderEntity.CancellationDate = time.Now().Unix()
}
case NoticeStatusAwarded:
if award := cn.GetAwardInformation(); award != nil {
if award.AwardDate != "" {
tenderEntity.AwardDate = s.parseDateToUnix(award.AwardDate)
}
if award.AwardedValue.Text != "" {
if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil {
tenderEntity.AwardedValue = value
}
}
tenderEntity.ContractNumber = award.ContractNumber
if award.WinningTenderer != nil && award.WinningTenderer.Company != nil {
tenderEntity.WinningTenderer = s.mapOrganization(&Organization{
Company: award.WinningTenderer.Company,
}, "winner")
}
}
case NoticeStatusSuspended:
if suspension := cn.GetSuspensionReason(); suspension != nil {
tenderEntity.SuspensionReason = suspension.Description
if suspension.ReasonCode != "" {
tenderEntity.SuspensionReason = fmt.Sprintf("%s: %s", suspension.ReasonCode, suspension.Description)
}
tenderEntity.SuspensionDate = time.Now().Unix()
}
case NoticeStatusModified:
if modifications := cn.GetModifications(); modifications != nil {
for _, mod := range modifications {
tenderMod := tender.TenderModification{
ModificationDate: s.parseDateToUnix(mod.ModificationDate),
ModificationReason: mod.ModificationReason,
Description: mod.Description,
LanguageID: mod.LanguageID,
}
tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod)
}
}
}
}
// applyContractAwardNoticeStatusInfo applies status-specific information from ContractAwardNotice
func (s *TEDScraper) applyContractAwardNoticeStatusInfo(tenderEntity *tender.Tender, can *ContractAwardNotice, status NoticeStatus) {
// Contract award notices are typically awarded status
tenderEntity.Status = tender.TenderStatusAwarded
// Set award date from tender result
if can.TenderResult != nil && can.TenderResult.AwardDate != "" {
tenderEntity.AwardDate = s.parseDateToUnix(can.TenderResult.AwardDate)
}
// Get award information from extensions if available
if award := can.GetAwardInformation(); award != nil {
if award.AwardedValue.Text != "" {
if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil {
tenderEntity.AwardedValue = value
}
}
tenderEntity.ContractNumber = award.ContractNumber
}
// Get total award value from notice result
if amount, currency := can.GetTotalAwardValue(); amount != "" {
if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
tenderEntity.AwardedValue = parsedAmount
tenderEntity.Currency = currency
}
}
// Extract detailed awarded contractors information
s.extractAwardedEntities(tenderEntity, can)
// Handle other statuses if applicable
switch status {
case NoticeStatusCancelled:
if cancellation := can.GetCancellationReason(); cancellation != nil {
tenderEntity.Status = tender.TenderStatusCancelled
tenderEntity.CancellationReason = cancellation.Description
if cancellation.ReasonCode != "" {
tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description)
}
tenderEntity.CancellationDate = time.Now().Unix()
}
case NoticeStatusModified:
if modifications := can.GetModifications(); modifications != nil {
tenderEntity.Status = tender.TenderStatusModified
for _, mod := range modifications {
tenderMod := tender.TenderModification{
ModificationDate: s.parseDateToUnix(mod.ModificationDate),
ModificationReason: mod.ModificationReason,
Description: mod.Description,
LanguageID: mod.LanguageID,
}
tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod)
}
}
}
}
// extractAwardedEntities extracts detailed information about awarded entities from a contract award notice
func (s *TEDScraper) extractAwardedEntities(tenderEntity *tender.Tender, can *ContractAwardNotice) {
awardedContractors := can.GetAwardedContractors()
if len(awardedContractors) == 0 {
return
}
s.logger.Info("Extracting awarded entities", map[string]interface{}{
"contract_notice_id": can.ID,
"num_contractors": len(awardedContractors),
})
for _, contractor := range awardedContractors {
entity := tender.AwardedEntity{
Name: contractor.Name,
Address: contractor.Address,
Country: contractor.Country,
Currency: contractor.Currency,
ContractID: contractor.ContractID,
TenderID: contractor.TenderID,
LotID: contractor.LotID,
CompanyID: contractor.CompanyID,
OrganizationID: contractor.OrganizationID,
}
// Parse amount
if contractor.Amount != "" {
if amount, err := strconv.ParseFloat(contractor.Amount, 64); err == nil {
entity.Amount = amount
} else {
s.logger.Warn("Failed to parse award amount", map[string]interface{}{
"amount_string": contractor.Amount,
"contractor": contractor.Name,
"error": err.Error(),
})
}
}
// Parse award date
if contractor.AwardDate != "" {
entity.AwardDate = s.parseDateToUnix(contractor.AwardDate)
} else if contractor.IssueDate != "" {
entity.AwardDate = s.parseDateToUnix(contractor.IssueDate)
}
// Calculate share percentage if there are multiple contractors
totalAmount := tenderEntity.AwardedValue
if totalAmount > 0 && entity.Amount > 0 {
entity.Share = (entity.Amount / totalAmount) * 100
}
tenderEntity.AwardedEntities = append(tenderEntity.AwardedEntities, entity)
s.logger.Debug("Added awarded entity", map[string]interface{}{
"name": entity.Name,
"amount": entity.Amount,
"currency": entity.Currency,
"share": entity.Share,
"contract_id": entity.ContractID,
})
}
s.logger.Info("Successfully extracted awarded entities", map[string]interface{}{
"total_entities": len(tenderEntity.AwardedEntities),
"total_value": tenderEntity.AwardedValue,
"currency": tenderEntity.Currency,
})
}
// ScrapeByDateRange scrapes TED XML files within a specified date range
func (s *TEDScraper) ScrapeByDateRange(ctx context.Context, config DateRangeConfig) error {
startTime := time.Now()
s.logger.Info("Starting TED XML scraping by date range", map[string]interface{}{
"start_date": config.StartDate.Format("2006-01-02"),
"end_date": config.EndDate.Format("2006-01-02"),
"mode": string(config.Mode),
"timestamp": startTime.Unix(),
})
// Validate date range
if config.EndDate.Before(config.StartDate) {
return fmt.Errorf("end date cannot be before start date")
}
// Calculate the date range in days
daysInRange := int(config.EndDate.Sub(config.StartDate).Hours()/24) + 1
if daysInRange > 365 {
return fmt.Errorf("date range too large: maximum 365 days allowed")
}
// Create scraping job
job := &ScrapingJob{
Type: fmt.Sprintf("ted_%s", config.Mode),
Status: "running",
StartedAt: startTime.Unix(),
Progress: JobProgress{},
Results: JobResults{},
Config: map[string]interface{}{
"base_url": s.config.BaseURL,
"start_date": config.StartDate.Format("2006-01-02"),
"end_date": config.EndDate.Format("2006-01-02"),
"mode": string(config.Mode),
},
}
// Save job
if err := s.saveScrapingJob(ctx, job); err != nil {
s.logger.Error("Failed to save scraping job", map[string]interface{}{
"error": err.Error(),
})
}
// Calculate files to download based on date range
filesToDownload := s.calculateFilesInDateRange(config.StartDate, config.EndDate)
if len(filesToDownload) == 0 {
s.logger.Info("No files to download in date range", map[string]interface{}{
"start_date": config.StartDate.Format("2006-01-02"),
"end_date": config.EndDate.Format("2006-01-02"),
})
// Complete the job
s.completeJob(ctx, job, nil, nil)
return nil
}
// Update job progress
job.Progress.TotalFiles = len(filesToDownload)
s.saveScrapingJob(ctx, job)
var lastSuccessfulFile *FileInfo
// Download and process files
for i, fileInfo := range filesToDownload {
job.Progress.CurrentFile = fmt.Sprintf("%d%05d", fileInfo.Year, fileInfo.Number)
s.saveScrapingJob(ctx, job)
fileResult, err := s.downloadAndProcess(ctx, fileInfo)
if err != nil {
s.logger.Error("Failed to download and process file", map[string]interface{}{
"year": fileInfo.Year,
"number": fileInfo.Number,
"error": err.Error(),
})
job.Progress.FailedFiles++
job.Results.ErrorCount++
} else {
job.Progress.ProcessedFiles++
job.Results.TotalTenders += fileResult.ProcessedCount
job.Results.NewTenders += fileResult.NewTenders
job.Results.UpdatedTenders += fileResult.UpdatedTenders
if len(fileResult.Errors) > 0 {
job.Results.ErrorCount += len(fileResult.Errors)
}
lastSuccessfulFile = &fileInfo
}
// Update progress percentage
job.Progress.PercentComplete = float64(i+1) / float64(len(filesToDownload)) * 100
s.saveScrapingJob(ctx, job)
// Check context for cancellation
select {
case <-ctx.Done():
s.completeJob(ctx, job, nil, lastSuccessfulFile)
return ctx.Err()
default:
// Continue processing
}
}
// Complete the job
s.completeJob(ctx, job, nil, lastSuccessfulFile)
s.logger.Info("TED XML scraping by date range completed", map[string]interface{}{
"files_processed": job.Progress.ProcessedFiles,
"tenders_created": job.Results.NewTenders,
"tenders_updated": job.Results.UpdatedTenders,
"error_count": job.Results.ErrorCount,
"success_rate": job.Results.SuccessRate,
})
return nil
}
// calculateFilesInDateRange calculates which files need to be downloaded for a date range
func (s *TEDScraper) calculateFilesInDateRange(startDate, endDate time.Time) []FileInfo {
var files []FileInfo
// TED files are published roughly once per working day
// We estimate file numbers based on working days from start of year
for year := startDate.Year(); year <= endDate.Year(); year++ {
yearStart := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC)
yearEnd := time.Date(year, 12, 31, 23, 59, 59, 0, time.UTC)
// Determine the effective start and end dates for this year
effectiveStart := startDate
if yearStart.After(startDate) {
effectiveStart = yearStart
}
effectiveEnd := endDate
if yearEnd.Before(endDate) {
effectiveEnd = yearEnd
}
// Skip if no overlap with this year
if effectiveStart.After(effectiveEnd) {
continue
}
// Calculate approximate file range for this year
startDayOfYear := effectiveStart.YearDay()
endDayOfYear := effectiveEnd.YearDay()
// Estimate file numbers (roughly 0.7 files per day, accounting for weekends)
startFileNum := int(float64(startDayOfYear) * 0.7)
endFileNum := int(float64(endDayOfYear) * 0.7)
// Ensure minimum range
if startFileNum < 1 {
startFileNum = 1
}
if endFileNum < startFileNum {
endFileNum = startFileNum
}
// Add some buffer
startFileNum = max(1, startFileNum-2)
endFileNum = min(366, endFileNum+2) // Max ~366 files per year
s.logger.Info("Calculating file range for year", map[string]interface{}{
"year": year,
"start_file_num": startFileNum,
"end_file_num": endFileNum,
"start_date": effectiveStart.Format("2006-01-02"),
"end_date": effectiveEnd.Format("2006-01-02"),
})
for number := startFileNum; number <= endFileNum; number++ {
fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number)
ojsID := fmt.Sprintf("S %03d / %d", number, year)
files = append(files, FileInfo{
Year: year,
Number: number,
URL: fileURL,
OJS: ojsID,
})
}
}
s.logger.Info("Calculated files for date range", map[string]interface{}{
"total_files": len(files),
"start_date": startDate.Format("2006-01-02"),
"end_date": endDate.Format("2006-01-02"),
})
return files
}
// Helper function for max
func max(a, b int) int {
if a > b {
return a
}
return b
}
// Helper function for min
func min(a, b int) int {
if a < b {
return a
}
return b
}
// applyAwardLogic applies award-specific logic based on document type and content
func (s *TEDScraper) applyAwardLogic(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) {
if tenderEntity == nil || parsedDoc == nil {
return
}
// Only set award information if the tender has a confirmed winner
hasConfirmedWinner := s.hasConfirmedWinner(parsedDoc)
s.logger.Debug("Applying award logic", map[string]interface{}{
"document_type": string(parsedDoc.Type),
"status": string(parsedDoc.Status),
"has_confirmed_winner": hasConfirmedWinner,
})
if !hasConfirmedWinner {
// Clear any award date and winner information for tenders without confirmed winners
tenderEntity.AwardDate = 0
tenderEntity.AwardedValue = 0
tenderEntity.WinningTenderer = nil
tenderEntity.AwardedEntities = nil
tenderEntity.ContractNumber = ""
s.logger.Debug("Cleared award information for tender without confirmed winner", map[string]interface{}{
"contract_notice_id": tenderEntity.ContractNoticeID,
})
return
}
// Process award information for tenders with confirmed winners
s.logger.Info("Processing award information for tender with confirmed winner", map[string]interface{}{
"contract_notice_id": tenderEntity.ContractNoticeID,
"document_type": string(parsedDoc.Type),
})
}
// hasConfirmedWinner checks if the tender has a confirmed winner based on the document content
func (s *TEDScraper) hasConfirmedWinner(parsedDoc *ParsedDocument) bool {
if parsedDoc == nil {
return false
}
// Check for award-specific document types
switch parsedDoc.Type {
case DocumentTypeContractAwardNotice, DocumentTypeResultNotice:
// These document types typically indicate awarded tenders
if parsedDoc.Status == NoticeStatusAwarded {
return true
}
// Check for specific award indicators in the document
document := parsedDoc.GetDocument()
if document != nil {
return s.checkForAwardIndicators(document, parsedDoc.Type)
}
case DocumentTypeContractNotice:
// Contract notices may have award information if they're updated with results
if parsedDoc.Status == NoticeStatusAwarded {
if cn := parsedDoc.ContractNotice; cn != nil {
return s.checkForAwardIndicators(cn, parsedDoc.Type)
}
}
}
return false
}
// checkForAwardIndicators checks for specific award indicators in the document
func (s *TEDScraper) checkForAwardIndicators(document TEDDocument, docType DocumentType) bool {
switch docType {
case DocumentTypeContractAwardNotice:
if can, ok := document.(*ContractAwardNotice); ok {
// Check for tender result with award date
if can.TenderResult != nil && can.TenderResult.AwardDate != "" {
return true
}
// Check for awarded contractors
awardedContractors := can.GetAwardedContractors()
if len(awardedContractors) > 0 {
return true
}
// Check for award information in extensions
if award := can.GetAwardInformation(); award != nil && award.AwardDate != "" {
return true
}
}
case DocumentTypeContractNotice:
if cn, ok := document.(*ContractNotice); ok {
// Check for award information
if award := cn.GetAwardInformation(); award != nil && award.AwardDate != "" {
return true
}
}
case DocumentTypeResultNotice:
// Result notices typically contain award information
return true
}
return false
}
// mergeExistingTenderData merges new data with existing tender data, preserving important existing information
func (s *TEDScraper) mergeExistingTenderData(existing *tender.Tender, updated *tender.Tender) {
// Preserve creation timestamp and original metadata
updated.CreatedAt = existing.CreatedAt
// Merge processing metadata
if existing.ProcessingMetadata.ScrapedAt != 0 {
// Keep original scraped time, update processed time
updated.ProcessingMetadata.ScrapedAt = existing.ProcessingMetadata.ScrapedAt
}
// Preserve existing award information if the new document doesn't have confirmed winners
// but existing data does (to avoid losing award data on subsequent updates)
if existing.AwardDate != 0 && updated.AwardDate == 0 && len(existing.AwardedEntities) > 0 {
s.logger.Info("Preserving existing award information", map[string]interface{}{
"contract_notice_id": existing.ContractNoticeID,
"existing_award_date": existing.AwardDate,
"awarded_entities_count": len(existing.AwardedEntities),
})
updated.AwardDate = existing.AwardDate
updated.AwardedValue = existing.AwardedValue
updated.WinningTenderer = existing.WinningTenderer
updated.AwardedEntities = existing.AwardedEntities
updated.ContractNumber = existing.ContractNumber
}
// Merge modifications (append new modifications to existing ones)
if len(existing.Modifications) > 0 {
// Create a map to avoid duplicate modifications
modMap := make(map[string]tender.TenderModification)
// Add existing modifications
for _, mod := range existing.Modifications {
key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason)
modMap[key] = mod
}
// Add new modifications
for _, mod := range updated.Modifications {
key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason)
modMap[key] = mod
}
// Convert back to slice
updated.Modifications = make([]tender.TenderModification, 0, len(modMap))
for _, mod := range modMap {
updated.Modifications = append(updated.Modifications, mod)
}
}
}
+408 -84
View File
@@ -3,6 +3,7 @@ package ted
import (
"encoding/xml"
"fmt"
"strings"
"time"
)
@@ -27,9 +28,24 @@ const (
DocumentTypeSubcontractNotice DocumentType = "SubcontractNotice"
)
// NoticeStatus represents the status of a TED notice
type NoticeStatus string
// Notice status constants for all possible TED notice statuses
const (
NoticeStatusDraft NoticeStatus = "Draft"
NoticeStatusPublished NoticeStatus = "Published"
NoticeStatusCancelled NoticeStatus = "Cancelled"
NoticeStatusAwarded NoticeStatus = "Awarded"
NoticeStatusClosed NoticeStatus = "Closed"
NoticeStatusModified NoticeStatus = "Modified"
NoticeStatusSuspended NoticeStatus = "Suspended"
)
// ParsedDocument wraps any TED document with type information
type ParsedDocument struct {
Type DocumentType
Type DocumentType `json:"type"`
Status NoticeStatus `json:"status"`
ContractNotice *ContractNotice `json:"contract_notice,omitempty"`
ContractAwardNotice *ContractAwardNotice `json:"contract_award_notice,omitempty"`
PriorInformationNotice *PriorInformationNotice `json:"prior_information_notice,omitempty"`
@@ -78,6 +94,7 @@ type TEDDocument interface {
GetID() string
GetNoticeType() string
GetLanguage() string
GetNoticeStatus() NoticeStatus
Validate() error
}
@@ -123,6 +140,11 @@ func (pin PriorInformationNotice) GetLanguage() string {
return pin.NoticeLanguageCode
}
func (pin PriorInformationNotice) GetNoticeStatus() NoticeStatus {
// Prior information notices are typically published
return NoticeStatusPublished
}
func (pin PriorInformationNotice) Validate() error {
if pin.ID == "" {
return fmt.Errorf("prior information notice ID is required")
@@ -163,9 +185,10 @@ type DesignContest struct {
}
// Interface methods for DesignContest
func (dc DesignContest) GetID() string { return dc.ID }
func (dc DesignContest) GetNoticeType() string { return dc.NoticeTypeCode }
func (dc DesignContest) GetLanguage() string { return dc.NoticeLanguageCode }
func (dc DesignContest) GetID() string { return dc.ID }
func (dc DesignContest) GetNoticeType() string { return dc.NoticeTypeCode }
func (dc DesignContest) GetLanguage() string { return dc.NoticeLanguageCode }
func (dc DesignContest) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
func (dc DesignContest) Validate() error {
if dc.ID == "" {
return fmt.Errorf("design contest ID is required")
@@ -205,9 +228,10 @@ type QualificationSystemNotice struct {
}
// Interface methods for QualificationSystemNotice
func (qsn QualificationSystemNotice) GetID() string { return qsn.ID }
func (qsn QualificationSystemNotice) GetNoticeType() string { return qsn.NoticeTypeCode }
func (qsn QualificationSystemNotice) GetLanguage() string { return qsn.NoticeLanguageCode }
func (qsn QualificationSystemNotice) GetID() string { return qsn.ID }
func (qsn QualificationSystemNotice) GetNoticeType() string { return qsn.NoticeTypeCode }
func (qsn QualificationSystemNotice) GetLanguage() string { return qsn.NoticeLanguageCode }
func (qsn QualificationSystemNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
func (qsn QualificationSystemNotice) Validate() error {
if qsn.ID == "" {
return fmt.Errorf("qualification system notice ID is required")
@@ -249,9 +273,10 @@ type ConcessionNotice struct {
}
// Interface methods for ConcessionNotice
func (cn ConcessionNotice) GetID() string { return cn.ID }
func (cn ConcessionNotice) GetNoticeType() string { return cn.NoticeTypeCode }
func (cn ConcessionNotice) GetLanguage() string { return cn.NoticeLanguageCode }
func (cn ConcessionNotice) GetID() string { return cn.ID }
func (cn ConcessionNotice) GetNoticeType() string { return cn.NoticeTypeCode }
func (cn ConcessionNotice) GetLanguage() string { return cn.NoticeLanguageCode }
func (cn ConcessionNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
func (cn ConcessionNotice) Validate() error {
if cn.ID == "" {
return fmt.Errorf("concession notice ID is required")
@@ -291,9 +316,10 @@ type PlanningNotice struct {
}
// Interface methods for PlanningNotice
func (pn PlanningNotice) GetID() string { return pn.ID }
func (pn PlanningNotice) GetNoticeType() string { return pn.NoticeTypeCode }
func (pn PlanningNotice) GetLanguage() string { return pn.NoticeLanguageCode }
func (pn PlanningNotice) GetID() string { return pn.ID }
func (pn PlanningNotice) GetNoticeType() string { return pn.NoticeTypeCode }
func (pn PlanningNotice) GetLanguage() string { return pn.NoticeLanguageCode }
func (pn PlanningNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
func (pn PlanningNotice) Validate() error {
if pn.ID == "" {
return fmt.Errorf("planning notice ID is required")
@@ -333,9 +359,10 @@ type CompetitionNotice struct {
}
// Interface methods for CompetitionNotice
func (cn CompetitionNotice) GetID() string { return cn.ID }
func (cn CompetitionNotice) GetNoticeType() string { return cn.NoticeTypeCode }
func (cn CompetitionNotice) GetLanguage() string { return cn.NoticeLanguageCode }
func (cn CompetitionNotice) GetID() string { return cn.ID }
func (cn CompetitionNotice) GetNoticeType() string { return cn.NoticeTypeCode }
func (cn CompetitionNotice) GetLanguage() string { return cn.NoticeLanguageCode }
func (cn CompetitionNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
func (cn CompetitionNotice) Validate() error {
if cn.ID == "" {
return fmt.Errorf("competition notice ID is required")
@@ -376,9 +403,10 @@ type ResultNotice struct {
}
// Interface methods for ResultNotice
func (rn ResultNotice) GetID() string { return rn.ID }
func (rn ResultNotice) GetNoticeType() string { return rn.NoticeTypeCode }
func (rn ResultNotice) GetLanguage() string { return rn.NoticeLanguageCode }
func (rn ResultNotice) GetID() string { return rn.ID }
func (rn ResultNotice) GetNoticeType() string { return rn.NoticeTypeCode }
func (rn ResultNotice) GetLanguage() string { return rn.NoticeLanguageCode }
func (rn ResultNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusAwarded }
func (rn ResultNotice) Validate() error {
if rn.ID == "" {
return fmt.Errorf("result notice ID is required")
@@ -420,9 +448,10 @@ type ChangeNotice struct {
}
// Interface methods for ChangeNotice
func (cn ChangeNotice) GetID() string { return cn.ID }
func (cn ChangeNotice) GetNoticeType() string { return cn.NoticeTypeCode }
func (cn ChangeNotice) GetLanguage() string { return cn.NoticeLanguageCode }
func (cn ChangeNotice) GetID() string { return cn.ID }
func (cn ChangeNotice) GetNoticeType() string { return cn.NoticeTypeCode }
func (cn ChangeNotice) GetLanguage() string { return cn.NoticeLanguageCode }
func (cn ChangeNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusModified }
func (cn ChangeNotice) Validate() error {
if cn.ID == "" {
return fmt.Errorf("change notice ID is required")
@@ -463,9 +492,10 @@ type SubcontractNotice struct {
}
// Interface methods for SubcontractNotice
func (sn SubcontractNotice) GetID() string { return sn.ID }
func (sn SubcontractNotice) GetNoticeType() string { return sn.NoticeTypeCode }
func (sn SubcontractNotice) GetLanguage() string { return sn.NoticeLanguageCode }
func (sn SubcontractNotice) GetID() string { return sn.ID }
func (sn SubcontractNotice) GetNoticeType() string { return sn.NoticeTypeCode }
func (sn SubcontractNotice) GetLanguage() string { return sn.NoticeLanguageCode }
func (sn SubcontractNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
func (sn SubcontractNotice) Validate() error {
if sn.ID == "" {
return fmt.Errorf("subcontract notice ID is required")
@@ -665,67 +695,123 @@ type SubcontractingTerm struct {
SubcontractingDescription string `xml:"SubcontractingDescription,omitempty"`
}
// NoticeStatusInformation contains status-specific fields
type NoticeStatusInformation struct {
Status NoticeStatus `xml:"NoticeStatus,omitempty"`
StatusCode string `xml:"StatusCode,omitempty"`
CancellationReason *CancellationReason `xml:"CancellationReason,omitempty"`
AwardInformation *AwardInformation `xml:"AwardInformation,omitempty"`
Modifications []NoticeModification `xml:"Modifications>Modification,omitempty"`
SuspensionReason *SuspensionReason `xml:"SuspensionReason,omitempty"`
}
// CancellationReason contains cancellation information
type CancellationReason struct {
XMLName xml.Name `xml:"CancellationReason"`
ReasonCode string `xml:"ReasonCode"`
Description string `xml:"Description,omitempty"`
LanguageID string `xml:"languageID,attr,omitempty"`
}
// AwardInformation contains award-specific information
type AwardInformation struct {
XMLName xml.Name `xml:"AwardInformation"`
AwardDate string `xml:"AwardDate"`
AwardedValue CurrencyText `xml:"AwardedValue,omitempty"`
WinningTenderer *WinningTenderer `xml:"WinningTenderer,omitempty"`
ContractNumber string `xml:"ContractNumber,omitempty"`
}
// WinningTenderer contains information about the winning tenderer
type WinningTenderer struct {
XMLName xml.Name `xml:"WinningTenderer"`
NaturalPersonIndicator string `xml:"NaturalPersonIndicator,omitempty"`
Company *Company `xml:"Company,omitempty"`
}
// NoticeModification contains modification information
type NoticeModification struct {
XMLName xml.Name `xml:"Modification"`
ModificationDate string `xml:"ModificationDate"`
ModificationReason string `xml:"ModificationReason"`
Description string `xml:"Description,omitempty"`
LanguageID string `xml:"languageID,attr,omitempty"`
}
// SuspensionReason contains suspension information
type SuspensionReason struct {
XMLName xml.Name `xml:"SuspensionReason"`
ReasonCode string `xml:"ReasonCode"`
Description string `xml:"Description,omitempty"`
LanguageID string `xml:"languageID,attr,omitempty"`
}
// ContractNotice represents the main TED XML contract notice structure
type ContractNotice struct {
XMLName xml.Name `xml:"ContractNotice"`
Xmlns string `xml:"xmlns,attr,omitempty"`
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
XMLName xml.Name `xml:"ContractNotice"`
Xmlns string `xml:"xmlns,attr,omitempty"`
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
StatusInformation *NoticeStatusInformation `xml:"StatusInformation,omitempty"`
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
}
// ContractAwardNotice represents the main TED XML contract award notice structure
type ContractAwardNotice struct {
XMLName xml.Name `xml:"ContractAwardNotice"`
Xmlns string `xml:"xmlns,attr,omitempty"`
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
ProcurementProjectLot []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
TenderResult *TenderResult `xml:"TenderResult,omitempty"`
XMLName xml.Name `xml:"ContractAwardNotice"`
Xmlns string `xml:"xmlns,attr,omitempty"`
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
UBLVersionID string `xml:"UBLVersionID"`
CustomizationID string `xml:"CustomizationID"`
ID string `xml:"ID"`
ContractFolderID string `xml:"ContractFolderID"`
IssueDate string `xml:"IssueDate"`
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
StatusInformation *NoticeStatusInformation `xml:"StatusInformation,omitempty"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
ProcurementProjectLot []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
TenderResult *TenderResult `xml:"TenderResult,omitempty"`
}
// NoticeResult contains notice result information from extensions
type NoticeResult struct {
XMLName xml.Name `xml:"NoticeResult"`
TotalAmount *CurrencyText `xml:"TotalAmount,omitempty"`
LotResult []LotResult `xml:"LotResult,omitempty"`
XMLName xml.Name `xml:"NoticeResult"`
TotalAmount *CurrencyText `xml:"TotalAmount,omitempty"`
LotResult []LotResult `xml:"LotResult,omitempty"`
LotTender []LotTender `xml:"LotTender,omitempty"`
SettledContract []SettledContract `xml:"SettledContract,omitempty"`
TenderingParty []TenderingParty `xml:"TenderingParty,omitempty"`
}
// LotResult contains lot result information
@@ -761,7 +847,14 @@ type LegalMonetaryTotal struct {
// TenderingParty contains tendering party information
type TenderingParty struct {
XMLName xml.Name `xml:"TenderingParty"`
XMLName xml.Name `xml:"TenderingParty"`
ID string `xml:"ID"`
Tenderer []Tenderer `xml:"Tenderer,omitempty"`
}
// Tenderer contains tenderer information
type Tenderer struct {
XMLName xml.Name `xml:"Tenderer"`
ID string `xml:"ID"`
}
@@ -786,13 +879,16 @@ type ReceivedSubmissionsStatistics struct {
// SettledContract contains settled contract information
type SettledContract struct {
XMLName xml.Name `xml:"SettledContract"`
ID string `xml:"ID"`
IssueDate string `xml:"IssueDate,omitempty"`
Title string `xml:"Title,omitempty"`
SignatoryParty *SignatoryParty `xml:"SignatoryParty,omitempty"`
ContractReference *ContractReference `xml:"ContractReference,omitempty"`
LotTender *LotTender `xml:"LotTender,omitempty"`
XMLName xml.Name `xml:"SettledContract"`
ID string `xml:"ID"`
IssueDate string `xml:"IssueDate,omitempty"`
AwardDate string `xml:"AwardDate,omitempty"`
Title string `xml:"Title,omitempty"`
ContractFrameworkIndicator string `xml:"ContractFrameworkIndicator,omitempty"`
SignatoryParty *SignatoryParty `xml:"SignatoryParty,omitempty"`
ContractReference *ContractReference `xml:"ContractReference,omitempty"`
LotTender *LotTender `xml:"LotTender,omitempty"`
NoticeDocumentReference *NoticeDocumentReference `xml:"NoticeDocumentReference,omitempty"`
}
// SignatoryParty contains signatory party information
@@ -1439,6 +1535,62 @@ func (cn ContractNotice) GetLanguage() string {
return cn.NoticeLanguageCode
}
// GetNoticeStatus returns the notice status
func (cn ContractNotice) GetNoticeStatus() NoticeStatus {
if cn.StatusInformation != nil {
if cn.StatusInformation.Status != "" {
return cn.StatusInformation.Status
}
// Check for specific status indicators
if cn.StatusInformation.CancellationReason != nil {
return NoticeStatusCancelled
}
if cn.StatusInformation.AwardInformation != nil {
return NoticeStatusAwarded
}
if cn.StatusInformation.SuspensionReason != nil {
return NoticeStatusSuspended
}
if len(cn.StatusInformation.Modifications) > 0 {
return NoticeStatusModified
}
}
// Default to Published if no status information is available
return NoticeStatusPublished
}
// GetCancellationReason returns the cancellation reason if notice is cancelled
func (cn ContractNotice) GetCancellationReason() *CancellationReason {
if cn.StatusInformation != nil {
return cn.StatusInformation.CancellationReason
}
return nil
}
// GetAwardInformation returns the award information if notice is awarded
func (cn ContractNotice) GetAwardInformation() *AwardInformation {
if cn.StatusInformation != nil {
return cn.StatusInformation.AwardInformation
}
return nil
}
// GetModifications returns the list of modifications if notice is modified
func (cn ContractNotice) GetModifications() []NoticeModification {
if cn.StatusInformation != nil {
return cn.StatusInformation.Modifications
}
return nil
}
// GetSuspensionReason returns the suspension reason if notice is suspended
func (cn ContractNotice) GetSuspensionReason() *SuspensionReason {
if cn.StatusInformation != nil {
return cn.StatusInformation.SuspensionReason
}
return nil
}
// GetPublicationInfo returns publication information if available
func (cn ContractNotice) GetPublicationInfo() *Publication {
if cn.Extensions != nil &&
@@ -1759,6 +1911,47 @@ func (can ContractAwardNotice) GetLanguage() string {
return can.NoticeLanguageCode
}
// GetNoticeStatus returns the notice status
func (can ContractAwardNotice) GetNoticeStatus() NoticeStatus {
if can.StatusInformation != nil && can.StatusInformation.Status != "" {
return can.StatusInformation.Status
}
// Award notices are typically awarded status
return NoticeStatusAwarded
}
// GetCancellationReason returns the cancellation reason if notice is cancelled
func (can ContractAwardNotice) GetCancellationReason() *CancellationReason {
if can.StatusInformation != nil {
return can.StatusInformation.CancellationReason
}
return nil
}
// GetAwardInformation returns the award information if notice is awarded
func (can ContractAwardNotice) GetAwardInformation() *AwardInformation {
if can.StatusInformation != nil {
return can.StatusInformation.AwardInformation
}
return nil
}
// GetModifications returns the list of modifications if notice is modified
func (can ContractAwardNotice) GetModifications() []NoticeModification {
if can.StatusInformation != nil {
return can.StatusInformation.Modifications
}
return nil
}
// GetSuspensionReason returns the suspension reason if notice is suspended
func (can ContractAwardNotice) GetSuspensionReason() *SuspensionReason {
if can.StatusInformation != nil {
return can.StatusInformation.SuspensionReason
}
return nil
}
// GetPublicationInfo returns publication information if available
func (can ContractAwardNotice) GetPublicationInfo() *Publication {
if can.Extensions != nil &&
@@ -1915,3 +2108,134 @@ func (can ContractAwardNotice) GetOrganizationByID(orgID string) *Organization {
}
return nil
}
// GetAwardedContractors returns detailed information about awarded contractors
func (can ContractAwardNotice) GetAwardedContractors() []AwardedContractorInfo {
var contractors []AwardedContractorInfo
noticeResult := can.GetNoticeResult()
if noticeResult == nil {
return contractors
}
// Create a map of tender ID to tender info for quick lookup
tenderMap := make(map[string]*LotTender)
for _, lotTender := range noticeResult.LotTender {
tenderMap[lotTender.ID] = &lotTender
}
// Create a map of tendering party ID to organization ID
partyToOrgMap := make(map[string]string)
for _, tenderingParty := range noticeResult.TenderingParty {
for _, tenderer := range tenderingParty.Tenderer {
partyToOrgMap[tenderingParty.ID] = tenderer.ID
}
}
// Process settled contracts to extract award information
for _, contract := range noticeResult.SettledContract {
if contract.LotTender != nil {
// Get tender information
tender := tenderMap[contract.LotTender.ID]
if tender == nil {
continue
}
// Get organization information
var org *Organization
if tender.TenderingParty != nil {
orgID := partyToOrgMap[tender.TenderingParty.ID]
if orgID != "" {
org = can.GetOrganizationByID(orgID)
}
}
// Create awarded contractor info
contractor := AwardedContractorInfo{
ContractID: contract.ID,
TenderID: contract.LotTender.ID,
AwardDate: contract.AwardDate,
IssueDate: contract.IssueDate,
}
// Set contract reference if available
if contract.ContractReference != nil {
contractor.ContractReference = contract.ContractReference.ID
}
// Set amount and currency from tender
if tender.LegalMonetaryTotal != nil {
contractor.Amount = tender.LegalMonetaryTotal.PayableAmount.Text
contractor.Currency = tender.LegalMonetaryTotal.PayableAmount.CurrencyID
}
// Set organization information
if org != nil && org.Company != nil {
if org.Company.PartyName != nil {
contractor.Name = org.Company.PartyName.Name
}
if org.Company.PartyLegalEntity != nil {
contractor.CompanyID = org.Company.PartyLegalEntity.CompanyID
}
if org.Company.PartyIdentification != nil {
contractor.OrganizationID = org.Company.PartyIdentification.ID
}
// Set address information
if org.Company.PostalAddress != nil {
addressParts := []string{}
if org.Company.PostalAddress.StreetName != "" {
addressParts = append(addressParts, org.Company.PostalAddress.StreetName)
}
if org.Company.PostalAddress.CityName != "" {
addressParts = append(addressParts, org.Company.PostalAddress.CityName)
}
if org.Company.PostalAddress.PostalZone != "" {
addressParts = append(addressParts, org.Company.PostalAddress.PostalZone)
}
contractor.Address = strings.Join(addressParts, ", ")
if org.Company.PostalAddress.Country != nil {
contractor.Country = org.Company.PostalAddress.Country.IdentificationCode
}
}
// Set contact information
if org.Company.Contact != nil {
contractor.ContactName = org.Company.Contact.Name
contractor.ContactEmail = org.Company.Contact.ElectronicMail
contractor.ContactPhone = org.Company.Contact.Telephone
}
}
// Set lot information
if tender.TenderLot != nil {
contractor.LotID = tender.TenderLot.ID
}
contractors = append(contractors, contractor)
}
}
return contractors
}
// AwardedContractorInfo contains detailed information about an awarded contractor
type AwardedContractorInfo struct {
ContractID string `json:"contract_id"`
TenderID string `json:"tender_id"`
LotID string `json:"lot_id,omitempty"`
Name string `json:"name"`
CompanyID string `json:"company_id,omitempty"`
OrganizationID string `json:"organization_id,omitempty"`
Address string `json:"address,omitempty"`
Country string `json:"country,omitempty"`
Amount string `json:"amount"`
Currency string `json:"currency,omitempty"`
AwardDate string `json:"award_date,omitempty"`
IssueDate string `json:"issue_date,omitempty"`
ContractReference string `json:"contract_reference,omitempty"`
ContactName string `json:"contact_name,omitempty"`
ContactEmail string `json:"contact_email,omitempty"`
ContactPhone string `json:"contact_phone,omitempty"`
}