Enhance Tender Entity and TED Parser with Status Management and Upsert Logic
- Added new fields to the Tender entity for handling cancellation, award, and suspension details, improving the entity's capability to manage tender statuses. - Implemented a DetectNoticeStatus method in the TEDParser to identify the status of TED notices from XML data, enhancing the parser's functionality. - Updated the ParseXML method to include notice status detection, ensuring accurate status representation in parsed documents. - Introduced upsert logic in the TED scraper to handle tender records more effectively, allowing for updates or creation based on existing ContractIDs. - Enhanced logging and error handling throughout the scraping process to improve traceability and robustness. - Added a comprehensive usage example document to illustrate the new features and usage patterns for the TED XML parser and scraper.
This commit is contained in:
+150
-1
@@ -121,6 +121,141 @@ func (p *TEDParser) DetectDocumentType(xmlData []byte) (DocumentType, error) {
|
||||
return "", fmt.Errorf("unknown document type")
|
||||
}
|
||||
|
||||
// DetectNoticeStatus detects the status of a TED notice from XML data
|
||||
func (p *TEDParser) DetectNoticeStatus(xmlData []byte) NoticeStatus {
|
||||
// Convert to string for fast text search
|
||||
xmlStr := string(xmlData)
|
||||
|
||||
// Check for common status indicators in XML content
|
||||
// Many TED XMLs contain status information in extensions or specific tags
|
||||
|
||||
// Check for cancellation indicators
|
||||
if strings.Contains(xmlStr, "<CancellationReason") ||
|
||||
strings.Contains(xmlStr, "cancelled") ||
|
||||
strings.Contains(xmlStr, "canceled") ||
|
||||
strings.Contains(xmlStr, "CANCELLED") ||
|
||||
strings.Contains(xmlStr, "CANCELED") {
|
||||
return NoticeStatusCancelled
|
||||
}
|
||||
|
||||
// Check for award indicators
|
||||
if strings.Contains(xmlStr, "<AwardDate") ||
|
||||
strings.Contains(xmlStr, "<TenderResult") ||
|
||||
strings.Contains(xmlStr, "<SettledContract") ||
|
||||
strings.Contains(xmlStr, "<ContractAwardNotice") ||
|
||||
strings.Contains(xmlStr, "awarded") ||
|
||||
strings.Contains(xmlStr, "AWARDED") {
|
||||
return NoticeStatusAwarded
|
||||
}
|
||||
|
||||
// Check for modification indicators
|
||||
if strings.Contains(xmlStr, "<ChangeNotice") ||
|
||||
strings.Contains(xmlStr, "<ChangeReason") ||
|
||||
strings.Contains(xmlStr, "<Modification") ||
|
||||
strings.Contains(xmlStr, "modified") ||
|
||||
strings.Contains(xmlStr, "MODIFIED") {
|
||||
return NoticeStatusModified
|
||||
}
|
||||
|
||||
// Check for suspension indicators
|
||||
if strings.Contains(xmlStr, "<SuspensionReason") ||
|
||||
strings.Contains(xmlStr, "suspended") ||
|
||||
strings.Contains(xmlStr, "SUSPENDED") {
|
||||
return NoticeStatusSuspended
|
||||
}
|
||||
|
||||
// Check for closed indicators
|
||||
if strings.Contains(xmlStr, "closed") ||
|
||||
strings.Contains(xmlStr, "CLOSED") ||
|
||||
strings.Contains(xmlStr, "<ClosedDate") {
|
||||
return NoticeStatusClosed
|
||||
}
|
||||
|
||||
// Check for draft indicators
|
||||
if strings.Contains(xmlStr, "draft") ||
|
||||
strings.Contains(xmlStr, "DRAFT") {
|
||||
return NoticeStatusDraft
|
||||
}
|
||||
|
||||
// Check for explicit status in XML using XML parsing
|
||||
status := p.extractStatusFromXML(xmlData)
|
||||
if status != "" {
|
||||
switch strings.ToLower(status) {
|
||||
case "draft":
|
||||
return NoticeStatusDraft
|
||||
case "published":
|
||||
return NoticeStatusPublished
|
||||
case "cancelled", "canceled":
|
||||
return NoticeStatusCancelled
|
||||
case "awarded":
|
||||
return NoticeStatusAwarded
|
||||
case "closed":
|
||||
return NoticeStatusClosed
|
||||
case "modified":
|
||||
return NoticeStatusModified
|
||||
case "suspended":
|
||||
return NoticeStatusSuspended
|
||||
}
|
||||
}
|
||||
|
||||
// Default to Published for most TED notices
|
||||
return NoticeStatusPublished
|
||||
}
|
||||
|
||||
// extractStatusFromXML extracts status information using XML parsing
|
||||
func (p *TEDParser) extractStatusFromXML(xmlData []byte) string {
|
||||
decoder := xml.NewDecoder(bytes.NewReader(xmlData))
|
||||
|
||||
for {
|
||||
token, err := decoder.Token()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
if startElement, ok := token.(xml.StartElement); ok {
|
||||
// Look for common status element names
|
||||
switch startElement.Name.Local {
|
||||
case "NoticeStatus", "Status", "StatusCode", "DocumentStatus":
|
||||
// Read the content of this element
|
||||
if content, err := p.readElementContent(decoder); err == nil {
|
||||
return strings.TrimSpace(content)
|
||||
}
|
||||
}
|
||||
|
||||
// Check attributes for status information
|
||||
for _, attr := range startElement.Attr {
|
||||
if strings.Contains(strings.ToLower(attr.Name.Local), "status") {
|
||||
return strings.TrimSpace(attr.Value)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// readElementContent reads the text content of the current XML element
|
||||
func (p *TEDParser) readElementContent(decoder *xml.Decoder) (string, error) {
|
||||
var content strings.Builder
|
||||
|
||||
for {
|
||||
token, err := decoder.Token()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
switch t := token.(type) {
|
||||
case xml.CharData:
|
||||
content.Write(t)
|
||||
case xml.EndElement:
|
||||
return content.String(), nil
|
||||
case xml.StartElement:
|
||||
// Skip nested elements
|
||||
decoder.Skip()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ParseXML automatically detects and parses any supported TED XML document type
|
||||
func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
|
||||
docType, err := p.DetectDocumentType(xmlData)
|
||||
@@ -128,7 +263,13 @@ func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
|
||||
return nil, fmt.Errorf("failed to detect document type: %w", err)
|
||||
}
|
||||
|
||||
parsedDoc := &ParsedDocument{Type: docType}
|
||||
// Detect notice status
|
||||
noticeStatus := p.DetectNoticeStatus(xmlData)
|
||||
|
||||
parsedDoc := &ParsedDocument{
|
||||
Type: docType,
|
||||
Status: noticeStatus,
|
||||
}
|
||||
|
||||
switch docType {
|
||||
case DocumentTypeContractNotice:
|
||||
@@ -137,6 +278,10 @@ func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
|
||||
return nil, fmt.Errorf("failed to parse contract notice: %w", err)
|
||||
}
|
||||
parsedDoc.ContractNotice = parsed
|
||||
// Override status based on document content
|
||||
if parsed != nil {
|
||||
parsedDoc.Status = parsed.GetNoticeStatus()
|
||||
}
|
||||
|
||||
case DocumentTypeContractAwardNotice:
|
||||
parsed, err := p.contractAwardNoticeParser.ParseBytes(xmlData)
|
||||
@@ -144,6 +289,10 @@ func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
|
||||
return nil, fmt.Errorf("failed to parse contract award notice: %w", err)
|
||||
}
|
||||
parsedDoc.ContractAwardNotice = parsed
|
||||
// Override status based on document content
|
||||
if parsed != nil {
|
||||
parsedDoc.Status = parsed.GetNoticeStatus()
|
||||
}
|
||||
|
||||
case DocumentTypePriorInformationNotice:
|
||||
parsed, err := p.priorInformationNoticeParser.ParseBytes(xmlData)
|
||||
|
||||
+706
-36
@@ -64,11 +64,11 @@ type JobResults struct {
|
||||
// ScrapingState represents the current state of scraping
|
||||
type ScrapingState struct {
|
||||
mongo.Model
|
||||
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
|
||||
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
|
||||
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"`
|
||||
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"`
|
||||
IsRunning bool `bson:"is_running" json:"is_running"`
|
||||
LastProcessedYear int `bson:"last_processed_year" json:"last_processed_year"`
|
||||
LastProcessedNumber int `bson:"last_processed_number" json:"last_processed_number"`
|
||||
LastRunAt int64 `bson:"last_run_at" json:"last_run_at"`
|
||||
NextRunAt int64 `bson:"next_run_at" json:"next_run_at"`
|
||||
IsRunning bool `bson:"is_running" json:"is_running"`
|
||||
CurrentJobID bson.ObjectID `bson:"current_job_id,omitempty" json:"current_job_id,omitempty"`
|
||||
}
|
||||
|
||||
@@ -85,6 +85,22 @@ type Config struct {
|
||||
ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
|
||||
}
|
||||
|
||||
// ScrapingMode represents the type of scraping operation
|
||||
type ScrapingMode string
|
||||
|
||||
const (
|
||||
ScrapingModeDaily ScrapingMode = "daily" // Daily bulk fetch mode
|
||||
ScrapingModeManual ScrapingMode = "manual" // Manual date range mode
|
||||
ScrapingModeRealtime ScrapingMode = "realtime" // Real-time processing mode
|
||||
)
|
||||
|
||||
// DateRangeConfig holds configuration for manual date range scraping
|
||||
type DateRangeConfig struct {
|
||||
StartDate time.Time `json:"start_date"`
|
||||
EndDate time.Time `json:"end_date"`
|
||||
Mode ScrapingMode `json:"mode"`
|
||||
}
|
||||
|
||||
// NewTEDScraper creates a new TED scraper instance
|
||||
func NewTEDScraper(
|
||||
config *Config,
|
||||
@@ -245,7 +261,8 @@ func (s *TEDScraper) ScrapeLatest(ctx context.Context) error {
|
||||
} else {
|
||||
job.Progress.ProcessedFiles++
|
||||
job.Results.TotalTenders += fileResult.ProcessedCount
|
||||
job.Results.NewTenders += fileResult.SuccessCount
|
||||
job.Results.NewTenders += fileResult.NewTenders
|
||||
job.Results.UpdatedTenders += fileResult.UpdatedTenders
|
||||
if len(fileResult.Errors) > 0 {
|
||||
job.Results.ErrorCount += len(fileResult.Errors)
|
||||
}
|
||||
@@ -272,6 +289,8 @@ func (s *TEDScraper) ScrapeLatest(ctx context.Context) error {
|
||||
s.logger.Info("TED XML scraping completed", map[string]interface{}{
|
||||
"files_processed": job.Progress.ProcessedFiles,
|
||||
"tenders_created": job.Results.NewTenders,
|
||||
"tenders_updated": job.Results.UpdatedTenders,
|
||||
"total_tenders": job.Results.TotalTenders,
|
||||
"error_count": job.Results.ErrorCount,
|
||||
"success_rate": job.Results.SuccessRate,
|
||||
})
|
||||
@@ -347,6 +366,8 @@ type FileProcessingResult struct {
|
||||
Number int `json:"number"`
|
||||
ProcessedCount int `json:"processed_count"`
|
||||
SuccessCount int `json:"success_count"`
|
||||
NewTenders int `json:"new_tenders"`
|
||||
UpdatedTenders int `json:"updated_tenders"`
|
||||
Errors []string `json:"errors"`
|
||||
ProcessedAt int64 `json:"processed_at"`
|
||||
}
|
||||
@@ -527,9 +548,17 @@ func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fil
|
||||
}
|
||||
|
||||
// Parse and process XML content
|
||||
if err := s.processXMLContent(ctx, content, fileName, fileInfo, result); err != nil {
|
||||
processResult, err := s.processXMLContent(ctx, content, fileName, fileInfo)
|
||||
if err != nil {
|
||||
errMsg := fmt.Sprintf("failed to process XML file %s: %v", fileName, err)
|
||||
result.Errors = append(result.Errors, errMsg)
|
||||
} else {
|
||||
result.SuccessCount++
|
||||
if processResult.IsNew {
|
||||
result.NewTenders++
|
||||
} else {
|
||||
result.UpdatedTenders++
|
||||
}
|
||||
}
|
||||
|
||||
result.ProcessedCount++
|
||||
@@ -556,8 +585,15 @@ func (s *TEDScraper) processTarGzFile(ctx context.Context, reader io.Reader, fil
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// XMLProcessResult represents the result of processing a single XML file
|
||||
type XMLProcessResult struct {
|
||||
IsNew bool `json:"is_new"`
|
||||
TenderID string `json:"tender_id"`
|
||||
ContractID string `json:"contract_id"`
|
||||
}
|
||||
|
||||
// processXMLContent parses XML content and creates/updates tender
|
||||
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo, result *FileProcessingResult) error {
|
||||
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, fileInfo FileInfo) (*XMLProcessResult, error) {
|
||||
s.logger.Info("Starting XML processing", map[string]interface{}{
|
||||
"file_name": fileName,
|
||||
"file_size": len(content),
|
||||
@@ -570,7 +606,7 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
|
||||
"file_name": fileName,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Errorf("failed to parse XML: %w", err)
|
||||
return nil, fmt.Errorf("failed to parse XML: %w", err)
|
||||
}
|
||||
|
||||
// Get the underlying document for ID and validation
|
||||
@@ -612,33 +648,45 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
|
||||
sourceURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, fileInfo.Year, fileInfo.Number)
|
||||
tenderEntity := s.mapToTenderFromParsedDoc(parsedDoc, sourceURL, fileName)
|
||||
|
||||
// Handle status-specific information and award logic
|
||||
s.applyStatusSpecificInformation(tenderEntity, parsedDoc)
|
||||
s.applyAwardLogic(tenderEntity, parsedDoc)
|
||||
|
||||
s.logger.Info("Mapped to tender entity", map[string]interface{}{
|
||||
"file_name": fileName,
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"title": tenderEntity.Title,
|
||||
"status": string(tenderEntity.Status),
|
||||
})
|
||||
|
||||
// Try to find existing tender by contract notice ID
|
||||
// Try to find existing tender by contract notice ID (ContractID)
|
||||
s.logger.Info("Checking for existing tender", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
})
|
||||
|
||||
existingTender, err := s.findTenderByContractNoticeID(ctx, tenderEntity.ContractNoticeID)
|
||||
if err != nil {
|
||||
if err != nil && err.Error() != "document not found" {
|
||||
s.logger.Warn("Error checking for existing tender", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
var processResult *XMLProcessResult
|
||||
|
||||
if err == nil && existingTender != nil {
|
||||
// Update existing tender
|
||||
s.logger.Info("Found existing tender, updating", map[string]interface{}{
|
||||
"existing_tender_id": existingTender.ID,
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"old_status": string(existingTender.Status),
|
||||
"new_status": string(tenderEntity.Status),
|
||||
})
|
||||
|
||||
// Merge updates with existing data
|
||||
s.mergeExistingTenderData(existingTender, tenderEntity)
|
||||
|
||||
tenderEntity.ID = existingTender.ID
|
||||
tenderEntity.CreatedAt = existingTender.CreatedAt
|
||||
tenderEntity.UpdatedAt = time.Now().Unix()
|
||||
@@ -649,17 +697,24 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
|
||||
"tender_id": tenderEntity.ID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Errorf("failed to update existing tender: %w", err)
|
||||
return nil, fmt.Errorf("failed to update existing tender: %w", err)
|
||||
}
|
||||
|
||||
s.logger.Info("Successfully updated tender", map[string]interface{}{
|
||||
"tender_id": tenderEntity.ID,
|
||||
})
|
||||
|
||||
processResult = &XMLProcessResult{
|
||||
IsNew: false,
|
||||
TenderID: tenderEntity.TenderID,
|
||||
ContractID: tenderEntity.ContractNoticeID,
|
||||
}
|
||||
} else {
|
||||
// Create new tender
|
||||
s.logger.Info("Creating new tender", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"status": string(tenderEntity.Status),
|
||||
})
|
||||
|
||||
tenderEntity.CreatedAt = time.Now().Unix()
|
||||
@@ -672,25 +727,30 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Errorf("failed to create tender: %w", err)
|
||||
return nil, fmt.Errorf("failed to create tender: %w", err)
|
||||
}
|
||||
|
||||
s.logger.Info("Successfully created new tender", map[string]interface{}{
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
})
|
||||
}
|
||||
|
||||
result.SuccessCount++
|
||||
processResult = &XMLProcessResult{
|
||||
IsNew: true,
|
||||
TenderID: tenderEntity.TenderID,
|
||||
ContractID: tenderEntity.ContractNoticeID,
|
||||
}
|
||||
}
|
||||
|
||||
s.logger.Debug("Successfully processed XML file", map[string]interface{}{
|
||||
"file_name": fileName,
|
||||
"document_type": string(parsedDoc.Type),
|
||||
"document_id": documentID,
|
||||
"tender_id": tenderEntity.TenderID,
|
||||
"is_new": processResult.IsNew,
|
||||
})
|
||||
|
||||
return nil
|
||||
return processResult, nil
|
||||
}
|
||||
|
||||
// Helper methods for database operations
|
||||
@@ -940,8 +1000,8 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
|
||||
NoticeTypeCode: cn.GetNoticeSubType(),
|
||||
NoticeSubTypeCode: cn.GetNoticeSubType(),
|
||||
NoticeLanguageCode: cn.NoticeLanguageCode,
|
||||
IssueDate: s.parseDateToUnixMilli(cn.IssueDate),
|
||||
IssueTime: s.parseTimeToUnixMilli(cn.IssueTime),
|
||||
IssueDate: s.parseDateToUnix(cn.IssueDate),
|
||||
IssueTime: s.parseTimeToUnix(cn.IssueTime),
|
||||
GazetteID: cn.GetOJSID(),
|
||||
Title: cn.GetProcurementTitle(),
|
||||
Description: cn.GetProcurementDescription(),
|
||||
@@ -964,7 +1024,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
|
||||
// Set publication information
|
||||
if pubInfo := cn.GetPublicationInfo(); pubInfo != nil {
|
||||
t.NoticePublicationID = pubInfo.NoticePublicationID
|
||||
t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate)
|
||||
t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate)
|
||||
t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
|
||||
}
|
||||
|
||||
@@ -993,17 +1053,17 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
|
||||
|
||||
// Set tender deadline
|
||||
if deadline := cn.GetTenderDeadline(); deadline != "" {
|
||||
t.TenderDeadline = s.parseDateToUnixMilli(deadline)
|
||||
t.TenderDeadline = s.parseDateToUnix(deadline)
|
||||
|
||||
if deadlineTime, err := s.parseDate(deadline); err == nil {
|
||||
applicationDeadline := s.calculateApplicationDeadline(deadlineTime)
|
||||
t.ApplicationDeadline = applicationDeadline.UnixMilli()
|
||||
t.ApplicationDeadline = applicationDeadline.Unix()
|
||||
}
|
||||
}
|
||||
|
||||
if pubDate, err := s.parseDate(cn.GetPublicationDate()); err == nil {
|
||||
submissionDeadline := s.calculateSubmissionDeadline(pubDate)
|
||||
t.SubmissionDeadline = submissionDeadline.UnixMilli()
|
||||
t.SubmissionDeadline = submissionDeadline.Unix()
|
||||
}
|
||||
|
||||
// Set SubmissionURL from TenderingTerms
|
||||
@@ -1089,8 +1149,8 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
|
||||
NoticeTypeCode: can.NoticeTypeCode,
|
||||
NoticeSubTypeCode: can.GetNoticeSubType(),
|
||||
NoticeLanguageCode: can.NoticeLanguageCode,
|
||||
IssueDate: s.parseDateToUnixMilli(can.IssueDate),
|
||||
IssueTime: s.parseTimeToUnixMilli(can.IssueTime),
|
||||
IssueDate: s.parseDateToUnix(can.IssueDate),
|
||||
IssueTime: s.parseTimeToUnix(can.IssueTime),
|
||||
GazetteID: can.GetOJSID(),
|
||||
Title: can.GetProcurementTitle(),
|
||||
Description: can.GetProcurementDescription(),
|
||||
@@ -1111,7 +1171,7 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
|
||||
// Set publication information
|
||||
if pubInfo := can.GetPublicationInfo(); pubInfo != nil {
|
||||
t.NoticePublicationID = pubInfo.NoticePublicationID
|
||||
t.PublicationDate = s.parseDateToUnixMilli(pubInfo.PublicationDate)
|
||||
t.PublicationDate = s.parseDateToUnix(pubInfo.PublicationDate)
|
||||
t.TenderURL = s.generateTenderURL(pubInfo.NoticePublicationID)
|
||||
}
|
||||
|
||||
@@ -1234,7 +1294,7 @@ func (s *TEDScraper) generateTenderID(t *tender.Tender) string {
|
||||
// parts = append(parts, buyerCode)
|
||||
|
||||
// Date (YYYYMMDD format from issue date)
|
||||
parts = append(parts, s.padOrTruncate(s.unixMilliToDateString(t.IssueDate), 4))
|
||||
parts = append(parts, s.padOrTruncate(s.UnixToDateString(t.IssueDate), 4))
|
||||
|
||||
// Location code (country code + region code, padded to 6 chars)
|
||||
locationCode := "EU"
|
||||
@@ -1310,21 +1370,21 @@ func (s *TEDScraper) parseDate(dateStr string) (time.Time, error) {
|
||||
return time.Time{}, fmt.Errorf("unable to parse date: %s", dateStr)
|
||||
}
|
||||
|
||||
// parseDateToUnixMilli converts string date to Unix milliseconds (int64)
|
||||
func (s *TEDScraper) parseDateToUnixMilli(dateStr string) int64 {
|
||||
// parseDateToUnix converts string date to Unix milliseconds (int64)
|
||||
func (s *TEDScraper) parseDateToUnix(dateStr string) int64 {
|
||||
if dateStr == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
if parsedTime, err := s.parseDate(dateStr); err == nil {
|
||||
return parsedTime.UnixMilli()
|
||||
return parsedTime.Unix()
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// parseTimeToUnixMilli converts string time to Unix milliseconds (int64)
|
||||
func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 {
|
||||
// parseTimeToUnix converts string time to Unix milliseconds (int64)
|
||||
func (s *TEDScraper) parseTimeToUnix(timeStr string) int64 {
|
||||
if timeStr == "" {
|
||||
return 0
|
||||
}
|
||||
@@ -1337,19 +1397,19 @@ func (s *TEDScraper) parseTimeToUnixMilli(timeStr string) int64 {
|
||||
}
|
||||
|
||||
if parsedTime, err := s.parseDate(timeStr); err == nil {
|
||||
return parsedTime.UnixMilli()
|
||||
return parsedTime.Unix()
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
// unixMilliToDateString converts Unix milliseconds to date string for TenderID generation
|
||||
func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string {
|
||||
if unixMilli == 0 {
|
||||
// UnixToDateString converts Unix milliseconds to date string for TenderID generation
|
||||
func (s *TEDScraper) UnixToDateString(u int64) string {
|
||||
if u == 0 {
|
||||
return "00000000"
|
||||
}
|
||||
|
||||
return time.UnixMilli(unixMilli).Format("20060102")
|
||||
return time.Unix(u, 0).Format("20060102")
|
||||
}
|
||||
|
||||
func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time {
|
||||
@@ -1395,3 +1455,613 @@ func (s *TEDScraper) adjustToWorkingDay(targetDate time.Time, forward bool) time
|
||||
|
||||
return current
|
||||
}
|
||||
|
||||
// mapTEDStatusToTenderStatus maps TED notice status to tender status
|
||||
func (s *TEDScraper) mapTEDStatusToTenderStatus(tedStatus NoticeStatus) tender.TenderStatus {
|
||||
switch tedStatus {
|
||||
case NoticeStatusDraft:
|
||||
return tender.TenderStatusDraft
|
||||
case NoticeStatusPublished:
|
||||
return tender.TenderStatusPublished
|
||||
case NoticeStatusCancelled:
|
||||
return tender.TenderStatusCancelled
|
||||
case NoticeStatusAwarded:
|
||||
return tender.TenderStatusAwarded
|
||||
case NoticeStatusClosed:
|
||||
return tender.TenderStatusClosed
|
||||
case NoticeStatusModified:
|
||||
return tender.TenderStatusModified
|
||||
case NoticeStatusSuspended:
|
||||
return tender.TenderStatusSuspended
|
||||
default:
|
||||
// Default to active for unknown statuses, but log a warning
|
||||
s.logger.Warn("Unknown TED notice status, defaulting to active", map[string]interface{}{
|
||||
"ted_status": string(tedStatus),
|
||||
})
|
||||
return tender.TenderStatusActive
|
||||
}
|
||||
}
|
||||
|
||||
// applyStatusSpecificInformation applies status-specific information to the tender entity
|
||||
func (s *TEDScraper) applyStatusSpecificInformation(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) {
|
||||
if tenderEntity == nil || parsedDoc == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Map TED status to tender status
|
||||
tenderEntity.Status = s.mapTEDStatusToTenderStatus(parsedDoc.Status)
|
||||
|
||||
// Get the document to access status-specific methods
|
||||
document := parsedDoc.GetDocument()
|
||||
if document == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Apply status-specific information based on document type
|
||||
switch parsedDoc.Type {
|
||||
case DocumentTypeContractNotice:
|
||||
if cn, ok := document.(*ContractNotice); ok {
|
||||
s.applyContractNoticeStatusInfo(tenderEntity, cn, parsedDoc.Status)
|
||||
}
|
||||
case DocumentTypeContractAwardNotice:
|
||||
if can, ok := document.(*ContractAwardNotice); ok {
|
||||
s.applyContractAwardNoticeStatusInfo(tenderEntity, can, parsedDoc.Status)
|
||||
}
|
||||
default:
|
||||
// For other document types, we rely on the parsed status
|
||||
s.logger.Debug("Applied basic status information for document type", map[string]interface{}{
|
||||
"document_type": string(parsedDoc.Type),
|
||||
"status": string(parsedDoc.Status),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// applyContractNoticeStatusInfo applies status-specific information from ContractNotice
|
||||
func (s *TEDScraper) applyContractNoticeStatusInfo(tenderEntity *tender.Tender, cn *ContractNotice, status NoticeStatus) {
|
||||
switch status {
|
||||
case NoticeStatusCancelled:
|
||||
if cancellation := cn.GetCancellationReason(); cancellation != nil {
|
||||
tenderEntity.CancellationReason = cancellation.Description
|
||||
if cancellation.ReasonCode != "" {
|
||||
tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description)
|
||||
}
|
||||
// Note: TED XML doesn't typically have cancellation date, so we use current time
|
||||
tenderEntity.CancellationDate = time.Now().Unix()
|
||||
}
|
||||
|
||||
case NoticeStatusAwarded:
|
||||
if award := cn.GetAwardInformation(); award != nil {
|
||||
if award.AwardDate != "" {
|
||||
tenderEntity.AwardDate = s.parseDateToUnix(award.AwardDate)
|
||||
}
|
||||
if award.AwardedValue.Text != "" {
|
||||
if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil {
|
||||
tenderEntity.AwardedValue = value
|
||||
}
|
||||
}
|
||||
tenderEntity.ContractNumber = award.ContractNumber
|
||||
if award.WinningTenderer != nil && award.WinningTenderer.Company != nil {
|
||||
tenderEntity.WinningTenderer = s.mapOrganization(&Organization{
|
||||
Company: award.WinningTenderer.Company,
|
||||
}, "winner")
|
||||
}
|
||||
}
|
||||
|
||||
case NoticeStatusSuspended:
|
||||
if suspension := cn.GetSuspensionReason(); suspension != nil {
|
||||
tenderEntity.SuspensionReason = suspension.Description
|
||||
if suspension.ReasonCode != "" {
|
||||
tenderEntity.SuspensionReason = fmt.Sprintf("%s: %s", suspension.ReasonCode, suspension.Description)
|
||||
}
|
||||
tenderEntity.SuspensionDate = time.Now().Unix()
|
||||
}
|
||||
|
||||
case NoticeStatusModified:
|
||||
if modifications := cn.GetModifications(); modifications != nil {
|
||||
for _, mod := range modifications {
|
||||
tenderMod := tender.TenderModification{
|
||||
ModificationDate: s.parseDateToUnix(mod.ModificationDate),
|
||||
ModificationReason: mod.ModificationReason,
|
||||
Description: mod.Description,
|
||||
LanguageID: mod.LanguageID,
|
||||
}
|
||||
tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// applyContractAwardNoticeStatusInfo applies status-specific information from ContractAwardNotice
|
||||
func (s *TEDScraper) applyContractAwardNoticeStatusInfo(tenderEntity *tender.Tender, can *ContractAwardNotice, status NoticeStatus) {
|
||||
// Contract award notices are typically awarded status
|
||||
tenderEntity.Status = tender.TenderStatusAwarded
|
||||
|
||||
// Set award date from tender result
|
||||
if can.TenderResult != nil && can.TenderResult.AwardDate != "" {
|
||||
tenderEntity.AwardDate = s.parseDateToUnix(can.TenderResult.AwardDate)
|
||||
}
|
||||
|
||||
// Get award information from extensions if available
|
||||
if award := can.GetAwardInformation(); award != nil {
|
||||
if award.AwardedValue.Text != "" {
|
||||
if value, err := strconv.ParseFloat(award.AwardedValue.Text, 64); err == nil {
|
||||
tenderEntity.AwardedValue = value
|
||||
}
|
||||
}
|
||||
tenderEntity.ContractNumber = award.ContractNumber
|
||||
}
|
||||
|
||||
// Get total award value from notice result
|
||||
if amount, currency := can.GetTotalAwardValue(); amount != "" {
|
||||
if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
|
||||
tenderEntity.AwardedValue = parsedAmount
|
||||
tenderEntity.Currency = currency
|
||||
}
|
||||
}
|
||||
|
||||
// Extract detailed awarded contractors information
|
||||
s.extractAwardedEntities(tenderEntity, can)
|
||||
|
||||
// Handle other statuses if applicable
|
||||
switch status {
|
||||
case NoticeStatusCancelled:
|
||||
if cancellation := can.GetCancellationReason(); cancellation != nil {
|
||||
tenderEntity.Status = tender.TenderStatusCancelled
|
||||
tenderEntity.CancellationReason = cancellation.Description
|
||||
if cancellation.ReasonCode != "" {
|
||||
tenderEntity.CancellationReason = fmt.Sprintf("%s: %s", cancellation.ReasonCode, cancellation.Description)
|
||||
}
|
||||
tenderEntity.CancellationDate = time.Now().Unix()
|
||||
}
|
||||
|
||||
case NoticeStatusModified:
|
||||
if modifications := can.GetModifications(); modifications != nil {
|
||||
tenderEntity.Status = tender.TenderStatusModified
|
||||
for _, mod := range modifications {
|
||||
tenderMod := tender.TenderModification{
|
||||
ModificationDate: s.parseDateToUnix(mod.ModificationDate),
|
||||
ModificationReason: mod.ModificationReason,
|
||||
Description: mod.Description,
|
||||
LanguageID: mod.LanguageID,
|
||||
}
|
||||
tenderEntity.Modifications = append(tenderEntity.Modifications, tenderMod)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// extractAwardedEntities extracts detailed information about awarded entities from a contract award notice
|
||||
func (s *TEDScraper) extractAwardedEntities(tenderEntity *tender.Tender, can *ContractAwardNotice) {
|
||||
awardedContractors := can.GetAwardedContractors()
|
||||
if len(awardedContractors) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
s.logger.Info("Extracting awarded entities", map[string]interface{}{
|
||||
"contract_notice_id": can.ID,
|
||||
"num_contractors": len(awardedContractors),
|
||||
})
|
||||
|
||||
for _, contractor := range awardedContractors {
|
||||
entity := tender.AwardedEntity{
|
||||
Name: contractor.Name,
|
||||
Address: contractor.Address,
|
||||
Country: contractor.Country,
|
||||
Currency: contractor.Currency,
|
||||
ContractID: contractor.ContractID,
|
||||
TenderID: contractor.TenderID,
|
||||
LotID: contractor.LotID,
|
||||
CompanyID: contractor.CompanyID,
|
||||
OrganizationID: contractor.OrganizationID,
|
||||
}
|
||||
|
||||
// Parse amount
|
||||
if contractor.Amount != "" {
|
||||
if amount, err := strconv.ParseFloat(contractor.Amount, 64); err == nil {
|
||||
entity.Amount = amount
|
||||
} else {
|
||||
s.logger.Warn("Failed to parse award amount", map[string]interface{}{
|
||||
"amount_string": contractor.Amount,
|
||||
"contractor": contractor.Name,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Parse award date
|
||||
if contractor.AwardDate != "" {
|
||||
entity.AwardDate = s.parseDateToUnix(contractor.AwardDate)
|
||||
} else if contractor.IssueDate != "" {
|
||||
entity.AwardDate = s.parseDateToUnix(contractor.IssueDate)
|
||||
}
|
||||
|
||||
// Calculate share percentage if there are multiple contractors
|
||||
totalAmount := tenderEntity.AwardedValue
|
||||
if totalAmount > 0 && entity.Amount > 0 {
|
||||
entity.Share = (entity.Amount / totalAmount) * 100
|
||||
}
|
||||
|
||||
tenderEntity.AwardedEntities = append(tenderEntity.AwardedEntities, entity)
|
||||
|
||||
s.logger.Debug("Added awarded entity", map[string]interface{}{
|
||||
"name": entity.Name,
|
||||
"amount": entity.Amount,
|
||||
"currency": entity.Currency,
|
||||
"share": entity.Share,
|
||||
"contract_id": entity.ContractID,
|
||||
})
|
||||
}
|
||||
|
||||
s.logger.Info("Successfully extracted awarded entities", map[string]interface{}{
|
||||
"total_entities": len(tenderEntity.AwardedEntities),
|
||||
"total_value": tenderEntity.AwardedValue,
|
||||
"currency": tenderEntity.Currency,
|
||||
})
|
||||
}
|
||||
|
||||
// ScrapeByDateRange scrapes TED XML files within a specified date range
|
||||
func (s *TEDScraper) ScrapeByDateRange(ctx context.Context, config DateRangeConfig) error {
|
||||
startTime := time.Now()
|
||||
|
||||
s.logger.Info("Starting TED XML scraping by date range", map[string]interface{}{
|
||||
"start_date": config.StartDate.Format("2006-01-02"),
|
||||
"end_date": config.EndDate.Format("2006-01-02"),
|
||||
"mode": string(config.Mode),
|
||||
"timestamp": startTime.Unix(),
|
||||
})
|
||||
|
||||
// Validate date range
|
||||
if config.EndDate.Before(config.StartDate) {
|
||||
return fmt.Errorf("end date cannot be before start date")
|
||||
}
|
||||
|
||||
// Calculate the date range in days
|
||||
daysInRange := int(config.EndDate.Sub(config.StartDate).Hours()/24) + 1
|
||||
if daysInRange > 365 {
|
||||
return fmt.Errorf("date range too large: maximum 365 days allowed")
|
||||
}
|
||||
|
||||
// Create scraping job
|
||||
job := &ScrapingJob{
|
||||
Type: fmt.Sprintf("ted_%s", config.Mode),
|
||||
Status: "running",
|
||||
StartedAt: startTime.Unix(),
|
||||
Progress: JobProgress{},
|
||||
Results: JobResults{},
|
||||
Config: map[string]interface{}{
|
||||
"base_url": s.config.BaseURL,
|
||||
"start_date": config.StartDate.Format("2006-01-02"),
|
||||
"end_date": config.EndDate.Format("2006-01-02"),
|
||||
"mode": string(config.Mode),
|
||||
},
|
||||
}
|
||||
|
||||
// Save job
|
||||
if err := s.saveScrapingJob(ctx, job); err != nil {
|
||||
s.logger.Error("Failed to save scraping job", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Calculate files to download based on date range
|
||||
filesToDownload := s.calculateFilesInDateRange(config.StartDate, config.EndDate)
|
||||
if len(filesToDownload) == 0 {
|
||||
s.logger.Info("No files to download in date range", map[string]interface{}{
|
||||
"start_date": config.StartDate.Format("2006-01-02"),
|
||||
"end_date": config.EndDate.Format("2006-01-02"),
|
||||
})
|
||||
|
||||
// Complete the job
|
||||
s.completeJob(ctx, job, nil, nil)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Update job progress
|
||||
job.Progress.TotalFiles = len(filesToDownload)
|
||||
s.saveScrapingJob(ctx, job)
|
||||
|
||||
var lastSuccessfulFile *FileInfo
|
||||
|
||||
// Download and process files
|
||||
for i, fileInfo := range filesToDownload {
|
||||
job.Progress.CurrentFile = fmt.Sprintf("%d%05d", fileInfo.Year, fileInfo.Number)
|
||||
s.saveScrapingJob(ctx, job)
|
||||
|
||||
fileResult, err := s.downloadAndProcess(ctx, fileInfo)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to download and process file", map[string]interface{}{
|
||||
"year": fileInfo.Year,
|
||||
"number": fileInfo.Number,
|
||||
"error": err.Error(),
|
||||
})
|
||||
job.Progress.FailedFiles++
|
||||
job.Results.ErrorCount++
|
||||
} else {
|
||||
job.Progress.ProcessedFiles++
|
||||
job.Results.TotalTenders += fileResult.ProcessedCount
|
||||
job.Results.NewTenders += fileResult.NewTenders
|
||||
job.Results.UpdatedTenders += fileResult.UpdatedTenders
|
||||
if len(fileResult.Errors) > 0 {
|
||||
job.Results.ErrorCount += len(fileResult.Errors)
|
||||
}
|
||||
lastSuccessfulFile = &fileInfo
|
||||
}
|
||||
|
||||
// Update progress percentage
|
||||
job.Progress.PercentComplete = float64(i+1) / float64(len(filesToDownload)) * 100
|
||||
s.saveScrapingJob(ctx, job)
|
||||
|
||||
// Check context for cancellation
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
s.completeJob(ctx, job, nil, lastSuccessfulFile)
|
||||
return ctx.Err()
|
||||
default:
|
||||
// Continue processing
|
||||
}
|
||||
}
|
||||
|
||||
// Complete the job
|
||||
s.completeJob(ctx, job, nil, lastSuccessfulFile)
|
||||
|
||||
s.logger.Info("TED XML scraping by date range completed", map[string]interface{}{
|
||||
"files_processed": job.Progress.ProcessedFiles,
|
||||
"tenders_created": job.Results.NewTenders,
|
||||
"tenders_updated": job.Results.UpdatedTenders,
|
||||
"error_count": job.Results.ErrorCount,
|
||||
"success_rate": job.Results.SuccessRate,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// calculateFilesInDateRange calculates which files need to be downloaded for a date range
|
||||
func (s *TEDScraper) calculateFilesInDateRange(startDate, endDate time.Time) []FileInfo {
|
||||
var files []FileInfo
|
||||
|
||||
// TED files are published roughly once per working day
|
||||
// We estimate file numbers based on working days from start of year
|
||||
|
||||
for year := startDate.Year(); year <= endDate.Year(); year++ {
|
||||
yearStart := time.Date(year, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
yearEnd := time.Date(year, 12, 31, 23, 59, 59, 0, time.UTC)
|
||||
|
||||
// Determine the effective start and end dates for this year
|
||||
effectiveStart := startDate
|
||||
if yearStart.After(startDate) {
|
||||
effectiveStart = yearStart
|
||||
}
|
||||
|
||||
effectiveEnd := endDate
|
||||
if yearEnd.Before(endDate) {
|
||||
effectiveEnd = yearEnd
|
||||
}
|
||||
|
||||
// Skip if no overlap with this year
|
||||
if effectiveStart.After(effectiveEnd) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Calculate approximate file range for this year
|
||||
startDayOfYear := effectiveStart.YearDay()
|
||||
endDayOfYear := effectiveEnd.YearDay()
|
||||
|
||||
// Estimate file numbers (roughly 0.7 files per day, accounting for weekends)
|
||||
startFileNum := int(float64(startDayOfYear) * 0.7)
|
||||
endFileNum := int(float64(endDayOfYear) * 0.7)
|
||||
|
||||
// Ensure minimum range
|
||||
if startFileNum < 1 {
|
||||
startFileNum = 1
|
||||
}
|
||||
if endFileNum < startFileNum {
|
||||
endFileNum = startFileNum
|
||||
}
|
||||
|
||||
// Add some buffer
|
||||
startFileNum = max(1, startFileNum-2)
|
||||
endFileNum = min(366, endFileNum+2) // Max ~366 files per year
|
||||
|
||||
s.logger.Info("Calculating file range for year", map[string]interface{}{
|
||||
"year": year,
|
||||
"start_file_num": startFileNum,
|
||||
"end_file_num": endFileNum,
|
||||
"start_date": effectiveStart.Format("2006-01-02"),
|
||||
"end_date": effectiveEnd.Format("2006-01-02"),
|
||||
})
|
||||
|
||||
for number := startFileNum; number <= endFileNum; number++ {
|
||||
fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number)
|
||||
ojsID := fmt.Sprintf("S %03d / %d", number, year)
|
||||
|
||||
files = append(files, FileInfo{
|
||||
Year: year,
|
||||
Number: number,
|
||||
URL: fileURL,
|
||||
OJS: ojsID,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
s.logger.Info("Calculated files for date range", map[string]interface{}{
|
||||
"total_files": len(files),
|
||||
"start_date": startDate.Format("2006-01-02"),
|
||||
"end_date": endDate.Format("2006-01-02"),
|
||||
})
|
||||
|
||||
return files
|
||||
}
|
||||
|
||||
// Helper function for max
|
||||
func max(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Helper function for min
|
||||
func min(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// applyAwardLogic applies award-specific logic based on document type and content
|
||||
func (s *TEDScraper) applyAwardLogic(tenderEntity *tender.Tender, parsedDoc *ParsedDocument) {
|
||||
if tenderEntity == nil || parsedDoc == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Only set award information if the tender has a confirmed winner
|
||||
hasConfirmedWinner := s.hasConfirmedWinner(parsedDoc)
|
||||
|
||||
s.logger.Debug("Applying award logic", map[string]interface{}{
|
||||
"document_type": string(parsedDoc.Type),
|
||||
"status": string(parsedDoc.Status),
|
||||
"has_confirmed_winner": hasConfirmedWinner,
|
||||
})
|
||||
|
||||
if !hasConfirmedWinner {
|
||||
// Clear any award date and winner information for tenders without confirmed winners
|
||||
tenderEntity.AwardDate = 0
|
||||
tenderEntity.AwardedValue = 0
|
||||
tenderEntity.WinningTenderer = nil
|
||||
tenderEntity.AwardedEntities = nil
|
||||
tenderEntity.ContractNumber = ""
|
||||
|
||||
s.logger.Debug("Cleared award information for tender without confirmed winner", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// Process award information for tenders with confirmed winners
|
||||
s.logger.Info("Processing award information for tender with confirmed winner", map[string]interface{}{
|
||||
"contract_notice_id": tenderEntity.ContractNoticeID,
|
||||
"document_type": string(parsedDoc.Type),
|
||||
})
|
||||
}
|
||||
|
||||
// hasConfirmedWinner checks if the tender has a confirmed winner based on the document content
|
||||
func (s *TEDScraper) hasConfirmedWinner(parsedDoc *ParsedDocument) bool {
|
||||
if parsedDoc == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// Check for award-specific document types
|
||||
switch parsedDoc.Type {
|
||||
case DocumentTypeContractAwardNotice, DocumentTypeResultNotice:
|
||||
// These document types typically indicate awarded tenders
|
||||
if parsedDoc.Status == NoticeStatusAwarded {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for specific award indicators in the document
|
||||
document := parsedDoc.GetDocument()
|
||||
if document != nil {
|
||||
return s.checkForAwardIndicators(document, parsedDoc.Type)
|
||||
}
|
||||
|
||||
case DocumentTypeContractNotice:
|
||||
// Contract notices may have award information if they're updated with results
|
||||
if parsedDoc.Status == NoticeStatusAwarded {
|
||||
if cn := parsedDoc.ContractNotice; cn != nil {
|
||||
return s.checkForAwardIndicators(cn, parsedDoc.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// checkForAwardIndicators checks for specific award indicators in the document
|
||||
func (s *TEDScraper) checkForAwardIndicators(document TEDDocument, docType DocumentType) bool {
|
||||
switch docType {
|
||||
case DocumentTypeContractAwardNotice:
|
||||
if can, ok := document.(*ContractAwardNotice); ok {
|
||||
// Check for tender result with award date
|
||||
if can.TenderResult != nil && can.TenderResult.AwardDate != "" {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for awarded contractors
|
||||
awardedContractors := can.GetAwardedContractors()
|
||||
if len(awardedContractors) > 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
// Check for award information in extensions
|
||||
if award := can.GetAwardInformation(); award != nil && award.AwardDate != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
case DocumentTypeContractNotice:
|
||||
if cn, ok := document.(*ContractNotice); ok {
|
||||
// Check for award information
|
||||
if award := cn.GetAwardInformation(); award != nil && award.AwardDate != "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
case DocumentTypeResultNotice:
|
||||
// Result notices typically contain award information
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// mergeExistingTenderData merges new data with existing tender data, preserving important existing information
|
||||
func (s *TEDScraper) mergeExistingTenderData(existing *tender.Tender, updated *tender.Tender) {
|
||||
// Preserve creation timestamp and original metadata
|
||||
updated.CreatedAt = existing.CreatedAt
|
||||
|
||||
// Merge processing metadata
|
||||
if existing.ProcessingMetadata.ScrapedAt != 0 {
|
||||
// Keep original scraped time, update processed time
|
||||
updated.ProcessingMetadata.ScrapedAt = existing.ProcessingMetadata.ScrapedAt
|
||||
}
|
||||
|
||||
// Preserve existing award information if the new document doesn't have confirmed winners
|
||||
// but existing data does (to avoid losing award data on subsequent updates)
|
||||
if existing.AwardDate != 0 && updated.AwardDate == 0 && len(existing.AwardedEntities) > 0 {
|
||||
s.logger.Info("Preserving existing award information", map[string]interface{}{
|
||||
"contract_notice_id": existing.ContractNoticeID,
|
||||
"existing_award_date": existing.AwardDate,
|
||||
"awarded_entities_count": len(existing.AwardedEntities),
|
||||
})
|
||||
|
||||
updated.AwardDate = existing.AwardDate
|
||||
updated.AwardedValue = existing.AwardedValue
|
||||
updated.WinningTenderer = existing.WinningTenderer
|
||||
updated.AwardedEntities = existing.AwardedEntities
|
||||
updated.ContractNumber = existing.ContractNumber
|
||||
}
|
||||
|
||||
// Merge modifications (append new modifications to existing ones)
|
||||
if len(existing.Modifications) > 0 {
|
||||
// Create a map to avoid duplicate modifications
|
||||
modMap := make(map[string]tender.TenderModification)
|
||||
|
||||
// Add existing modifications
|
||||
for _, mod := range existing.Modifications {
|
||||
key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason)
|
||||
modMap[key] = mod
|
||||
}
|
||||
|
||||
// Add new modifications
|
||||
for _, mod := range updated.Modifications {
|
||||
key := fmt.Sprintf("%d_%s", mod.ModificationDate, mod.ModificationReason)
|
||||
modMap[key] = mod
|
||||
}
|
||||
|
||||
// Convert back to slice
|
||||
updated.Modifications = make([]tender.TenderModification, 0, len(modMap))
|
||||
for _, mod := range modMap {
|
||||
updated.Modifications = append(updated.Modifications, mod)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+408
-84
@@ -3,6 +3,7 @@ package ted
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -27,9 +28,24 @@ const (
|
||||
DocumentTypeSubcontractNotice DocumentType = "SubcontractNotice"
|
||||
)
|
||||
|
||||
// NoticeStatus represents the status of a TED notice
|
||||
type NoticeStatus string
|
||||
|
||||
// Notice status constants for all possible TED notice statuses
|
||||
const (
|
||||
NoticeStatusDraft NoticeStatus = "Draft"
|
||||
NoticeStatusPublished NoticeStatus = "Published"
|
||||
NoticeStatusCancelled NoticeStatus = "Cancelled"
|
||||
NoticeStatusAwarded NoticeStatus = "Awarded"
|
||||
NoticeStatusClosed NoticeStatus = "Closed"
|
||||
NoticeStatusModified NoticeStatus = "Modified"
|
||||
NoticeStatusSuspended NoticeStatus = "Suspended"
|
||||
)
|
||||
|
||||
// ParsedDocument wraps any TED document with type information
|
||||
type ParsedDocument struct {
|
||||
Type DocumentType
|
||||
Type DocumentType `json:"type"`
|
||||
Status NoticeStatus `json:"status"`
|
||||
ContractNotice *ContractNotice `json:"contract_notice,omitempty"`
|
||||
ContractAwardNotice *ContractAwardNotice `json:"contract_award_notice,omitempty"`
|
||||
PriorInformationNotice *PriorInformationNotice `json:"prior_information_notice,omitempty"`
|
||||
@@ -78,6 +94,7 @@ type TEDDocument interface {
|
||||
GetID() string
|
||||
GetNoticeType() string
|
||||
GetLanguage() string
|
||||
GetNoticeStatus() NoticeStatus
|
||||
Validate() error
|
||||
}
|
||||
|
||||
@@ -123,6 +140,11 @@ func (pin PriorInformationNotice) GetLanguage() string {
|
||||
return pin.NoticeLanguageCode
|
||||
}
|
||||
|
||||
func (pin PriorInformationNotice) GetNoticeStatus() NoticeStatus {
|
||||
// Prior information notices are typically published
|
||||
return NoticeStatusPublished
|
||||
}
|
||||
|
||||
func (pin PriorInformationNotice) Validate() error {
|
||||
if pin.ID == "" {
|
||||
return fmt.Errorf("prior information notice ID is required")
|
||||
@@ -163,9 +185,10 @@ type DesignContest struct {
|
||||
}
|
||||
|
||||
// Interface methods for DesignContest
|
||||
func (dc DesignContest) GetID() string { return dc.ID }
|
||||
func (dc DesignContest) GetNoticeType() string { return dc.NoticeTypeCode }
|
||||
func (dc DesignContest) GetLanguage() string { return dc.NoticeLanguageCode }
|
||||
func (dc DesignContest) GetID() string { return dc.ID }
|
||||
func (dc DesignContest) GetNoticeType() string { return dc.NoticeTypeCode }
|
||||
func (dc DesignContest) GetLanguage() string { return dc.NoticeLanguageCode }
|
||||
func (dc DesignContest) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
|
||||
func (dc DesignContest) Validate() error {
|
||||
if dc.ID == "" {
|
||||
return fmt.Errorf("design contest ID is required")
|
||||
@@ -205,9 +228,10 @@ type QualificationSystemNotice struct {
|
||||
}
|
||||
|
||||
// Interface methods for QualificationSystemNotice
|
||||
func (qsn QualificationSystemNotice) GetID() string { return qsn.ID }
|
||||
func (qsn QualificationSystemNotice) GetNoticeType() string { return qsn.NoticeTypeCode }
|
||||
func (qsn QualificationSystemNotice) GetLanguage() string { return qsn.NoticeLanguageCode }
|
||||
func (qsn QualificationSystemNotice) GetID() string { return qsn.ID }
|
||||
func (qsn QualificationSystemNotice) GetNoticeType() string { return qsn.NoticeTypeCode }
|
||||
func (qsn QualificationSystemNotice) GetLanguage() string { return qsn.NoticeLanguageCode }
|
||||
func (qsn QualificationSystemNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
|
||||
func (qsn QualificationSystemNotice) Validate() error {
|
||||
if qsn.ID == "" {
|
||||
return fmt.Errorf("qualification system notice ID is required")
|
||||
@@ -249,9 +273,10 @@ type ConcessionNotice struct {
|
||||
}
|
||||
|
||||
// Interface methods for ConcessionNotice
|
||||
func (cn ConcessionNotice) GetID() string { return cn.ID }
|
||||
func (cn ConcessionNotice) GetNoticeType() string { return cn.NoticeTypeCode }
|
||||
func (cn ConcessionNotice) GetLanguage() string { return cn.NoticeLanguageCode }
|
||||
func (cn ConcessionNotice) GetID() string { return cn.ID }
|
||||
func (cn ConcessionNotice) GetNoticeType() string { return cn.NoticeTypeCode }
|
||||
func (cn ConcessionNotice) GetLanguage() string { return cn.NoticeLanguageCode }
|
||||
func (cn ConcessionNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
|
||||
func (cn ConcessionNotice) Validate() error {
|
||||
if cn.ID == "" {
|
||||
return fmt.Errorf("concession notice ID is required")
|
||||
@@ -291,9 +316,10 @@ type PlanningNotice struct {
|
||||
}
|
||||
|
||||
// Interface methods for PlanningNotice
|
||||
func (pn PlanningNotice) GetID() string { return pn.ID }
|
||||
func (pn PlanningNotice) GetNoticeType() string { return pn.NoticeTypeCode }
|
||||
func (pn PlanningNotice) GetLanguage() string { return pn.NoticeLanguageCode }
|
||||
func (pn PlanningNotice) GetID() string { return pn.ID }
|
||||
func (pn PlanningNotice) GetNoticeType() string { return pn.NoticeTypeCode }
|
||||
func (pn PlanningNotice) GetLanguage() string { return pn.NoticeLanguageCode }
|
||||
func (pn PlanningNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
|
||||
func (pn PlanningNotice) Validate() error {
|
||||
if pn.ID == "" {
|
||||
return fmt.Errorf("planning notice ID is required")
|
||||
@@ -333,9 +359,10 @@ type CompetitionNotice struct {
|
||||
}
|
||||
|
||||
// Interface methods for CompetitionNotice
|
||||
func (cn CompetitionNotice) GetID() string { return cn.ID }
|
||||
func (cn CompetitionNotice) GetNoticeType() string { return cn.NoticeTypeCode }
|
||||
func (cn CompetitionNotice) GetLanguage() string { return cn.NoticeLanguageCode }
|
||||
func (cn CompetitionNotice) GetID() string { return cn.ID }
|
||||
func (cn CompetitionNotice) GetNoticeType() string { return cn.NoticeTypeCode }
|
||||
func (cn CompetitionNotice) GetLanguage() string { return cn.NoticeLanguageCode }
|
||||
func (cn CompetitionNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
|
||||
func (cn CompetitionNotice) Validate() error {
|
||||
if cn.ID == "" {
|
||||
return fmt.Errorf("competition notice ID is required")
|
||||
@@ -376,9 +403,10 @@ type ResultNotice struct {
|
||||
}
|
||||
|
||||
// Interface methods for ResultNotice
|
||||
func (rn ResultNotice) GetID() string { return rn.ID }
|
||||
func (rn ResultNotice) GetNoticeType() string { return rn.NoticeTypeCode }
|
||||
func (rn ResultNotice) GetLanguage() string { return rn.NoticeLanguageCode }
|
||||
func (rn ResultNotice) GetID() string { return rn.ID }
|
||||
func (rn ResultNotice) GetNoticeType() string { return rn.NoticeTypeCode }
|
||||
func (rn ResultNotice) GetLanguage() string { return rn.NoticeLanguageCode }
|
||||
func (rn ResultNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusAwarded }
|
||||
func (rn ResultNotice) Validate() error {
|
||||
if rn.ID == "" {
|
||||
return fmt.Errorf("result notice ID is required")
|
||||
@@ -420,9 +448,10 @@ type ChangeNotice struct {
|
||||
}
|
||||
|
||||
// Interface methods for ChangeNotice
|
||||
func (cn ChangeNotice) GetID() string { return cn.ID }
|
||||
func (cn ChangeNotice) GetNoticeType() string { return cn.NoticeTypeCode }
|
||||
func (cn ChangeNotice) GetLanguage() string { return cn.NoticeLanguageCode }
|
||||
func (cn ChangeNotice) GetID() string { return cn.ID }
|
||||
func (cn ChangeNotice) GetNoticeType() string { return cn.NoticeTypeCode }
|
||||
func (cn ChangeNotice) GetLanguage() string { return cn.NoticeLanguageCode }
|
||||
func (cn ChangeNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusModified }
|
||||
func (cn ChangeNotice) Validate() error {
|
||||
if cn.ID == "" {
|
||||
return fmt.Errorf("change notice ID is required")
|
||||
@@ -463,9 +492,10 @@ type SubcontractNotice struct {
|
||||
}
|
||||
|
||||
// Interface methods for SubcontractNotice
|
||||
func (sn SubcontractNotice) GetID() string { return sn.ID }
|
||||
func (sn SubcontractNotice) GetNoticeType() string { return sn.NoticeTypeCode }
|
||||
func (sn SubcontractNotice) GetLanguage() string { return sn.NoticeLanguageCode }
|
||||
func (sn SubcontractNotice) GetID() string { return sn.ID }
|
||||
func (sn SubcontractNotice) GetNoticeType() string { return sn.NoticeTypeCode }
|
||||
func (sn SubcontractNotice) GetLanguage() string { return sn.NoticeLanguageCode }
|
||||
func (sn SubcontractNotice) GetNoticeStatus() NoticeStatus { return NoticeStatusPublished }
|
||||
func (sn SubcontractNotice) Validate() error {
|
||||
if sn.ID == "" {
|
||||
return fmt.Errorf("subcontract notice ID is required")
|
||||
@@ -665,67 +695,123 @@ type SubcontractingTerm struct {
|
||||
SubcontractingDescription string `xml:"SubcontractingDescription,omitempty"`
|
||||
}
|
||||
|
||||
// NoticeStatusInformation contains status-specific fields
|
||||
type NoticeStatusInformation struct {
|
||||
Status NoticeStatus `xml:"NoticeStatus,omitempty"`
|
||||
StatusCode string `xml:"StatusCode,omitempty"`
|
||||
CancellationReason *CancellationReason `xml:"CancellationReason,omitempty"`
|
||||
AwardInformation *AwardInformation `xml:"AwardInformation,omitempty"`
|
||||
Modifications []NoticeModification `xml:"Modifications>Modification,omitempty"`
|
||||
SuspensionReason *SuspensionReason `xml:"SuspensionReason,omitempty"`
|
||||
}
|
||||
|
||||
// CancellationReason contains cancellation information
|
||||
type CancellationReason struct {
|
||||
XMLName xml.Name `xml:"CancellationReason"`
|
||||
ReasonCode string `xml:"ReasonCode"`
|
||||
Description string `xml:"Description,omitempty"`
|
||||
LanguageID string `xml:"languageID,attr,omitempty"`
|
||||
}
|
||||
|
||||
// AwardInformation contains award-specific information
|
||||
type AwardInformation struct {
|
||||
XMLName xml.Name `xml:"AwardInformation"`
|
||||
AwardDate string `xml:"AwardDate"`
|
||||
AwardedValue CurrencyText `xml:"AwardedValue,omitempty"`
|
||||
WinningTenderer *WinningTenderer `xml:"WinningTenderer,omitempty"`
|
||||
ContractNumber string `xml:"ContractNumber,omitempty"`
|
||||
}
|
||||
|
||||
// WinningTenderer contains information about the winning tenderer
|
||||
type WinningTenderer struct {
|
||||
XMLName xml.Name `xml:"WinningTenderer"`
|
||||
NaturalPersonIndicator string `xml:"NaturalPersonIndicator,omitempty"`
|
||||
Company *Company `xml:"Company,omitempty"`
|
||||
}
|
||||
|
||||
// NoticeModification contains modification information
|
||||
type NoticeModification struct {
|
||||
XMLName xml.Name `xml:"Modification"`
|
||||
ModificationDate string `xml:"ModificationDate"`
|
||||
ModificationReason string `xml:"ModificationReason"`
|
||||
Description string `xml:"Description,omitempty"`
|
||||
LanguageID string `xml:"languageID,attr,omitempty"`
|
||||
}
|
||||
|
||||
// SuspensionReason contains suspension information
|
||||
type SuspensionReason struct {
|
||||
XMLName xml.Name `xml:"SuspensionReason"`
|
||||
ReasonCode string `xml:"ReasonCode"`
|
||||
Description string `xml:"Description,omitempty"`
|
||||
LanguageID string `xml:"languageID,attr,omitempty"`
|
||||
}
|
||||
|
||||
// ContractNotice represents the main TED XML contract notice structure
|
||||
type ContractNotice struct {
|
||||
XMLName xml.Name `xml:"ContractNotice"`
|
||||
Xmlns string `xml:"xmlns,attr,omitempty"`
|
||||
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
|
||||
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
|
||||
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
|
||||
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
|
||||
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
|
||||
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
|
||||
UBLVersionID string `xml:"UBLVersionID"`
|
||||
CustomizationID string `xml:"CustomizationID"`
|
||||
ID string `xml:"ID"`
|
||||
ContractFolderID string `xml:"ContractFolderID"`
|
||||
IssueDate string `xml:"IssueDate"`
|
||||
IssueTime string `xml:"IssueTime"`
|
||||
VersionID string `xml:"VersionID"`
|
||||
RegulatoryDomain string `xml:"RegulatoryDomain"`
|
||||
NoticeTypeCode string `xml:"NoticeTypeCode"`
|
||||
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
|
||||
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
|
||||
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
|
||||
TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"`
|
||||
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
|
||||
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
|
||||
ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
|
||||
XMLName xml.Name `xml:"ContractNotice"`
|
||||
Xmlns string `xml:"xmlns,attr,omitempty"`
|
||||
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
|
||||
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
|
||||
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
|
||||
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
|
||||
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
|
||||
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
|
||||
UBLVersionID string `xml:"UBLVersionID"`
|
||||
CustomizationID string `xml:"CustomizationID"`
|
||||
ID string `xml:"ID"`
|
||||
ContractFolderID string `xml:"ContractFolderID"`
|
||||
IssueDate string `xml:"IssueDate"`
|
||||
IssueTime string `xml:"IssueTime"`
|
||||
VersionID string `xml:"VersionID"`
|
||||
RegulatoryDomain string `xml:"RegulatoryDomain"`
|
||||
NoticeTypeCode string `xml:"NoticeTypeCode"`
|
||||
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
|
||||
StatusInformation *NoticeStatusInformation `xml:"StatusInformation,omitempty"`
|
||||
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
|
||||
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
|
||||
TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"`
|
||||
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
|
||||
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
|
||||
ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
|
||||
}
|
||||
|
||||
// ContractAwardNotice represents the main TED XML contract award notice structure
|
||||
type ContractAwardNotice struct {
|
||||
XMLName xml.Name `xml:"ContractAwardNotice"`
|
||||
Xmlns string `xml:"xmlns,attr,omitempty"`
|
||||
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
|
||||
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
|
||||
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
|
||||
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
|
||||
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
|
||||
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
|
||||
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
|
||||
UBLVersionID string `xml:"UBLVersionID"`
|
||||
CustomizationID string `xml:"CustomizationID"`
|
||||
ID string `xml:"ID"`
|
||||
ContractFolderID string `xml:"ContractFolderID"`
|
||||
IssueDate string `xml:"IssueDate"`
|
||||
IssueTime string `xml:"IssueTime"`
|
||||
VersionID string `xml:"VersionID"`
|
||||
RegulatoryDomain string `xml:"RegulatoryDomain"`
|
||||
NoticeTypeCode string `xml:"NoticeTypeCode"`
|
||||
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
|
||||
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
|
||||
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
|
||||
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
|
||||
ProcurementProjectLot []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
|
||||
TenderResult *TenderResult `xml:"TenderResult,omitempty"`
|
||||
XMLName xml.Name `xml:"ContractAwardNotice"`
|
||||
Xmlns string `xml:"xmlns,attr,omitempty"`
|
||||
XmlnsCac string `xml:"xmlns:cac,attr,omitempty"`
|
||||
XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"`
|
||||
XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"`
|
||||
XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"`
|
||||
XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"`
|
||||
XmlnsExt string `xml:"xmlns:ext,attr,omitempty"`
|
||||
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
|
||||
UBLVersionID string `xml:"UBLVersionID"`
|
||||
CustomizationID string `xml:"CustomizationID"`
|
||||
ID string `xml:"ID"`
|
||||
ContractFolderID string `xml:"ContractFolderID"`
|
||||
IssueDate string `xml:"IssueDate"`
|
||||
IssueTime string `xml:"IssueTime"`
|
||||
VersionID string `xml:"VersionID"`
|
||||
RegulatoryDomain string `xml:"RegulatoryDomain"`
|
||||
NoticeTypeCode string `xml:"NoticeTypeCode"`
|
||||
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
|
||||
StatusInformation *NoticeStatusInformation `xml:"StatusInformation,omitempty"`
|
||||
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
|
||||
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
|
||||
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
|
||||
ProcurementProjectLot []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
|
||||
TenderResult *TenderResult `xml:"TenderResult,omitempty"`
|
||||
}
|
||||
|
||||
// NoticeResult contains notice result information from extensions
|
||||
type NoticeResult struct {
|
||||
XMLName xml.Name `xml:"NoticeResult"`
|
||||
TotalAmount *CurrencyText `xml:"TotalAmount,omitempty"`
|
||||
LotResult []LotResult `xml:"LotResult,omitempty"`
|
||||
XMLName xml.Name `xml:"NoticeResult"`
|
||||
TotalAmount *CurrencyText `xml:"TotalAmount,omitempty"`
|
||||
LotResult []LotResult `xml:"LotResult,omitempty"`
|
||||
LotTender []LotTender `xml:"LotTender,omitempty"`
|
||||
SettledContract []SettledContract `xml:"SettledContract,omitempty"`
|
||||
TenderingParty []TenderingParty `xml:"TenderingParty,omitempty"`
|
||||
}
|
||||
|
||||
// LotResult contains lot result information
|
||||
@@ -761,7 +847,14 @@ type LegalMonetaryTotal struct {
|
||||
|
||||
// TenderingParty contains tendering party information
|
||||
type TenderingParty struct {
|
||||
XMLName xml.Name `xml:"TenderingParty"`
|
||||
XMLName xml.Name `xml:"TenderingParty"`
|
||||
ID string `xml:"ID"`
|
||||
Tenderer []Tenderer `xml:"Tenderer,omitempty"`
|
||||
}
|
||||
|
||||
// Tenderer contains tenderer information
|
||||
type Tenderer struct {
|
||||
XMLName xml.Name `xml:"Tenderer"`
|
||||
ID string `xml:"ID"`
|
||||
}
|
||||
|
||||
@@ -786,13 +879,16 @@ type ReceivedSubmissionsStatistics struct {
|
||||
|
||||
// SettledContract contains settled contract information
|
||||
type SettledContract struct {
|
||||
XMLName xml.Name `xml:"SettledContract"`
|
||||
ID string `xml:"ID"`
|
||||
IssueDate string `xml:"IssueDate,omitempty"`
|
||||
Title string `xml:"Title,omitempty"`
|
||||
SignatoryParty *SignatoryParty `xml:"SignatoryParty,omitempty"`
|
||||
ContractReference *ContractReference `xml:"ContractReference,omitempty"`
|
||||
LotTender *LotTender `xml:"LotTender,omitempty"`
|
||||
XMLName xml.Name `xml:"SettledContract"`
|
||||
ID string `xml:"ID"`
|
||||
IssueDate string `xml:"IssueDate,omitempty"`
|
||||
AwardDate string `xml:"AwardDate,omitempty"`
|
||||
Title string `xml:"Title,omitempty"`
|
||||
ContractFrameworkIndicator string `xml:"ContractFrameworkIndicator,omitempty"`
|
||||
SignatoryParty *SignatoryParty `xml:"SignatoryParty,omitempty"`
|
||||
ContractReference *ContractReference `xml:"ContractReference,omitempty"`
|
||||
LotTender *LotTender `xml:"LotTender,omitempty"`
|
||||
NoticeDocumentReference *NoticeDocumentReference `xml:"NoticeDocumentReference,omitempty"`
|
||||
}
|
||||
|
||||
// SignatoryParty contains signatory party information
|
||||
@@ -1439,6 +1535,62 @@ func (cn ContractNotice) GetLanguage() string {
|
||||
return cn.NoticeLanguageCode
|
||||
}
|
||||
|
||||
// GetNoticeStatus returns the notice status
|
||||
func (cn ContractNotice) GetNoticeStatus() NoticeStatus {
|
||||
if cn.StatusInformation != nil {
|
||||
if cn.StatusInformation.Status != "" {
|
||||
return cn.StatusInformation.Status
|
||||
}
|
||||
// Check for specific status indicators
|
||||
if cn.StatusInformation.CancellationReason != nil {
|
||||
return NoticeStatusCancelled
|
||||
}
|
||||
if cn.StatusInformation.AwardInformation != nil {
|
||||
return NoticeStatusAwarded
|
||||
}
|
||||
if cn.StatusInformation.SuspensionReason != nil {
|
||||
return NoticeStatusSuspended
|
||||
}
|
||||
if len(cn.StatusInformation.Modifications) > 0 {
|
||||
return NoticeStatusModified
|
||||
}
|
||||
}
|
||||
// Default to Published if no status information is available
|
||||
return NoticeStatusPublished
|
||||
}
|
||||
|
||||
// GetCancellationReason returns the cancellation reason if notice is cancelled
|
||||
func (cn ContractNotice) GetCancellationReason() *CancellationReason {
|
||||
if cn.StatusInformation != nil {
|
||||
return cn.StatusInformation.CancellationReason
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAwardInformation returns the award information if notice is awarded
|
||||
func (cn ContractNotice) GetAwardInformation() *AwardInformation {
|
||||
if cn.StatusInformation != nil {
|
||||
return cn.StatusInformation.AwardInformation
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetModifications returns the list of modifications if notice is modified
|
||||
func (cn ContractNotice) GetModifications() []NoticeModification {
|
||||
if cn.StatusInformation != nil {
|
||||
return cn.StatusInformation.Modifications
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetSuspensionReason returns the suspension reason if notice is suspended
|
||||
func (cn ContractNotice) GetSuspensionReason() *SuspensionReason {
|
||||
if cn.StatusInformation != nil {
|
||||
return cn.StatusInformation.SuspensionReason
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetPublicationInfo returns publication information if available
|
||||
func (cn ContractNotice) GetPublicationInfo() *Publication {
|
||||
if cn.Extensions != nil &&
|
||||
@@ -1759,6 +1911,47 @@ func (can ContractAwardNotice) GetLanguage() string {
|
||||
return can.NoticeLanguageCode
|
||||
}
|
||||
|
||||
// GetNoticeStatus returns the notice status
|
||||
func (can ContractAwardNotice) GetNoticeStatus() NoticeStatus {
|
||||
if can.StatusInformation != nil && can.StatusInformation.Status != "" {
|
||||
return can.StatusInformation.Status
|
||||
}
|
||||
// Award notices are typically awarded status
|
||||
return NoticeStatusAwarded
|
||||
}
|
||||
|
||||
// GetCancellationReason returns the cancellation reason if notice is cancelled
|
||||
func (can ContractAwardNotice) GetCancellationReason() *CancellationReason {
|
||||
if can.StatusInformation != nil {
|
||||
return can.StatusInformation.CancellationReason
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAwardInformation returns the award information if notice is awarded
|
||||
func (can ContractAwardNotice) GetAwardInformation() *AwardInformation {
|
||||
if can.StatusInformation != nil {
|
||||
return can.StatusInformation.AwardInformation
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetModifications returns the list of modifications if notice is modified
|
||||
func (can ContractAwardNotice) GetModifications() []NoticeModification {
|
||||
if can.StatusInformation != nil {
|
||||
return can.StatusInformation.Modifications
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetSuspensionReason returns the suspension reason if notice is suspended
|
||||
func (can ContractAwardNotice) GetSuspensionReason() *SuspensionReason {
|
||||
if can.StatusInformation != nil {
|
||||
return can.StatusInformation.SuspensionReason
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetPublicationInfo returns publication information if available
|
||||
func (can ContractAwardNotice) GetPublicationInfo() *Publication {
|
||||
if can.Extensions != nil &&
|
||||
@@ -1915,3 +2108,134 @@ func (can ContractAwardNotice) GetOrganizationByID(orgID string) *Organization {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetAwardedContractors returns detailed information about awarded contractors
|
||||
func (can ContractAwardNotice) GetAwardedContractors() []AwardedContractorInfo {
|
||||
var contractors []AwardedContractorInfo
|
||||
|
||||
noticeResult := can.GetNoticeResult()
|
||||
if noticeResult == nil {
|
||||
return contractors
|
||||
}
|
||||
|
||||
// Create a map of tender ID to tender info for quick lookup
|
||||
tenderMap := make(map[string]*LotTender)
|
||||
for _, lotTender := range noticeResult.LotTender {
|
||||
tenderMap[lotTender.ID] = &lotTender
|
||||
}
|
||||
|
||||
// Create a map of tendering party ID to organization ID
|
||||
partyToOrgMap := make(map[string]string)
|
||||
for _, tenderingParty := range noticeResult.TenderingParty {
|
||||
for _, tenderer := range tenderingParty.Tenderer {
|
||||
partyToOrgMap[tenderingParty.ID] = tenderer.ID
|
||||
}
|
||||
}
|
||||
|
||||
// Process settled contracts to extract award information
|
||||
for _, contract := range noticeResult.SettledContract {
|
||||
if contract.LotTender != nil {
|
||||
// Get tender information
|
||||
tender := tenderMap[contract.LotTender.ID]
|
||||
if tender == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Get organization information
|
||||
var org *Organization
|
||||
if tender.TenderingParty != nil {
|
||||
orgID := partyToOrgMap[tender.TenderingParty.ID]
|
||||
if orgID != "" {
|
||||
org = can.GetOrganizationByID(orgID)
|
||||
}
|
||||
}
|
||||
|
||||
// Create awarded contractor info
|
||||
contractor := AwardedContractorInfo{
|
||||
ContractID: contract.ID,
|
||||
TenderID: contract.LotTender.ID,
|
||||
AwardDate: contract.AwardDate,
|
||||
IssueDate: contract.IssueDate,
|
||||
}
|
||||
|
||||
// Set contract reference if available
|
||||
if contract.ContractReference != nil {
|
||||
contractor.ContractReference = contract.ContractReference.ID
|
||||
}
|
||||
|
||||
// Set amount and currency from tender
|
||||
if tender.LegalMonetaryTotal != nil {
|
||||
contractor.Amount = tender.LegalMonetaryTotal.PayableAmount.Text
|
||||
contractor.Currency = tender.LegalMonetaryTotal.PayableAmount.CurrencyID
|
||||
}
|
||||
|
||||
// Set organization information
|
||||
if org != nil && org.Company != nil {
|
||||
if org.Company.PartyName != nil {
|
||||
contractor.Name = org.Company.PartyName.Name
|
||||
}
|
||||
if org.Company.PartyLegalEntity != nil {
|
||||
contractor.CompanyID = org.Company.PartyLegalEntity.CompanyID
|
||||
}
|
||||
if org.Company.PartyIdentification != nil {
|
||||
contractor.OrganizationID = org.Company.PartyIdentification.ID
|
||||
}
|
||||
|
||||
// Set address information
|
||||
if org.Company.PostalAddress != nil {
|
||||
addressParts := []string{}
|
||||
if org.Company.PostalAddress.StreetName != "" {
|
||||
addressParts = append(addressParts, org.Company.PostalAddress.StreetName)
|
||||
}
|
||||
if org.Company.PostalAddress.CityName != "" {
|
||||
addressParts = append(addressParts, org.Company.PostalAddress.CityName)
|
||||
}
|
||||
if org.Company.PostalAddress.PostalZone != "" {
|
||||
addressParts = append(addressParts, org.Company.PostalAddress.PostalZone)
|
||||
}
|
||||
contractor.Address = strings.Join(addressParts, ", ")
|
||||
|
||||
if org.Company.PostalAddress.Country != nil {
|
||||
contractor.Country = org.Company.PostalAddress.Country.IdentificationCode
|
||||
}
|
||||
}
|
||||
|
||||
// Set contact information
|
||||
if org.Company.Contact != nil {
|
||||
contractor.ContactName = org.Company.Contact.Name
|
||||
contractor.ContactEmail = org.Company.Contact.ElectronicMail
|
||||
contractor.ContactPhone = org.Company.Contact.Telephone
|
||||
}
|
||||
}
|
||||
|
||||
// Set lot information
|
||||
if tender.TenderLot != nil {
|
||||
contractor.LotID = tender.TenderLot.ID
|
||||
}
|
||||
|
||||
contractors = append(contractors, contractor)
|
||||
}
|
||||
}
|
||||
|
||||
return contractors
|
||||
}
|
||||
|
||||
// AwardedContractorInfo contains detailed information about an awarded contractor
|
||||
type AwardedContractorInfo struct {
|
||||
ContractID string `json:"contract_id"`
|
||||
TenderID string `json:"tender_id"`
|
||||
LotID string `json:"lot_id,omitempty"`
|
||||
Name string `json:"name"`
|
||||
CompanyID string `json:"company_id,omitempty"`
|
||||
OrganizationID string `json:"organization_id,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
Country string `json:"country,omitempty"`
|
||||
Amount string `json:"amount"`
|
||||
Currency string `json:"currency,omitempty"`
|
||||
AwardDate string `json:"award_date,omitempty"`
|
||||
IssueDate string `json:"issue_date,omitempty"`
|
||||
ContractReference string `json:"contract_reference,omitempty"`
|
||||
ContactName string `json:"contact_name,omitempty"`
|
||||
ContactEmail string `json:"contact_email,omitempty"`
|
||||
ContactPhone string `json:"contact_phone,omitempty"`
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user