Enhance Tender Entity and TED Parser with Status Management and Upsert Logic

- Added new fields to the Tender entity for handling cancellation, award, and suspension details, improving the entity's capability to manage tender statuses.
- Implemented a DetectNoticeStatus method in the TEDParser to identify the status of TED notices from XML data, enhancing the parser's functionality.
- Updated the ParseXML method to include notice status detection, ensuring accurate status representation in parsed documents.
- Introduced upsert logic in the TED scraper to handle tender records more effectively, allowing for updates or creation based on existing ContractIDs.
- Enhanced logging and error handling throughout the scraping process to improve traceability and robustness.
- Added a comprehensive usage example document to illustrate the new features and usage patterns for the TED XML parser and scraper.

ted/parser.go

@@ -121,6 +121,141 @@ func (p *TEDParser) DetectDocumentType(xmlData []byte) (DocumentType, error) {
 	return "", fmt.Errorf("unknown document type")
 }
 
+// DetectNoticeStatus detects the status of a TED notice from XML data
+func (p *TEDParser) DetectNoticeStatus(xmlData []byte) NoticeStatus {
+	// Convert to string for fast text search
+	xmlStr := string(xmlData)
+
+	// Check for common status indicators in XML content
+	// Many TED XMLs contain status information in extensions or specific tags
+
+	// Check for cancellation indicators
+	if strings.Contains(xmlStr, "<CancellationReason") ||
+		strings.Contains(xmlStr, "cancelled") ||
+		strings.Contains(xmlStr, "canceled") ||
+		strings.Contains(xmlStr, "CANCELLED") ||
+		strings.Contains(xmlStr, "CANCELED") {
+		return NoticeStatusCancelled
+	}
+
+	// Check for award indicators
+	if strings.Contains(xmlStr, "<AwardDate") ||
+		strings.Contains(xmlStr, "<TenderResult") ||
+		strings.Contains(xmlStr, "<SettledContract") ||
+		strings.Contains(xmlStr, "<ContractAwardNotice") ||
+		strings.Contains(xmlStr, "awarded") ||
+		strings.Contains(xmlStr, "AWARDED") {
+		return NoticeStatusAwarded
+	}
+
+	// Check for modification indicators
+	if strings.Contains(xmlStr, "<ChangeNotice") ||
+		strings.Contains(xmlStr, "<ChangeReason") ||
+		strings.Contains(xmlStr, "<Modification") ||
+		strings.Contains(xmlStr, "modified") ||
+		strings.Contains(xmlStr, "MODIFIED") {
+		return NoticeStatusModified
+	}
+
+	// Check for suspension indicators
+	if strings.Contains(xmlStr, "<SuspensionReason") ||
+		strings.Contains(xmlStr, "suspended") ||
+		strings.Contains(xmlStr, "SUSPENDED") {
+		return NoticeStatusSuspended
+	}
+
+	// Check for closed indicators
+	if strings.Contains(xmlStr, "closed") ||
+		strings.Contains(xmlStr, "CLOSED") ||
+		strings.Contains(xmlStr, "<ClosedDate") {
+		return NoticeStatusClosed
+	}
+
+	// Check for draft indicators
+	if strings.Contains(xmlStr, "draft") ||
+		strings.Contains(xmlStr, "DRAFT") {
+		return NoticeStatusDraft
+	}
+
+	// Check for explicit status in XML using XML parsing
+	status := p.extractStatusFromXML(xmlData)
+	if status != "" {
+		switch strings.ToLower(status) {
+		case "draft":
+			return NoticeStatusDraft
+		case "published":
+			return NoticeStatusPublished
+		case "cancelled", "canceled":
+			return NoticeStatusCancelled
+		case "awarded":
+			return NoticeStatusAwarded
+		case "closed":
+			return NoticeStatusClosed
+		case "modified":
+			return NoticeStatusModified
+		case "suspended":
+			return NoticeStatusSuspended
+		}
+	}
+
+	// Default to Published for most TED notices
+	return NoticeStatusPublished
+}
+
+// extractStatusFromXML extracts status information using XML parsing
+func (p *TEDParser) extractStatusFromXML(xmlData []byte) string {
+	decoder := xml.NewDecoder(bytes.NewReader(xmlData))
+
+	for {
+		token, err := decoder.Token()
+		if err != nil {
+			break
+		}
+
+		if startElement, ok := token.(xml.StartElement); ok {
+			// Look for common status element names
+			switch startElement.Name.Local {
+			case "NoticeStatus", "Status", "StatusCode", "DocumentStatus":
+				// Read the content of this element
+				if content, err := p.readElementContent(decoder); err == nil {
+					return strings.TrimSpace(content)
+				}
+			}
+
+			// Check attributes for status information
+			for _, attr := range startElement.Attr {
+				if strings.Contains(strings.ToLower(attr.Name.Local), "status") {
+					return strings.TrimSpace(attr.Value)
+				}
+			}
+		}
+	}
+
+	return ""
+}
+
+// readElementContent reads the text content of the current XML element
+func (p *TEDParser) readElementContent(decoder *xml.Decoder) (string, error) {
+	var content strings.Builder
+
+	for {
+		token, err := decoder.Token()
+		if err != nil {
+			return "", err
+		}
+
+		switch t := token.(type) {
+		case xml.CharData:
+			content.Write(t)
+		case xml.EndElement:
+			return content.String(), nil
+		case xml.StartElement:
+			// Skip nested elements
+			decoder.Skip()
+		}
+	}
+}
+
 // ParseXML automatically detects and parses any supported TED XML document type
 func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
 	docType, err := p.DetectDocumentType(xmlData)
@@ -128,7 +263,13 @@ func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
 		return nil, fmt.Errorf("failed to detect document type: %w", err)
 	}
 
-	parsedDoc := &ParsedDocument{Type: docType}
+	// Detect notice status
+	noticeStatus := p.DetectNoticeStatus(xmlData)
+
+	parsedDoc := &ParsedDocument{
+		Type:   docType,
+		Status: noticeStatus,
+	}
 
 	switch docType {
 	case DocumentTypeContractNotice:
@@ -137,6 +278,10 @@ func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
 			return nil, fmt.Errorf("failed to parse contract notice: %w", err)
 		}
 		parsedDoc.ContractNotice = parsed
+		// Override status based on document content
+		if parsed != nil {
+			parsedDoc.Status = parsed.GetNoticeStatus()
+		}
 
 	case DocumentTypeContractAwardNotice:
 		parsed, err := p.contractAwardNoticeParser.ParseBytes(xmlData)
@@ -144,6 +289,10 @@ func (p *TEDParser) ParseXML(xmlData []byte) (*ParsedDocument, error) {
 			return nil, fmt.Errorf("failed to parse contract award notice: %w", err)
 		}
 		parsedDoc.ContractAwardNotice = parsed
+		// Override status based on document content
+		if parsed != nil {
+			parsedDoc.Status = parsed.GetNoticeStatus()
+		}
 
 	case DocumentTypePriorInformationNotice:
 		parsed, err := p.priorInformationNoticeParser.ParseBytes(xmlData)