Enhance Tender Entity and TED Parser with Status Management and Upsert Logic

- Added new fields to the Tender entity for handling cancellation, award, and suspension details, improving the entity's capability to manage tender statuses.
- Implemented a DetectNoticeStatus method in the TEDParser to identify the status of TED notices from XML data, enhancing the parser's functionality.
- Updated the ParseXML method to include notice status detection, ensuring accurate status representation in parsed documents.
- Introduced upsert logic in the TED scraper to handle tender records more effectively, allowing for updates or creation based on existing ContractIDs.
- Enhanced logging and error handling throughout the scraping process to improve traceability and robustness.
- Added a comprehensive usage example document to illustrate the new features and usage patterns for the TED XML parser and scraper.

internal/tender/entity.go

@@ -54,6 +54,17 @@ type Tender struct {
 	ProcessingMetadata        ProcessingMetadata   `bson:"processing_metadata" json:"processing_metadata"`
 	TenderID                  string               `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
 
+	// Status-specific fields
+	CancellationReason string               `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"`
+	CancellationDate   int64                `bson:"cancellation_date,omitempty" json:"cancellation_date,omitempty"` // Unix milliseconds
+	AwardDate          int64                `bson:"award_date,omitempty" json:"award_date,omitempty"`               // Unix milliseconds
+	AwardedValue       float64              `bson:"awarded_value,omitempty" json:"awarded_value,omitempty"`
+	WinningTenderer    *Organization        `bson:"winning_tenderer,omitempty" json:"winning_tenderer,omitempty"`
+	ContractNumber     string               `bson:"contract_number,omitempty" json:"contract_number,omitempty"`
+	SuspensionReason   string               `bson:"suspension_reason,omitempty" json:"suspension_reason,omitempty"`
+	SuspensionDate     int64                `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"` // Unix milliseconds
+	Modifications      []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"`
+	AwardedEntities    []AwardedEntity      `bson:"awarded_entities,omitempty" json:"awarded_entities,omitempty"`
 }
 
 // Organization represents organization information
@@ -98,6 +109,30 @@ type ProcessingMetadata struct {
 	EnrichmentData    map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
 }
 
+// TenderModification represents a modification made to a tender
+type TenderModification struct {
+	ModificationDate   int64  `bson:"modification_date" json:"modification_date"` // Unix milliseconds
+	ModificationReason string `bson:"modification_reason" json:"modification_reason"`
+	Description        string `bson:"description,omitempty" json:"description,omitempty"`
+	LanguageID         string `bson:"language_id,omitempty" json:"language_id,omitempty"`
+}
+
+// AwardedEntity represents an entity that has been awarded a contract
+type AwardedEntity struct {
+	Name           string  `bson:"name" json:"name"`
+	Address        string  `bson:"address,omitempty" json:"address,omitempty"`
+	Country        string  `bson:"country,omitempty" json:"country,omitempty"`
+	Amount         float64 `bson:"amount" json:"amount"`
+	Currency       string  `bson:"currency,omitempty" json:"currency,omitempty"`
+	Share          float64 `bson:"share,omitempty" json:"share,omitempty"`                     // Percentage share for multiple winners
+	AwardDate      int64   `bson:"award_date" json:"award_date"`                               // Unix milliseconds
+	ContractID     string  `bson:"contract_id,omitempty" json:"contract_id,omitempty"`         // Contract reference ID
+	TenderID       string  `bson:"tender_id,omitempty" json:"tender_id,omitempty"`             // Tender reference ID
+	LotID          string  `bson:"lot_id,omitempty" json:"lot_id,omitempty"`                   // Lot reference ID
+	CompanyID      string  `bson:"company_id,omitempty" json:"company_id,omitempty"`           // Company registration ID
+	OrganizationID string  `bson:"organization_id,omitempty" json:"organization_id,omitempty"` // Organization reference ID
+}
+
 // TenderStatus represents the status of a tender
 type TenderStatus string
 
@@ -107,6 +142,10 @@ const (
 	TenderStatusCancelled TenderStatus = "cancelled"
 	TenderStatusAwarded   TenderStatus = "awarded"
 	TenderStatusDraft     TenderStatus = "draft"
+	TenderStatusClosed    TenderStatus = "closed"
+	TenderStatusModified  TenderStatus = "modified"
+	TenderStatusSuspended TenderStatus = "suspended"
+	TenderStatusPublished TenderStatus = "published"
 )
 
 // TenderSource represents the source of tender data
@@ -153,7 +192,7 @@ func (t *Tender) IsExpired() bool {
 	}
 
 	// Check if deadline (in milliseconds) is in the past
-	now := time.Now().UnixMilli()
+	now := time.Now().Unix()
 	return t.TenderDeadline < now
 }
 
@@ -214,7 +253,7 @@ func (t *Tender) GetFormattedEstimatedValue() string {
 // UpdateStatus updates the tender status and updated timestamp
 func (t *Tender) UpdateStatus(status TenderStatus) {
 	t.Status = status
-	t.UpdatedAt = time.Now().UnixMilli()
+	t.UpdatedAt = time.Now().Unix()
 }
 
 // GetDaysUntilDeadline returns the number of days until the tender deadline
@@ -223,7 +262,7 @@ func (t *Tender) GetDaysUntilDeadline() int {
 		return 0
 	}
 
-	now := time.Now().UnixMilli()
+	now := time.Now().Unix()
 	if t.TenderDeadline <= now {
 		return 0 // Already expired
 	}