n.nakhostin
f57e53bd9b
- Added new fields to the Tender entity for handling cancellation, award, and suspension details, improving the entity's capability to manage tender statuses. - Implemented a DetectNoticeStatus method in the TEDParser to identify the status of TED notices from XML data, enhancing the parser's functionality. - Updated the ParseXML method to include notice status detection, ensuring accurate status representation in parsed documents. - Introduced upsert logic in the TED scraper to handle tender records more effectively, allowing for updates or creation based on existing ContractIDs. - Enhanced logging and error handling throughout the scraping process to improve traceability and robustness. - Added a comprehensive usage example document to illustrate the new features and usage patterns for the TED XML parser and scraper.
274 lines
14 KiB
Go
274 lines
14 KiB
Go
package tender
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"tm/pkg/mongo"
|
|
)
|
|
|
|
// Tender represents a tender/contract notice entity
|
|
type Tender struct {
|
|
mongo.Model `bson:",inline"`
|
|
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
|
|
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
|
|
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
|
|
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
|
|
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
|
|
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
|
|
IssueDate int64 `bson:"issue_date" json:"issue_date"` // Unix milliseconds
|
|
IssueTime int64 `bson:"issue_time" json:"issue_time"` // Unix milliseconds
|
|
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds
|
|
PublicationDate int64 `bson:"publication_date" json:"publication_date"` // Unix milliseconds
|
|
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds
|
|
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` // Unix milliseconds
|
|
GazetteID string `bson:"gazette_id" json:"gazette_id"`
|
|
Title string `bson:"title" json:"title"`
|
|
Description string `bson:"description" json:"description"`
|
|
ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"`
|
|
ProcedureCode string `bson:"procedure_code" json:"procedure_code"`
|
|
MainClassification string `bson:"main_classification" json:"main_classification"`
|
|
AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"`
|
|
EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"`
|
|
Currency string `bson:"currency" json:"currency"`
|
|
Duration string `bson:"duration" json:"duration"`
|
|
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
|
|
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
|
|
CountryCode string `bson:"country_code" json:"country_code"`
|
|
RegionCode string `bson:"region_code" json:"region_code"`
|
|
CityName string `bson:"city_name" json:"city_name"`
|
|
PostalCode string `bson:"postal_code" json:"postal_code"`
|
|
DocumentURI string `bson:"document_uri" json:"document_uri"`
|
|
TenderURL string `bson:"tender_url" json:"tender_url"`
|
|
SubmissionURL string `bson:"submission_url" json:"submission_url"`
|
|
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
|
|
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
|
|
Organizations []Organization `bson:"organizations" json:"organizations"`
|
|
SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"`
|
|
OfficialLanguages []string `bson:"official_languages" json:"official_languages"`
|
|
Status TenderStatus `bson:"status" json:"status"`
|
|
Source TenderSource `bson:"source" json:"source"`
|
|
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
|
|
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
|
|
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
|
|
TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
|
|
|
|
// Status-specific fields
|
|
CancellationReason string `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"`
|
|
CancellationDate int64 `bson:"cancellation_date,omitempty" json:"cancellation_date,omitempty"` // Unix milliseconds
|
|
AwardDate int64 `bson:"award_date,omitempty" json:"award_date,omitempty"` // Unix milliseconds
|
|
AwardedValue float64 `bson:"awarded_value,omitempty" json:"awarded_value,omitempty"`
|
|
WinningTenderer *Organization `bson:"winning_tenderer,omitempty" json:"winning_tenderer,omitempty"`
|
|
ContractNumber string `bson:"contract_number,omitempty" json:"contract_number,omitempty"`
|
|
SuspensionReason string `bson:"suspension_reason,omitempty" json:"suspension_reason,omitempty"`
|
|
SuspensionDate int64 `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"` // Unix milliseconds
|
|
Modifications []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"`
|
|
AwardedEntities []AwardedEntity `bson:"awarded_entities,omitempty" json:"awarded_entities,omitempty"`
|
|
}
|
|
|
|
// Organization represents organization information
|
|
type Organization struct {
|
|
ID string `bson:"id" json:"id"`
|
|
Name string `bson:"name" json:"name"`
|
|
CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"`
|
|
WebsiteURI string `bson:"website_uri,omitempty" json:"website_uri,omitempty"`
|
|
ContactName string `bson:"contact_name,omitempty" json:"contact_name,omitempty"`
|
|
ContactTelephone string `bson:"contact_telephone,omitempty" json:"contact_telephone,omitempty"`
|
|
ContactEmail string `bson:"contact_email,omitempty" json:"contact_email,omitempty"`
|
|
ContactFax string `bson:"contact_fax,omitempty" json:"contact_fax,omitempty"`
|
|
Address Address `bson:"address" json:"address"`
|
|
Role string `bson:"role,omitempty" json:"role,omitempty"`
|
|
}
|
|
|
|
// Address represents address information
|
|
type Address struct {
|
|
StreetName string `bson:"street_name,omitempty" json:"street_name,omitempty"`
|
|
CityName string `bson:"city_name,omitempty" json:"city_name,omitempty"`
|
|
PostalZone string `bson:"postal_zone,omitempty" json:"postal_zone,omitempty"`
|
|
CountrySubentityCode string `bson:"country_subentity_code,omitempty" json:"country_subentity_code,omitempty"`
|
|
Department string `bson:"department,omitempty" json:"department,omitempty"`
|
|
Region string `bson:"region,omitempty" json:"region,omitempty"`
|
|
CountryCode string `bson:"country_code" json:"country_code"`
|
|
}
|
|
|
|
// SelectionCriterion represents selection criteria
|
|
type SelectionCriterion struct {
|
|
TypeCode string `bson:"type_code" json:"type_code"`
|
|
Description string `bson:"description,omitempty" json:"description,omitempty"`
|
|
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
|
|
}
|
|
|
|
// ProcessingMetadata contains metadata about how the tender was processed
|
|
type ProcessingMetadata struct {
|
|
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
|
|
ProcessedAt int64 `bson:"processed_at" json:"processed_at"` // Unix milliseconds
|
|
ProcessingVersion string `bson:"processing_version" json:"processing_version"`
|
|
ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
|
|
ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
|
|
EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
|
|
}
|
|
|
|
// TenderModification represents a modification made to a tender
|
|
type TenderModification struct {
|
|
ModificationDate int64 `bson:"modification_date" json:"modification_date"` // Unix milliseconds
|
|
ModificationReason string `bson:"modification_reason" json:"modification_reason"`
|
|
Description string `bson:"description,omitempty" json:"description,omitempty"`
|
|
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
|
|
}
|
|
|
|
// AwardedEntity represents an entity that has been awarded a contract
|
|
type AwardedEntity struct {
|
|
Name string `bson:"name" json:"name"`
|
|
Address string `bson:"address,omitempty" json:"address,omitempty"`
|
|
Country string `bson:"country,omitempty" json:"country,omitempty"`
|
|
Amount float64 `bson:"amount" json:"amount"`
|
|
Currency string `bson:"currency,omitempty" json:"currency,omitempty"`
|
|
Share float64 `bson:"share,omitempty" json:"share,omitempty"` // Percentage share for multiple winners
|
|
AwardDate int64 `bson:"award_date" json:"award_date"` // Unix milliseconds
|
|
ContractID string `bson:"contract_id,omitempty" json:"contract_id,omitempty"` // Contract reference ID
|
|
TenderID string `bson:"tender_id,omitempty" json:"tender_id,omitempty"` // Tender reference ID
|
|
LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` // Lot reference ID
|
|
CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"` // Company registration ID
|
|
OrganizationID string `bson:"organization_id,omitempty" json:"organization_id,omitempty"` // Organization reference ID
|
|
}
|
|
|
|
// TenderStatus represents the status of a tender
|
|
type TenderStatus string
|
|
|
|
const (
|
|
TenderStatusActive TenderStatus = "active"
|
|
TenderStatusExpired TenderStatus = "expired"
|
|
TenderStatusCancelled TenderStatus = "cancelled"
|
|
TenderStatusAwarded TenderStatus = "awarded"
|
|
TenderStatusDraft TenderStatus = "draft"
|
|
TenderStatusClosed TenderStatus = "closed"
|
|
TenderStatusModified TenderStatus = "modified"
|
|
TenderStatusSuspended TenderStatus = "suspended"
|
|
TenderStatusPublished TenderStatus = "published"
|
|
)
|
|
|
|
// TenderSource represents the source of tender data
|
|
type TenderSource string
|
|
|
|
const (
|
|
TenderSourceTEDScraper TenderSource = "ted_scraper"
|
|
TenderSourceManual TenderSource = "manual"
|
|
TenderSourceAPI TenderSource = "api"
|
|
TenderSourceBulkImport TenderSource = "bulk_import"
|
|
)
|
|
|
|
// TenderStatistics represents tender statistics
|
|
type TenderStatistics struct {
|
|
TotalTenders int64 `json:"total_tenders"`
|
|
ActiveTenders int64 `json:"active_tenders"`
|
|
ExpiredTenders int64 `json:"expired_tenders"`
|
|
TendersByCountry map[string]int64 `json:"tenders_by_country"`
|
|
TendersByType map[string]int64 `json:"tenders_by_type"`
|
|
TendersByClassification map[string]int64 `json:"tenders_by_classification"`
|
|
AverageEstimatedValue float64 `json:"average_estimated_value"`
|
|
TotalEstimatedValue float64 `json:"total_estimated_value"`
|
|
LastUpdated int64 `json:"last_updated"` // Unix milliseconds
|
|
}
|
|
|
|
// GetID returns the tender ID
|
|
func (t *Tender) GetID() string {
|
|
return t.ID.Hex()
|
|
}
|
|
|
|
// IsActive returns true if the tender is active
|
|
func (t *Tender) IsActive() bool {
|
|
return t.Status == TenderStatusActive
|
|
}
|
|
|
|
// IsExpired returns true if the tender deadline has passed
|
|
func (t *Tender) IsExpired() bool {
|
|
if t.Status == TenderStatusExpired {
|
|
return true
|
|
}
|
|
|
|
if t.TenderDeadline == 0 {
|
|
return false
|
|
}
|
|
|
|
// Check if deadline (in milliseconds) is in the past
|
|
now := time.Now().Unix()
|
|
return t.TenderDeadline < now
|
|
}
|
|
|
|
// GetTenderURL returns the TED tender URL
|
|
func (t *Tender) GetTenderURL() string {
|
|
if t.TenderURL != "" {
|
|
return t.TenderURL
|
|
}
|
|
|
|
if t.NoticePublicationID != "" {
|
|
return generateTenderURL(t.NoticePublicationID)
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// generateTenderURL generates the TED tender URL from NoticePublicationID
|
|
func generateTenderURL(noticePublicationID string) string {
|
|
// Remove leading zeros and format for URL
|
|
// Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025
|
|
parts := strings.Split(noticePublicationID, "-")
|
|
if len(parts) != 2 {
|
|
return ""
|
|
}
|
|
|
|
// Convert to int and back to string to remove leading zeros
|
|
if num, err := strconv.Atoi(parts[0]); err == nil {
|
|
cleanID := fmt.Sprintf("%d-%s", num, parts[1])
|
|
return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID)
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// GetMainOrganization returns the buyer organization
|
|
func (t *Tender) GetMainOrganization() *Organization {
|
|
return t.BuyerOrganization
|
|
}
|
|
|
|
// HasEstimatedValue returns true if the tender has an estimated value
|
|
func (t *Tender) HasEstimatedValue() bool {
|
|
return t.EstimatedValue > 0
|
|
}
|
|
|
|
// GetFormattedEstimatedValue returns formatted estimated value with currency
|
|
func (t *Tender) GetFormattedEstimatedValue() string {
|
|
if !t.HasEstimatedValue() {
|
|
return ""
|
|
}
|
|
|
|
if t.Currency != "" {
|
|
return fmt.Sprintf("%.2f %s", t.EstimatedValue, t.Currency)
|
|
}
|
|
|
|
return fmt.Sprintf("%.2f", t.EstimatedValue)
|
|
}
|
|
|
|
// UpdateStatus updates the tender status and updated timestamp
|
|
func (t *Tender) UpdateStatus(status TenderStatus) {
|
|
t.Status = status
|
|
t.UpdatedAt = time.Now().Unix()
|
|
}
|
|
|
|
// GetDaysUntilDeadline returns the number of days until the tender deadline
|
|
func (t *Tender) GetDaysUntilDeadline() int {
|
|
if t.TenderDeadline == 0 {
|
|
return 0
|
|
}
|
|
|
|
now := time.Now().Unix()
|
|
if t.TenderDeadline <= now {
|
|
return 0 // Already expired
|
|
}
|
|
|
|
diffMillis := t.TenderDeadline - now
|
|
diffDays := diffMillis / (24 * 60 * 60 * 1000)
|
|
return int(diffDays)
|
|
}
|