n.nakhostin
f67e179ffc
- Added 'ted_samples/' to .gitignore to exclude TED sample files from version control. - Updated the TED scraper configuration to extend the cleanup duration from 24 hours to 72 hours, allowing for better resource management. - Enhanced the Tender entity by adding new fields: TenderDeadline, SubmissionDeadline, ApplicationDeadline, and SubmissionURL, improving the data model for tender management. - Updated the TenderResponse structure to include the new fields, ensuring consistency in API responses. - Implemented logic in the TED scraper to calculate submission and application deadlines based on tender deadlines, enhancing the scraping functionality.
256 lines
12 KiB
Go
256 lines
12 KiB
Go
package tender
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"tm/pkg/mongo"
|
|
)
|
|
|
|
// Tender represents a tender/contract notice entity
|
|
type Tender struct {
|
|
mongo.Model `bson:",inline"`
|
|
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
|
|
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
|
|
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
|
|
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
|
|
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
|
|
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
|
|
IssueDate int64 `bson:"issue_date" json:"issue_date"` // Unix milliseconds
|
|
IssueTime int64 `bson:"issue_time" json:"issue_time"` // Unix milliseconds
|
|
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds
|
|
PublicationDate int64 `bson:"publication_date" json:"publication_date"` // Unix milliseconds
|
|
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds
|
|
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` // Unix milliseconds
|
|
GazetteID string `bson:"gazette_id" json:"gazette_id"`
|
|
Title string `bson:"title" json:"title"`
|
|
Description string `bson:"description" json:"description"`
|
|
ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"`
|
|
ProcedureCode string `bson:"procedure_code" json:"procedure_code"`
|
|
MainClassification string `bson:"main_classification" json:"main_classification"`
|
|
AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"`
|
|
EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"`
|
|
Currency string `bson:"currency" json:"currency"`
|
|
Duration string `bson:"duration" json:"duration"`
|
|
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
|
|
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
|
|
CountryCode string `bson:"country_code" json:"country_code"`
|
|
RegionCode string `bson:"region_code" json:"region_code"`
|
|
CityName string `bson:"city_name" json:"city_name"`
|
|
PostalCode string `bson:"postal_code" json:"postal_code"`
|
|
DocumentURI string `bson:"document_uri" json:"document_uri"`
|
|
TenderURL string `bson:"tender_url" json:"tender_url"`
|
|
SubmissionURL string `bson:"submission_url" json:"submission_url"`
|
|
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
|
|
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
|
|
Organizations []Organization `bson:"organizations" json:"organizations"`
|
|
SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"`
|
|
OfficialLanguages []string `bson:"official_languages" json:"official_languages"`
|
|
Status TenderStatus `bson:"status" json:"status"`
|
|
Source TenderSource `bson:"source" json:"source"`
|
|
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
|
|
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
|
|
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
|
|
TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
|
|
|
|
}
|
|
|
|
// Organization represents organization information
|
|
type Organization struct {
|
|
ID string `bson:"id" json:"id"`
|
|
Name string `bson:"name" json:"name"`
|
|
CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"`
|
|
WebsiteURI string `bson:"website_uri,omitempty" json:"website_uri,omitempty"`
|
|
ContactName string `bson:"contact_name,omitempty" json:"contact_name,omitempty"`
|
|
ContactTelephone string `bson:"contact_telephone,omitempty" json:"contact_telephone,omitempty"`
|
|
ContactEmail string `bson:"contact_email,omitempty" json:"contact_email,omitempty"`
|
|
ContactFax string `bson:"contact_fax,omitempty" json:"contact_fax,omitempty"`
|
|
Address Address `bson:"address" json:"address"`
|
|
Role string `bson:"role,omitempty" json:"role,omitempty"`
|
|
}
|
|
|
|
// Address represents address information
|
|
type Address struct {
|
|
StreetName string `bson:"street_name,omitempty" json:"street_name,omitempty"`
|
|
CityName string `bson:"city_name,omitempty" json:"city_name,omitempty"`
|
|
PostalZone string `bson:"postal_zone,omitempty" json:"postal_zone,omitempty"`
|
|
CountrySubentityCode string `bson:"country_subentity_code,omitempty" json:"country_subentity_code,omitempty"`
|
|
Department string `bson:"department,omitempty" json:"department,omitempty"`
|
|
Region string `bson:"region,omitempty" json:"region,omitempty"`
|
|
CountryCode string `bson:"country_code" json:"country_code"`
|
|
}
|
|
|
|
// SelectionCriterion represents selection criteria
|
|
type SelectionCriterion struct {
|
|
TypeCode string `bson:"type_code" json:"type_code"`
|
|
Description string `bson:"description,omitempty" json:"description,omitempty"`
|
|
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
|
|
}
|
|
|
|
// ProcessingMetadata contains metadata about how the tender was processed
|
|
type ProcessingMetadata struct {
|
|
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
|
|
ProcessedAt int64 `bson:"processed_at" json:"processed_at"` // Unix milliseconds
|
|
ProcessingVersion string `bson:"processing_version" json:"processing_version"`
|
|
ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
|
|
ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
|
|
EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
|
|
}
|
|
|
|
// TenderStatus represents the status of a tender
|
|
type TenderStatus string
|
|
|
|
const (
|
|
TenderStatusActive TenderStatus = "active"
|
|
TenderStatusExpired TenderStatus = "expired"
|
|
TenderStatusCancelled TenderStatus = "cancelled"
|
|
TenderStatusAwarded TenderStatus = "awarded"
|
|
TenderStatusDraft TenderStatus = "draft"
|
|
)
|
|
|
|
// TenderSource represents the source of tender data
|
|
type TenderSource string
|
|
|
|
const (
|
|
TenderSourceTEDScraper TenderSource = "ted_scraper"
|
|
TenderSourceManual TenderSource = "manual"
|
|
TenderSourceAPI TenderSource = "api"
|
|
TenderSourceBulkImport TenderSource = "bulk_import"
|
|
)
|
|
|
|
// TenderSearchCriteria represents search criteria for tenders
|
|
type TenderSearchCriteria struct {
|
|
Query string `json:"query,omitempty"`
|
|
NoticeType string `json:"notice_type,omitempty"`
|
|
ProcurementType string `json:"procurement_type,omitempty"`
|
|
CountryCodes []string `json:"country_codes,omitempty"`
|
|
RegionCodes []string `json:"region_codes,omitempty"`
|
|
Classifications []string `json:"classifications,omitempty"`
|
|
MinEstimatedValue *float64 `json:"min_estimated_value,omitempty"`
|
|
MaxEstimatedValue *float64 `json:"max_estimated_value,omitempty"`
|
|
Currency string `json:"currency,omitempty"`
|
|
DeadlineFrom *int64 `json:"deadline_from,omitempty"`
|
|
DeadlineTo *int64 `json:"deadline_to,omitempty"`
|
|
PublicationDateFrom *int64 `json:"publication_date_from,omitempty"`
|
|
PublicationDateTo *int64 `json:"publication_date_to,omitempty"`
|
|
Status []TenderStatus `json:"status,omitempty"`
|
|
Source []TenderSource `json:"source,omitempty"`
|
|
BuyerOrganizationID string `json:"buyer_organization_id,omitempty"`
|
|
Languages []string `json:"languages,omitempty"`
|
|
}
|
|
|
|
// TenderStatistics represents tender statistics
|
|
type TenderStatistics struct {
|
|
TotalTenders int64 `json:"total_tenders"`
|
|
ActiveTenders int64 `json:"active_tenders"`
|
|
ExpiredTenders int64 `json:"expired_tenders"`
|
|
TendersByCountry map[string]int64 `json:"tenders_by_country"`
|
|
TendersByType map[string]int64 `json:"tenders_by_type"`
|
|
TendersByClassification map[string]int64 `json:"tenders_by_classification"`
|
|
AverageEstimatedValue float64 `json:"average_estimated_value"`
|
|
TotalEstimatedValue float64 `json:"total_estimated_value"`
|
|
LastUpdated int64 `json:"last_updated"` // Unix milliseconds
|
|
}
|
|
|
|
// GetID returns the tender ID
|
|
func (t *Tender) GetID() string {
|
|
return t.ID.Hex()
|
|
}
|
|
|
|
// IsActive returns true if the tender is active
|
|
func (t *Tender) IsActive() bool {
|
|
return t.Status == TenderStatusActive
|
|
}
|
|
|
|
// IsExpired returns true if the tender deadline has passed
|
|
func (t *Tender) IsExpired() bool {
|
|
if t.Status == TenderStatusExpired {
|
|
return true
|
|
}
|
|
|
|
if t.TenderDeadline == 0 {
|
|
return false
|
|
}
|
|
|
|
// Check if deadline (in milliseconds) is in the past
|
|
now := time.Now().UnixMilli()
|
|
return t.TenderDeadline < now
|
|
}
|
|
|
|
// GetTenderURL returns the TED tender URL
|
|
func (t *Tender) GetTenderURL() string {
|
|
if t.TenderURL != "" {
|
|
return t.TenderURL
|
|
}
|
|
|
|
if t.NoticePublicationID != "" {
|
|
return generateTenderURL(t.NoticePublicationID)
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// generateTenderURL generates the TED tender URL from NoticePublicationID
|
|
func generateTenderURL(noticePublicationID string) string {
|
|
// Remove leading zeros and format for URL
|
|
// Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025
|
|
parts := strings.Split(noticePublicationID, "-")
|
|
if len(parts) != 2 {
|
|
return ""
|
|
}
|
|
|
|
// Convert to int and back to string to remove leading zeros
|
|
if num, err := strconv.Atoi(parts[0]); err == nil {
|
|
cleanID := fmt.Sprintf("%d-%s", num, parts[1])
|
|
return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID)
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
// GetMainOrganization returns the buyer organization
|
|
func (t *Tender) GetMainOrganization() *Organization {
|
|
return t.BuyerOrganization
|
|
}
|
|
|
|
// HasEstimatedValue returns true if the tender has an estimated value
|
|
func (t *Tender) HasEstimatedValue() bool {
|
|
return t.EstimatedValue > 0
|
|
}
|
|
|
|
// GetFormattedEstimatedValue returns formatted estimated value with currency
|
|
func (t *Tender) GetFormattedEstimatedValue() string {
|
|
if !t.HasEstimatedValue() {
|
|
return ""
|
|
}
|
|
|
|
if t.Currency != "" {
|
|
return fmt.Sprintf("%.2f %s", t.EstimatedValue, t.Currency)
|
|
}
|
|
|
|
return fmt.Sprintf("%.2f", t.EstimatedValue)
|
|
}
|
|
|
|
// UpdateStatus updates the tender status and updated timestamp
|
|
func (t *Tender) UpdateStatus(status TenderStatus) {
|
|
t.Status = status
|
|
t.UpdatedAt = time.Now().UnixMilli()
|
|
}
|
|
|
|
// GetDaysUntilDeadline returns the number of days until the tender deadline
|
|
func (t *Tender) GetDaysUntilDeadline() int {
|
|
if t.TenderDeadline == 0 {
|
|
return 0
|
|
}
|
|
|
|
now := time.Now().UnixMilli()
|
|
if t.TenderDeadline <= now {
|
|
return 0 // Already expired
|
|
}
|
|
|
|
diffMillis := t.TenderDeadline - now
|
|
diffDays := diffMillis / (24 * 60 * 60 * 1000)
|
|
return int(diffDays)
|
|
}
|