Files
tm_back/internal/tender/entity.go
T
Mazyar c46a8d54f4
continuous-integration/drone/push Build is passing
Refactor document scraper to utilize active publication windows for tender retrieval
- Updated the ListPendingTenders method to filter tenders based on their active publication submission window instead of just deadlines, enhancing the accuracy of tender listings.
- Introduced a new HasActivePublicationWindow method in the Tender entity to determine if the publication-based submission window is still open.
- Modified the GetTenderByNoticeID method to reflect the new logic for filtering tenders based on their publication status.
- Enhanced the documentation for relevant API endpoints to clarify the changes in tender retrieval criteria.

This update improves the precision of tender scraping by ensuring only relevant tenders with active submission windows are processed, contributing to better data management and scraping efficiency.
2026-07-01 19:22:45 +03:30

428 lines
22 KiB
Go

package tender
import (
"fmt"
"strconv"
"strings"
"time"
"tm/pkg/mongo"
"tm/ted"
)
// DocumentSummary represents a summary of a tender document
type DocumentSummary struct {
DocumentName string `bson:"document_name" json:"document_name"`
ObjectName string `bson:"object_name" json:"object_name"`
BucketName string `bson:"bucket_name" json:"bucket_name"`
Summary string `bson:"summary" json:"summary"`
SummaryLanguage string `bson:"summary_language" json:"summary_language"`
DocumentType string `bson:"document_type" json:"document_type"`
SummarizedAt int64 `bson:"summarized_at" json:"summarized_at"`
SummaryModel string `bson:"summary_model" json:"summary_model"`
Error string `bson:"error,omitempty" json:"error,omitempty"`
}
// ScrapedDocument represents information about a document that was scraped and uploaded to MinIO
type ScrapedDocument struct {
Filename string `bson:"filename" json:"filename"`
ObjectName string `bson:"object_name" json:"object_name"`
BucketName string `bson:"bucket_name" json:"bucket_name"`
Size int64 `bson:"size" json:"size"`
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"`
}
// Tender represents a tender/contract notice entity
type Tender struct {
mongo.Model `bson:",inline"`
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
// RelatedNoticePublicationIDs lists every TED notice publication id that has contributed to this tender
// (same procedure / contract_folder_id). Ordered oldest-first; the latest is also reflected in NoticePublicationID.
RelatedNoticePublicationIDs []string `bson:"related_notice_publication_ids,omitempty" json:"related_notice_publication_ids,omitempty"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
// ProcurementProjectID is UBL ProcurementProject/ID — merges per-lot CAN/CN notices into one tender.
ProcurementProjectID string `bson:"procurement_project_id,omitempty" json:"procurement_project_id,omitempty"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"`
PublicationDate int64 `bson:"publication_date" json:"publication_date"`
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"`
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"`
GazetteID string `bson:"gazette_id" json:"gazette_id"`
Title string `bson:"title" json:"title"`
Description string `bson:"description" json:"description"`
ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"`
ProcedureCode string `bson:"procedure_code" json:"procedure_code"`
MainClassification string `bson:"main_classification" json:"main_classification"`
MainClassificationDescription string `bson:"main_classification_description,omitempty" json:"main_classification_description,omitempty"`
AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"`
EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"`
Currency string `bson:"currency" json:"currency"`
Duration string `bson:"duration" json:"duration"`
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
CountryCode string `bson:"country_code" json:"country_code"`
RegionCode string `bson:"region_code" json:"region_code"`
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
Organizations []Organization `bson:"organizations" json:"organizations"`
SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"`
OfficialLanguages []string `bson:"official_languages" json:"official_languages"`
Status TenderStatus `bson:"status" json:"status"`
Source TenderSource `bson:"source" json:"source"`
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"`
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
// AIOverallSummary optional denormalized overall summary for search/list (e.g. synced from AI pipeline storage).
AIOverallSummary string `bson:"ai_overall_summary,omitempty" json:"ai_overall_summary,omitempty"`
TenderID string `bson:"tender_id" json:"tender_id"` // PBL Project Number (SCDYYNNN format)
ProjectName string `bson:"project_name" json:"project_name"` // PBL Project Name (<InternalProjectNumber>_<Client-OpportunityName>_<PartnerCode>)
// Status-specific fields
CancellationReason string `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"`
CancellationDate int64 `bson:"cancellation_date,omitempty" json:"cancellation_date,omitempty"` // Unix milliseconds
AwardDate int64 `bson:"award_date,omitempty" json:"award_date,omitempty"` // Unix milliseconds
AwardedValue float64 `bson:"awarded_value,omitempty" json:"awarded_value,omitempty"`
WinningTenderer *Organization `bson:"winning_tenderer,omitempty" json:"winning_tenderer,omitempty"`
ContractNumber string `bson:"contract_number,omitempty" json:"contract_number,omitempty"`
SuspensionReason string `bson:"suspension_reason,omitempty" json:"suspension_reason,omitempty"`
SuspensionDate int64 `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"` // Unix milliseconds
Modifications []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"`
AwardedEntities []Awarded `bson:"awarded,omitempty" json:"awarded,omitempty"`
ScrapedDocuments []ScrapedDocument `bson:"scraped_documents,omitempty" json:"scraped_documents,omitempty"`
DocumentSummaries []DocumentSummary `bson:"document_summaries,omitempty" json:"document_summaries,omitempty"`
Translations map[string]TranslationEntry `bson:"translations,omitempty" json:"translations,omitempty"`
}
// TranslationEntry stores translated tender content for a target language.
type TranslationEntry struct {
Title string `bson:"title" json:"title"`
Description string `bson:"description" json:"description"`
Language string `bson:"language" json:"language"`
TranslatedAt int64 `bson:"translated_at" json:"translated_at"`
}
// Organization represents organization information
type Organization struct {
ID string `bson:"id" json:"id"`
Name string `bson:"name" json:"name"`
CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"`
WebsiteURI string `bson:"website_uri,omitempty" json:"website_uri,omitempty"`
ContactName string `bson:"contact_name,omitempty" json:"contact_name,omitempty"`
ContactTelephone string `bson:"contact_telephone,omitempty" json:"contact_telephone,omitempty"`
ContactEmail string `bson:"contact_email,omitempty" json:"contact_email,omitempty"`
ContactFax string `bson:"contact_fax,omitempty" json:"contact_fax,omitempty"`
Address Address `bson:"address" json:"address"`
Role string `bson:"role,omitempty" json:"role,omitempty"`
}
// Address represents address information
type Address struct {
StreetName string `bson:"street_name,omitempty" json:"street_name,omitempty"`
CityName string `bson:"city_name,omitempty" json:"city_name,omitempty"`
PostalZone string `bson:"postal_zone,omitempty" json:"postal_zone,omitempty"`
CountrySubentityCode string `bson:"country_subentity_code,omitempty" json:"country_subentity_code,omitempty"`
Department string `bson:"department,omitempty" json:"department,omitempty"`
Region string `bson:"region,omitempty" json:"region,omitempty"`
CountryCode string `bson:"country_code" json:"country_code"`
}
// SelectionCriterion represents selection criteria
type SelectionCriterion struct {
TypeCode string `bson:"type_code" json:"type_code"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
}
// ProcurementLot captures per-lot procurement fields from TED notices.
type ProcurementLot struct {
LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"`
Title string `bson:"title,omitempty" json:"title,omitempty"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"`
MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"`
MainClassificationDescription string `bson:"main_classification_description,omitempty" json:"main_classification_description,omitempty"`
AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"`
Duration string `bson:"duration,omitempty" json:"duration,omitempty"`
EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"`
Currency string `bson:"currency,omitempty" json:"currency,omitempty"`
}
// ProcessingMetadata contains metadata about how the tender was processed
type ProcessingMetadata struct {
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
ProcessedAt int64 `bson:"processed_at" json:"processed_at"` // Unix milliseconds
ProcessingVersion string `bson:"processing_version" json:"processing_version"`
ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
TranslatedData map[string]string `bson:"translated_data,omitempty" json:"translated_data,omitempty"`
TranslatedAt int64 `bson:"translated_at,omitempty" json:"translated_at,omitempty"`
Processed bool `bson:"processed" json:"processed"`
DocumentsScraped bool `bson:"documents_scraped" json:"documents_scraped"` // Whether documents have been scraped
DocumentsScrapedAt int64 `bson:"documents_scraped_at" json:"documents_scraped_at"` // Unix timestamp (seconds) when documents were scraped
DocumentsSummarized bool `bson:"documents_summarized" json:"documents_summarized"` // Whether documents have been summarized
DocumentsSummarizedAt int64 `bson:"documents_summarized_at" json:"documents_summarized_at"` // Unix timestamp (seconds) when documents were summarized
}
// TenderModification represents a modification made to a tender
type TenderModification struct {
ModificationDate int64 `bson:"modification_date" json:"modification_date"` // Unix milliseconds
ModificationReason string `bson:"modification_reason" json:"modification_reason"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
}
// AwardedEntity represents an entity that has been awarded a contract
type Awarded struct {
Name string `bson:"name" json:"name"`
Address string `bson:"address,omitempty" json:"address,omitempty"`
Country string `bson:"country,omitempty" json:"country,omitempty"`
Amount float64 `bson:"amount" json:"amount"`
Currency string `bson:"currency,omitempty" json:"currency,omitempty"`
Share float64 `bson:"share,omitempty" json:"share,omitempty"` // Percentage share for multiple winners
AwardDate int64 `bson:"award_date" json:"award_date"` // Unix milliseconds
ContractID string `bson:"contract_id,omitempty" json:"contract_id,omitempty"` // Contract reference ID
TenderID string `bson:"tender_id,omitempty" json:"tender_id,omitempty"` // Tender reference ID
LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` // Lot reference ID
CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"` // Company registration ID
OrganizationID string `bson:"organization_id,omitempty" json:"organization_id,omitempty"` // Organization reference ID
}
// TenderStatus represents the status of a tender
type TenderStatus string
const (
TenderStatusActive TenderStatus = "active"
TenderStatusExpired TenderStatus = "expired"
TenderStatusCancelled TenderStatus = "cancelled"
TenderStatusAwarded TenderStatus = "awarded"
TenderStatusDraft TenderStatus = "draft"
TenderStatusClosed TenderStatus = "closed"
TenderStatusModified TenderStatus = "modified"
TenderStatusSuspended TenderStatus = "suspended"
TenderStatusPublished TenderStatus = "published"
)
// TenderSource represents the source of tender data
type TenderSource string
const (
TenderSourceTEDScraper TenderSource = "ted_scraper"
TenderSourceManual TenderSource = "manual"
TenderSourceAPI TenderSource = "api"
TenderSourceBulkImport TenderSource = "bulk_import"
)
// TenderStatistics represents tender statistics
type TenderStatistics struct {
TotalTenders int64 `json:"total_tenders"`
ActiveTenders int64 `json:"active_tenders"`
ExpiredTenders int64 `json:"expired_tenders"`
TendersByCountry map[string]int64 `json:"tenders_by_country"`
TendersByType map[string]int64 `json:"tenders_by_type"`
TendersByClassification map[string]int64 `json:"tenders_by_classification"`
AverageEstimatedValue float64 `json:"average_estimated_value"`
TotalEstimatedValue float64 `json:"total_estimated_value"`
LastUpdated int64 `json:"last_updated"` // Unix milliseconds
}
// GetID returns the tender ID
func (t *Tender) GetID() string {
return t.ID.Hex()
}
// IsActive returns true if the tender is active
func (t *Tender) IsActive() bool {
return t.Status == TenderStatusActive
}
// IsExpired returns true if the tender deadline has passed
func (t *Tender) IsExpired() bool {
if t.Status == TenderStatusExpired {
return true
}
if t.TenderDeadline == 0 {
return false
}
now := time.Now().Unix()
return t.TenderDeadline < now
}
// PublicationSubmissionDeadline returns the submission window end derived from publication_date.
// Stored submission_deadline is preferred; otherwise it is calculated as 48 working hours after publication.
func (t *Tender) PublicationSubmissionDeadline() int64 {
if t.SubmissionDeadline > 0 {
return t.SubmissionDeadline
}
if t.PublicationDate <= 0 {
return 0
}
return ted.CalculateSubmissionDeadline(time.Unix(t.PublicationDate, 0)).Unix()
}
// HasActivePublicationWindow reports whether the publication-based submission window is still open.
func (t *Tender) HasActivePublicationWindow(now int64) bool {
deadline := t.PublicationSubmissionDeadline()
return deadline > 0 && deadline > now
}
// IsRecommendable reports whether a tender should appear in customer AI recommendation results.
// Recommendations use the publication-based submission window, not tender_deadline.
func (t *Tender) IsRecommendable(now int64) bool {
switch t.Status {
case TenderStatusExpired, TenderStatusCancelled, TenderStatusAwarded, TenderStatusClosed:
return false
}
return t.HasActivePublicationWindow(now)
}
// GetTenderURL returns the TED tender URL
func (t *Tender) GetTenderURL() string {
if t.TenderURL != "" {
return t.TenderURL
}
if t.NoticePublicationID != "" {
return generateTenderURL(t.NoticePublicationID)
}
return ""
}
// generateTenderURL generates the TED tender URL from NoticePublicationID
func generateTenderURL(noticePublicationID string) string {
// Remove leading zeros and format for URL
// Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025
parts := strings.Split(noticePublicationID, "-")
if len(parts) != 2 {
return ""
}
// Convert to int and back to string to remove leading zeros
if num, err := strconv.Atoi(parts[0]); err == nil {
cleanID := fmt.Sprintf("%d-%s", num, parts[1])
return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID)
}
return ""
}
// GetMainOrganization returns the buyer organization
func (t *Tender) GetMainOrganization() *Organization {
return t.BuyerOrganization
}
// HasEstimatedValue returns true if the tender has an estimated value
func (t *Tender) HasEstimatedValue() bool {
value, _ := t.ResolvedEstimatedValueAndCurrency()
return value > 0
}
// AggregateProcurementLotEstimatedValue sums lot-level estimated values when all non-empty
// lot currencies match. Conflicting currencies are not aggregated.
func AggregateProcurementLotEstimatedValue(lots []ProcurementLot) (value float64, currency string) {
var refCurrency string
for _, lot := range lots {
if lot.EstimatedValue <= 0 {
continue
}
c := strings.TrimSpace(lot.Currency)
if c == "" {
continue
}
if refCurrency == "" {
refCurrency = c
continue
}
if c != refCurrency {
return 0, ""
}
}
for _, lot := range lots {
if lot.EstimatedValue <= 0 {
continue
}
value += lot.EstimatedValue
}
if value <= 0 {
return 0, ""
}
return value, refCurrency
}
// ResolvedEstimatedValueAndCurrency returns the tender-level estimated value when set,
// otherwise aggregates procurement lot values for API and display use.
func (t *Tender) ResolvedEstimatedValueAndCurrency() (float64, string) {
if t == nil {
return 0, ""
}
if t.EstimatedValue > 0 {
return t.EstimatedValue, strings.TrimSpace(t.Currency)
}
value, currency := AggregateProcurementLotEstimatedValue(t.ProcurementLots)
if value > 0 {
if currency == "" {
currency = strings.TrimSpace(t.Currency)
}
return value, currency
}
return 0, strings.TrimSpace(t.Currency)
}
// GetFormattedEstimatedValue returns formatted estimated value with currency
func (t *Tender) GetFormattedEstimatedValue() string {
value, currency := t.ResolvedEstimatedValueAndCurrency()
if value <= 0 {
return ""
}
if currency != "" {
return fmt.Sprintf("%.2f %s", value, currency)
}
return fmt.Sprintf("%.2f", value)
}
// UpdateStatus updates the tender status and updated timestamp
func (t *Tender) UpdateStatus(status TenderStatus) {
t.Status = status
t.UpdatedAt = time.Now().Unix()
}
// GetDaysUntilDeadline returns the number of days until the tender deadline
func (t *Tender) GetDaysUntilDeadline() int {
if t.TenderDeadline == 0 {
return 0
}
now := time.Now().Unix()
if t.TenderDeadline <= now {
return 0 // Already expired
}
diffMillis := t.TenderDeadline - now
diffDays := diffMillis / (24 * 60 * 60 * 1000)
return int(diffDays)
}