diff --git a/cmd/worker/bootstrap/bootstrap.go b/cmd/worker/bootstrap/bootstrap.go index 302e48e..1ba6bfa 100644 --- a/cmd/worker/bootstrap/bootstrap.go +++ b/cmd/worker/bootstrap/bootstrap.go @@ -96,7 +96,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger appLogger.Info("Starting notice worker", map[string]interface{}{ "processing_limit": config.Worker.NoticeProcessingLimit, }) - noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, ¬ify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit) + noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, ¬ify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit, config.Worker.DeleteProcessedNotices) noticeWorker.Run() // Initialize Queue-based scraping if enabled @@ -181,7 +181,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger scheduler.AddJob(schedule.Job{ Name: "Notice Worker Job", Func: func() { - worker := workers.NewNoticeWorker(mongoManager, appLogger, ¬ify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit) + worker := workers.NewNoticeWorker(mongoManager, appLogger, ¬ify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit, config.Worker.DeleteProcessedNotices) worker.Run() }, Expr: workerInterval, diff --git a/cmd/worker/bootstrap/config.go b/cmd/worker/bootstrap/config.go index 3fc8abc..349ece3 100644 --- a/cmd/worker/bootstrap/config.go +++ b/cmd/worker/bootstrap/config.go @@ -36,10 +36,11 @@ type QueueConfig struct { } type WorkerConfig struct { - Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"` - NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"` - TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"` - TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"` + Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"` + NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"` + DeleteProcessedNotices bool `env:"WORKER_DELETE_PROCESSED_NOTICES" envDefault:"true"` + TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"` + TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"` } // AISummarizerConfig holds configuration for the external AI summarizer service. diff --git a/cmd/worker/workers/notice.go b/cmd/worker/workers/notice.go index 9cb068c..764692c 100644 --- a/cmd/worker/workers/notice.go +++ b/cmd/worker/workers/notice.go @@ -17,27 +17,29 @@ import ( ) type NoticeWorker struct { - Mongo *mongo.ConnectionManager - Logger logger.Logger - Notify *notification.SDK - NoticeRepo notice.Repository - TenderRepo tender.TenderRepository - GLM *glm.SDK - ProcessingLimit int + Mongo *mongo.ConnectionManager + Logger logger.Logger + Notify *notification.SDK + NoticeRepo notice.Repository + TenderRepo tender.TenderRepository + GLM *glm.SDK + ProcessingLimit int + DeleteProcessedNotices bool } -func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int) *NoticeWorker { +func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int, deleteProcessedNotices bool) *NoticeWorker { if processingLimit < 0 { processingLimit = 5 // Default limit } return &NoticeWorker{ - Mongo: mongo, - Logger: logger, - Notify: notify, - NoticeRepo: noticeRepo, - TenderRepo: tenderRepo, - GLM: glmService, - ProcessingLimit: processingLimit, + Mongo: mongo, + Logger: logger, + Notify: notify, + NoticeRepo: noticeRepo, + TenderRepo: tenderRepo, + GLM: glmService, + ProcessingLimit: processingLimit, + DeleteProcessedNotices: deleteProcessedNotices, } } @@ -197,8 +199,12 @@ func (w *NoticeWorker) Run() { } } - // Clean up: delete notices that were marked as processed in previous runs - w.cleanupProcessedNotices() + // Clean up: delete notices that were marked as processed in previous runs (optional to retain XML for audit/backfill) + if w.DeleteProcessedNotices { + w.cleanupProcessedNotices() + } else { + w.Logger.Info("Skipping deletion of processed notices (WORKER_DELETE_PROCESSED_NOTICES=false)", map[string]interface{}{}) + } } func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { @@ -211,6 +217,12 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { t = new(tender.Tender) } + var preserveTenderID, preserveProjectName string + if t != nil && !t.ID.IsZero() { + preserveTenderID = t.TenderID + preserveProjectName = t.ProjectName + } + // Buyer Organization buyerOrg := &tender.Organization{} if n.BuyerOrganization != nil { @@ -347,6 +359,21 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { } } + procurementLots := make([]tender.ProcurementLot, len(n.ProcurementLots)) + for i, l := range n.ProcurementLots { + procurementLots[i] = tender.ProcurementLot{ + LotID: l.LotID, + Title: l.Title, + Description: l.Description, + MainNatureOfContract: l.MainNatureOfContract, + MainClassification: l.MainClassification, + AdditionalClassifications: append([]string(nil), l.AdditionalClassifications...), + Duration: l.Duration, + EstimatedValue: l.EstimatedValue, + Currency: l.Currency, + } + } + var title string if w.GLM != nil { title, err = w.GLM.Translate(context.Background(), n.Title, n.NoticeLanguageCode, "en") @@ -402,8 +429,14 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { t.WinningTenderer = winningTenderer t.Modifications = modifications t.AwardedEntities = awardedEntities + t.ProcurementLots = procurementLots t.NoticeTypeCode = n.NoticeTypeCode + t.FormType = n.FormType t.NoticeSubTypeCode = n.NoticeSubTypeCode + t.NoticeIdentifier = n.NoticeIdentifier + t.NoticeVersion = n.NoticeVersion + t.NoticeDispatchAt = n.NoticeDispatchAt + t.ESenderDispatchAt = n.ESenderDispatchAt t.NoticeLanguageCode = n.NoticeLanguageCode t.IssueDate = n.IssueDate t.IssueTime = n.IssueTime @@ -417,6 +450,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { t.CityName = n.CityName t.PostalCode = n.PostalCode t.DocumentURI = n.DocumentURI + t.BuyerProfileURL = n.BuyerProfileURL t.TenderURL = n.TenderURL t.MainClassification = n.MainClassification t.AdditionalClassifications = n.AdditionalClassifications @@ -427,6 +461,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { t.ContractFolderID = n.ContractFolderID t.SourceFileURL = n.SourceFileURL t.SourceFileName = n.SourceFileName + t.ContentXML = n.ContentXML // Preserve document-scraping/summarization flags on the tender (managed by // downstream workers), but refresh the notice-side processing metadata. t.ProcessingMetadata.ScrapedAt = n.ProcessingMetadata.ScrapedAt @@ -448,8 +483,13 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { t.WinningTenderer = winningTenderer t.Modifications = modifications t.AwardedEntities = awardedEntities - t.TenderID = w.GenerateTenderID(context.Background(), n) - t.ProjectName = w.GenerateProjectName(t) + if preserveTenderID != "" { + t.TenderID = preserveTenderID + t.ProjectName = preserveProjectName + } else { + t.TenderID = w.GenerateTenderID(context.Background(), n) + t.ProjectName = w.GenerateProjectName(t) + } return t, nil } diff --git a/internal/notice/entity.go b/internal/notice/entity.go index 0b917f3..ce67e11 100644 --- a/internal/notice/entity.go +++ b/internal/notice/entity.go @@ -15,7 +15,12 @@ type Notice struct { NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"` ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"` NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"` + FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"` NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"` + NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"` + NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"` + NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"` + ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"` NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"` IssueDate int64 `bson:"issue_date" json:"issue_date"` IssueTime int64 `bson:"issue_time" json:"issue_time"` @@ -40,6 +45,8 @@ type Notice struct { CityName string `bson:"city_name" json:"city_name"` PostalCode string `bson:"postal_code" json:"postal_code"` DocumentURI string `bson:"document_uri" json:"document_uri"` + BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"` + ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"` TenderURL string `bson:"tender_url" json:"tender_url"` SubmissionURL string `bson:"submission_url" json:"submission_url"` BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"` @@ -99,6 +106,19 @@ type SelectionCriterion struct { LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"` } +// ProcurementLot captures per-lot procurement fields from TED notices. +type ProcurementLot struct { + LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` + Title string `bson:"title,omitempty" json:"title,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"` + MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"` + AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"` + Duration string `bson:"duration,omitempty" json:"duration,omitempty"` + EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"` + Currency string `bson:"currency,omitempty" json:"currency,omitempty"` +} + // ProcessingMetadata contains metadata about how the tender was processed type ProcessingMetadata struct { ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` diff --git a/internal/notice/repository.go b/internal/notice/repository.go index fe04486..60b1cd8 100644 --- a/internal/notice/repository.go +++ b/internal/notice/repository.go @@ -16,6 +16,7 @@ type Repository interface { BulkImport(ctx context.Context, notices []Notice) error GetByID(ctx context.Context, id string) (*Notice, error) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error) + FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error) GetUnProcessedNotices(ctx context.Context, ProcessingLimit int, skip int) ([]Notice, int64, error) GetProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) Delete(ctx context.Context, id string) error @@ -39,6 +40,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re // processing_metadata.processed *orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}), *orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}), + *orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}), } // Create indexes @@ -144,6 +146,27 @@ func (r *noticeRepository) GetByContractNoticeID(ctx context.Context, contractNo return &result.Items[0], nil } +// FindNoticesWithContentXML returns notices that have non-empty stored TED XML (for backfill / remapping). +func (r *noticeRepository) FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error) { + filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}} + pagination := orm.Pagination{ + Limit: limit, + Skip: skip, + SortField: "_id", + SortOrder: 1, + } + + result, err := r.ormRepo.FindAll(ctx, filter, pagination) + if err != nil { + r.logger.Error("Failed to list notices with content_xml", map[string]interface{}{ + "error": err.Error(), + }) + return nil, err + } + + return result.Items, nil +} + // GetUnProcessedNotices retrieves unprocessed notices func (r *noticeRepository) GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) { diff --git a/internal/tender/entity.go b/internal/tender/entity.go index bbfc7da..b4e891b 100644 --- a/internal/tender/entity.go +++ b/internal/tender/entity.go @@ -37,7 +37,12 @@ type Tender struct { NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"` ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"` NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"` + FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"` NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"` + NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"` + NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"` + NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"` + ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"` NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"` IssueDate int64 `bson:"issue_date" json:"issue_date"` IssueTime int64 `bson:"issue_time" json:"issue_time"` @@ -62,6 +67,8 @@ type Tender struct { CityName string `bson:"city_name" json:"city_name"` PostalCode string `bson:"postal_code" json:"postal_code"` DocumentURI string `bson:"document_uri" json:"document_uri"` + BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"` + ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"` TenderURL string `bson:"tender_url" json:"tender_url"` SubmissionURL string `bson:"submission_url" json:"submission_url"` BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"` @@ -73,6 +80,7 @@ type Tender struct { Source TenderSource `bson:"source" json:"source"` SourceFileURL string `bson:"source_file_url" json:"source_file_url"` SourceFileName string `bson:"source_file_name" json:"source_file_name"` + ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"` ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` TenderID string `bson:"tender_id" json:"tender_id"` // PBL Project Number (SCDYYNNN format) ProjectName string `bson:"project_name" json:"project_name"` // PBL Project Name (__) @@ -133,6 +141,19 @@ type SelectionCriterion struct { LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"` } +// ProcurementLot captures per-lot procurement fields from TED notices. +type ProcurementLot struct { + LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` + Title string `bson:"title,omitempty" json:"title,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"` + MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"` + AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"` + Duration string `bson:"duration,omitempty" json:"duration,omitempty"` + EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"` + Currency string `bson:"currency,omitempty" json:"currency,omitempty"` +} + // ProcessingMetadata contains metadata about how the tender was processed type ProcessingMetadata struct { ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds diff --git a/internal/tender/repository.go b/internal/tender/repository.go index b46cd57..e1074d9 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -18,6 +18,7 @@ type TenderRepository interface { GetByID(ctx context.Context, id string) (*Tender, error) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error) GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error) + FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) GetTenderCountByCountry(ctx context.Context) (map[string]int64, error) GetTenderCountByType(ctx context.Context) (map[string]int64, error) GetTenderCountByClassification(ctx context.Context) (map[string]int64, error) @@ -175,6 +176,27 @@ func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticeP return &result.Items[0], nil } +// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted). +func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) { + filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}} + pagination := orm.Pagination{ + Limit: limit, + Skip: skip, + SortField: "_id", + SortOrder: 1, + } + + result, err := r.ormRepo.FindAll(ctx, filter, pagination) + if err != nil { + r.logger.Error("Failed to list tenders with content_xml", map[string]interface{}{ + "error": err.Error(), + }) + return nil, err + } + + return result.Items, nil +} + // Update updates an existing tender func (r *tenderRepository) Update(ctx context.Context, tender *Tender) error { tender.UpdatedAt = time.Now().Unix() diff --git a/ted/mapper.go b/ted/mapper.go index 3aaa4d2..6e05b03 100644 --- a/ted/mapper.go +++ b/ted/mapper.go @@ -1,12 +1,63 @@ package ted import ( - "strconv" "strings" "time" "tm/internal/notice" ) +func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot { + if lot == nil { + return notice.ProcurementLot{} + } + out := notice.ProcurementLot{LotID: lot.ID} + if lot.ProcurementProject == nil { + return out + } + pp := lot.ProcurementProject + out.Title = pp.Name + out.Description = pp.Description + out.MainNatureOfContract = pp.ProcurementTypeCode + if pp.MainCommodityClassification != nil { + out.MainClassification = strings.TrimSpace(pp.MainCommodityClassification.ItemClassificationCode) + } + for _, ac := range pp.AdditionalCommodityClassification { + c := strings.TrimSpace(ac.ItemClassificationCode) + if c != "" { + out.AdditionalClassifications = append(out.AdditionalClassifications, c) + } + } + if pp.RequestedTenderTotal != nil { + a := pp.RequestedTenderTotal.EstimatedOverallContractAmount + if v, ok := ParseEuropeanAmount(a.Text); ok { + out.EstimatedValue = v + } + out.Currency = a.CurrencyID + } + if pp.PlannedPeriod != nil && pp.PlannedPeriod.DurationMeasure != nil { + dm := pp.PlannedPeriod.DurationMeasure + txt := strings.TrimSpace(dm.Text) + unit := strings.TrimSpace(dm.UnitCode) + switch unit { + case "MONTH": + out.Duration = txt + " Months" + case "DAY": + out.Duration = txt + " Days" + case "WEEK": + out.Duration = txt + " Weeks" + case "YEAR": + out.Duration = txt + " Years" + default: + if unit != "" && txt != "" { + out.Duration = txt + " " + unit + } else { + out.Duration = txt + } + } + } + return out +} + // mapToTenderFromParsedDoc maps a parsed TED document to tender entity func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice { switch parsedDoc.Type { @@ -89,8 +140,13 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa t := ¬ice.Notice{ ContractNoticeID: cn.ID, ContractFolderID: cn.ContractFolderID, - NoticeTypeCode: cn.GetNoticeSubType(), + NoticeTypeCode: strings.TrimSpace(cn.GetNoticeType()), + FormType: cn.GetNoticeFormType(), NoticeSubTypeCode: cn.GetNoticeSubType(), + NoticeIdentifier: cn.ID, + NoticeVersion: cn.VersionID, + NoticeDispatchAt: ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime), + ESenderDispatchAt: TransmissionDispatchUnix(cn.Extensions), NoticeLanguageCode: cn.NoticeLanguageCode, IssueDate: ParseDateToUnix(cn.IssueDate), IssueTime: ParseTimeToUnix(cn.IssueTime), @@ -131,12 +187,20 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa // Set estimated value and currency if amount, currency := cn.GetBudget(); amount != "" { - if value, err := strconv.ParseFloat(amount, 64); err == nil { + if value, ok := ParseEuropeanAmount(amount); ok { t.EstimatedValue = value t.Currency = currency } } + if cn.ContractingParty != nil { + t.BuyerProfileURL = cn.ContractingParty.BuyerProfileURI + } + + for i := range cn.ProcurementProjectLots { + t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&cn.ProcurementProjectLots[i])) + } + // Set duration information if duration, unit := cn.GetDuration(); duration != "" { t.Duration = duration @@ -159,8 +223,8 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa } // Set SubmissionURL from TenderingTerms - if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil { - if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil { + if lot := cn.firstLot(); lot != nil && lot.TenderingTerms != nil { + if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil { if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" { if strings.HasPrefix(endpointID, "http") { t.SubmissionURL = endpointID @@ -235,8 +299,13 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so t := ¬ice.Notice{ ContractNoticeID: can.ID, ContractFolderID: can.ContractFolderID, - NoticeTypeCode: can.NoticeTypeCode, + NoticeTypeCode: strings.TrimSpace(can.GetNoticeType()), + FormType: can.GetNoticeFormType(), NoticeSubTypeCode: can.GetNoticeSubType(), + NoticeIdentifier: can.ID, + NoticeVersion: can.VersionID, + NoticeDispatchAt: ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime), + ESenderDispatchAt: TransmissionDispatchUnix(can.Extensions), NoticeLanguageCode: can.NoticeLanguageCode, IssueDate: ParseDateToUnix(can.IssueDate), IssueTime: ParseTimeToUnix(can.IssueTime), @@ -266,21 +335,75 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so // Set total award value if amount, currency := can.GetTotalAwardValue(); amount != "" { - if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil { + if parsedAmount, ok := ParseEuropeanAmount(amount); ok { t.EstimatedValue = parsedAmount + t.AwardedValue = parsedAmount } t.Currency = currency } - // Set location information from main procurement project + if ad := can.GetAwardDate(); ad != "" { + t.AwardDate = ParseDateToUnix(ad) + } + + if can.ContractingParty != nil { + t.BuyerProfileURL = can.ContractingParty.BuyerProfileURI + } + + for i := range can.ProcurementProjectLot { + t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&can.ProcurementProjectLot[i])) + } + + // Set location information from main procurement project or first lot + var realizedAddr *Address if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil { - if addr := can.ProcurementProject.RealizedLocation.Address; addr != nil { - t.CityName = addr.CityName - t.PostalCode = addr.PostalZone - t.RegionCode = addr.CountrySubentityCode - if addr.Country != nil { - t.CountryCode = addr.Country.IdentificationCode + realizedAddr = can.ProcurementProject.RealizedLocation.Address + } else if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil && + lot.ProcurementProject.RealizedLocation != nil { + realizedAddr = lot.ProcurementProject.RealizedLocation.Address + } + if realizedAddr != nil { + t.CityName = realizedAddr.CityName + t.PostalCode = realizedAddr.PostalZone + t.RegionCode = realizedAddr.CountrySubentityCode + t.PlaceOfPerformance = realizedAddr.Region + if realizedAddr.Country != nil { + t.CountryCode = realizedAddr.Country.IdentificationCode + } + } + + if wOrg := can.GetWinningTenderOrganization(); wOrg != nil { + t.WinningTenderer = s.mapOrganization(wOrg, "winner") + } + + if nr := can.GetNoticeResult(); nr != nil { + for _, lr := range nr.LotResult { + winner := pickWinningLotTender(lr.LotTender) + if winner == nil || winner.TenderingParty == nil { + continue } + org := can.GetOrganizationByID(winner.TenderingParty.ID) + aw := notice.Awarded{ + LotID: lr.ID, + OrganizationID: winner.TenderingParty.ID, + AwardDate: t.AwardDate, + } + if org != nil && org.Company != nil { + if org.Company.PartyName != nil { + aw.Name = org.Company.PartyName.Name + } + if org.Company.PartyLegalEntity != nil { + aw.CompanyID = org.Company.PartyLegalEntity.CompanyID + } + } + if winner.LegalMonetaryTotal != nil { + pa := winner.LegalMonetaryTotal.PayableAmount + if v, ok := ParseEuropeanAmount(pa.Text); ok { + aw.Amount = v + } + aw.Currency = pa.CurrencyID + } + t.AwardedEntities = append(t.AwardedEntities, aw) } } @@ -348,6 +471,7 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice. PostalZone: company.PostalAddress.PostalZone, CountrySubentityCode: company.PostalAddress.CountrySubentityCode, Department: company.PostalAddress.Department, + Region: strings.TrimSpace(company.PostalAddress.CountrySubentity), } if company.PostalAddress.Country != nil { org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode diff --git a/ted/model.go b/ted/model.go index 9315b30..16987aa 100644 --- a/ted/model.go +++ b/ted/model.go @@ -3,9 +3,16 @@ package ted import ( "encoding/xml" "fmt" + "strings" "time" ) +// NoticeTypeCodeField captures cbc:NoticeTypeCode: inner text (notice type, BT-02) and listName (form type, BT-03). +type NoticeTypeCodeField struct { + ListName string `xml:"listName,attr,omitempty"` + Text string `xml:",chardata"` +} + // DocumentType represents the type of TED document type DocumentType string @@ -634,30 +641,37 @@ type SubcontractingTerm struct { // ContractNotice represents the main TED XML contract notice structure type ContractNotice struct { - XMLName xml.Name `xml:"ContractNotice"` - Xmlns string `xml:"xmlns,attr,omitempty"` - XmlnsCac string `xml:"xmlns:cac,attr,omitempty"` - XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"` - XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"` - XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"` - XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"` - XmlnsExt string `xml:"xmlns:ext,attr,omitempty"` - UBLVersionID string `xml:"UBLVersionID"` - CustomizationID string `xml:"CustomizationID"` - ID string `xml:"ID"` - ContractFolderID string `xml:"ContractFolderID"` - IssueDate string `xml:"IssueDate"` - IssueTime string `xml:"IssueTime"` - VersionID string `xml:"VersionID"` - RegulatoryDomain string `xml:"RegulatoryDomain"` - NoticeTypeCode string `xml:"NoticeTypeCode"` - NoticeLanguageCode string `xml:"NoticeLanguageCode"` - Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"` - ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"` - TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"` - TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"` - ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"` - ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"` + XMLName xml.Name `xml:"ContractNotice"` + Xmlns string `xml:"xmlns,attr,omitempty"` + XmlnsCac string `xml:"xmlns:cac,attr,omitempty"` + XmlnsCbc string `xml:"xmlns:cbc,attr,omitempty"` + XmlnsEfac string `xml:"xmlns:efac,attr,omitempty"` + XmlnsEfbc string `xml:"xmlns:efbc,attr,omitempty"` + XmlnsEfext string `xml:"xmlns:efext,attr,omitempty"` + XmlnsExt string `xml:"xmlns:ext,attr,omitempty"` + UBLVersionID string `xml:"UBLVersionID"` + CustomizationID string `xml:"CustomizationID"` + ID string `xml:"ID"` + ContractFolderID string `xml:"ContractFolderID"` + IssueDate string `xml:"IssueDate"` + IssueTime string `xml:"IssueTime"` + VersionID string `xml:"VersionID"` + RegulatoryDomain string `xml:"RegulatoryDomain"` + NoticeTypeCode NoticeTypeCodeField `xml:"NoticeTypeCode,omitempty"` + NoticeLanguageCode string `xml:"NoticeLanguageCode"` + Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"` + ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"` + TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"` + TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"` + ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"` + ProcurementProjectLots []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"` +} + +func (cn ContractNotice) firstLot() *ProcurementProjectLot { + if len(cn.ProcurementProjectLots) == 0 { + return nil + } + return &cn.ProcurementProjectLots[0] } // ContractAwardNotice represents the main TED XML contract award notice structure @@ -679,7 +693,7 @@ type ContractAwardNotice struct { IssueTime string `xml:"IssueTime"` VersionID string `xml:"VersionID"` RegulatoryDomain string `xml:"RegulatoryDomain"` - NoticeTypeCode string `xml:"NoticeTypeCode"` + NoticeTypeCode NoticeTypeCodeField `xml:"NoticeTypeCode,omitempty"` NoticeLanguageCode string `xml:"NoticeLanguageCode"` ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"` TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"` @@ -688,6 +702,25 @@ type ContractAwardNotice struct { TenderResult *TenderResult `xml:"TenderResult,omitempty"` } +func (can ContractAwardNotice) firstAwardLot() *ProcurementProjectLot { + if len(can.ProcurementProjectLot) == 0 { + return nil + } + return &can.ProcurementProjectLot[0] +} + +func pickWinningLotTender(lotTenders []LotTender) *LotTender { + if len(lotTenders) == 0 { + return nil + } + for i := range lotTenders { + if strings.TrimSpace(lotTenders[i].RankCode) == "1" { + return &lotTenders[i] + } + } + return &lotTenders[0] +} + // NoticeResult contains notice result information from extensions type NoticeResult struct { XMLName xml.Name `xml:"NoticeResult"` @@ -819,8 +852,10 @@ type UBLExtension struct { // ExtensionContent contains eForms extension data type ExtensionContent struct { - XMLName xml.Name `xml:"ExtensionContent"` - EformsExtension *EformsExtension `xml:"EformsExtension,omitempty"` + XMLName xml.Name `xml:"ExtensionContent"` + EformsExtension *EformsExtension `xml:"EformsExtension,omitempty"` + TransmissionDate string `xml:"TransmissionDate,omitempty"` + TransmissionTime string `xml:"TransmissionTime,omitempty"` } // EformsExtension contains eForms specific data @@ -835,6 +870,8 @@ type EformsExtension struct { OfficialLanguages *OfficialLanguages `xml:"OfficialLanguages,omitempty"` AwardCriterionParameter *AwardCriterionParameter `xml:"AwardCriterionParameter,omitempty"` AccessToolName string `xml:"AccessToolName,omitempty"` + TransmissionDate string `xml:"TransmissionDate,omitempty"` + TransmissionTime string `xml:"TransmissionTime,omitempty"` } // NoticeSubType contains notice subtype information @@ -908,6 +945,7 @@ type PostalAddress struct { StreetName string `xml:"StreetName,omitempty"` CityName string `xml:"CityName,omitempty"` PostalZone string `xml:"PostalZone,omitempty"` + CountrySubentity string `xml:"CountrySubentity,omitempty"` CountrySubentityCode string `xml:"CountrySubentityCode,omitempty"` Department string `xml:"Department,omitempty"` Country *Country `xml:"Country,omitempty"` @@ -1338,11 +1376,11 @@ func (cn ContractNotice) Validate() error { } // Validate tender submission deadline format if present - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.TenderingProcess != nil && - cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil { + if cn.firstLot() != nil && + cn.firstLot().TenderingProcess != nil && + cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil { - deadline := cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate + deadline := cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate if deadline != "" { var validDeadline bool for _, format := range validFormats { @@ -1358,23 +1396,23 @@ func (cn ContractNotice) Validate() error { } // Validate currency code if provided in budget - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.ProcurementProject != nil && - cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil { + if cn.firstLot() != nil && + cn.firstLot().ProcurementProject != nil && + cn.firstLot().ProcurementProject.RequestedTenderTotal != nil { - currency := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID + currency := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID if currency != "" && len(currency) != 3 { return fmt.Errorf("invalid currency code: %s (expected 3-letter ISO code)", currency) } } // Validate duration unit code if provided - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.ProcurementProject != nil && - cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil && - cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil { + if cn.firstLot() != nil && + cn.firstLot().ProcurementProject != nil && + cn.firstLot().ProcurementProject.PlannedPeriod != nil && + cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil { - unitCode := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode + unitCode := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode if unitCode != "" { validUnits := map[string]bool{ "MONTH": true, @@ -1396,9 +1434,14 @@ func (cn ContractNotice) GetID() string { return cn.ID } -// GetNoticeType returns the notice type +// GetNoticeType returns the notice type code (BT-02, cbc:NoticeTypeCode text). func (cn ContractNotice) GetNoticeType() string { - return cn.NoticeTypeCode + return strings.TrimSpace(cn.NoticeTypeCode.Text) +} + +// GetNoticeFormType returns the form type from listName on cbc:NoticeTypeCode (BT-03), e.g. "competition". +func (cn ContractNotice) GetNoticeFormType() string { + return strings.TrimSpace(cn.NoticeTypeCode.ListName) } // GetLanguage returns the notice language @@ -1496,12 +1539,12 @@ func (cn ContractNotice) GetBuyerID() string { // GetReviewOrganizationID returns the review organization ID from appeal terms func (cn ContractNotice) GetReviewOrganizationID() string { - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.TenderingTerms != nil && - cn.ProcurementProjectLot.TenderingTerms.AppealTerms != nil && - cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty != nil && - cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil { - return cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID + if cn.firstLot() != nil && + cn.firstLot().TenderingTerms != nil && + cn.firstLot().TenderingTerms.AppealTerms != nil && + cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty != nil && + cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil { + return cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID } return "" } @@ -1516,18 +1559,18 @@ func (cn ContractNotice) GetProcedureCode() string { // GetProcurementTitle returns the procurement project title func (cn ContractNotice) GetProcurementTitle() string { - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.ProcurementProject != nil { - return cn.ProcurementProjectLot.ProcurementProject.Name + if cn.firstLot() != nil && + cn.firstLot().ProcurementProject != nil { + return cn.firstLot().ProcurementProject.Name } return "" } // GetProcurementDescription returns the procurement project description func (cn ContractNotice) GetProcurementDescription() string { - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.ProcurementProject != nil { - return cn.ProcurementProjectLot.ProcurementProject.Description + if cn.firstLot() != nil && + cn.firstLot().ProcurementProject != nil { + return cn.firstLot().ProcurementProject.Description } return "" } @@ -1561,8 +1604,8 @@ func (cn ContractNotice) GetPlaceOfPerformance() string { // GetLotID returns the procurement lot ID func (cn ContractNotice) GetLotID() string { - if cn.ProcurementProjectLot != nil { - return cn.ProcurementProjectLot.ID + if cn.firstLot() != nil { + return cn.firstLot().ID } return "" } @@ -1580,17 +1623,17 @@ func (cn ContractNotice) GetSelectionCriteria() []SelectionCriteria { // GetOfficialLanguages returns official languages from tender terms extensions func (cn ContractNotice) GetOfficialLanguages() []string { - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.TenderingTerms != nil && - len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 && - cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil && - cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil && - cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil && - cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil && - cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil { + if cn.firstLot() != nil && + cn.firstLot().TenderingTerms != nil && + len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 && + cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil && + cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil && + cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil && + cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil && + cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil { - languages := make([]string, len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language)) - for i, lang := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language { + languages := make([]string, len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language)) + for i, lang := range cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language { languages[i] = lang.ID } return languages @@ -1600,12 +1643,12 @@ func (cn ContractNotice) GetOfficialLanguages() []string { // GetDocumentURI returns the document URI from call for tenders document reference func (cn ContractNotice) GetDocumentURI() string { - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.TenderingTerms != nil && - len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 && - cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil && - cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil { - return cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI + if cn.firstLot() != nil && + cn.firstLot().TenderingTerms != nil && + len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 && + cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil && + cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil { + return cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI } return "" } @@ -1613,10 +1656,12 @@ func (cn ContractNotice) GetDocumentURI() string { // GetAllDocumentURIs returns all document URIs from call for tenders document references func (cn ContractNotice) GetAllDocumentURIs() []string { var uris []string - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.TenderingTerms != nil { - - for _, docRef := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference { + for i := range cn.ProcurementProjectLots { + lot := &cn.ProcurementProjectLots[i] + if lot.TenderingTerms == nil { + continue + } + for _, docRef := range lot.TenderingTerms.CallForTendersDocumentReference { if docRef.Attachment != nil && docRef.Attachment.ExternalReference != nil && docRef.Attachment.ExternalReference.URI != "" { @@ -1629,21 +1674,21 @@ func (cn ContractNotice) GetAllDocumentURIs() []string { // GetTenderDeadline returns the tender submission deadline func (cn ContractNotice) GetTenderDeadline() string { - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.TenderingProcess != nil && - cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil { - return cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate + if cn.firstLot() != nil && + cn.firstLot().TenderingProcess != nil && + cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil { + return cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate } return "" } // GetBudget returns the estimated contract amount and currency func (cn ContractNotice) GetBudget() (amount string, currency string) { - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.ProcurementProject != nil && - cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil { + if cn.firstLot() != nil && + cn.firstLot().ProcurementProject != nil && + cn.firstLot().ProcurementProject.RequestedTenderTotal != nil { - contractAmount := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount + contractAmount := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount return contractAmount.Text, contractAmount.CurrencyID } return "", "" @@ -1651,12 +1696,12 @@ func (cn ContractNotice) GetBudget() (amount string, currency string) { // GetDuration returns the contract duration and unit func (cn ContractNotice) GetDuration() (duration string, unit string) { - if cn.ProcurementProjectLot != nil && - cn.ProcurementProjectLot.ProcurementProject != nil && - cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil && - cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil { + if cn.firstLot() != nil && + cn.firstLot().ProcurementProject != nil && + cn.firstLot().ProcurementProject.PlannedPeriod != nil && + cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil { - durationMeasure := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure + durationMeasure := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure return durationMeasure.Text, durationMeasure.UnitCode } return "", "" @@ -1716,9 +1761,14 @@ func (can ContractAwardNotice) GetID() string { return can.ID } -// GetNoticeType returns the notice type +// GetNoticeType returns the notice type code (BT-02). func (can ContractAwardNotice) GetNoticeType() string { - return can.NoticeTypeCode + return strings.TrimSpace(can.NoticeTypeCode.Text) +} + +// GetNoticeFormType returns the form type from listName (BT-03). +func (can ContractAwardNotice) GetNoticeFormType() string { + return strings.TrimSpace(can.NoticeTypeCode.ListName) } // GetLanguage returns the notice language @@ -1818,25 +1868,34 @@ func (can ContractAwardNotice) GetProcedureCode() string { // GetProcurementTitle returns the procurement project title func (can ContractAwardNotice) GetProcurementTitle() string { - if can.ProcurementProject != nil { + if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Name) != "" { return can.ProcurementProject.Name } + if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil { + return lot.ProcurementProject.Name + } return "" } // GetProcurementDescription returns the procurement project description func (can ContractAwardNotice) GetProcurementDescription() string { - if can.ProcurementProject != nil { + if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Description) != "" { return can.ProcurementProject.Description } + if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil { + return lot.ProcurementProject.Description + } return "" } // GetContractNature returns the main nature of the contract func (can ContractAwardNotice) GetContractNature() string { - if can.ProcurementProject != nil { + if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.ProcurementTypeCode) != "" { return can.ProcurementProject.ProcurementTypeCode } + if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil { + return lot.ProcurementProject.ProcurementTypeCode + } return "" } @@ -1846,6 +1905,11 @@ func (can ContractAwardNotice) GetMainClassification() string { can.ProcurementProject.MainCommodityClassification != nil { return can.ProcurementProject.MainCommodityClassification.ItemClassificationCode } + if lot := can.firstAwardLot(); lot != nil && + lot.ProcurementProject != nil && + lot.ProcurementProject.MainCommodityClassification != nil { + return lot.ProcurementProject.MainCommodityClassification.ItemClassificationCode + } return "" } @@ -1882,3 +1946,27 @@ func (can ContractAwardNotice) GetOrganizationByID(orgID string) *Organization { } return nil } + +// GetWinningTenderPartyID resolves the tendering party reference for the primary winning bid from NoticeResult. +func (can ContractAwardNotice) GetWinningTenderPartyID() string { + nr := can.GetNoticeResult() + if nr == nil { + return "" + } + for _, lr := range nr.LotResult { + winner := pickWinningLotTender(lr.LotTender) + if winner != nil && winner.TenderingParty != nil && winner.TenderingParty.ID != "" { + return winner.TenderingParty.ID + } + } + return "" +} + +// GetWinningTenderOrganization resolves the winning tenderer organization from NoticeResult via Organizations. +func (can ContractAwardNotice) GetWinningTenderOrganization() *Organization { + id := can.GetWinningTenderPartyID() + if id == "" { + return nil + } + return can.GetOrganizationByID(id) +} diff --git a/ted/notice_upsert.go b/ted/notice_upsert.go new file mode 100644 index 0000000..6ad9ed9 --- /dev/null +++ b/ted/notice_upsert.go @@ -0,0 +1,80 @@ +package ted + +import ( + "strconv" + "strings" + "time" + "tm/internal/notice" +) + +// noticeNeedsRefresh reports whether incoming scraped data should replace an existing notice row. +func noticeNeedsRefresh(existing, incoming *notice.Notice) bool { + if incoming == nil || existing == nil { + return true + } + if strings.TrimSpace(incoming.ContentXML) != "" && + strings.TrimSpace(existing.ContentXML) != strings.TrimSpace(incoming.ContentXML) { + return true + } + if normalizePublicationID(existing.NoticePublicationID) != normalizePublicationID(incoming.NoticePublicationID) { + return true + } + if compareNoticeVersions(incoming.NoticeVersion, existing.NoticeVersion) > 0 { + return true + } + return false +} + +func normalizePublicationID(s string) string { + return strings.TrimSpace(s) +} + +// compareNoticeVersions compares BT-757 notice versions (e.g. "01", "02"). Empty parses as 0. +func compareNoticeVersions(incoming, existing string) int { + vi := parseNoticeVersion(incoming) + ve := parseNoticeVersion(existing) + switch { + case vi > ve: + return 1 + case vi < ve: + return -1 + default: + return 0 + } +} + +func parseNoticeVersion(s string) int { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + n, err := strconv.Atoi(s) + if err != nil { + return 0 + } + return n +} + +// MergeNoticeUpdate replaces scrape-derived fields from src into dst while preserving identity, +// creation time, and optional enrichment metadata already stored on dst. +func MergeNoticeUpdate(dst *notice.Notice, src *notice.Notice) { + preservedID := dst.ID + preservedCreated := dst.CreatedAt + oldEnrichment := dst.ProcessingMetadata.EnrichmentData + oldTranslated := dst.ProcessingMetadata.TranslatedData + oldTranslatedAt := dst.ProcessingMetadata.TranslatedAt + + *dst = *src + dst.ID = preservedID + dst.CreatedAt = preservedCreated + dst.UpdatedAt = time.Now().Unix() + dst.ProcessingMetadata.Processed = false + + if len(dst.ProcessingMetadata.EnrichmentData) == 0 && len(oldEnrichment) > 0 { + dst.ProcessingMetadata.EnrichmentData = oldEnrichment + } + if len(dst.ProcessingMetadata.TranslatedData) == 0 && len(oldTranslated) > 0 { + dst.ProcessingMetadata.TranslatedData = oldTranslated + dst.ProcessingMetadata.TranslatedAt = oldTranslatedAt + } +} diff --git a/ted/scraper.go b/ted/scraper.go index cc4762d..f177b1c 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -4,6 +4,7 @@ import ( "archive/tar" "compress/gzip" "context" + "errors" "fmt" "io" "net/http" @@ -12,7 +13,7 @@ import ( "time" "tm/internal/notice" "tm/pkg/logger" - "tm/pkg/mongo" + orm "tm/pkg/mongo" "tm/pkg/notification" ) @@ -38,16 +39,16 @@ type TEDScraper struct { logger logger.Logger httpClient *http.Client xmlParser *TEDParser - mongoManager *mongo.ConnectionManager - tenderRepo notice.Repository + mongoManager *orm.ConnectionManager + noticeRepo notice.Repository } // NewTEDScraper creates a new TED scraper instance func NewTEDScraper( config *Config, logger logger.Logger, - mongoManager *mongo.ConnectionManager, - tenderRepo notice.Repository, + mongoManager *orm.ConnectionManager, + noticeRepo notice.Repository, notify notification.SDK, ) *TEDScraper { httpClient := &http.Client{ @@ -61,7 +62,7 @@ func NewTEDScraper( httpClient: httpClient, xmlParser: NewTEDParser(), mongoManager: mongoManager, - tenderRepo: tenderRepo, + noticeRepo: noticeRepo, } } @@ -289,16 +290,18 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o return result, nil } -func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) { - existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID) +func (s *TEDScraper) findNoticeByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) { + if strings.TrimSpace(contractNoticeID) == "" { + return nil, nil + } + existing, err := s.noticeRepo.GetByContractNoticeID(ctx, contractNoticeID) if err != nil { - if err.Error() == "document not found" { + if errors.Is(err, orm.ErrDocumentNotFound) { return nil, nil } return nil, err } - - return existingTender, nil + return existing, nil } func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, result *FileProcessingResult) error { @@ -331,45 +334,66 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file } t.ContentXML = string(content) - // Check if tender already exists to prevent duplicates - existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID) + existingNotice, err := s.findNoticeByContractNoticeID(ctx, t.ContractNoticeID) if err != nil { - s.logger.Error("Failed to check existing tender", map[string]interface{}{ + s.logger.Error("Failed to check existing notice", map[string]interface{}{ "contract_notice_id": t.ContractNoticeID, "error": err.Error(), }) - return fmt.Errorf("failed to check existing tender: %w", err) + return fmt.Errorf("failed to check existing notice: %w", err) } - if existingTender != nil { - s.logger.Info("Tender already exists, skipping duplicate", map[string]interface{}{ - "contract_notice_id": t.ContractNoticeID, - "existing_id": existingTender.ID, + if existingNotice != nil { + if !noticeNeedsRefresh(existingNotice, t) { + s.logger.Debug("Notice unchanged, skipping", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "notice_doc_id": existingNotice.ID.Hex(), + }) + return nil + } + + MergeNoticeUpdate(existingNotice, t) + err = s.noticeRepo.Update(ctx, existingNotice) + if err != nil { + s.logger.Error("Failed to update existing notice", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "notice_doc_id": existingNotice.ID.Hex(), + "error": err.Error(), + }) + return fmt.Errorf("failed to update notice: %w", err) + } + + s.logger.Info("Notice updated (refresh)", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "notice_publication_id": t.NoticePublicationID, + "notice_version": t.NoticeVersion, + "notice_doc_id": existingNotice.ID.Hex(), + "processing_reset_worker": true, }) - return nil - } - - // Create new tender - s.logger.Info("Creating new tender", map[string]interface{}{ - "contract_notice_id": t.ContractNoticeID, - }) - - t.CreatedAt = time.Now().Unix() - t.UpdatedAt = t.CreatedAt - - err = s.tenderRepo.Import(ctx, t) - if err != nil { - s.logger.Error("Failed to create new tender", map[string]interface{}{ + result.SuccessCount++ + } else { + s.logger.Info("Creating new notice", map[string]interface{}{ "contract_notice_id": t.ContractNoticeID, - "error": err.Error(), }) - return fmt.Errorf("failed to create tender: %w", err) - } - s.logger.Info("Successfully created new tender", map[string]interface{}{ - "contract_notice_id": t.ContractNoticeID, - }) - // } + now := time.Now().Unix() + t.CreatedAt = now + t.UpdatedAt = now + + err = s.noticeRepo.Import(ctx, t) + if err != nil { + s.logger.Error("Failed to import notice", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "error": err.Error(), + }) + return fmt.Errorf("failed to import notice: %w", err) + } + + s.logger.Info("Successfully imported new notice", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + }) + result.SuccessCount++ + } // Log detailed parsed information based on notice type s.logger.Info("Successfully parsed notice", map[string]interface{}{ @@ -379,7 +403,6 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file "tender_url": t.TenderURL, }) - result.SuccessCount++ return nil } diff --git a/ted/service.go b/ted/service.go index 8f75eed..f614730 100644 --- a/ted/service.go +++ b/ted/service.go @@ -7,6 +7,57 @@ import ( "time" ) +// ParseIssueDateTimeToUnix combines cbc:IssueDate and cbc:IssueTime (with timezone offsets) into one Unix timestamp. +func ParseIssueDateTimeToUnix(issueDate, issueTime string) int64 { + issueDate = strings.TrimSpace(issueDate) + issueTime = strings.TrimSpace(issueTime) + if issueDate == "" { + return 0 + } + if issueTime == "" { + return ParseDateToUnix(issueDate) + } + dateCore := issueDate + if len(issueDate) >= 10 && issueDate[4] == '-' && issueDate[7] == '-' { + dateCore = issueDate[:10] + } + combined := dateCore + "T" + issueTime + if t, err := ParseDate(combined); err == nil { + return t.Unix() + } + return ParseDateToUnix(issueDate) +} + +// TransmissionDispatchUnix reads BT-803 (eSender) transmission date/time from extension content (root or eForms extension). +func TransmissionDispatchUnix(ext *UBLExtensions) int64 { + if ext == nil || ext.UBLExtension == nil || ext.UBLExtension.ExtensionContent == nil { + return 0 + } + ec := ext.UBLExtension.ExtensionContent + if u := ParseIssueDateTimeToUnix(ec.TransmissionDate, ec.TransmissionTime); u != 0 { + return u + } + if ec.EformsExtension != nil { + if u := ParseIssueDateTimeToUnix(ec.EformsExtension.TransmissionDate, ec.EformsExtension.TransmissionTime); u != 0 { + return u + } + } + return 0 +} + +// ParseEuropeanAmount normalizes TED amount strings (spaces, comma decimal) for strconv.ParseFloat. +func ParseEuropeanAmount(raw string) (float64, bool) { + s := strings.TrimSpace(raw) + s = strings.ReplaceAll(s, "\u00a0", "") + s = strings.ReplaceAll(s, " ", "") + s = strings.ReplaceAll(s, ",", ".") + if s == "" { + return 0, false + } + v, err := strconv.ParseFloat(s, 64) + return v, err == nil +} + // parseDate parses various date formats used in TED XML func ParseDate(dateStr string) (time.Time, error) { if dateStr == "" {