mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added missing fields - scraper and worker fixes

Browse Source
This commit is contained in:
Mazyar
2026-05-11 00:09:37 +03:30
parent e136f0eaa7
commit eb6706e061
12 changed files with 667 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/bootstrap/bootstrap.go
+2 -2
View File
@@ -96,7 +96,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
appLogger.Info("Starting notice worker", map[string]interface{}{
"processing_limit": config.Worker.NoticeProcessingLimit,
})
noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit)
noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit, config.Worker.DeleteProcessedNotices)
noticeWorker.Run()
// Initialize Queue-based scraping if enabled
@@ -181,7 +181,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
scheduler.AddJob(schedule.Job{
Name: "Notice Worker Job",
Func: func() {
worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit)
worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit, config.Worker.DeleteProcessedNotices)
worker.Run()
},
Expr: workerInterval,
cmd/worker/bootstrap/config.go
+1
View File
@@ -38,6 +38,7 @@ type QueueConfig struct {
type WorkerConfig struct {
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
DeleteProcessedNotices bool `env:"WORKER_DELETE_PROCESSED_NOTICES" envDefault:"true"`
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
}
cmd/worker/workers/notice.go
+42 -2
View File
@@ -24,9 +24,10 @@ type NoticeWorker struct {
TenderRepo tender.TenderRepository
GLM *glm.SDK
ProcessingLimit int
DeleteProcessedNotices bool
}
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int) *NoticeWorker {
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int, deleteProcessedNotices bool) *NoticeWorker {
if processingLimit < 0 {
processingLimit = 5 // Default limit
}
@@ -38,6 +39,7 @@ func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notif
TenderRepo: tenderRepo,
GLM: glmService,
ProcessingLimit: processingLimit,
DeleteProcessedNotices: deleteProcessedNotices,
}
}
@@ -197,8 +199,12 @@ func (w *NoticeWorker) Run() {
}
}
// Clean up: delete notices that were marked as processed in previous runs
// Clean up: delete notices that were marked as processed in previous runs (optional to retain XML for audit/backfill)
if w.DeleteProcessedNotices {
w.cleanupProcessedNotices()
} else {
w.Logger.Info("Skipping deletion of processed notices (WORKER_DELETE_PROCESSED_NOTICES=false)", map[string]interface{}{})
}
}
func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
@@ -211,6 +217,12 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t = new(tender.Tender)
}
var preserveTenderID, preserveProjectName string
if t != nil && !t.ID.IsZero() {
preserveTenderID = t.TenderID
preserveProjectName = t.ProjectName
}
// Buyer Organization
buyerOrg := &tender.Organization{}
if n.BuyerOrganization != nil {
@@ -347,6 +359,21 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
}
}
procurementLots := make([]tender.ProcurementLot, len(n.ProcurementLots))
for i, l := range n.ProcurementLots {
procurementLots[i] = tender.ProcurementLot{
LotID: l.LotID,
Title: l.Title,
Description: l.Description,
MainNatureOfContract: l.MainNatureOfContract,
MainClassification: l.MainClassification,
AdditionalClassifications: append([]string(nil), l.AdditionalClassifications...),
Duration: l.Duration,
EstimatedValue: l.EstimatedValue,
Currency: l.Currency,
}
}
var title string
if w.GLM != nil {
title, err = w.GLM.Translate(context.Background(), n.Title, n.NoticeLanguageCode, "en")
@@ -402,8 +429,14 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t.WinningTenderer = winningTenderer
t.Modifications = modifications
t.AwardedEntities = awardedEntities
t.ProcurementLots = procurementLots
t.NoticeTypeCode = n.NoticeTypeCode
t.FormType = n.FormType
t.NoticeSubTypeCode = n.NoticeSubTypeCode
t.NoticeIdentifier = n.NoticeIdentifier
t.NoticeVersion = n.NoticeVersion
t.NoticeDispatchAt = n.NoticeDispatchAt
t.ESenderDispatchAt = n.ESenderDispatchAt
t.NoticeLanguageCode = n.NoticeLanguageCode
t.IssueDate = n.IssueDate
t.IssueTime = n.IssueTime
@@ -417,6 +450,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t.CityName = n.CityName
t.PostalCode = n.PostalCode
t.DocumentURI = n.DocumentURI
t.BuyerProfileURL = n.BuyerProfileURL
t.TenderURL = n.TenderURL
t.MainClassification = n.MainClassification
t.AdditionalClassifications = n.AdditionalClassifications
@@ -427,6 +461,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t.ContractFolderID = n.ContractFolderID
t.SourceFileURL = n.SourceFileURL
t.SourceFileName = n.SourceFileName
t.ContentXML = n.ContentXML
// Preserve document-scraping/summarization flags on the tender (managed by
// downstream workers), but refresh the notice-side processing metadata.
t.ProcessingMetadata.ScrapedAt = n.ProcessingMetadata.ScrapedAt
@@ -448,8 +483,13 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t.WinningTenderer = winningTenderer
t.Modifications = modifications
t.AwardedEntities = awardedEntities
if preserveTenderID != "" {
t.TenderID = preserveTenderID
t.ProjectName = preserveProjectName
} else {
t.TenderID = w.GenerateTenderID(context.Background(), n)
t.ProjectName = w.GenerateProjectName(t)
}
return t, nil
}
internal/notice/entity.go
+20
View File
@@ -15,7 +15,12 @@ type Notice struct {
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
@@ -40,6 +45,8 @@ type Notice struct {
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
@@ -99,6 +106,19 @@ type SelectionCriterion struct {
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
}
// ProcurementLot captures per-lot procurement fields from TED notices.
type ProcurementLot struct {
LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"`
Title string `bson:"title,omitempty" json:"title,omitempty"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"`
MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"`
AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"`
Duration string `bson:"duration,omitempty" json:"duration,omitempty"`
EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"`
Currency string `bson:"currency,omitempty" json:"currency,omitempty"`
}
// ProcessingMetadata contains metadata about how the tender was processed
type ProcessingMetadata struct {
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"`
internal/notice/repository.go
+23
View File
@@ -16,6 +16,7 @@ type Repository interface {
BulkImport(ctx context.Context, notices []Notice) error
GetByID(ctx context.Context, id string) (*Notice, error)
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error)
FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error)
GetUnProcessedNotices(ctx context.Context, ProcessingLimit int, skip int) ([]Notice, int64, error)
GetProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error)
Delete(ctx context.Context, id string) error
@@ -39,6 +40,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
// processing_metadata.processed
*orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}),
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
}
// Create indexes
@@ -144,6 +146,27 @@ func (r *noticeRepository) GetByContractNoticeID(ctx context.Context, contractNo
return &result.Items[0], nil
}
// FindNoticesWithContentXML returns notices that have non-empty stored TED XML (for backfill / remapping).
func (r *noticeRepository) FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error) {
filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
pagination := orm.Pagination{
Limit: limit,
Skip: skip,
SortField: "_id",
SortOrder: 1,
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to list notices with content_xml", map[string]interface{}{
"error": err.Error(),
})
return nil, err
}
return result.Items, nil
}
// GetUnProcessedNotices retrieves unprocessed notices
func (r *noticeRepository) GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) {
internal/tender/entity.go
+21
View File
@@ -37,7 +37,12 @@ type Tender struct {
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
@@ -62,6 +67,8 @@ type Tender struct {
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
@@ -73,6 +80,7 @@ type Tender struct {
Source TenderSource `bson:"source" json:"source"`
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"`
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
TenderID string `bson:"tender_id" json:"tender_id"` // PBL Project Number (SCDYYNNN format)
ProjectName string `bson:"project_name" json:"project_name"` // PBL Project Name (<InternalProjectNumber>_<Client-OpportunityName>_<PartnerCode>)
@@ -133,6 +141,19 @@ type SelectionCriterion struct {
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
}
// ProcurementLot captures per-lot procurement fields from TED notices.
type ProcurementLot struct {
LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"`
Title string `bson:"title,omitempty" json:"title,omitempty"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"`
MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"`
AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"`
Duration string `bson:"duration,omitempty" json:"duration,omitempty"`
EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"`
Currency string `bson:"currency,omitempty" json:"currency,omitempty"`
}
// ProcessingMetadata contains metadata about how the tender was processed
type ProcessingMetadata struct {
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
internal/tender/repository.go
+22
View File
@@ -18,6 +18,7 @@ type TenderRepository interface {
GetByID(ctx context.Context, id string) (*Tender, error)
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error)
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
@@ -175,6 +176,27 @@ func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticeP
return &result.Items[0], nil
}
// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted).
func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) {
filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
pagination := orm.Pagination{
Limit: limit,
Skip: skip,
SortField: "_id",
SortOrder: 1,
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to list tenders with content_xml", map[string]interface{}{
"error": err.Error(),
})
return nil, err
}
return result.Items, nil
}
// Update updates an existing tender
func (r *tenderRepository) Update(ctx context.Context, tender *Tender) error {
tender.UpdatedAt = time.Now().Unix()
ted/mapper.go
+139 -15
View File
@@ -1,12 +1,63 @@
package ted
import (
"strconv"
"strings"
"time"
"tm/internal/notice"
)
func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot {
if lot == nil {
return notice.ProcurementLot{}
}
out := notice.ProcurementLot{LotID: lot.ID}
if lot.ProcurementProject == nil {
return out
}
pp := lot.ProcurementProject
out.Title = pp.Name
out.Description = pp.Description
out.MainNatureOfContract = pp.ProcurementTypeCode
if pp.MainCommodityClassification != nil {
out.MainClassification = strings.TrimSpace(pp.MainCommodityClassification.ItemClassificationCode)
}
for _, ac := range pp.AdditionalCommodityClassification {
c := strings.TrimSpace(ac.ItemClassificationCode)
if c != "" {
out.AdditionalClassifications = append(out.AdditionalClassifications, c)
}
}
if pp.RequestedTenderTotal != nil {
a := pp.RequestedTenderTotal.EstimatedOverallContractAmount
if v, ok := ParseEuropeanAmount(a.Text); ok {
out.EstimatedValue = v
}
out.Currency = a.CurrencyID
}
if pp.PlannedPeriod != nil && pp.PlannedPeriod.DurationMeasure != nil {
dm := pp.PlannedPeriod.DurationMeasure
txt := strings.TrimSpace(dm.Text)
unit := strings.TrimSpace(dm.UnitCode)
switch unit {
case "MONTH":
out.Duration = txt + " Months"
case "DAY":
out.Duration = txt + " Days"
case "WEEK":
out.Duration = txt + " Weeks"
case "YEAR":
out.Duration = txt + " Years"
default:
if unit != "" && txt != "" {
out.Duration = txt + " " + unit
} else {
out.Duration = txt
}
}
}
return out
}
// mapToTenderFromParsedDoc maps a parsed TED document to tender entity
func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice {
switch parsedDoc.Type {
@@ -89,8 +140,13 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
t := &notice.Notice{
ContractNoticeID: cn.ID,
ContractFolderID: cn.ContractFolderID,
NoticeTypeCode: cn.GetNoticeSubType(),
NoticeTypeCode: strings.TrimSpace(cn.GetNoticeType()),
FormType: cn.GetNoticeFormType(),
NoticeSubTypeCode: cn.GetNoticeSubType(),
NoticeIdentifier: cn.ID,
NoticeVersion: cn.VersionID,
NoticeDispatchAt: ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime),
ESenderDispatchAt: TransmissionDispatchUnix(cn.Extensions),
NoticeLanguageCode: cn.NoticeLanguageCode,
IssueDate: ParseDateToUnix(cn.IssueDate),
IssueTime: ParseTimeToUnix(cn.IssueTime),
@@ -131,12 +187,20 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
// Set estimated value and currency
if amount, currency := cn.GetBudget(); amount != "" {
if value, err := strconv.ParseFloat(amount, 64); err == nil {
if value, ok := ParseEuropeanAmount(amount); ok {
t.EstimatedValue = value
t.Currency = currency
}
}
if cn.ContractingParty != nil {
t.BuyerProfileURL = cn.ContractingParty.BuyerProfileURI
}
for i := range cn.ProcurementProjectLots {
t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&cn.ProcurementProjectLots[i]))
}
// Set duration information
if duration, unit := cn.GetDuration(); duration != "" {
t.Duration = duration
@@ -159,8 +223,8 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
}
// Set SubmissionURL from TenderingTerms
if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil {
if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
if lot := cn.firstLot(); lot != nil && lot.TenderingTerms != nil {
if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
if strings.HasPrefix(endpointID, "http") {
t.SubmissionURL = endpointID
@@ -235,8 +299,13 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
t := &notice.Notice{
ContractNoticeID: can.ID,
ContractFolderID: can.ContractFolderID,
NoticeTypeCode: can.NoticeTypeCode,
NoticeTypeCode: strings.TrimSpace(can.GetNoticeType()),
FormType: can.GetNoticeFormType(),
NoticeSubTypeCode: can.GetNoticeSubType(),
NoticeIdentifier: can.ID,
NoticeVersion: can.VersionID,
NoticeDispatchAt: ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime),
ESenderDispatchAt: TransmissionDispatchUnix(can.Extensions),
NoticeLanguageCode: can.NoticeLanguageCode,
IssueDate: ParseDateToUnix(can.IssueDate),
IssueTime: ParseTimeToUnix(can.IssueTime),
@@ -266,21 +335,75 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
// Set total award value
if amount, currency := can.GetTotalAwardValue(); amount != "" {
if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
if parsedAmount, ok := ParseEuropeanAmount(amount); ok {
t.EstimatedValue = parsedAmount
t.AwardedValue = parsedAmount
}
t.Currency = currency
}
// Set location information from main procurement project
if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
if addr := can.ProcurementProject.RealizedLocation.Address; addr != nil {
t.CityName = addr.CityName
t.PostalCode = addr.PostalZone
t.RegionCode = addr.CountrySubentityCode
if addr.Country != nil {
t.CountryCode = addr.Country.IdentificationCode
if ad := can.GetAwardDate(); ad != "" {
t.AwardDate = ParseDateToUnix(ad)
}
if can.ContractingParty != nil {
t.BuyerProfileURL = can.ContractingParty.BuyerProfileURI
}
for i := range can.ProcurementProjectLot {
t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&can.ProcurementProjectLot[i]))
}
// Set location information from main procurement project or first lot
var realizedAddr *Address
if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
realizedAddr = can.ProcurementProject.RealizedLocation.Address
} else if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil &&
lot.ProcurementProject.RealizedLocation != nil {
realizedAddr = lot.ProcurementProject.RealizedLocation.Address
}
if realizedAddr != nil {
t.CityName = realizedAddr.CityName
t.PostalCode = realizedAddr.PostalZone
t.RegionCode = realizedAddr.CountrySubentityCode
t.PlaceOfPerformance = realizedAddr.Region
if realizedAddr.Country != nil {
t.CountryCode = realizedAddr.Country.IdentificationCode
}
}
if wOrg := can.GetWinningTenderOrganization(); wOrg != nil {
t.WinningTenderer = s.mapOrganization(wOrg, "winner")
}
if nr := can.GetNoticeResult(); nr != nil {
for _, lr := range nr.LotResult {
winner := pickWinningLotTender(lr.LotTender)
if winner == nil || winner.TenderingParty == nil {
continue
}
org := can.GetOrganizationByID(winner.TenderingParty.ID)
aw := notice.Awarded{
LotID: lr.ID,
OrganizationID: winner.TenderingParty.ID,
AwardDate: t.AwardDate,
}
if org != nil && org.Company != nil {
if org.Company.PartyName != nil {
aw.Name = org.Company.PartyName.Name
}
if org.Company.PartyLegalEntity != nil {
aw.CompanyID = org.Company.PartyLegalEntity.CompanyID
}
}
if winner.LegalMonetaryTotal != nil {
pa := winner.LegalMonetaryTotal.PayableAmount
if v, ok := ParseEuropeanAmount(pa.Text); ok {
aw.Amount = v
}
aw.Currency = pa.CurrencyID
}
t.AwardedEntities = append(t.AwardedEntities, aw)
}
}
@@ -348,6 +471,7 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.
PostalZone: company.PostalAddress.PostalZone,
CountrySubentityCode: company.PostalAddress.CountrySubentityCode,
Department: company.PostalAddress.Department,
Region: strings.TrimSpace(company.PostalAddress.CountrySubentity),
}
if company.PostalAddress.Country != nil {
org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode
ted/model.go
+158 -70
View File
@@ -3,9 +3,16 @@ package ted
import (
"encoding/xml"
"fmt"
"strings"
"time"
)
// NoticeTypeCodeField captures cbc:NoticeTypeCode: inner text (notice type, BT-02) and listName (form type, BT-03).
type NoticeTypeCodeField struct {
ListName string `xml:"listName,attr,omitempty"`
Text string `xml:",chardata"`
}
// DocumentType represents the type of TED document
type DocumentType string
@@ -650,14 +657,21 @@ type ContractNotice struct {
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeTypeCode NoticeTypeCodeField `xml:"NoticeTypeCode,omitempty"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
Extensions *UBLExtensions `xml:"UBLExtensions,omitempty"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingTerms *TenderingTerms `xml:"TenderingTerms,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
ProcurementProject *ProcurementProject `xml:"ProcurementProject,omitempty"`
ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
ProcurementProjectLots []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
}
func (cn ContractNotice) firstLot() *ProcurementProjectLot {
if len(cn.ProcurementProjectLots) == 0 {
return nil
}
return &cn.ProcurementProjectLots[0]
}
// ContractAwardNotice represents the main TED XML contract award notice structure
@@ -679,7 +693,7 @@ type ContractAwardNotice struct {
IssueTime string `xml:"IssueTime"`
VersionID string `xml:"VersionID"`
RegulatoryDomain string `xml:"RegulatoryDomain"`
NoticeTypeCode string `xml:"NoticeTypeCode"`
NoticeTypeCode NoticeTypeCodeField `xml:"NoticeTypeCode,omitempty"`
NoticeLanguageCode string `xml:"NoticeLanguageCode"`
ContractingParty *ContractingParty `xml:"ContractingParty,omitempty"`
TenderingProcess *TenderingProcess `xml:"TenderingProcess,omitempty"`
@@ -688,6 +702,25 @@ type ContractAwardNotice struct {
TenderResult *TenderResult `xml:"TenderResult,omitempty"`
}
func (can ContractAwardNotice) firstAwardLot() *ProcurementProjectLot {
if len(can.ProcurementProjectLot) == 0 {
return nil
}
return &can.ProcurementProjectLot[0]
}
func pickWinningLotTender(lotTenders []LotTender) *LotTender {
if len(lotTenders) == 0 {
return nil
}
for i := range lotTenders {
if strings.TrimSpace(lotTenders[i].RankCode) == "1" {
return &lotTenders[i]
}
}
return &lotTenders[0]
}
// NoticeResult contains notice result information from extensions
type NoticeResult struct {
XMLName xml.Name `xml:"NoticeResult"`
@@ -821,6 +854,8 @@ type UBLExtension struct {
type ExtensionContent struct {
XMLName xml.Name `xml:"ExtensionContent"`
EformsExtension *EformsExtension `xml:"EformsExtension,omitempty"`
TransmissionDate string `xml:"TransmissionDate,omitempty"`
TransmissionTime string `xml:"TransmissionTime,omitempty"`
}
// EformsExtension contains eForms specific data
@@ -835,6 +870,8 @@ type EformsExtension struct {
OfficialLanguages *OfficialLanguages `xml:"OfficialLanguages,omitempty"`
AwardCriterionParameter *AwardCriterionParameter `xml:"AwardCriterionParameter,omitempty"`
AccessToolName string `xml:"AccessToolName,omitempty"`
TransmissionDate string `xml:"TransmissionDate,omitempty"`
TransmissionTime string `xml:"TransmissionTime,omitempty"`
}
// NoticeSubType contains notice subtype information
@@ -908,6 +945,7 @@ type PostalAddress struct {
StreetName string `xml:"StreetName,omitempty"`
CityName string `xml:"CityName,omitempty"`
PostalZone string `xml:"PostalZone,omitempty"`
CountrySubentity string `xml:"CountrySubentity,omitempty"`
CountrySubentityCode string `xml:"CountrySubentityCode,omitempty"`
Department string `xml:"Department,omitempty"`
Country *Country `xml:"Country,omitempty"`
@@ -1338,11 +1376,11 @@ func (cn ContractNotice) Validate() error {
}
// Validate tender submission deadline format if present
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingProcess != nil &&
cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
if cn.firstLot() != nil &&
cn.firstLot().TenderingProcess != nil &&
cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
deadline := cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
deadline := cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
if deadline != "" {
var validDeadline bool
for _, format := range validFormats {
@@ -1358,23 +1396,23 @@ func (cn ContractNotice) Validate() error {
}
// Validate currency code if provided in budget
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
currency := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID
currency := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID
if currency != "" && len(currency) != 3 {
return fmt.Errorf("invalid currency code: %s (expected 3-letter ISO code)", currency)
}
}
// Validate duration unit code if provided
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil {
unitCode := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode
unitCode := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode
if unitCode != "" {
validUnits := map[string]bool{
"MONTH": true,
@@ -1396,9 +1434,14 @@ func (cn ContractNotice) GetID() string {
return cn.ID
}
// GetNoticeType returns the notice type
// GetNoticeType returns the notice type code (BT-02, cbc:NoticeTypeCode text).
func (cn ContractNotice) GetNoticeType() string {
return cn.NoticeTypeCode
return strings.TrimSpace(cn.NoticeTypeCode.Text)
}
// GetNoticeFormType returns the form type from listName on cbc:NoticeTypeCode (BT-03), e.g. "competition".
func (cn ContractNotice) GetNoticeFormType() string {
return strings.TrimSpace(cn.NoticeTypeCode.ListName)
}
// GetLanguage returns the notice language
@@ -1496,12 +1539,12 @@ func (cn ContractNotice) GetBuyerID() string {
// GetReviewOrganizationID returns the review organization ID from appeal terms
func (cn ContractNotice) GetReviewOrganizationID() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil &&
cn.ProcurementProjectLot.TenderingTerms.AppealTerms != nil &&
cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty != nil &&
cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil {
return cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID
if cn.firstLot() != nil &&
cn.firstLot().TenderingTerms != nil &&
cn.firstLot().TenderingTerms.AppealTerms != nil &&
cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty != nil &&
cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil {
return cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID
}
return ""
}
@@ -1516,18 +1559,18 @@ func (cn ContractNotice) GetProcedureCode() string {
// GetProcurementTitle returns the procurement project title
func (cn ContractNotice) GetProcurementTitle() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil {
return cn.ProcurementProjectLot.ProcurementProject.Name
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil {
return cn.firstLot().ProcurementProject.Name
}
return ""
}
// GetProcurementDescription returns the procurement project description
func (cn ContractNotice) GetProcurementDescription() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil {
return cn.ProcurementProjectLot.ProcurementProject.Description
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil {
return cn.firstLot().ProcurementProject.Description
}
return ""
}
@@ -1561,8 +1604,8 @@ func (cn ContractNotice) GetPlaceOfPerformance() string {
// GetLotID returns the procurement lot ID
func (cn ContractNotice) GetLotID() string {
if cn.ProcurementProjectLot != nil {
return cn.ProcurementProjectLot.ID
if cn.firstLot() != nil {
return cn.firstLot().ID
}
return ""
}
@@ -1580,17 +1623,17 @@ func (cn ContractNotice) GetSelectionCriteria() []SelectionCriteria {
// GetOfficialLanguages returns official languages from tender terms extensions
func (cn ContractNotice) GetOfficialLanguages() []string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil &&
len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil {
if cn.firstLot() != nil &&
cn.firstLot().TenderingTerms != nil &&
len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil {
languages := make([]string, len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language))
for i, lang := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language {
languages := make([]string, len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language))
for i, lang := range cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language {
languages[i] = lang.ID
}
return languages
@@ -1600,12 +1643,12 @@ func (cn ContractNotice) GetOfficialLanguages() []string {
// GetDocumentURI returns the document URI from call for tenders document reference
func (cn ContractNotice) GetDocumentURI() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil &&
len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil &&
cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil {
return cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI
if cn.firstLot() != nil &&
cn.firstLot().TenderingTerms != nil &&
len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil &&
cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil {
return cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI
}
return ""
}
@@ -1613,10 +1656,12 @@ func (cn ContractNotice) GetDocumentURI() string {
// GetAllDocumentURIs returns all document URIs from call for tenders document references
func (cn ContractNotice) GetAllDocumentURIs() []string {
var uris []string
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingTerms != nil {
for _, docRef := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference {
for i := range cn.ProcurementProjectLots {
lot := &cn.ProcurementProjectLots[i]
if lot.TenderingTerms == nil {
continue
}
for _, docRef := range lot.TenderingTerms.CallForTendersDocumentReference {
if docRef.Attachment != nil &&
docRef.Attachment.ExternalReference != nil &&
docRef.Attachment.ExternalReference.URI != "" {
@@ -1629,21 +1674,21 @@ func (cn ContractNotice) GetAllDocumentURIs() []string {
// GetTenderDeadline returns the tender submission deadline
func (cn ContractNotice) GetTenderDeadline() string {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.TenderingProcess != nil &&
cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
return cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
if cn.firstLot() != nil &&
cn.firstLot().TenderingProcess != nil &&
cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
return cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
}
return ""
}
// GetBudget returns the estimated contract amount and currency
func (cn ContractNotice) GetBudget() (amount string, currency string) {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
contractAmount := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
contractAmount := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
return contractAmount.Text, contractAmount.CurrencyID
}
return "", ""
@@ -1651,12 +1696,12 @@ func (cn ContractNotice) GetBudget() (amount string, currency string) {
// GetDuration returns the contract duration and unit
func (cn ContractNotice) GetDuration() (duration string, unit string) {
if cn.ProcurementProjectLot != nil &&
cn.ProcurementProjectLot.ProcurementProject != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil &&
cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil {
if cn.firstLot() != nil &&
cn.firstLot().ProcurementProject != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod != nil &&
cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil {
durationMeasure := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure
durationMeasure := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure
return durationMeasure.Text, durationMeasure.UnitCode
}
return "", ""
@@ -1716,9 +1761,14 @@ func (can ContractAwardNotice) GetID() string {
return can.ID
}
// GetNoticeType returns the notice type
// GetNoticeType returns the notice type code (BT-02).
func (can ContractAwardNotice) GetNoticeType() string {
return can.NoticeTypeCode
return strings.TrimSpace(can.NoticeTypeCode.Text)
}
// GetNoticeFormType returns the form type from listName (BT-03).
func (can ContractAwardNotice) GetNoticeFormType() string {
return strings.TrimSpace(can.NoticeTypeCode.ListName)
}
// GetLanguage returns the notice language
@@ -1818,25 +1868,34 @@ func (can ContractAwardNotice) GetProcedureCode() string {
// GetProcurementTitle returns the procurement project title
func (can ContractAwardNotice) GetProcurementTitle() string {
if can.ProcurementProject != nil {
if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Name) != "" {
return can.ProcurementProject.Name
}
if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
return lot.ProcurementProject.Name
}
return ""
}
// GetProcurementDescription returns the procurement project description
func (can ContractAwardNotice) GetProcurementDescription() string {
if can.ProcurementProject != nil {
if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Description) != "" {
return can.ProcurementProject.Description
}
if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
return lot.ProcurementProject.Description
}
return ""
}
// GetContractNature returns the main nature of the contract
func (can ContractAwardNotice) GetContractNature() string {
if can.ProcurementProject != nil {
if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.ProcurementTypeCode) != "" {
return can.ProcurementProject.ProcurementTypeCode
}
if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
return lot.ProcurementProject.ProcurementTypeCode
}
return ""
}
@@ -1846,6 +1905,11 @@ func (can ContractAwardNotice) GetMainClassification() string {
can.ProcurementProject.MainCommodityClassification != nil {
return can.ProcurementProject.MainCommodityClassification.ItemClassificationCode
}
if lot := can.firstAwardLot(); lot != nil &&
lot.ProcurementProject != nil &&
lot.ProcurementProject.MainCommodityClassification != nil {
return lot.ProcurementProject.MainCommodityClassification.ItemClassificationCode
}
return ""
}
@@ -1882,3 +1946,27 @@ func (can ContractAwardNotice) GetOrganizationByID(orgID string) *Organization {
}
return nil
}
// GetWinningTenderPartyID resolves the tendering party reference for the primary winning bid from NoticeResult.
func (can ContractAwardNotice) GetWinningTenderPartyID() string {
nr := can.GetNoticeResult()
if nr == nil {
return ""
}
for _, lr := range nr.LotResult {
winner := pickWinningLotTender(lr.LotTender)
if winner != nil && winner.TenderingParty != nil && winner.TenderingParty.ID != "" {
return winner.TenderingParty.ID
}
}
return ""
}
// GetWinningTenderOrganization resolves the winning tenderer organization from NoticeResult via Organizations.
func (can ContractAwardNotice) GetWinningTenderOrganization() *Organization {
id := can.GetWinningTenderPartyID()
if id == "" {
return nil
}
return can.GetOrganizationByID(id)
}
ted/notice_upsert.go
+80
View File
@@ -0,0 +1,80 @@
package ted
import (
"strconv"
"strings"
"time"
"tm/internal/notice"
)
// noticeNeedsRefresh reports whether incoming scraped data should replace an existing notice row.
func noticeNeedsRefresh(existing, incoming *notice.Notice) bool {
if incoming == nil || existing == nil {
return true
}
if strings.TrimSpace(incoming.ContentXML) != "" &&
strings.TrimSpace(existing.ContentXML) != strings.TrimSpace(incoming.ContentXML) {
return true
}
if normalizePublicationID(existing.NoticePublicationID) != normalizePublicationID(incoming.NoticePublicationID) {
return true
}
if compareNoticeVersions(incoming.NoticeVersion, existing.NoticeVersion) > 0 {
return true
}
return false
}
func normalizePublicationID(s string) string {
return strings.TrimSpace(s)
}
// compareNoticeVersions compares BT-757 notice versions (e.g. "01", "02"). Empty parses as 0.
func compareNoticeVersions(incoming, existing string) int {
vi := parseNoticeVersion(incoming)
ve := parseNoticeVersion(existing)
switch {
case vi > ve:
return 1
case vi < ve:
return -1
default:
return 0
}
}
func parseNoticeVersion(s string) int {
s = strings.TrimSpace(s)
if s == "" {
return 0
}
n, err := strconv.Atoi(s)
if err != nil {
return 0
}
return n
}
// MergeNoticeUpdate replaces scrape-derived fields from src into dst while preserving identity,
// creation time, and optional enrichment metadata already stored on dst.
func MergeNoticeUpdate(dst *notice.Notice, src *notice.Notice) {
preservedID := dst.ID
preservedCreated := dst.CreatedAt
oldEnrichment := dst.ProcessingMetadata.EnrichmentData
oldTranslated := dst.ProcessingMetadata.TranslatedData
oldTranslatedAt := dst.ProcessingMetadata.TranslatedAt
*dst = *src
dst.ID = preservedID
dst.CreatedAt = preservedCreated
dst.UpdatedAt = time.Now().Unix()
dst.ProcessingMetadata.Processed = false
if len(dst.ProcessingMetadata.EnrichmentData) == 0 && len(oldEnrichment) > 0 {
dst.ProcessingMetadata.EnrichmentData = oldEnrichment
}
if len(dst.ProcessingMetadata.TranslatedData) == 0 && len(oldTranslated) > 0 {
dst.ProcessingMetadata.TranslatedData = oldTranslated
dst.ProcessingMetadata.TranslatedAt = oldTranslatedAt
}
}
ted/scraper.go
+51 -28
View File
@@ -4,6 +4,7 @@ import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"net/http"
@@ -12,7 +13,7 @@ import (
"time"
"tm/internal/notice"
"tm/pkg/logger"
"tm/pkg/mongo"
orm "tm/pkg/mongo"
"tm/pkg/notification"
)
@@ -38,16 +39,16 @@ type TEDScraper struct {
logger logger.Logger
httpClient *http.Client
xmlParser *TEDParser
mongoManager *mongo.ConnectionManager
tenderRepo notice.Repository
mongoManager *orm.ConnectionManager
noticeRepo notice.Repository
}
// NewTEDScraper creates a new TED scraper instance
func NewTEDScraper(
config *Config,
logger logger.Logger,
mongoManager *mongo.ConnectionManager,
tenderRepo notice.Repository,
mongoManager *orm.ConnectionManager,
noticeRepo notice.Repository,
notify notification.SDK,
) *TEDScraper {
httpClient := &http.Client{
@@ -61,7 +62,7 @@ func NewTEDScraper(
httpClient: httpClient,
xmlParser: NewTEDParser(),
mongoManager: mongoManager,
tenderRepo: tenderRepo,
noticeRepo: noticeRepo,
}
}
@@ -289,16 +290,18 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
return result, nil
}
func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID)
func (s *TEDScraper) findNoticeByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
if strings.TrimSpace(contractNoticeID) == "" {
return nil, nil
}
existing, err := s.noticeRepo.GetByContractNoticeID(ctx, contractNoticeID)
if err != nil {
if err.Error() == "document not found" {
if errors.Is(err, orm.ErrDocumentNotFound) {
return nil, nil
}
return nil, err
}
return existingTender, nil
return existing, nil
}
func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, result *FileProcessingResult) error {
@@ -331,45 +334,66 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
}
t.ContentXML = string(content)
// Check if tender already exists to prevent duplicates
existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID)
existingNotice, err := s.findNoticeByContractNoticeID(ctx, t.ContractNoticeID)
if err != nil {
s.logger.Error("Failed to check existing tender", map[string]interface{}{
s.logger.Error("Failed to check existing notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"error": err.Error(),
})
return fmt.Errorf("failed to check existing tender: %w", err)
return fmt.Errorf("failed to check existing notice: %w", err)
}
if existingTender != nil {
s.logger.Info("Tender already exists, skipping duplicate", map[string]interface{}{
if existingNotice != nil {
if !noticeNeedsRefresh(existingNotice, t) {
s.logger.Debug("Notice unchanged, skipping", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"existing_id": existingTender.ID,
"notice_doc_id": existingNotice.ID.Hex(),
})
return nil
}
// Create new tender
s.logger.Info("Creating new tender", map[string]interface{}{
MergeNoticeUpdate(existingNotice, t)
err = s.noticeRepo.Update(ctx, existingNotice)
if err != nil {
s.logger.Error("Failed to update existing notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"notice_doc_id": existingNotice.ID.Hex(),
"error": err.Error(),
})
return fmt.Errorf("failed to update notice: %w", err)
}
s.logger.Info("Notice updated (refresh)", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"notice_publication_id": t.NoticePublicationID,
"notice_version": t.NoticeVersion,
"notice_doc_id": existingNotice.ID.Hex(),
"processing_reset_worker": true,
})
result.SuccessCount++
} else {
s.logger.Info("Creating new notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
})
t.CreatedAt = time.Now().Unix()
t.UpdatedAt = t.CreatedAt
now := time.Now().Unix()
t.CreatedAt = now
t.UpdatedAt = now
err = s.tenderRepo.Import(ctx, t)
err = s.noticeRepo.Import(ctx, t)
if err != nil {
s.logger.Error("Failed to create new tender", map[string]interface{}{
s.logger.Error("Failed to import notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
"error": err.Error(),
})
return fmt.Errorf("failed to create tender: %w", err)
return fmt.Errorf("failed to import notice: %w", err)
}
s.logger.Info("Successfully created new tender", map[string]interface{}{
s.logger.Info("Successfully imported new notice", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID,
})
// }
result.SuccessCount++
}
// Log detailed parsed information based on notice type
s.logger.Info("Successfully parsed notice", map[string]interface{}{
@@ -379,7 +403,6 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
"tender_url": t.TenderURL,
})
result.SuccessCount++
return nil
}
ted/service.go
+51
View File
@@ -7,6 +7,57 @@ import (
"time"
)
// ParseIssueDateTimeToUnix combines cbc:IssueDate and cbc:IssueTime (with timezone offsets) into one Unix timestamp.
func ParseIssueDateTimeToUnix(issueDate, issueTime string) int64 {
issueDate = strings.TrimSpace(issueDate)
issueTime = strings.TrimSpace(issueTime)
if issueDate == "" {
return 0
}
if issueTime == "" {
return ParseDateToUnix(issueDate)
}
dateCore := issueDate
if len(issueDate) >= 10 && issueDate[4] == '-' && issueDate[7] == '-' {
dateCore = issueDate[:10]
}
combined := dateCore + "T" + issueTime
if t, err := ParseDate(combined); err == nil {
return t.Unix()
}
return ParseDateToUnix(issueDate)
}
// TransmissionDispatchUnix reads BT-803 (eSender) transmission date/time from extension content (root or eForms extension).
func TransmissionDispatchUnix(ext *UBLExtensions) int64 {
if ext == nil || ext.UBLExtension == nil || ext.UBLExtension.ExtensionContent == nil {
return 0
}
ec := ext.UBLExtension.ExtensionContent
if u := ParseIssueDateTimeToUnix(ec.TransmissionDate, ec.TransmissionTime); u != 0 {
return u
}
if ec.EformsExtension != nil {
if u := ParseIssueDateTimeToUnix(ec.EformsExtension.TransmissionDate, ec.EformsExtension.TransmissionTime); u != 0 {
return u
}
}
return 0
}
// ParseEuropeanAmount normalizes TED amount strings (spaces, comma decimal) for strconv.ParseFloat.
func ParseEuropeanAmount(raw string) (float64, bool) {
s := strings.TrimSpace(raw)
s = strings.ReplaceAll(s, "\u00a0", "")
s = strings.ReplaceAll(s, " ", "")
s = strings.ReplaceAll(s, ",", ".")
if s == "" {
return 0, false
}
v, err := strconv.ParseFloat(s, 64)
return v, err == nil
}
// parseDate parses various date formats used in TED XML
func ParseDate(dateStr string) (time.Time, error) {
if dateStr == "" {