Added missing fields - scraper and worker fixes

cmd/worker/bootstrap/bootstrap.go

@@ -96,7 +96,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 	appLogger.Info("Starting notice worker", map[string]interface{}{
 		"processing_limit": config.Worker.NoticeProcessingLimit,
 	})
-	noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit)
+	noticeWorker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit, config.Worker.DeleteProcessedNotices)
 	noticeWorker.Run()
 
 	// Initialize Queue-based scraping if enabled
@@ -181,7 +181,7 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
 	scheduler.AddJob(schedule.Job{
 		Name: "Notice Worker Job",
 		Func: func() {
-			worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit)
+			worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService, config.Worker.NoticeProcessingLimit, config.Worker.DeleteProcessedNotices)
 			worker.Run()
 		},
 		Expr: workerInterval,

cmd/worker/bootstrap/config.go

@@ -38,6 +38,7 @@ type QueueConfig struct {
 type WorkerConfig struct {
 	Interval                 string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
 	NoticeProcessingLimit    int    `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
+	DeleteProcessedNotices   bool   `env:"WORKER_DELETE_PROCESSED_NOTICES" envDefault:"true"`
 	TranslationInterval      string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
 	TranslationBatchSize       int    `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
 }

cmd/worker/workers/notice.go

@@ -24,9 +24,10 @@ type NoticeWorker struct {
 	TenderRepo              tender.TenderRepository
 	GLM                     *glm.SDK
 	ProcessingLimit         int
+	DeleteProcessedNotices  bool
 }
 
-func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int) *NoticeWorker {
+func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int, deleteProcessedNotices bool) *NoticeWorker {
 	if processingLimit < 0 {
 		processingLimit = 5 // Default limit
 	}
@@ -38,6 +39,7 @@ func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notif
 		TenderRepo:             tenderRepo,
 		GLM:                    glmService,
 		ProcessingLimit:        processingLimit,
+		DeleteProcessedNotices: deleteProcessedNotices,
 	}
 }
 
@@ -197,8 +199,12 @@ func (w *NoticeWorker) Run() {
 		}
 	}
 
-	// Clean up: delete notices that were marked as processed in previous runs
+	// Clean up: delete notices that were marked as processed in previous runs (optional to retain XML for audit/backfill)
+	if w.DeleteProcessedNotices {
 		w.cleanupProcessedNotices()
+	} else {
+		w.Logger.Info("Skipping deletion of processed notices (WORKER_DELETE_PROCESSED_NOTICES=false)", map[string]interface{}{})
+	}
 }
 
 func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
@@ -211,6 +217,12 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 		t = new(tender.Tender)
 	}
 
+	var preserveTenderID, preserveProjectName string
+	if t != nil && !t.ID.IsZero() {
+		preserveTenderID = t.TenderID
+		preserveProjectName = t.ProjectName
+	}
+
 	// Buyer Organization
 	buyerOrg := &tender.Organization{}
 	if n.BuyerOrganization != nil {
@@ -347,6 +359,21 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 		}
 	}
 
+	procurementLots := make([]tender.ProcurementLot, len(n.ProcurementLots))
+	for i, l := range n.ProcurementLots {
+		procurementLots[i] = tender.ProcurementLot{
+			LotID:                     l.LotID,
+			Title:                     l.Title,
+			Description:               l.Description,
+			MainNatureOfContract:      l.MainNatureOfContract,
+			MainClassification:        l.MainClassification,
+			AdditionalClassifications: append([]string(nil), l.AdditionalClassifications...),
+			Duration:                  l.Duration,
+			EstimatedValue:            l.EstimatedValue,
+			Currency:                  l.Currency,
+		}
+	}
+
 	var title string
 	if w.GLM != nil {
 		title, err = w.GLM.Translate(context.Background(), n.Title, n.NoticeLanguageCode, "en")
@@ -402,8 +429,14 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 	t.WinningTenderer = winningTenderer
 	t.Modifications = modifications
 	t.AwardedEntities = awardedEntities
+	t.ProcurementLots = procurementLots
 	t.NoticeTypeCode = n.NoticeTypeCode
+	t.FormType = n.FormType
 	t.NoticeSubTypeCode = n.NoticeSubTypeCode
+	t.NoticeIdentifier = n.NoticeIdentifier
+	t.NoticeVersion = n.NoticeVersion
+	t.NoticeDispatchAt = n.NoticeDispatchAt
+	t.ESenderDispatchAt = n.ESenderDispatchAt
 	t.NoticeLanguageCode = n.NoticeLanguageCode
 	t.IssueDate = n.IssueDate
 	t.IssueTime = n.IssueTime
@@ -417,6 +450,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 	t.CityName = n.CityName
 	t.PostalCode = n.PostalCode
 	t.DocumentURI = n.DocumentURI
+	t.BuyerProfileURL = n.BuyerProfileURL
 	t.TenderURL = n.TenderURL
 	t.MainClassification = n.MainClassification
 	t.AdditionalClassifications = n.AdditionalClassifications
@@ -427,6 +461,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 	t.ContractFolderID = n.ContractFolderID
 	t.SourceFileURL = n.SourceFileURL
 	t.SourceFileName = n.SourceFileName
+	t.ContentXML = n.ContentXML
 	// Preserve document-scraping/summarization flags on the tender (managed by
 	// downstream workers), but refresh the notice-side processing metadata.
 	t.ProcessingMetadata.ScrapedAt = n.ProcessingMetadata.ScrapedAt
@@ -448,8 +483,13 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 	t.WinningTenderer = winningTenderer
 	t.Modifications = modifications
 	t.AwardedEntities = awardedEntities
+	if preserveTenderID != "" {
+		t.TenderID = preserveTenderID
+		t.ProjectName = preserveProjectName
+	} else {
 		t.TenderID = w.GenerateTenderID(context.Background(), n)
 		t.ProjectName = w.GenerateProjectName(t)
+	}
 
 	return t, nil
 }

internal/notice/entity.go

@@ -15,7 +15,12 @@ type Notice struct {
 	NoticePublicationID       string               `bson:"notice_publication_id" json:"notice_publication_id"`
 	ContractFolderID          string               `bson:"contract_folder_id" json:"contract_folder_id"`
 	NoticeTypeCode            string               `bson:"notice_type_code" json:"notice_type_code"`
+	FormType                  string               `bson:"form_type,omitempty" json:"form_type,omitempty"`
 	NoticeSubTypeCode         string               `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
+	NoticeIdentifier          string               `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
+	NoticeVersion             string               `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
+	NoticeDispatchAt          int64                `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
+	ESenderDispatchAt         int64                `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
 	NoticeLanguageCode        string               `bson:"notice_language_code" json:"notice_language_code"`
 	IssueDate                 int64                `bson:"issue_date" json:"issue_date"`
 	IssueTime                 int64                `bson:"issue_time" json:"issue_time"`
@@ -40,6 +45,8 @@ type Notice struct {
 	CityName                  string               `bson:"city_name" json:"city_name"`
 	PostalCode                string               `bson:"postal_code" json:"postal_code"`
 	DocumentURI               string               `bson:"document_uri" json:"document_uri"`
+	BuyerProfileURL           string               `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
+	ProcurementLots           []ProcurementLot     `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
 	TenderURL                 string               `bson:"tender_url" json:"tender_url"`
 	SubmissionURL             string               `bson:"submission_url" json:"submission_url"`
 	BuyerOrganization         *Organization        `bson:"buyer_organization" json:"buyer_organization"`
@@ -99,6 +106,19 @@ type SelectionCriterion struct {
 	LanguageID  string `bson:"language_id,omitempty" json:"language_id,omitempty"`
 }
 
+// ProcurementLot captures per-lot procurement fields from TED notices.
+type ProcurementLot struct {
+	LotID                     string   `bson:"lot_id,omitempty" json:"lot_id,omitempty"`
+	Title                     string   `bson:"title,omitempty" json:"title,omitempty"`
+	Description               string   `bson:"description,omitempty" json:"description,omitempty"`
+	MainNatureOfContract      string   `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"`
+	MainClassification        string   `bson:"main_classification,omitempty" json:"main_classification,omitempty"`
+	AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"`
+	Duration                  string   `bson:"duration,omitempty" json:"duration,omitempty"`
+	EstimatedValue            float64  `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"`
+	Currency                  string   `bson:"currency,omitempty" json:"currency,omitempty"`
+}
+
 // ProcessingMetadata contains metadata about how the tender was processed
 type ProcessingMetadata struct {
 	ScrapedAt         int64             `bson:"scraped_at" json:"scraped_at"`

internal/notice/repository.go

@@ -16,6 +16,7 @@ type Repository interface {
 	BulkImport(ctx context.Context, notices []Notice) error
 	GetByID(ctx context.Context, id string) (*Notice, error)
 	GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error)
+	FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error)
 	GetUnProcessedNotices(ctx context.Context, ProcessingLimit int, skip int) ([]Notice, int64, error)
 	GetProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error)
 	Delete(ctx context.Context, id string) error
@@ -39,6 +40,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
 		// processing_metadata.processed
 		*orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}),
 		*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
+		*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
 	}
 
 	// Create indexes
@@ -144,6 +146,27 @@ func (r *noticeRepository) GetByContractNoticeID(ctx context.Context, contractNo
 	return &result.Items[0], nil
 }
 
+// FindNoticesWithContentXML returns notices that have non-empty stored TED XML (for backfill / remapping).
+func (r *noticeRepository) FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error) {
+	filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
+	pagination := orm.Pagination{
+		Limit:     limit,
+		Skip:      skip,
+		SortField: "_id",
+		SortOrder: 1,
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to list notices with content_xml", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, err
+	}
+
+	return result.Items, nil
+}
+
 // GetUnProcessedNotices retrieves unprocessed notices
 
 func (r *noticeRepository) GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) {

internal/tender/entity.go

@@ -37,7 +37,12 @@ type Tender struct {
 	NoticePublicationID       string               `bson:"notice_publication_id" json:"notice_publication_id"`
 	ContractFolderID          string               `bson:"contract_folder_id" json:"contract_folder_id"`
 	NoticeTypeCode            string               `bson:"notice_type_code" json:"notice_type_code"`
+	FormType                  string               `bson:"form_type,omitempty" json:"form_type,omitempty"`
 	NoticeSubTypeCode         string               `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
+	NoticeIdentifier          string               `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
+	NoticeVersion             string               `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
+	NoticeDispatchAt          int64                `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
+	ESenderDispatchAt         int64                `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
 	NoticeLanguageCode        string               `bson:"notice_language_code" json:"notice_language_code"`
 	IssueDate                 int64                `bson:"issue_date" json:"issue_date"`
 	IssueTime                 int64                `bson:"issue_time" json:"issue_time"`
@@ -62,6 +67,8 @@ type Tender struct {
 	CityName                  string               `bson:"city_name" json:"city_name"`
 	PostalCode                string               `bson:"postal_code" json:"postal_code"`
 	DocumentURI               string               `bson:"document_uri" json:"document_uri"`
+	BuyerProfileURL           string               `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
+	ProcurementLots           []ProcurementLot     `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
 	TenderURL                 string               `bson:"tender_url" json:"tender_url"`
 	SubmissionURL             string               `bson:"submission_url" json:"submission_url"`
 	BuyerOrganization         *Organization        `bson:"buyer_organization" json:"buyer_organization"`
@@ -73,6 +80,7 @@ type Tender struct {
 	Source                    TenderSource         `bson:"source" json:"source"`
 	SourceFileURL             string               `bson:"source_file_url" json:"source_file_url"`
 	SourceFileName            string               `bson:"source_file_name" json:"source_file_name"`
+	ContentXML                string               `bson:"content_xml,omitempty" json:"content_xml,omitempty"`
 	ProcessingMetadata        ProcessingMetadata   `bson:"processing_metadata" json:"processing_metadata"`
 	TenderID                  string               `bson:"tender_id" json:"tender_id"`       // PBL Project Number (SCDYYNNN format)
 	ProjectName               string               `bson:"project_name" json:"project_name"` // PBL Project Name (<InternalProjectNumber>_<Client-OpportunityName>_<PartnerCode>)
@@ -133,6 +141,19 @@ type SelectionCriterion struct {
 	LanguageID  string `bson:"language_id,omitempty" json:"language_id,omitempty"`
 }
 
+// ProcurementLot captures per-lot procurement fields from TED notices.
+type ProcurementLot struct {
+	LotID                     string   `bson:"lot_id,omitempty" json:"lot_id,omitempty"`
+	Title                     string   `bson:"title,omitempty" json:"title,omitempty"`
+	Description               string   `bson:"description,omitempty" json:"description,omitempty"`
+	MainNatureOfContract      string   `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"`
+	MainClassification        string   `bson:"main_classification,omitempty" json:"main_classification,omitempty"`
+	AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"`
+	Duration                  string   `bson:"duration,omitempty" json:"duration,omitempty"`
+	EstimatedValue            float64  `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"`
+	Currency                  string   `bson:"currency,omitempty" json:"currency,omitempty"`
+}
+
 // ProcessingMetadata contains metadata about how the tender was processed
 type ProcessingMetadata struct {
 	ScrapedAt             int64             `bson:"scraped_at" json:"scraped_at"`     // Unix milliseconds

internal/tender/repository.go

@@ -18,6 +18,7 @@ type TenderRepository interface {
 	GetByID(ctx context.Context, id string) (*Tender, error)
 	GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
 	GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
+	FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error)
 	GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
 	GetTenderCountByType(ctx context.Context) (map[string]int64, error)
 	GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
@@ -175,6 +176,27 @@ func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticeP
 	return &result.Items[0], nil
 }
 
+// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted).
+func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) {
+	filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
+	pagination := orm.Pagination{
+		Limit:     limit,
+		Skip:      skip,
+		SortField: "_id",
+		SortOrder: 1,
+	}
+
+	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+	if err != nil {
+		r.logger.Error("Failed to list tenders with content_xml", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, err
+	}
+
+	return result.Items, nil
+}
+
 // Update updates an existing tender
 func (r *tenderRepository) Update(ctx context.Context, tender *Tender) error {
 	tender.UpdatedAt = time.Now().Unix()

ted/mapper.go

@@ -1,12 +1,63 @@
 package ted
 
 import (
-	"strconv"
 	"strings"
 	"time"
 	"tm/internal/notice"
 )
 
+func mapProcurementLotFromTED(lot *ProcurementProjectLot) notice.ProcurementLot {
+	if lot == nil {
+		return notice.ProcurementLot{}
+	}
+	out := notice.ProcurementLot{LotID: lot.ID}
+	if lot.ProcurementProject == nil {
+		return out
+	}
+	pp := lot.ProcurementProject
+	out.Title = pp.Name
+	out.Description = pp.Description
+	out.MainNatureOfContract = pp.ProcurementTypeCode
+	if pp.MainCommodityClassification != nil {
+		out.MainClassification = strings.TrimSpace(pp.MainCommodityClassification.ItemClassificationCode)
+	}
+	for _, ac := range pp.AdditionalCommodityClassification {
+		c := strings.TrimSpace(ac.ItemClassificationCode)
+		if c != "" {
+			out.AdditionalClassifications = append(out.AdditionalClassifications, c)
+		}
+	}
+	if pp.RequestedTenderTotal != nil {
+		a := pp.RequestedTenderTotal.EstimatedOverallContractAmount
+		if v, ok := ParseEuropeanAmount(a.Text); ok {
+			out.EstimatedValue = v
+		}
+		out.Currency = a.CurrencyID
+	}
+	if pp.PlannedPeriod != nil && pp.PlannedPeriod.DurationMeasure != nil {
+		dm := pp.PlannedPeriod.DurationMeasure
+		txt := strings.TrimSpace(dm.Text)
+		unit := strings.TrimSpace(dm.UnitCode)
+		switch unit {
+		case "MONTH":
+			out.Duration = txt + " Months"
+		case "DAY":
+			out.Duration = txt + " Days"
+		case "WEEK":
+			out.Duration = txt + " Weeks"
+		case "YEAR":
+			out.Duration = txt + " Years"
+		default:
+			if unit != "" && txt != "" {
+				out.Duration = txt + " " + unit
+			} else {
+				out.Duration = txt
+			}
+		}
+	}
+	return out
+}
+
 // mapToTenderFromParsedDoc maps a parsed TED document to tender entity
 func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice {
 	switch parsedDoc.Type {
@@ -89,8 +140,13 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 	t := &notice.Notice{
 		ContractNoticeID:    cn.ID,
 		ContractFolderID:    cn.ContractFolderID,
-		NoticeTypeCode:      cn.GetNoticeSubType(),
+		NoticeTypeCode:      strings.TrimSpace(cn.GetNoticeType()),
+		FormType:            cn.GetNoticeFormType(),
 		NoticeSubTypeCode:   cn.GetNoticeSubType(),
+		NoticeIdentifier:    cn.ID,
+		NoticeVersion:       cn.VersionID,
+		NoticeDispatchAt:    ParseIssueDateTimeToUnix(cn.IssueDate, cn.IssueTime),
+		ESenderDispatchAt:   TransmissionDispatchUnix(cn.Extensions),
 		NoticeLanguageCode:  cn.NoticeLanguageCode,
 		IssueDate:           ParseDateToUnix(cn.IssueDate),
 		IssueTime:           ParseTimeToUnix(cn.IssueTime),
@@ -131,12 +187,20 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 
 	// Set estimated value and currency
 	if amount, currency := cn.GetBudget(); amount != "" {
-		if value, err := strconv.ParseFloat(amount, 64); err == nil {
+		if value, ok := ParseEuropeanAmount(amount); ok {
 			t.EstimatedValue = value
 			t.Currency = currency
 		}
 	}
 
+	if cn.ContractingParty != nil {
+		t.BuyerProfileURL = cn.ContractingParty.BuyerProfileURI
+	}
+
+	for i := range cn.ProcurementProjectLots {
+		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&cn.ProcurementProjectLots[i]))
+	}
+
 	// Set duration information
 	if duration, unit := cn.GetDuration(); duration != "" {
 		t.Duration = duration
@@ -159,8 +223,8 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 	}
 
 	// Set SubmissionURL from TenderingTerms
-	if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil {
+	if lot := cn.firstLot(); lot != nil && lot.TenderingTerms != nil {
-		if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
+		if tenderingTerms := lot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
 			if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
 				if strings.HasPrefix(endpointID, "http") {
 					t.SubmissionURL = endpointID
@@ -235,8 +299,13 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
 	t := &notice.Notice{
 		ContractNoticeID:    can.ID,
 		ContractFolderID:    can.ContractFolderID,
-		NoticeTypeCode:      can.NoticeTypeCode,
+		NoticeTypeCode:      strings.TrimSpace(can.GetNoticeType()),
+		FormType:            can.GetNoticeFormType(),
 		NoticeSubTypeCode:   can.GetNoticeSubType(),
+		NoticeIdentifier:    can.ID,
+		NoticeVersion:       can.VersionID,
+		NoticeDispatchAt:    ParseIssueDateTimeToUnix(can.IssueDate, can.IssueTime),
+		ESenderDispatchAt:   TransmissionDispatchUnix(can.Extensions),
 		NoticeLanguageCode:  can.NoticeLanguageCode,
 		IssueDate:           ParseDateToUnix(can.IssueDate),
 		IssueTime:           ParseTimeToUnix(can.IssueTime),
@@ -266,21 +335,75 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
 
 	// Set total award value
 	if amount, currency := can.GetTotalAwardValue(); amount != "" {
-		if parsedAmount, err := strconv.ParseFloat(amount, 64); err == nil {
+		if parsedAmount, ok := ParseEuropeanAmount(amount); ok {
 			t.EstimatedValue = parsedAmount
+			t.AwardedValue = parsedAmount
 		}
 		t.Currency = currency
 	}
 
-	// Set location information from main procurement project
+	if ad := can.GetAwardDate(); ad != "" {
-	if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
+		t.AwardDate = ParseDateToUnix(ad)
-		if addr := can.ProcurementProject.RealizedLocation.Address; addr != nil {
-			t.CityName = addr.CityName
-			t.PostalCode = addr.PostalZone
-			t.RegionCode = addr.CountrySubentityCode
-			if addr.Country != nil {
-				t.CountryCode = addr.Country.IdentificationCode
 	}
+
+	if can.ContractingParty != nil {
+		t.BuyerProfileURL = can.ContractingParty.BuyerProfileURI
+	}
+
+	for i := range can.ProcurementProjectLot {
+		t.ProcurementLots = append(t.ProcurementLots, mapProcurementLotFromTED(&can.ProcurementProjectLot[i]))
+	}
+
+	// Set location information from main procurement project or first lot
+	var realizedAddr *Address
+	if can.ProcurementProject != nil && can.ProcurementProject.RealizedLocation != nil {
+		realizedAddr = can.ProcurementProject.RealizedLocation.Address
+	} else if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil &&
+		lot.ProcurementProject.RealizedLocation != nil {
+		realizedAddr = lot.ProcurementProject.RealizedLocation.Address
+	}
+	if realizedAddr != nil {
+		t.CityName = realizedAddr.CityName
+		t.PostalCode = realizedAddr.PostalZone
+		t.RegionCode = realizedAddr.CountrySubentityCode
+		t.PlaceOfPerformance = realizedAddr.Region
+		if realizedAddr.Country != nil {
+			t.CountryCode = realizedAddr.Country.IdentificationCode
+		}
+	}
+
+	if wOrg := can.GetWinningTenderOrganization(); wOrg != nil {
+		t.WinningTenderer = s.mapOrganization(wOrg, "winner")
+	}
+
+	if nr := can.GetNoticeResult(); nr != nil {
+		for _, lr := range nr.LotResult {
+			winner := pickWinningLotTender(lr.LotTender)
+			if winner == nil || winner.TenderingParty == nil {
+				continue
+			}
+			org := can.GetOrganizationByID(winner.TenderingParty.ID)
+			aw := notice.Awarded{
+				LotID:          lr.ID,
+				OrganizationID: winner.TenderingParty.ID,
+				AwardDate:      t.AwardDate,
+			}
+			if org != nil && org.Company != nil {
+				if org.Company.PartyName != nil {
+					aw.Name = org.Company.PartyName.Name
+				}
+				if org.Company.PartyLegalEntity != nil {
+					aw.CompanyID = org.Company.PartyLegalEntity.CompanyID
+				}
+			}
+			if winner.LegalMonetaryTotal != nil {
+				pa := winner.LegalMonetaryTotal.PayableAmount
+				if v, ok := ParseEuropeanAmount(pa.Text); ok {
+					aw.Amount = v
+				}
+				aw.Currency = pa.CurrencyID
+			}
+			t.AwardedEntities = append(t.AwardedEntities, aw)
 		}
 	}
 
@@ -348,6 +471,7 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.
 			PostalZone:           company.PostalAddress.PostalZone,
 			CountrySubentityCode: company.PostalAddress.CountrySubentityCode,
 			Department:           company.PostalAddress.Department,
+			Region:               strings.TrimSpace(company.PostalAddress.CountrySubentity),
 		}
 		if company.PostalAddress.Country != nil {
 			org.Address.CountryCode = company.PostalAddress.Country.IdentificationCode

ted/model.go

@@ -3,9 +3,16 @@ package ted
 import (
 	"encoding/xml"
 	"fmt"
+	"strings"
 	"time"
 )
 
+// NoticeTypeCodeField captures cbc:NoticeTypeCode: inner text (notice type, BT-02) and listName (form type, BT-03).
+type NoticeTypeCodeField struct {
+	ListName string `xml:"listName,attr,omitempty"`
+	Text     string `xml:",chardata"`
+}
+
 // DocumentType represents the type of TED document
 type DocumentType string
 
@@ -650,14 +657,21 @@ type ContractNotice struct {
 	IssueTime              string                  `xml:"IssueTime"`
 	VersionID              string                  `xml:"VersionID"`
 	RegulatoryDomain       string                  `xml:"RegulatoryDomain"`
-	NoticeTypeCode        string                 `xml:"NoticeTypeCode"`
+	NoticeTypeCode         NoticeTypeCodeField     `xml:"NoticeTypeCode,omitempty"`
 	NoticeLanguageCode     string                  `xml:"NoticeLanguageCode"`
 	Extensions             *UBLExtensions          `xml:"UBLExtensions,omitempty"`
 	ContractingParty       *ContractingParty       `xml:"ContractingParty,omitempty"`
 	TenderingTerms         *TenderingTerms         `xml:"TenderingTerms,omitempty"`
 	TenderingProcess       *TenderingProcess       `xml:"TenderingProcess,omitempty"`
 	ProcurementProject     *ProcurementProject     `xml:"ProcurementProject,omitempty"`
-	ProcurementProjectLot *ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
+	ProcurementProjectLots []ProcurementProjectLot `xml:"ProcurementProjectLot,omitempty"`
+}
+
+func (cn ContractNotice) firstLot() *ProcurementProjectLot {
+	if len(cn.ProcurementProjectLots) == 0 {
+		return nil
+	}
+	return &cn.ProcurementProjectLots[0]
 }
 
 // ContractAwardNotice represents the main TED XML contract award notice structure
@@ -679,7 +693,7 @@ type ContractAwardNotice struct {
 	IssueTime             string                  `xml:"IssueTime"`
 	VersionID             string                  `xml:"VersionID"`
 	RegulatoryDomain      string                  `xml:"RegulatoryDomain"`
-	NoticeTypeCode        string                  `xml:"NoticeTypeCode"`
+	NoticeTypeCode        NoticeTypeCodeField     `xml:"NoticeTypeCode,omitempty"`
 	NoticeLanguageCode    string                  `xml:"NoticeLanguageCode"`
 	ContractingParty      *ContractingParty       `xml:"ContractingParty,omitempty"`
 	TenderingProcess      *TenderingProcess       `xml:"TenderingProcess,omitempty"`
@@ -688,6 +702,25 @@ type ContractAwardNotice struct {
 	TenderResult          *TenderResult           `xml:"TenderResult,omitempty"`
 }
 
+func (can ContractAwardNotice) firstAwardLot() *ProcurementProjectLot {
+	if len(can.ProcurementProjectLot) == 0 {
+		return nil
+	}
+	return &can.ProcurementProjectLot[0]
+}
+
+func pickWinningLotTender(lotTenders []LotTender) *LotTender {
+	if len(lotTenders) == 0 {
+		return nil
+	}
+	for i := range lotTenders {
+		if strings.TrimSpace(lotTenders[i].RankCode) == "1" {
+			return &lotTenders[i]
+		}
+	}
+	return &lotTenders[0]
+}
+
 // NoticeResult contains notice result information from extensions
 type NoticeResult struct {
 	XMLName     xml.Name      `xml:"NoticeResult"`
@@ -821,6 +854,8 @@ type UBLExtension struct {
 type ExtensionContent struct {
 	XMLName          xml.Name         `xml:"ExtensionContent"`
 	EformsExtension  *EformsExtension `xml:"EformsExtension,omitempty"`
+	TransmissionDate string           `xml:"TransmissionDate,omitempty"`
+	TransmissionTime string           `xml:"TransmissionTime,omitempty"`
 }
 
 // EformsExtension contains eForms specific data
@@ -835,6 +870,8 @@ type EformsExtension struct {
 	OfficialLanguages       *OfficialLanguages       `xml:"OfficialLanguages,omitempty"`
 	AwardCriterionParameter *AwardCriterionParameter `xml:"AwardCriterionParameter,omitempty"`
 	AccessToolName          string                   `xml:"AccessToolName,omitempty"`
+	TransmissionDate        string                   `xml:"TransmissionDate,omitempty"`
+	TransmissionTime        string                   `xml:"TransmissionTime,omitempty"`
 }
 
 // NoticeSubType contains notice subtype information
@@ -908,6 +945,7 @@ type PostalAddress struct {
 	StreetName           string   `xml:"StreetName,omitempty"`
 	CityName             string   `xml:"CityName,omitempty"`
 	PostalZone           string   `xml:"PostalZone,omitempty"`
+	CountrySubentity     string   `xml:"CountrySubentity,omitempty"`
 	CountrySubentityCode string   `xml:"CountrySubentityCode,omitempty"`
 	Department           string   `xml:"Department,omitempty"`
 	Country              *Country `xml:"Country,omitempty"`
@@ -1338,11 +1376,11 @@ func (cn ContractNotice) Validate() error {
 	}
 
 	// Validate tender submission deadline format if present
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.TenderingProcess != nil &&
+		cn.firstLot().TenderingProcess != nil &&
-		cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
+		cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
 
-		deadline := cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
+		deadline := cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
 		if deadline != "" {
 			var validDeadline bool
 			for _, format := range validFormats {
@@ -1358,23 +1396,23 @@ func (cn ContractNotice) Validate() error {
 	}
 
 	// Validate currency code if provided in budget
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.ProcurementProject != nil &&
+		cn.firstLot().ProcurementProject != nil &&
-		cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil {
+		cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
 
-		currency := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID
+		currency := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount.CurrencyID
 		if currency != "" && len(currency) != 3 {
 			return fmt.Errorf("invalid currency code: %s (expected 3-letter ISO code)", currency)
 		}
 	}
 
 	// Validate duration unit code if provided
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.ProcurementProject != nil &&
+		cn.firstLot().ProcurementProject != nil &&
-		cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil &&
+		cn.firstLot().ProcurementProject.PlannedPeriod != nil &&
-		cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil {
+		cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil {
 
-		unitCode := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode
+		unitCode := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure.UnitCode
 		if unitCode != "" {
 			validUnits := map[string]bool{
 				"MONTH": true,
@@ -1396,9 +1434,14 @@ func (cn ContractNotice) GetID() string {
 	return cn.ID
 }
 
-// GetNoticeType returns the notice type
+// GetNoticeType returns the notice type code (BT-02, cbc:NoticeTypeCode text).
 func (cn ContractNotice) GetNoticeType() string {
-	return cn.NoticeTypeCode
+	return strings.TrimSpace(cn.NoticeTypeCode.Text)
+}
+
+// GetNoticeFormType returns the form type from listName on cbc:NoticeTypeCode (BT-03), e.g. "competition".
+func (cn ContractNotice) GetNoticeFormType() string {
+	return strings.TrimSpace(cn.NoticeTypeCode.ListName)
 }
 
 // GetLanguage returns the notice language
@@ -1496,12 +1539,12 @@ func (cn ContractNotice) GetBuyerID() string {
 
 // GetReviewOrganizationID returns the review organization ID from appeal terms
 func (cn ContractNotice) GetReviewOrganizationID() string {
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.TenderingTerms != nil &&
+		cn.firstLot().TenderingTerms != nil &&
-		cn.ProcurementProjectLot.TenderingTerms.AppealTerms != nil &&
+		cn.firstLot().TenderingTerms.AppealTerms != nil &&
-		cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty != nil &&
+		cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty != nil &&
-		cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil {
+		cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification != nil {
-		return cn.ProcurementProjectLot.TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID
+		return cn.firstLot().TenderingTerms.AppealTerms.AppealReceiverParty.PartyIdentification.ID
 	}
 	return ""
 }
@@ -1516,18 +1559,18 @@ func (cn ContractNotice) GetProcedureCode() string {
 
 // GetProcurementTitle returns the procurement project title
 func (cn ContractNotice) GetProcurementTitle() string {
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.ProcurementProject != nil {
+		cn.firstLot().ProcurementProject != nil {
-		return cn.ProcurementProjectLot.ProcurementProject.Name
+		return cn.firstLot().ProcurementProject.Name
 	}
 	return ""
 }
 
 // GetProcurementDescription returns the procurement project description
 func (cn ContractNotice) GetProcurementDescription() string {
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.ProcurementProject != nil {
+		cn.firstLot().ProcurementProject != nil {
-		return cn.ProcurementProjectLot.ProcurementProject.Description
+		return cn.firstLot().ProcurementProject.Description
 	}
 	return ""
 }
@@ -1561,8 +1604,8 @@ func (cn ContractNotice) GetPlaceOfPerformance() string {
 
 // GetLotID returns the procurement lot ID
 func (cn ContractNotice) GetLotID() string {
-	if cn.ProcurementProjectLot != nil {
+	if cn.firstLot() != nil {
-		return cn.ProcurementProjectLot.ID
+		return cn.firstLot().ID
 	}
 	return ""
 }
@@ -1580,17 +1623,17 @@ func (cn ContractNotice) GetSelectionCriteria() []SelectionCriteria {
 
 // GetOfficialLanguages returns official languages from tender terms extensions
 func (cn ContractNotice) GetOfficialLanguages() []string {
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.TenderingTerms != nil &&
+		cn.firstLot().TenderingTerms != nil &&
-		len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 &&
+		len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 &&
-		cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil &&
+		cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions != nil &&
-		cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil &&
+		cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension != nil &&
-		cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil &&
+		cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent != nil &&
-		cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil &&
+		cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension != nil &&
-		cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil {
+		cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages != nil {
 
-		languages := make([]string, len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language))
+		languages := make([]string, len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language))
-		for i, lang := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language {
+		for i, lang := range cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].UBLExtensions.UBLExtension.ExtensionContent.EformsExtension.OfficialLanguages.Language {
 			languages[i] = lang.ID
 		}
 		return languages
@@ -1600,12 +1643,12 @@ func (cn ContractNotice) GetOfficialLanguages() []string {
 
 // GetDocumentURI returns the document URI from call for tenders document reference
 func (cn ContractNotice) GetDocumentURI() string {
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.TenderingTerms != nil &&
+		cn.firstLot().TenderingTerms != nil &&
-		len(cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference) > 0 &&
+		len(cn.firstLot().TenderingTerms.CallForTendersDocumentReference) > 0 &&
-		cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil &&
+		cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment != nil &&
-		cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil {
+		cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference != nil {
-		return cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI
+		return cn.firstLot().TenderingTerms.CallForTendersDocumentReference[0].Attachment.ExternalReference.URI
 	}
 	return ""
 }
@@ -1613,10 +1656,12 @@ func (cn ContractNotice) GetDocumentURI() string {
 // GetAllDocumentURIs returns all document URIs from call for tenders document references
 func (cn ContractNotice) GetAllDocumentURIs() []string {
 	var uris []string
-	if cn.ProcurementProjectLot != nil &&
+	for i := range cn.ProcurementProjectLots {
-		cn.ProcurementProjectLot.TenderingTerms != nil {
+		lot := &cn.ProcurementProjectLots[i]
-
+		if lot.TenderingTerms == nil {
-		for _, docRef := range cn.ProcurementProjectLot.TenderingTerms.CallForTendersDocumentReference {
+			continue
+		}
+		for _, docRef := range lot.TenderingTerms.CallForTendersDocumentReference {
 			if docRef.Attachment != nil &&
 				docRef.Attachment.ExternalReference != nil &&
 				docRef.Attachment.ExternalReference.URI != "" {
@@ -1629,21 +1674,21 @@ func (cn ContractNotice) GetAllDocumentURIs() []string {
 
 // GetTenderDeadline returns the tender submission deadline
 func (cn ContractNotice) GetTenderDeadline() string {
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.TenderingProcess != nil &&
+		cn.firstLot().TenderingProcess != nil &&
-		cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
+		cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod != nil {
-		return cn.ProcurementProjectLot.TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
+		return cn.firstLot().TenderingProcess.TenderSubmissionDeadlinePeriod.EndDate
 	}
 	return ""
 }
 
 // GetBudget returns the estimated contract amount and currency
 func (cn ContractNotice) GetBudget() (amount string, currency string) {
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.ProcurementProject != nil &&
+		cn.firstLot().ProcurementProject != nil &&
-		cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal != nil {
+		cn.firstLot().ProcurementProject.RequestedTenderTotal != nil {
 
-		contractAmount := cn.ProcurementProjectLot.ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
+		contractAmount := cn.firstLot().ProcurementProject.RequestedTenderTotal.EstimatedOverallContractAmount
 		return contractAmount.Text, contractAmount.CurrencyID
 	}
 	return "", ""
@@ -1651,12 +1696,12 @@ func (cn ContractNotice) GetBudget() (amount string, currency string) {
 
 // GetDuration returns the contract duration and unit
 func (cn ContractNotice) GetDuration() (duration string, unit string) {
-	if cn.ProcurementProjectLot != nil &&
+	if cn.firstLot() != nil &&
-		cn.ProcurementProjectLot.ProcurementProject != nil &&
+		cn.firstLot().ProcurementProject != nil &&
-		cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod != nil &&
+		cn.firstLot().ProcurementProject.PlannedPeriod != nil &&
-		cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure != nil {
+		cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure != nil {
 
-		durationMeasure := cn.ProcurementProjectLot.ProcurementProject.PlannedPeriod.DurationMeasure
+		durationMeasure := cn.firstLot().ProcurementProject.PlannedPeriod.DurationMeasure
 		return durationMeasure.Text, durationMeasure.UnitCode
 	}
 	return "", ""
@@ -1716,9 +1761,14 @@ func (can ContractAwardNotice) GetID() string {
 	return can.ID
 }
 
-// GetNoticeType returns the notice type
+// GetNoticeType returns the notice type code (BT-02).
 func (can ContractAwardNotice) GetNoticeType() string {
-	return can.NoticeTypeCode
+	return strings.TrimSpace(can.NoticeTypeCode.Text)
+}
+
+// GetNoticeFormType returns the form type from listName (BT-03).
+func (can ContractAwardNotice) GetNoticeFormType() string {
+	return strings.TrimSpace(can.NoticeTypeCode.ListName)
 }
 
 // GetLanguage returns the notice language
@@ -1818,25 +1868,34 @@ func (can ContractAwardNotice) GetProcedureCode() string {
 
 // GetProcurementTitle returns the procurement project title
 func (can ContractAwardNotice) GetProcurementTitle() string {
-	if can.ProcurementProject != nil {
+	if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Name) != "" {
 		return can.ProcurementProject.Name
 	}
+	if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
+		return lot.ProcurementProject.Name
+	}
 	return ""
 }
 
 // GetProcurementDescription returns the procurement project description
 func (can ContractAwardNotice) GetProcurementDescription() string {
-	if can.ProcurementProject != nil {
+	if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.Description) != "" {
 		return can.ProcurementProject.Description
 	}
+	if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
+		return lot.ProcurementProject.Description
+	}
 	return ""
 }
 
 // GetContractNature returns the main nature of the contract
 func (can ContractAwardNotice) GetContractNature() string {
-	if can.ProcurementProject != nil {
+	if can.ProcurementProject != nil && strings.TrimSpace(can.ProcurementProject.ProcurementTypeCode) != "" {
 		return can.ProcurementProject.ProcurementTypeCode
 	}
+	if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
+		return lot.ProcurementProject.ProcurementTypeCode
+	}
 	return ""
 }
 
@@ -1846,6 +1905,11 @@ func (can ContractAwardNotice) GetMainClassification() string {
 		can.ProcurementProject.MainCommodityClassification != nil {
 		return can.ProcurementProject.MainCommodityClassification.ItemClassificationCode
 	}
+	if lot := can.firstAwardLot(); lot != nil &&
+		lot.ProcurementProject != nil &&
+		lot.ProcurementProject.MainCommodityClassification != nil {
+		return lot.ProcurementProject.MainCommodityClassification.ItemClassificationCode
+	}
 	return ""
 }
 
@@ -1882,3 +1946,27 @@ func (can ContractAwardNotice) GetOrganizationByID(orgID string) *Organization {
 	}
 	return nil
 }
+
+// GetWinningTenderPartyID resolves the tendering party reference for the primary winning bid from NoticeResult.
+func (can ContractAwardNotice) GetWinningTenderPartyID() string {
+	nr := can.GetNoticeResult()
+	if nr == nil {
+		return ""
+	}
+	for _, lr := range nr.LotResult {
+		winner := pickWinningLotTender(lr.LotTender)
+		if winner != nil && winner.TenderingParty != nil && winner.TenderingParty.ID != "" {
+			return winner.TenderingParty.ID
+		}
+	}
+	return ""
+}
+
+// GetWinningTenderOrganization resolves the winning tenderer organization from NoticeResult via Organizations.
+func (can ContractAwardNotice) GetWinningTenderOrganization() *Organization {
+	id := can.GetWinningTenderPartyID()
+	if id == "" {
+		return nil
+	}
+	return can.GetOrganizationByID(id)
+}

ted/notice_upsert.go

@@ -0,0 +1,80 @@
+package ted
+
+import (
+	"strconv"
+	"strings"
+	"time"
+	"tm/internal/notice"
+)
+
+// noticeNeedsRefresh reports whether incoming scraped data should replace an existing notice row.
+func noticeNeedsRefresh(existing, incoming *notice.Notice) bool {
+	if incoming == nil || existing == nil {
+		return true
+	}
+	if strings.TrimSpace(incoming.ContentXML) != "" &&
+		strings.TrimSpace(existing.ContentXML) != strings.TrimSpace(incoming.ContentXML) {
+		return true
+	}
+	if normalizePublicationID(existing.NoticePublicationID) != normalizePublicationID(incoming.NoticePublicationID) {
+		return true
+	}
+	if compareNoticeVersions(incoming.NoticeVersion, existing.NoticeVersion) > 0 {
+		return true
+	}
+	return false
+}
+
+func normalizePublicationID(s string) string {
+	return strings.TrimSpace(s)
+}
+
+// compareNoticeVersions compares BT-757 notice versions (e.g. "01", "02"). Empty parses as 0.
+func compareNoticeVersions(incoming, existing string) int {
+	vi := parseNoticeVersion(incoming)
+	ve := parseNoticeVersion(existing)
+	switch {
+	case vi > ve:
+		return 1
+	case vi < ve:
+		return -1
+	default:
+		return 0
+	}
+}
+
+func parseNoticeVersion(s string) int {
+	s = strings.TrimSpace(s)
+	if s == "" {
+		return 0
+	}
+	n, err := strconv.Atoi(s)
+	if err != nil {
+		return 0
+	}
+	return n
+}
+
+// MergeNoticeUpdate replaces scrape-derived fields from src into dst while preserving identity,
+// creation time, and optional enrichment metadata already stored on dst.
+func MergeNoticeUpdate(dst *notice.Notice, src *notice.Notice) {
+	preservedID := dst.ID
+	preservedCreated := dst.CreatedAt
+	oldEnrichment := dst.ProcessingMetadata.EnrichmentData
+	oldTranslated := dst.ProcessingMetadata.TranslatedData
+	oldTranslatedAt := dst.ProcessingMetadata.TranslatedAt
+
+	*dst = *src
+	dst.ID = preservedID
+	dst.CreatedAt = preservedCreated
+	dst.UpdatedAt = time.Now().Unix()
+	dst.ProcessingMetadata.Processed = false
+
+	if len(dst.ProcessingMetadata.EnrichmentData) == 0 && len(oldEnrichment) > 0 {
+		dst.ProcessingMetadata.EnrichmentData = oldEnrichment
+	}
+	if len(dst.ProcessingMetadata.TranslatedData) == 0 && len(oldTranslated) > 0 {
+		dst.ProcessingMetadata.TranslatedData = oldTranslated
+		dst.ProcessingMetadata.TranslatedAt = oldTranslatedAt
+	}
+}

ted/scraper.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"compress/gzip"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -12,7 +13,7 @@ import (
 	"time"
 	"tm/internal/notice"
 	"tm/pkg/logger"
-	"tm/pkg/mongo"
+	orm "tm/pkg/mongo"
 	"tm/pkg/notification"
 )
 
@@ -38,16 +39,16 @@ type TEDScraper struct {
 	logger       logger.Logger
 	httpClient   *http.Client
 	xmlParser    *TEDParser
-	mongoManager *mongo.ConnectionManager
+	mongoManager *orm.ConnectionManager
-	tenderRepo   notice.Repository
+	noticeRepo   notice.Repository
 }
 
 // NewTEDScraper creates a new TED scraper instance
 func NewTEDScraper(
 	config *Config,
 	logger logger.Logger,
-	mongoManager *mongo.ConnectionManager,
+	mongoManager *orm.ConnectionManager,
-	tenderRepo notice.Repository,
+	noticeRepo notice.Repository,
 	notify notification.SDK,
 ) *TEDScraper {
 	httpClient := &http.Client{
@@ -61,7 +62,7 @@ func NewTEDScraper(
 		httpClient:   httpClient,
 		xmlParser:    NewTEDParser(),
 		mongoManager: mongoManager,
-		tenderRepo:   tenderRepo,
+		noticeRepo:   noticeRepo,
 	}
 }
 
@@ -289,16 +290,18 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
 	return result, nil
 }
 
-func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
+func (s *TEDScraper) findNoticeByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
-	existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID)
+	if strings.TrimSpace(contractNoticeID) == "" {
+		return nil, nil
+	}
+	existing, err := s.noticeRepo.GetByContractNoticeID(ctx, contractNoticeID)
 	if err != nil {
-		if err.Error() == "document not found" {
+		if errors.Is(err, orm.ErrDocumentNotFound) {
 			return nil, nil
 		}
 		return nil, err
 	}
-
+	return existing, nil
-	return existingTender, nil
 }
 
 func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, result *FileProcessingResult) error {
@@ -331,45 +334,66 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 	}
 	t.ContentXML = string(content)
 
-	// Check if tender already exists to prevent duplicates
+	existingNotice, err := s.findNoticeByContractNoticeID(ctx, t.ContractNoticeID)
-	existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID)
 	if err != nil {
-		s.logger.Error("Failed to check existing tender", map[string]interface{}{
+		s.logger.Error("Failed to check existing notice", map[string]interface{}{
 			"contract_notice_id": t.ContractNoticeID,
 			"error":              err.Error(),
 		})
-		return fmt.Errorf("failed to check existing tender: %w", err)
+		return fmt.Errorf("failed to check existing notice: %w", err)
 	}
 
-	if existingTender != nil {
+	if existingNotice != nil {
-		s.logger.Info("Tender already exists, skipping duplicate", map[string]interface{}{
+		if !noticeNeedsRefresh(existingNotice, t) {
+			s.logger.Debug("Notice unchanged, skipping", map[string]interface{}{
 				"contract_notice_id": t.ContractNoticeID,
-			"existing_id":        existingTender.ID,
+				"notice_doc_id":      existingNotice.ID.Hex(),
 			})
 			return nil
 		}
 
-	// Create new tender
+		MergeNoticeUpdate(existingNotice, t)
-	s.logger.Info("Creating new tender", map[string]interface{}{
+		err = s.noticeRepo.Update(ctx, existingNotice)
+		if err != nil {
+			s.logger.Error("Failed to update existing notice", map[string]interface{}{
+				"contract_notice_id": t.ContractNoticeID,
+				"notice_doc_id":      existingNotice.ID.Hex(),
+				"error":              err.Error(),
+			})
+			return fmt.Errorf("failed to update notice: %w", err)
+		}
+
+		s.logger.Info("Notice updated (refresh)", map[string]interface{}{
+			"contract_notice_id":      t.ContractNoticeID,
+			"notice_publication_id":   t.NoticePublicationID,
+			"notice_version":          t.NoticeVersion,
+			"notice_doc_id":           existingNotice.ID.Hex(),
+			"processing_reset_worker": true,
+		})
+		result.SuccessCount++
+	} else {
+		s.logger.Info("Creating new notice", map[string]interface{}{
 			"contract_notice_id": t.ContractNoticeID,
 		})
 
-	t.CreatedAt = time.Now().Unix()
+		now := time.Now().Unix()
-	t.UpdatedAt = t.CreatedAt
+		t.CreatedAt = now
+		t.UpdatedAt = now
 
-	err = s.tenderRepo.Import(ctx, t)
+		err = s.noticeRepo.Import(ctx, t)
 		if err != nil {
-		s.logger.Error("Failed to create new tender", map[string]interface{}{
+			s.logger.Error("Failed to import notice", map[string]interface{}{
 				"contract_notice_id": t.ContractNoticeID,
 				"error":              err.Error(),
 			})
-		return fmt.Errorf("failed to create tender: %w", err)
+			return fmt.Errorf("failed to import notice: %w", err)
 		}
 
-	s.logger.Info("Successfully created new tender", map[string]interface{}{
+		s.logger.Info("Successfully imported new notice", map[string]interface{}{
 			"contract_notice_id": t.ContractNoticeID,
 		})
-	// }
+		result.SuccessCount++
+	}
 
 	// Log detailed parsed information based on notice type
 	s.logger.Info("Successfully parsed notice", map[string]interface{}{
@@ -379,7 +403,6 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 		"tender_url":  t.TenderURL,
 	})
 
-	result.SuccessCount++
 	return nil
 }
 

ted/service.go

@@ -7,6 +7,57 @@ import (
 	"time"
 )
 
+// ParseIssueDateTimeToUnix combines cbc:IssueDate and cbc:IssueTime (with timezone offsets) into one Unix timestamp.
+func ParseIssueDateTimeToUnix(issueDate, issueTime string) int64 {
+	issueDate = strings.TrimSpace(issueDate)
+	issueTime = strings.TrimSpace(issueTime)
+	if issueDate == "" {
+		return 0
+	}
+	if issueTime == "" {
+		return ParseDateToUnix(issueDate)
+	}
+	dateCore := issueDate
+	if len(issueDate) >= 10 && issueDate[4] == '-' && issueDate[7] == '-' {
+		dateCore = issueDate[:10]
+	}
+	combined := dateCore + "T" + issueTime
+	if t, err := ParseDate(combined); err == nil {
+		return t.Unix()
+	}
+	return ParseDateToUnix(issueDate)
+}
+
+// TransmissionDispatchUnix reads BT-803 (eSender) transmission date/time from extension content (root or eForms extension).
+func TransmissionDispatchUnix(ext *UBLExtensions) int64 {
+	if ext == nil || ext.UBLExtension == nil || ext.UBLExtension.ExtensionContent == nil {
+		return 0
+	}
+	ec := ext.UBLExtension.ExtensionContent
+	if u := ParseIssueDateTimeToUnix(ec.TransmissionDate, ec.TransmissionTime); u != 0 {
+		return u
+	}
+	if ec.EformsExtension != nil {
+		if u := ParseIssueDateTimeToUnix(ec.EformsExtension.TransmissionDate, ec.EformsExtension.TransmissionTime); u != 0 {
+			return u
+		}
+	}
+	return 0
+}
+
+// ParseEuropeanAmount normalizes TED amount strings (spaces, comma decimal) for strconv.ParseFloat.
+func ParseEuropeanAmount(raw string) (float64, bool) {
+	s := strings.TrimSpace(raw)
+	s = strings.ReplaceAll(s, "\u00a0", "")
+	s = strings.ReplaceAll(s, " ", "")
+	s = strings.ReplaceAll(s, ",", ".")
+	if s == "" {
+		return 0, false
+	}
+	v, err := strconv.ParseFloat(s, 64)
+	return v, err == nil
+}
+
 // parseDate parses various date formats used in TED XML
 func ParseDate(dateStr string) (time.Time, error) {
 	if dateStr == "" {