Added missing fields - scraper and worker fixes

cmd/worker/workers/notice.go

@@ -17,27 +17,29 @@ import (
 )
 
 type NoticeWorker struct {
-	Mongo           *mongo.ConnectionManager
-	Logger          logger.Logger
-	Notify          *notification.SDK
-	NoticeRepo      notice.Repository
-	TenderRepo      tender.TenderRepository
-	GLM             *glm.SDK
-	ProcessingLimit int
+	Mongo                   *mongo.ConnectionManager
+	Logger                  logger.Logger
+	Notify                  *notification.SDK
+	NoticeRepo              notice.Repository
+	TenderRepo              tender.TenderRepository
+	GLM                     *glm.SDK
+	ProcessingLimit         int
+	DeleteProcessedNotices  bool
 }
 
-func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int) *NoticeWorker {
+func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK, processingLimit int, deleteProcessedNotices bool) *NoticeWorker {
 	if processingLimit < 0 {
 		processingLimit = 5 // Default limit
 	}
 	return &NoticeWorker{
-		Mongo:           mongo,
-		Logger:          logger,
-		Notify:          notify,
-		NoticeRepo:      noticeRepo,
-		TenderRepo:      tenderRepo,
-		GLM:             glmService,
-		ProcessingLimit: processingLimit,
+		Mongo:                  mongo,
+		Logger:                 logger,
+		Notify:                 notify,
+		NoticeRepo:             noticeRepo,
+		TenderRepo:             tenderRepo,
+		GLM:                    glmService,
+		ProcessingLimit:        processingLimit,
+		DeleteProcessedNotices: deleteProcessedNotices,
 	}
 }
 
@@ -197,8 +199,12 @@ func (w *NoticeWorker) Run() {
 		}
 	}
 
-	// Clean up: delete notices that were marked as processed in previous runs
-	w.cleanupProcessedNotices()
+	// Clean up: delete notices that were marked as processed in previous runs (optional to retain XML for audit/backfill)
+	if w.DeleteProcessedNotices {
+		w.cleanupProcessedNotices()
+	} else {
+		w.Logger.Info("Skipping deletion of processed notices (WORKER_DELETE_PROCESSED_NOTICES=false)", map[string]interface{}{})
+	}
 }
 
 func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
@@ -211,6 +217,12 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 		t = new(tender.Tender)
 	}
 
+	var preserveTenderID, preserveProjectName string
+	if t != nil && !t.ID.IsZero() {
+		preserveTenderID = t.TenderID
+		preserveProjectName = t.ProjectName
+	}
+
 	// Buyer Organization
 	buyerOrg := &tender.Organization{}
 	if n.BuyerOrganization != nil {
@@ -347,6 +359,21 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 		}
 	}
 
+	procurementLots := make([]tender.ProcurementLot, len(n.ProcurementLots))
+	for i, l := range n.ProcurementLots {
+		procurementLots[i] = tender.ProcurementLot{
+			LotID:                     l.LotID,
+			Title:                     l.Title,
+			Description:               l.Description,
+			MainNatureOfContract:      l.MainNatureOfContract,
+			MainClassification:        l.MainClassification,
+			AdditionalClassifications: append([]string(nil), l.AdditionalClassifications...),
+			Duration:                  l.Duration,
+			EstimatedValue:            l.EstimatedValue,
+			Currency:                  l.Currency,
+		}
+	}
+
 	var title string
 	if w.GLM != nil {
 		title, err = w.GLM.Translate(context.Background(), n.Title, n.NoticeLanguageCode, "en")
@@ -402,8 +429,14 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 	t.WinningTenderer = winningTenderer
 	t.Modifications = modifications
 	t.AwardedEntities = awardedEntities
+	t.ProcurementLots = procurementLots
 	t.NoticeTypeCode = n.NoticeTypeCode
+	t.FormType = n.FormType
 	t.NoticeSubTypeCode = n.NoticeSubTypeCode
+	t.NoticeIdentifier = n.NoticeIdentifier
+	t.NoticeVersion = n.NoticeVersion
+	t.NoticeDispatchAt = n.NoticeDispatchAt
+	t.ESenderDispatchAt = n.ESenderDispatchAt
 	t.NoticeLanguageCode = n.NoticeLanguageCode
 	t.IssueDate = n.IssueDate
 	t.IssueTime = n.IssueTime
@@ -417,6 +450,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 	t.CityName = n.CityName
 	t.PostalCode = n.PostalCode
 	t.DocumentURI = n.DocumentURI
+	t.BuyerProfileURL = n.BuyerProfileURL
 	t.TenderURL = n.TenderURL
 	t.MainClassification = n.MainClassification
 	t.AdditionalClassifications = n.AdditionalClassifications
@@ -427,6 +461,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 	t.ContractFolderID = n.ContractFolderID
 	t.SourceFileURL = n.SourceFileURL
 	t.SourceFileName = n.SourceFileName
+	t.ContentXML = n.ContentXML
 	// Preserve document-scraping/summarization flags on the tender (managed by
 	// downstream workers), but refresh the notice-side processing metadata.
 	t.ProcessingMetadata.ScrapedAt = n.ProcessingMetadata.ScrapedAt
@@ -448,8 +483,13 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
 	t.WinningTenderer = winningTenderer
 	t.Modifications = modifications
 	t.AwardedEntities = awardedEntities
-	t.TenderID = w.GenerateTenderID(context.Background(), n)
-	t.ProjectName = w.GenerateProjectName(t)
+	if preserveTenderID != "" {
+		t.TenderID = preserveTenderID
+		t.ProjectName = preserveProjectName
+	} else {
+		t.TenderID = w.GenerateTenderID(context.Background(), n)
+		t.ProjectName = w.GenerateProjectName(t)
+	}
 
 	return t, nil
 }