tender duplication fix

ted/mapper.go

@@ -138,8 +138,9 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 	now := time.Now().Unix()
 
 	t := &notice.Notice{
-		ContractNoticeID:    cn.ID,
-		ContractFolderID:    cn.ContractFolderID,
+		ContractNoticeID:       cn.ID,
+		ContractFolderID:       cn.ContractFolderID,
+		ProcurementProjectID:   procurementProjectIDFromContractNotice(cn),
 		NoticeTypeCode:      strings.TrimSpace(cn.GetNoticeType()),
 		FormType:            cn.GetNoticeFormType(),
 		NoticeSubTypeCode:   cn.GetNoticeSubType(),
@@ -297,8 +298,9 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
 	now := time.Now().Unix()
 
 	t := &notice.Notice{
-		ContractNoticeID:    can.ID,
-		ContractFolderID:    can.ContractFolderID,
+		ContractNoticeID:       can.ID,
+		ContractFolderID:       can.ContractFolderID,
+		ProcurementProjectID:   procurementProjectIDFromContractAwardNotice(can),
 		NoticeTypeCode:      strings.TrimSpace(can.GetNoticeType()),
 		FormType:            can.GetNoticeFormType(),
 		NoticeSubTypeCode:   can.GetNoticeSubType(),
@@ -428,6 +430,36 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
 	return t
 }
 
+func procurementProjectIDFromContractNotice(cn *ContractNotice) string {
+	if cn == nil {
+		return ""
+	}
+	if cn.ProcurementProject != nil {
+		if id := strings.TrimSpace(cn.ProcurementProject.ID); id != "" {
+			return id
+		}
+	}
+	if lot := cn.firstLot(); lot != nil && lot.ProcurementProject != nil {
+		return strings.TrimSpace(lot.ProcurementProject.ID)
+	}
+	return ""
+}
+
+func procurementProjectIDFromContractAwardNotice(can *ContractAwardNotice) string {
+	if can == nil {
+		return ""
+	}
+	if can.ProcurementProject != nil {
+		if id := strings.TrimSpace(can.ProcurementProject.ID); id != "" {
+			return id
+		}
+	}
+	if lot := can.firstAwardLot(); lot != nil && lot.ProcurementProject != nil {
+		return strings.TrimSpace(lot.ProcurementProject.ID)
+	}
+	return ""
+}
+
 // mapOrganization maps a TED Organization to tender Organization
 func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.Organization {
 	if tedOrg == nil || tedOrg.Company == nil {

ted/scraper.go

@@ -304,6 +304,14 @@ func (s *TEDScraper) findNoticeByContractNoticeID(ctx context.Context, contractN
 	return existing, nil
 }
 
+func isMongoDupKeyOnContractNoticeID(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	return strings.Contains(msg, "E11000") && strings.Contains(msg, "contract_notice_id")
+}
+
 func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, fileName string, result *FileProcessingResult) error {
 	s.logger.Debug("Processing XML content", map[string]interface{}{
 		"file_name":        fileName,
@@ -382,6 +390,29 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 
 		err = s.noticeRepo.Import(ctx, t)
 		if err != nil {
+			if isMongoDupKeyOnContractNoticeID(err) {
+				existingAfterRace, findErr := s.findNoticeByContractNoticeID(ctx, t.ContractNoticeID)
+				if findErr != nil {
+					return fmt.Errorf("import duplicate key, re-fetch notice: %w", findErr)
+				}
+				if existingAfterRace == nil {
+					return fmt.Errorf("failed to import notice: %w", err)
+				}
+				MergeNoticeUpdate(existingAfterRace, t)
+				if upErr := s.noticeRepo.Update(ctx, existingAfterRace); upErr != nil {
+					s.logger.Error("Failed to update notice after duplicate-key race", map[string]interface{}{
+						"contract_notice_id": t.ContractNoticeID,
+						"error":              upErr.Error(),
+					})
+					return fmt.Errorf("failed to update notice after duplicate key: %w", upErr)
+				}
+				s.logger.Info("Notice merged after duplicate-key race (parallel ingest)", map[string]interface{}{
+					"contract_notice_id": t.ContractNoticeID,
+					"notice_doc_id":      existingAfterRace.ID.Hex(),
+				})
+				result.SuccessCount++
+				return nil
+			}
 			s.logger.Error("Failed to import notice", map[string]interface{}{
 				"contract_notice_id": t.ContractNoticeID,
 				"error":              err.Error(),