mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tender duplication fix

Browse Source
This commit is contained in:
Mazyar
2026-05-12 17:02:39 +03:30
parent 29b4aa50bd
commit 5e8d4f67b2
7 changed files with 290 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/workers/notice.go
+182 -18
View File
@@ -436,14 +436,44 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
}
}
title := n.Title
description := n.Description
estimatedVal := n.EstimatedValue
awardedVal := n.AwardedValue
currency := n.Currency
if !t.ID.IsZero() {
procurementLots = mergeProcurementLotsByLotID(t.ProcurementLots, procurementLots)
awardedEntities = mergeAwardedEntitiesByLotID(t.AwardedEntities, awardedEntities)
title, description = mergeTitleAndDescriptionForMultiLot(t.Title, t.Description, title, description)
if strings.TrimSpace(previousNoticePublicationID) != "" && previousNoticePublicationID != n.NoticePublicationID {
if n.EstimatedValue > 0 {
estimatedVal = t.EstimatedValue + n.EstimatedValue
} else {
estimatedVal = t.EstimatedValue
}
if n.AwardedValue > 0 {
awardedVal = t.AwardedValue + n.AwardedValue
} else {
awardedVal = t.AwardedValue
}
if n.Currency != "" {
currency = n.Currency
} else {
currency = t.Currency
}
}
}
// Title and description stay in the notice language; English translation is handled by the AI service (translation worker).
t.Title = n.Title
t.Description = n.Description
t.Title = title
t.Description = description
t.NoticePublicationID = n.NoticePublicationID
t.ProcurementTypeCode = n.ProcurementTypeCode
t.ProcedureCode = n.ProcedureCode
t.EstimatedValue = n.EstimatedValue
t.Currency = n.Currency
t.EstimatedValue = estimatedVal
t.Currency = currency
t.TenderDeadline = n.TenderDeadline
t.SubmissionURL = n.SubmissionURL
t.BuyerOrganization = buyerOrg
@@ -483,6 +513,7 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t.Source = tender.TenderSource(n.Source)
t.ContractNoticeID = n.ContractNoticeID
t.ContractFolderID = n.ContractFolderID
t.ProcurementProjectID = n.ProcurementProjectID
t.SourceFileURL = n.SourceFileURL
t.SourceFileName = n.SourceFileName
t.ContentXML = n.ContentXML
@@ -500,13 +531,10 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t.CancellationReason = n.CancellationReason
t.CancellationDate = n.CancellationDate
t.AwardDate = n.AwardDate
t.AwardedValue = n.AwardedValue
t.AwardedValue = awardedVal
t.ContractNumber = n.ContractNumber
t.SuspensionReason = n.SuspensionReason
t.SuspensionDate = n.SuspensionDate
t.WinningTenderer = winningTenderer
t.Modifications = modifications
t.AwardedEntities = awardedEntities
if preserveTenderID != "" {
t.TenderID = preserveTenderID
t.ProjectName = preserveProjectName
@@ -526,13 +554,18 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
// resolveExistingTender finds the tender that this notice should update, preferring the procedure-level
// ContractFolderID (so follow-up notices like a Result for an earlier Competition notice merge into the
// same tender) and falling back to NoticePublicationID for older rows or notices without a folder id.
// same tender), then UBL ProcurementProject/ID (merges per-lot CAN/CN notices), then NoticePublicationID.
func (w *NoticeWorker) resolveExistingTender(ctx context.Context, n *notice.Notice) *tender.Tender {
if strings.TrimSpace(n.ContractFolderID) != "" {
if t, err := w.TenderRepo.GetByContractFolderID(ctx, n.ContractFolderID); err == nil && t != nil {
return t
}
}
if strings.TrimSpace(n.ProcurementProjectID) != "" {
if t, err := w.TenderRepo.GetByProcurementProjectID(ctx, n.ProcurementProjectID); err == nil && t != nil {
return t
}
}
if strings.TrimSpace(n.NoticePublicationID) != "" {
if t, err := w.TenderRepo.GetByNoticePublicationID(ctx, n.NoticePublicationID); err == nil && t != nil {
return t
@@ -660,6 +693,80 @@ func mergeRelatedPublicationIDs(existing []string, previous, current string) []s
return out
}
func mergeProcurementLotsByLotID(existing, incoming []tender.ProcurementLot) []tender.ProcurementLot {
out := append([]tender.ProcurementLot(nil), existing...)
for _, l := range incoming {
id := strings.TrimSpace(l.LotID)
if id != "" {
replaced := false
for i := range out {
if strings.TrimSpace(out[i].LotID) == id {
out[i] = l
replaced = true
break
}
}
if !replaced {
out = append(out, l)
}
continue
}
out = append(out, l)
}
return out
}
func mergeAwardedEntitiesByLotID(existing, incoming []tender.Awarded) []tender.Awarded {
out := append([]tender.Awarded(nil), existing...)
for _, a := range incoming {
id := strings.TrimSpace(a.LotID)
if id != "" {
replaced := false
for i := range out {
if strings.TrimSpace(out[i].LotID) == id {
out[i] = a
replaced = true
break
}
}
if !replaced {
out = append(out, a)
}
continue
}
out = append(out, a)
}
return out
}
func mergeTitleAndDescriptionForMultiLot(existingTitle, existingDesc, incomingTitle, incomingDesc string) (string, string) {
et, it := strings.TrimSpace(existingTitle), strings.TrimSpace(incomingTitle)
title := existingTitle
if len(it) > len(et) {
title = incomingTitle
}
desc := mergeDescriptionsConcat(existingDesc, incomingDesc)
return title, desc
}
func mergeDescriptionsConcat(a, b string) string {
a = strings.TrimSpace(a)
b = strings.TrimSpace(b)
if a == "" {
return b
}
if b == "" {
return a
}
if strings.Contains(a, b) {
return a
}
if strings.Contains(b, a) {
return b
}
return a + "\n\n---\n\n" + b
}
// GenerateTenderID generates a unique tender ID using the PBL naming convention: SCDYYNNN
// SCD = Source Code (3 letters), YY = Year (2 digits), NNN = Sequential number (variable digits)
func (w *NoticeWorker) GenerateTenderID(ctx context.Context, t *notice.Notice) string {
@@ -866,6 +973,25 @@ func (w *NoticeWorker) persistTender(ctx context.Context, t *tender.Tender, n *n
return nil
}
lastErr = err
// Another goroutine inserted the same procedure/publication first: merge into the winner row.
if isDuplicateKeyError(err) && (isDuplicateTenderBusinessKeyError(err) || isDuplicateContractFolderIDKeyError(err) || isDuplicateProcurementProjectIDKeyError(err)) {
tMerged, convErr := w.ToTender(n)
if convErr != nil {
return convErr
}
if tMerged.ID.IsZero() {
return fmt.Errorf("duplicate key on insert but could not resolve existing tender: %w", err)
}
w.Logger.Info("Merged notice into existing tender after duplicate-key race", map[string]interface{}{
"existing_tender_id": tMerged.TenderID,
"notice_id": n.ID.Hex(),
"notice_publication_id": n.NoticePublicationID,
"contract_folder_id": n.ContractFolderID,
})
return w.TenderRepo.Update(ctx, tMerged)
}
if !isDuplicateTenderIDError(err) {
return err
}
@@ -886,18 +1012,12 @@ func (w *NoticeWorker) persistTender(ctx context.Context, t *tender.Tender, n *n
return fmt.Errorf("exhausted %d tender_id retries: %w", tenderIDMaxRetries, lastErr)
}
// isDuplicateTenderIDError returns true if err is the Mongo "E11000 duplicate key" error on the
// tender_id_1 index. We retry only that specific case; other write errors propagate.
func isDuplicateTenderIDError(err error) bool {
// isDuplicateKeyError is true for Mongo duplicate key (E11000) write failures.
func isDuplicateKeyError(err error) bool {
if err == nil {
return false
}
// Cheap, driver-version-agnostic check: look for the error code and the specific index.
msg := err.Error()
if !strings.Contains(msg, "E11000") {
return false
}
if strings.Contains(msg, "tender_id_1") || strings.Contains(msg, "tender_id:") {
if strings.Contains(err.Error(), "E11000") {
return true
}
var we mongodriver.WriteException
@@ -911,6 +1031,50 @@ func isDuplicateTenderIDError(err error) bool {
return false
}
func isDuplicateTenderBusinessKeyError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
return strings.Contains(msg, "notice_publication_id")
}
func isDuplicateContractFolderIDKeyError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
// Field name in dup key (avoid matching unrelated "contract" substrings).
return strings.Contains(msg, "contract_folder_id")
}
func isDuplicateProcurementProjectIDKeyError(err error) bool {
if err == nil {
return false
}
return strings.Contains(err.Error(), "procurement_project_id")
}
// isDuplicateTenderIDError returns true if err is the Mongo "E11000 duplicate key" error on the
// tender_id index. We retry only that specific case; other write errors propagate.
func isDuplicateTenderIDError(err error) bool {
if err == nil {
return false
}
msg := err.Error()
if !strings.Contains(msg, "E11000") {
return false
}
if strings.Contains(msg, "tender_id_1") || strings.Contains(msg, "tender_id:") {
return true
}
// Some drivers only include "dup key" and the field in the message.
if strings.Contains(msg, "dup key") && strings.Contains(msg, "tender_id") {
return true
}
return false
}
// padOrTruncate pads a string with zeros or truncates it to the specified length
func PadOrTruncate(str string, length int) string {
// Remove any non-alphanumeric characters and convert to uppercase