Enhance tender reference handling with improved deduplication and test coverage

- Updated the `markResolvedRecommendedTenderSeen` function to accept an additional `tenderRef` parameter, allowing for better tracking of tender references and preventing duplicates.
- Refactored the `GetByProcedureReferences` method in the repository to combine indexed notice lookups with contract-folder candidates, ensuring older notice references resolve correctly after tender merges.
- Introduced a new `findTendersByContractFolderIDs` method to streamline the retrieval of tenders by contract folder IDs, enhancing performance and maintainability.
- Added unit tests for new functionality in `ai_reference_test.go` and `recommendation_filter_test.go`, ensuring robust validation of tender reference mapping and deduplication logic.

This update significantly improves the handling of tender references and enhances the overall test coverage, ensuring more reliable and efficient processing of tender data.

internal/tender/ai_reference_test.go

@@ -1,6 +1,10 @@
 package tender
 
-import "testing"
+import (
+	"testing"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+)
 
 func TestParseAIProcedureRef(t *testing.T) {
 	tests := []struct {
@@ -84,3 +88,37 @@ func TestMapProcedureReferencesToTenders(t *testing.T) {
 		t.Fatalf("expected folder-2 ref to resolve, got %+v", got["PROC_folder-2/notice-2"])
 	}
 }
+
+func TestMapProcedureReferencesToTendersUsesFolderCandidatesWhenNoticeLookupMisses(t *testing.T) {
+	// Notice-old is only present on the merged tender loaded via contract_folder_id.
+	candidates := []Tender{
+		{
+			ContractFolderID:            "folder-1",
+			NoticePublicationID:         "notice-new",
+			RelatedNoticePublicationIDs: []string{"notice-old"},
+		},
+	}
+	refs := []ProcedureReference{
+		{Ref: "PROC_folder-1/notice-old", ContractFolderID: "folder-1", NoticePublicationID: "notice-old"},
+	}
+
+	got := mapProcedureReferencesToTenders(candidates, refs)
+	if len(got) != 1 {
+		t.Fatalf("mapProcedureReferencesToTenders() len = %d, want 1", len(got))
+	}
+	if got["PROC_folder-1/notice-old"].NoticePublicationID != "notice-new" {
+		t.Fatalf("expected folder candidate to resolve notice-old ref, got %+v", got["PROC_folder-1/notice-old"])
+	}
+}
+
+func TestMergeTenderCandidatesDedupesByMongoID(t *testing.T) {
+	first := Tender{ContractFolderID: "folder-1", NoticePublicationID: "notice-a"}
+	first.ID, _ = bson.ObjectIDFromHex("507f1f77bcf86cd799439011")
+	second := Tender{ContractFolderID: "folder-1", NoticePublicationID: "notice-b"}
+	second.ID = first.ID
+
+	got := mergeTenderCandidates([]Tender{first}, []Tender{second})
+	if len(got) != 1 {
+		t.Fatalf("mergeTenderCandidates() len = %d, want 1", len(got))
+	}
+}

internal/tender/recommendation_filter.go

@@ -97,19 +97,22 @@ func addTenderIDsToSet(set map[string]struct{}, ids []string) {
 // markResolvedRecommendedTenderSeen records a resolved tender and reports whether it was already seen.
 // Recommendations may reference the same tender via different AI refs (procedure ref vs Mongo ID,
 // or multiple notice publication IDs for one merged tender).
-func markResolvedRecommendedTenderSeen(seen map[string]struct{}, tender Tender) bool {
+func markResolvedRecommendedTenderSeen(seen map[string]struct{}, tender Tender, tenderRef string) bool {
 	if seen == nil {
 		return false
 	}
 
-	tenderID := strings.TrimSpace(tender.GetID())
-	if tenderID == "" {
+	key := strings.TrimSpace(tender.GetID())
+	if key == "" {
+		key = strings.TrimSpace(tenderRef)
+	}
+	if key == "" {
+		return false
+	}
+	if _, exists := seen[key]; exists {
 		return true
 	}
-	if _, exists := seen[tenderID]; exists {
-		return true
-	}
-	seen[tenderID] = struct{}{}
+	seen[key] = struct{}{}
 	return false
 }
 

internal/tender/recommendation_filter_test.go

@@ -132,8 +132,8 @@ func TestMarkResolvedRecommendedTenderSeen(t *testing.T) {
 	sample.ID, _ = bson.ObjectIDFromHex("507f1f77bcf86cd799439011")
 
 	seen := make(map[string]struct{})
-	first := markResolvedRecommendedTenderSeen(seen, sample)
-	second := markResolvedRecommendedTenderSeen(seen, sample)
+	first := markResolvedRecommendedTenderSeen(seen, sample, "PROC_folder/notice")
+	second := markResolvedRecommendedTenderSeen(seen, sample, "PROC_folder/notice")
 	if first {
 		t.Fatalf("first sighting should not be skipped, id=%q seen=%v", sample.GetID(), seen)
 	}
@@ -143,7 +143,7 @@ func TestMarkResolvedRecommendedTenderSeen(t *testing.T) {
 
 	otherRefSameTender := Tender{}
 	otherRefSameTender.ID = sample.ID
-	if !markResolvedRecommendedTenderSeen(seen, otherRefSameTender) {
+	if !markResolvedRecommendedTenderSeen(seen, otherRefSameTender, "PROC_folder/other-notice") {
 		t.Fatalf("duplicate tender via different ref should be skipped")
 	}
 }

internal/tender/recommendation_page.go

@@ -69,7 +69,7 @@ func (s *tenderService) buildRecommendationPage(
 			if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) {
 				continue
 			}
-			if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) {
+			if markResolvedRecommendedTenderSeen(seenTenderIDs, tender, tenderRef) {
 				continue
 			}
 

internal/tender/repository.go

@@ -362,13 +362,18 @@ func (r *tenderRepository) GetByProcedureReference(ctx context.Context, contract
 
 const procedureReferenceLookupBatchSize = 250
 
-// GetByProcedureReferences loads tenders for multiple AI procedure refs using indexed notice lookups.
+const contractFolderLookupBatchSize = 250
+
+// GetByProcedureReferences loads tenders for multiple AI procedure refs.
+// It combines indexed notice lookups with contract-folder candidates so older notice
+// references still resolve after tender merges (same behavior as before the perf refactor).
 func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error) {
 	out := make(map[string]Tender)
 	if len(refs) == 0 {
 		return out, nil
 	}
 
+	folderIDs := make([]string, 0, len(refs))
 	noticeIDs := make([]string, 0, len(refs))
 	for _, ref := range refs {
 		folder := strings.TrimSpace(ref.ContractFolderID)
@@ -376,15 +381,12 @@ func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []
 		if folder == "" || notice == "" {
 			continue
 		}
+		folderIDs = append(folderIDs, folder)
 		noticeIDs = append(noticeIDs, notice)
 	}
 
 	uniqueNotices := uniqueNonEmptyTrimmedStrings(noticeIDs)
-	if len(uniqueNotices) == 0 {
-		return out, nil
-	}
-
-	candidates := make([]Tender, 0, len(uniqueNotices))
+	noticeCandidates := make([]Tender, 0, len(uniqueNotices))
 	for start := 0; start < len(uniqueNotices); start += procedureReferenceLookupBatchSize {
 		end := start + procedureReferenceLookupBatchSize
 		if end > len(uniqueNotices) {
@@ -395,12 +397,79 @@ func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []
 		if err != nil {
 			return nil, err
 		}
-		candidates = append(candidates, batch...)
+		noticeCandidates = append(noticeCandidates, batch...)
 	}
 
+	folderCandidates, err := r.findTendersByContractFolderIDs(ctx, folderIDs)
+	if err != nil {
+		return nil, err
+	}
+
+	candidates := mergeTenderCandidates(noticeCandidates, folderCandidates)
 	return mapProcedureReferencesToTenders(candidates, refs), nil
 }
 
+func (r *tenderRepository) findTendersByContractFolderIDs(ctx context.Context, folderIDs []string) ([]Tender, error) {
+	uniqueFolders := uniqueNonEmptyTrimmedStrings(folderIDs)
+	if len(uniqueFolders) == 0 {
+		return nil, nil
+	}
+
+	out := make([]Tender, 0, len(uniqueFolders))
+	for start := 0; start < len(uniqueFolders); start += contractFolderLookupBatchSize {
+		end := start + contractFolderLookupBatchSize
+		if end > len(uniqueFolders) {
+			end = len(uniqueFolders)
+		}
+		batch := uniqueFolders[start:end]
+
+		findLimit := len(batch) * 5
+		if minLimit := len(batch) + 1; findLimit < minLimit {
+			findLimit = minLimit
+		}
+
+		filter := bson.M{"contract_folder_id": bson.M{"$in": batch}}
+		pagination := orm.Pagination{
+			Limit:     findLimit,
+			SortField: "updated_at",
+			SortOrder: -1,
+			SkipCount: true,
+		}
+
+		result, err := r.ormRepo.FindAll(ctx, filter, pagination)
+		if err != nil {
+			r.logger.Error("Failed to get tenders by contract folder IDs", map[string]interface{}{
+				"count": len(batch),
+				"error": err.Error(),
+			})
+			return nil, err
+		}
+		out = append(out, result.Items...)
+	}
+
+	return out, nil
+}
+
+func mergeTenderCandidates(groups ...[]Tender) []Tender {
+	seen := make(map[string]struct{})
+	out := make([]Tender, 0)
+	for _, group := range groups {
+		for i := range group {
+			tenderID := strings.TrimSpace(group[i].GetID())
+			if tenderID == "" {
+				out = append(out, group[i])
+				continue
+			}
+			if _, exists := seen[tenderID]; exists {
+				continue
+			}
+			seen[tenderID] = struct{}{}
+			out = append(out, group[i])
+		}
+	}
+	return out
+}
+
 func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context, noticeIDs []string) ([]Tender, error) {
 	if len(noticeIDs) == 0 {
 		return nil, nil
@@ -413,7 +482,7 @@ func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context
 		},
 	}
 	pagination := orm.Pagination{
-		Limit:     len(noticeIDs) * 2,
+		Limit:     len(noticeIDs) * 3,
 		SortField: "updated_at",
 		SortOrder: -1,
 		SkipCount: true,