Add deduplication and normalization for tender recommendations

- Introduced a new function `dedupeRecommendationResponsesByTenderID` to remove duplicate tender recommendations based on their IDs, ensuring unique entries in the recommendation list.
- Updated the `mergeRecommendedTenders` function to utilize the new deduplication logic, streamlining the merging process of recommendation lists.
- Enhanced the `fetchAIRecommendations` method to apply deduplication on the fetched recommendations, improving data integrity.
- Added unit tests for the new deduplication function to validate its behavior and ensure correct handling of various input scenarios.

This update enhances the recommendation handling by ensuring that duplicate tender entries are effectively managed, improving the overall quality of the recommendations provided to users.

internal/company/recommendation.go

@@ -44,23 +44,10 @@ func normalizeRecommendationCompanyIDs(companyIDs []string) []string {
 
 func mergeRecommendedTenders(recommendationLists ...[]RecommendedTenderResponse) []RecommendedTenderResponse {
 	merged := make([]RecommendedTenderResponse, 0)
-	seen := make(map[string]struct{})
 	for _, recommendations := range recommendationLists {
-		for _, recommendation := range recommendations {
-			tenderID := strings.TrimSpace(recommendation.TenderID)
-			if tenderID == "" {
-				continue
-			}
-			if _, ok := seen[tenderID]; ok {
-				continue
-			}
-			seen[tenderID] = struct{}{}
-			recommendation.TenderID = tenderID
-			recommendation.Analysis = strings.TrimSpace(recommendation.Analysis)
-			merged = append(merged, recommendation)
-		}
+		merged = append(merged, recommendations...)
 	}
-	return merged
+	return dedupeRecommendationResponsesByTenderID(merged)
 }
 
 func aiRecommendationCacheKey(companyID string) string {
@@ -382,6 +369,10 @@ func (s *companyService) fetchAIRecommendations(ctx context.Context, companyID s
 		return nil, fmt.Errorf("failed to fetch AI recommendations: %w", err)
 	}
 
+	return dedupeRecommendationResponsesByTenderID(itemsToRecommendationResponses(items)), nil
+}
+
+func itemsToRecommendationResponses(items []ai_summarizer.RecommendedTender) []RecommendedTenderResponse {
 	responses := make([]RecommendedTenderResponse, 0, len(items))
 	for _, item := range items {
 		responses = append(responses, RecommendedTenderResponse{
@@ -390,8 +381,26 @@ func (s *companyService) fetchAIRecommendations(ctx context.Context, companyID s
 			Analysis: item.Analysis,
 		})
 	}
+	return responses
+}
 
-	return responses, nil
+func dedupeRecommendationResponsesByTenderID(responses []RecommendedTenderResponse) []RecommendedTenderResponse {
+	seen := make(map[string]struct{}, len(responses))
+	out := make([]RecommendedTenderResponse, 0, len(responses))
+	for _, recommendation := range responses {
+		tenderID := strings.TrimSpace(recommendation.TenderID)
+		if tenderID == "" {
+			continue
+		}
+		if _, ok := seen[tenderID]; ok {
+			continue
+		}
+		seen[tenderID] = struct{}{}
+		recommendation.TenderID = tenderID
+		recommendation.Analysis = strings.TrimSpace(recommendation.Analysis)
+		out = append(out, recommendation)
+	}
+	return out
 }
 
 func (s *companyService) getCachedAIRecommendations(ctx context.Context, companyID string) ([]RecommendedTenderResponse, bool) {

internal/company/recommendation_test.go

@@ -2,6 +2,26 @@ package company
 
 import "testing"
 
+func TestDedupeRecommendationResponsesByTenderID(t *testing.T) {
+	input := []RecommendedTenderResponse{
+		{Rank: 1, TenderID: " PROC-1 ", Analysis: " first "},
+		{Rank: 2, TenderID: "PROC-2", Analysis: "second"},
+		{Rank: 1, TenderID: "PROC-2", Analysis: "duplicate"},
+		{Rank: 4, TenderID: "   ", Analysis: "ignored"},
+	}
+
+	got := dedupeRecommendationResponsesByTenderID(input)
+	if len(got) != 2 {
+		t.Fatalf("dedupeRecommendationResponsesByTenderID() len = %d, want 2", len(got))
+	}
+	if got[0].TenderID != "PROC-1" || got[0].Analysis != "first" {
+		t.Fatalf("first recommendation = %+v, want trimmed PROC-1/first", got[0])
+	}
+	if got[1].TenderID != "PROC-2" || got[1].Rank != 2 {
+		t.Fatalf("second recommendation = %+v, want original PROC-2 rank 2", got[1])
+	}
+}
+
 func TestMergeRecommendedTenders(t *testing.T) {
 	first := []RecommendedTenderResponse{
 		{Rank: 1, TenderID: " PROC-1 ", Analysis: " first "},

internal/tender/recommendation_filter.go

@@ -94,6 +94,25 @@ func addTenderIDsToSet(set map[string]struct{}, ids []string) {
 	}
 }
 
+// markResolvedRecommendedTenderSeen records a resolved tender and reports whether it was already seen.
+// Recommendations may reference the same tender via different AI refs (procedure ref vs Mongo ID,
+// or multiple notice publication IDs for one merged tender).
+func markResolvedRecommendedTenderSeen(seen map[string]struct{}, tender Tender) bool {
+	if seen == nil {
+		return false
+	}
+
+	tenderID := strings.TrimSpace(tender.GetID())
+	if tenderID == "" {
+		return true
+	}
+	if _, exists := seen[tenderID]; exists {
+		return true
+	}
+	seen[tenderID] = struct{}{}
+	return false
+}
+
 func isRecommendedTenderExcluded(tender Tender, tenderRef string, excluded map[string]struct{}) bool {
 	if len(excluded) == 0 {
 		return false

internal/tender/recommendation_filter_test.go

@@ -126,3 +126,24 @@ func TestIsRecommendedTenderExcludedForAllCompanies(t *testing.T) {
 		t.Fatalf("expected tender to be hidden when all companies rejected it")
 	}
 }
+
+func TestMarkResolvedRecommendedTenderSeen(t *testing.T) {
+	sample := Tender{}
+	sample.ID, _ = bson.ObjectIDFromHex("507f1f77bcf86cd799439011")
+
+	seen := make(map[string]struct{})
+	first := markResolvedRecommendedTenderSeen(seen, sample)
+	second := markResolvedRecommendedTenderSeen(seen, sample)
+	if first {
+		t.Fatalf("first sighting should not be skipped, id=%q seen=%v", sample.GetID(), seen)
+	}
+	if !second {
+		t.Fatalf("duplicate sighting should be skipped, id=%q seen=%v", sample.GetID(), seen)
+	}
+
+	otherRefSameTender := Tender{}
+	otherRefSameTender.ID = sample.ID
+	if !markResolvedRecommendedTenderSeen(seen, otherRefSameTender) {
+		t.Fatalf("duplicate tender via different ref should be skipped")
+	}
+}

internal/tender/service.go

@@ -1002,6 +1002,7 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat
 		tenderRef string
 	}
 	ordered := make([]rankedRecommendedTender, 0, len(recommendations))
+	seenTenderIDs := make(map[string]struct{}, len(recommendations))
 	for _, rec := range recommendations {
 		tenderRef := strings.TrimSpace(rec.TenderID)
 		tender, ok := tenderByRef[tenderRef]
@@ -1021,6 +1022,9 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat
 		if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) {
 			continue
 		}
+		if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) {
+			continue
+		}
 
 		ordered = append(ordered, rankedRecommendedTender{
 			tender:    tender,