From 4e5296d5dd2cf5ec5c60fadcb75b8ec65c79afdb Mon Sep 17 00:00:00 2001 From: Mazyar Date: Tue, 7 Jul 2026 23:00:16 +0330 Subject: [PATCH] Add deduplication and normalization for tender recommendations - Introduced a new function `dedupeRecommendationResponsesByTenderID` to remove duplicate tender recommendations based on their IDs, ensuring unique entries in the recommendation list. - Updated the `mergeRecommendedTenders` function to utilize the new deduplication logic, streamlining the merging process of recommendation lists. - Enhanced the `fetchAIRecommendations` method to apply deduplication on the fetched recommendations, improving data integrity. - Added unit tests for the new deduplication function to validate its behavior and ensure correct handling of various input scenarios. This update enhances the recommendation handling by ensuring that duplicate tender entries are effectively managed, improving the overall quality of the recommendations provided to users. --- internal/company/recommendation.go | 41 +++++++++++-------- internal/company/recommendation_test.go | 20 +++++++++ internal/tender/recommendation_filter.go | 19 +++++++++ internal/tender/recommendation_filter_test.go | 21 ++++++++++ internal/tender/service.go | 4 ++ 5 files changed, 89 insertions(+), 16 deletions(-) diff --git a/internal/company/recommendation.go b/internal/company/recommendation.go index 6fe20a6..47dc7ff 100644 --- a/internal/company/recommendation.go +++ b/internal/company/recommendation.go @@ -44,23 +44,10 @@ func normalizeRecommendationCompanyIDs(companyIDs []string) []string { func mergeRecommendedTenders(recommendationLists ...[]RecommendedTenderResponse) []RecommendedTenderResponse { merged := make([]RecommendedTenderResponse, 0) - seen := make(map[string]struct{}) for _, recommendations := range recommendationLists { - for _, recommendation := range recommendations { - tenderID := strings.TrimSpace(recommendation.TenderID) - if tenderID == "" { - continue - } - if _, ok := seen[tenderID]; ok { - continue - } - seen[tenderID] = struct{}{} - recommendation.TenderID = tenderID - recommendation.Analysis = strings.TrimSpace(recommendation.Analysis) - merged = append(merged, recommendation) - } + merged = append(merged, recommendations...) } - return merged + return dedupeRecommendationResponsesByTenderID(merged) } func aiRecommendationCacheKey(companyID string) string { @@ -382,6 +369,10 @@ func (s *companyService) fetchAIRecommendations(ctx context.Context, companyID s return nil, fmt.Errorf("failed to fetch AI recommendations: %w", err) } + return dedupeRecommendationResponsesByTenderID(itemsToRecommendationResponses(items)), nil +} + +func itemsToRecommendationResponses(items []ai_summarizer.RecommendedTender) []RecommendedTenderResponse { responses := make([]RecommendedTenderResponse, 0, len(items)) for _, item := range items { responses = append(responses, RecommendedTenderResponse{ @@ -390,8 +381,26 @@ func (s *companyService) fetchAIRecommendations(ctx context.Context, companyID s Analysis: item.Analysis, }) } + return responses +} - return responses, nil +func dedupeRecommendationResponsesByTenderID(responses []RecommendedTenderResponse) []RecommendedTenderResponse { + seen := make(map[string]struct{}, len(responses)) + out := make([]RecommendedTenderResponse, 0, len(responses)) + for _, recommendation := range responses { + tenderID := strings.TrimSpace(recommendation.TenderID) + if tenderID == "" { + continue + } + if _, ok := seen[tenderID]; ok { + continue + } + seen[tenderID] = struct{}{} + recommendation.TenderID = tenderID + recommendation.Analysis = strings.TrimSpace(recommendation.Analysis) + out = append(out, recommendation) + } + return out } func (s *companyService) getCachedAIRecommendations(ctx context.Context, companyID string) ([]RecommendedTenderResponse, bool) { diff --git a/internal/company/recommendation_test.go b/internal/company/recommendation_test.go index 4f02255..d00fc43 100644 --- a/internal/company/recommendation_test.go +++ b/internal/company/recommendation_test.go @@ -2,6 +2,26 @@ package company import "testing" +func TestDedupeRecommendationResponsesByTenderID(t *testing.T) { + input := []RecommendedTenderResponse{ + {Rank: 1, TenderID: " PROC-1 ", Analysis: " first "}, + {Rank: 2, TenderID: "PROC-2", Analysis: "second"}, + {Rank: 1, TenderID: "PROC-2", Analysis: "duplicate"}, + {Rank: 4, TenderID: " ", Analysis: "ignored"}, + } + + got := dedupeRecommendationResponsesByTenderID(input) + if len(got) != 2 { + t.Fatalf("dedupeRecommendationResponsesByTenderID() len = %d, want 2", len(got)) + } + if got[0].TenderID != "PROC-1" || got[0].Analysis != "first" { + t.Fatalf("first recommendation = %+v, want trimmed PROC-1/first", got[0]) + } + if got[1].TenderID != "PROC-2" || got[1].Rank != 2 { + t.Fatalf("second recommendation = %+v, want original PROC-2 rank 2", got[1]) + } +} + func TestMergeRecommendedTenders(t *testing.T) { first := []RecommendedTenderResponse{ {Rank: 1, TenderID: " PROC-1 ", Analysis: " first "}, diff --git a/internal/tender/recommendation_filter.go b/internal/tender/recommendation_filter.go index 6426c10..b6c2ed7 100644 --- a/internal/tender/recommendation_filter.go +++ b/internal/tender/recommendation_filter.go @@ -94,6 +94,25 @@ func addTenderIDsToSet(set map[string]struct{}, ids []string) { } } +// markResolvedRecommendedTenderSeen records a resolved tender and reports whether it was already seen. +// Recommendations may reference the same tender via different AI refs (procedure ref vs Mongo ID, +// or multiple notice publication IDs for one merged tender). +func markResolvedRecommendedTenderSeen(seen map[string]struct{}, tender Tender) bool { + if seen == nil { + return false + } + + tenderID := strings.TrimSpace(tender.GetID()) + if tenderID == "" { + return true + } + if _, exists := seen[tenderID]; exists { + return true + } + seen[tenderID] = struct{}{} + return false +} + func isRecommendedTenderExcluded(tender Tender, tenderRef string, excluded map[string]struct{}) bool { if len(excluded) == 0 { return false diff --git a/internal/tender/recommendation_filter_test.go b/internal/tender/recommendation_filter_test.go index 19e26df..0f16e65 100644 --- a/internal/tender/recommendation_filter_test.go +++ b/internal/tender/recommendation_filter_test.go @@ -126,3 +126,24 @@ func TestIsRecommendedTenderExcludedForAllCompanies(t *testing.T) { t.Fatalf("expected tender to be hidden when all companies rejected it") } } + +func TestMarkResolvedRecommendedTenderSeen(t *testing.T) { + sample := Tender{} + sample.ID, _ = bson.ObjectIDFromHex("507f1f77bcf86cd799439011") + + seen := make(map[string]struct{}) + first := markResolvedRecommendedTenderSeen(seen, sample) + second := markResolvedRecommendedTenderSeen(seen, sample) + if first { + t.Fatalf("first sighting should not be skipped, id=%q seen=%v", sample.GetID(), seen) + } + if !second { + t.Fatalf("duplicate sighting should be skipped, id=%q seen=%v", sample.GetID(), seen) + } + + otherRefSameTender := Tender{} + otherRefSameTender.ID = sample.ID + if !markResolvedRecommendedTenderSeen(seen, otherRefSameTender) { + t.Fatalf("duplicate tender via different ref should be skipped") + } +} diff --git a/internal/tender/service.go b/internal/tender/service.go index ecf874b..c3f110b 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -1002,6 +1002,7 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat tenderRef string } ordered := make([]rankedRecommendedTender, 0, len(recommendations)) + seenTenderIDs := make(map[string]struct{}, len(recommendations)) for _, rec := range recommendations { tenderRef := strings.TrimSpace(rec.TenderID) tender, ok := tenderByRef[tenderRef] @@ -1021,6 +1022,9 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) { continue } + if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) { + continue + } ordered = append(ordered, rankedRecommendedTender{ tender: tender,