From b28bc23975f4605eb6e9f9225d9f851e4210603e Mon Sep 17 00:00:00 2001 From: Mazyar Date: Wed, 8 Jul 2026 01:55:33 +0330 Subject: [PATCH] Enhance tender reference handling with improved deduplication and test coverage - Updated the `markResolvedRecommendedTenderSeen` function to accept an additional `tenderRef` parameter, allowing for better tracking of tender references and preventing duplicates. - Refactored the `GetByProcedureReferences` method in the repository to combine indexed notice lookups with contract-folder candidates, ensuring older notice references resolve correctly after tender merges. - Introduced a new `findTendersByContractFolderIDs` method to streamline the retrieval of tenders by contract folder IDs, enhancing performance and maintainability. - Added unit tests for new functionality in `ai_reference_test.go` and `recommendation_filter_test.go`, ensuring robust validation of tender reference mapping and deduplication logic. This update significantly improves the handling of tender references and enhances the overall test coverage, ensuring more reliable and efficient processing of tender data. --- internal/tender/ai_reference_test.go | 40 ++++++++- internal/tender/recommendation_filter.go | 17 ++-- internal/tender/recommendation_filter_test.go | 6 +- internal/tender/recommendation_page.go | 2 +- internal/tender/repository.go | 85 +++++++++++++++++-- 5 files changed, 130 insertions(+), 20 deletions(-) diff --git a/internal/tender/ai_reference_test.go b/internal/tender/ai_reference_test.go index 7dfc6ae..76c1ce6 100644 --- a/internal/tender/ai_reference_test.go +++ b/internal/tender/ai_reference_test.go @@ -1,6 +1,10 @@ package tender -import "testing" +import ( + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" +) func TestParseAIProcedureRef(t *testing.T) { tests := []struct { @@ -84,3 +88,37 @@ func TestMapProcedureReferencesToTenders(t *testing.T) { t.Fatalf("expected folder-2 ref to resolve, got %+v", got["PROC_folder-2/notice-2"]) } } + +func TestMapProcedureReferencesToTendersUsesFolderCandidatesWhenNoticeLookupMisses(t *testing.T) { + // Notice-old is only present on the merged tender loaded via contract_folder_id. + candidates := []Tender{ + { + ContractFolderID: "folder-1", + NoticePublicationID: "notice-new", + RelatedNoticePublicationIDs: []string{"notice-old"}, + }, + } + refs := []ProcedureReference{ + {Ref: "PROC_folder-1/notice-old", ContractFolderID: "folder-1", NoticePublicationID: "notice-old"}, + } + + got := mapProcedureReferencesToTenders(candidates, refs) + if len(got) != 1 { + t.Fatalf("mapProcedureReferencesToTenders() len = %d, want 1", len(got)) + } + if got["PROC_folder-1/notice-old"].NoticePublicationID != "notice-new" { + t.Fatalf("expected folder candidate to resolve notice-old ref, got %+v", got["PROC_folder-1/notice-old"]) + } +} + +func TestMergeTenderCandidatesDedupesByMongoID(t *testing.T) { + first := Tender{ContractFolderID: "folder-1", NoticePublicationID: "notice-a"} + first.ID, _ = bson.ObjectIDFromHex("507f1f77bcf86cd799439011") + second := Tender{ContractFolderID: "folder-1", NoticePublicationID: "notice-b"} + second.ID = first.ID + + got := mergeTenderCandidates([]Tender{first}, []Tender{second}) + if len(got) != 1 { + t.Fatalf("mergeTenderCandidates() len = %d, want 1", len(got)) + } +} diff --git a/internal/tender/recommendation_filter.go b/internal/tender/recommendation_filter.go index b6c2ed7..7483149 100644 --- a/internal/tender/recommendation_filter.go +++ b/internal/tender/recommendation_filter.go @@ -97,19 +97,22 @@ func addTenderIDsToSet(set map[string]struct{}, ids []string) { // markResolvedRecommendedTenderSeen records a resolved tender and reports whether it was already seen. // Recommendations may reference the same tender via different AI refs (procedure ref vs Mongo ID, // or multiple notice publication IDs for one merged tender). -func markResolvedRecommendedTenderSeen(seen map[string]struct{}, tender Tender) bool { +func markResolvedRecommendedTenderSeen(seen map[string]struct{}, tender Tender, tenderRef string) bool { if seen == nil { return false } - tenderID := strings.TrimSpace(tender.GetID()) - if tenderID == "" { + key := strings.TrimSpace(tender.GetID()) + if key == "" { + key = strings.TrimSpace(tenderRef) + } + if key == "" { + return false + } + if _, exists := seen[key]; exists { return true } - if _, exists := seen[tenderID]; exists { - return true - } - seen[tenderID] = struct{}{} + seen[key] = struct{}{} return false } diff --git a/internal/tender/recommendation_filter_test.go b/internal/tender/recommendation_filter_test.go index 0f16e65..1576357 100644 --- a/internal/tender/recommendation_filter_test.go +++ b/internal/tender/recommendation_filter_test.go @@ -132,8 +132,8 @@ func TestMarkResolvedRecommendedTenderSeen(t *testing.T) { sample.ID, _ = bson.ObjectIDFromHex("507f1f77bcf86cd799439011") seen := make(map[string]struct{}) - first := markResolvedRecommendedTenderSeen(seen, sample) - second := markResolvedRecommendedTenderSeen(seen, sample) + first := markResolvedRecommendedTenderSeen(seen, sample, "PROC_folder/notice") + second := markResolvedRecommendedTenderSeen(seen, sample, "PROC_folder/notice") if first { t.Fatalf("first sighting should not be skipped, id=%q seen=%v", sample.GetID(), seen) } @@ -143,7 +143,7 @@ func TestMarkResolvedRecommendedTenderSeen(t *testing.T) { otherRefSameTender := Tender{} otherRefSameTender.ID = sample.ID - if !markResolvedRecommendedTenderSeen(seen, otherRefSameTender) { + if !markResolvedRecommendedTenderSeen(seen, otherRefSameTender, "PROC_folder/other-notice") { t.Fatalf("duplicate tender via different ref should be skipped") } } diff --git a/internal/tender/recommendation_page.go b/internal/tender/recommendation_page.go index 2695e41..60f5340 100644 --- a/internal/tender/recommendation_page.go +++ b/internal/tender/recommendation_page.go @@ -69,7 +69,7 @@ func (s *tenderService) buildRecommendationPage( if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) { continue } - if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) { + if markResolvedRecommendedTenderSeen(seenTenderIDs, tender, tenderRef) { continue } diff --git a/internal/tender/repository.go b/internal/tender/repository.go index 8c5dca8..3987164 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -362,13 +362,18 @@ func (r *tenderRepository) GetByProcedureReference(ctx context.Context, contract const procedureReferenceLookupBatchSize = 250 -// GetByProcedureReferences loads tenders for multiple AI procedure refs using indexed notice lookups. +const contractFolderLookupBatchSize = 250 + +// GetByProcedureReferences loads tenders for multiple AI procedure refs. +// It combines indexed notice lookups with contract-folder candidates so older notice +// references still resolve after tender merges (same behavior as before the perf refactor). func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error) { out := make(map[string]Tender) if len(refs) == 0 { return out, nil } + folderIDs := make([]string, 0, len(refs)) noticeIDs := make([]string, 0, len(refs)) for _, ref := range refs { folder := strings.TrimSpace(ref.ContractFolderID) @@ -376,15 +381,12 @@ func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs [] if folder == "" || notice == "" { continue } + folderIDs = append(folderIDs, folder) noticeIDs = append(noticeIDs, notice) } uniqueNotices := uniqueNonEmptyTrimmedStrings(noticeIDs) - if len(uniqueNotices) == 0 { - return out, nil - } - - candidates := make([]Tender, 0, len(uniqueNotices)) + noticeCandidates := make([]Tender, 0, len(uniqueNotices)) for start := 0; start < len(uniqueNotices); start += procedureReferenceLookupBatchSize { end := start + procedureReferenceLookupBatchSize if end > len(uniqueNotices) { @@ -395,12 +397,79 @@ func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs [] if err != nil { return nil, err } - candidates = append(candidates, batch...) + noticeCandidates = append(noticeCandidates, batch...) } + folderCandidates, err := r.findTendersByContractFolderIDs(ctx, folderIDs) + if err != nil { + return nil, err + } + + candidates := mergeTenderCandidates(noticeCandidates, folderCandidates) return mapProcedureReferencesToTenders(candidates, refs), nil } +func (r *tenderRepository) findTendersByContractFolderIDs(ctx context.Context, folderIDs []string) ([]Tender, error) { + uniqueFolders := uniqueNonEmptyTrimmedStrings(folderIDs) + if len(uniqueFolders) == 0 { + return nil, nil + } + + out := make([]Tender, 0, len(uniqueFolders)) + for start := 0; start < len(uniqueFolders); start += contractFolderLookupBatchSize { + end := start + contractFolderLookupBatchSize + if end > len(uniqueFolders) { + end = len(uniqueFolders) + } + batch := uniqueFolders[start:end] + + findLimit := len(batch) * 5 + if minLimit := len(batch) + 1; findLimit < minLimit { + findLimit = minLimit + } + + filter := bson.M{"contract_folder_id": bson.M{"$in": batch}} + pagination := orm.Pagination{ + Limit: findLimit, + SortField: "updated_at", + SortOrder: -1, + SkipCount: true, + } + + result, err := r.ormRepo.FindAll(ctx, filter, pagination) + if err != nil { + r.logger.Error("Failed to get tenders by contract folder IDs", map[string]interface{}{ + "count": len(batch), + "error": err.Error(), + }) + return nil, err + } + out = append(out, result.Items...) + } + + return out, nil +} + +func mergeTenderCandidates(groups ...[]Tender) []Tender { + seen := make(map[string]struct{}) + out := make([]Tender, 0) + for _, group := range groups { + for i := range group { + tenderID := strings.TrimSpace(group[i].GetID()) + if tenderID == "" { + out = append(out, group[i]) + continue + } + if _, exists := seen[tenderID]; exists { + continue + } + seen[tenderID] = struct{}{} + out = append(out, group[i]) + } + } + return out +} + func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context, noticeIDs []string) ([]Tender, error) { if len(noticeIDs) == 0 { return nil, nil @@ -413,7 +482,7 @@ func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context }, } pagination := orm.Pagination{ - Limit: len(noticeIDs) * 2, + Limit: len(noticeIDs) * 3, SortField: "updated_at", SortOrder: -1, SkipCount: true,