From f108733c2ad03f0f82b38e2aecaec0d8b775e2ba Mon Sep 17 00:00:00 2001 From: Mazyar Date: Wed, 8 Jul 2026 01:32:27 +0330 Subject: [PATCH] Enhance tender recommendation service with improved caching and pagination - Introduced a context with a timeout for Redis cache retrieval in the `getCachedAIRecommendations` method, ensuring more robust handling of potential delays. - Added a new file `recommendation_page.go` to encapsulate the logic for building recommendation pages, improving code organization and readability. - Implemented a `buildRecommendationPage` method to handle pagination and batch processing of recommendations, enhancing performance and user experience. - Created a new test file `recommendation_page_test.go` to validate the functionality of the recommendation page building process, ensuring correct pagination and batch resolution. This update significantly improves the efficiency and maintainability of the tender recommendation service by optimizing caching and implementing structured pagination handling. --- internal/company/recommendation.go | 5 +- internal/tender/recommendation_page.go | 94 ++++++++++++++++++ internal/tender/recommendation_page_test.go | 86 +++++++++++++++++ internal/tender/repository.go | 100 ++++++++++++++------ internal/tender/service.go | 82 ++++------------ 5 files changed, 271 insertions(+), 96 deletions(-) create mode 100644 internal/tender/recommendation_page.go create mode 100644 internal/tender/recommendation_page_test.go diff --git a/internal/company/recommendation.go b/internal/company/recommendation.go index 5bf06ec..3420715 100644 --- a/internal/company/recommendation.go +++ b/internal/company/recommendation.go @@ -407,7 +407,10 @@ func (s *companyService) getCachedAIRecommendations(ctx context.Context, company return nil, false } - raw, err := s.redisClient.Get(ctx, aiRecommendationCacheKey(companyID)) + readCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + raw, err := s.redisClient.Get(readCtx, aiRecommendationCacheKey(companyID)) if err != nil { if err != redis.Nil { s.logger.Warn("Failed to read AI recommendation cache", map[string]interface{}{ diff --git a/internal/tender/recommendation_page.go b/internal/tender/recommendation_page.go new file mode 100644 index 0000000..2695e41 --- /dev/null +++ b/internal/tender/recommendation_page.go @@ -0,0 +1,94 @@ +package tender + +import ( + "context" + "strings" + + "tm/internal/company" +) + +const recommendationResolveBatchSize = 100 + +type recommendationPageResult struct { + items []rankedRecommendedTender + total int64 +} + +func (s *tenderService) buildRecommendationPage( + ctx context.Context, + recommendations []company.RecommendedTenderResponse, + form *SearchForm, + companyIDs []string, + excluded map[string]struct{}, + excludedByCompany map[string]map[string]struct{}, + now int64, + offset int, + limit int, +) (recommendationPageResult, error) { + seenTenderIDs := make(map[string]struct{}, len(recommendations)) + pageStart := offset + pageEnd := offset + limit + if pageStart < 0 { + pageStart = 0 + } + if pageEnd < pageStart { + pageEnd = pageStart + } + + result := recommendationPageResult{ + items: make([]rankedRecommendedTender, 0, limit), + } + + for start := 0; start < len(recommendations); start += recommendationResolveBatchSize { + end := start + recommendationResolveBatchSize + if end > len(recommendations) { + end = len(recommendations) + } + + tenderByRef, err := s.resolveRecommendedTenders(ctx, recommendations[start:end]) + if err != nil { + return recommendationPageResult{}, err + } + + for _, rec := range recommendations[start:end] { + tenderRef := strings.TrimSpace(rec.TenderID) + tender, ok := tenderByRef[tenderRef] + if !ok { + continue + } + if len(companyIDs) > 0 { + if isRecommendedTenderExcludedForAllCompanies(tender, tenderRef, excludedByCompany, companyIDs) { + continue + } + } else if isRecommendedTenderExcluded(tender, tenderRef, excluded) { + continue + } + if form.OnlyActiveDeadlines && !tender.IsRecommendable(now) { + continue + } + if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) { + continue + } + if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) { + continue + } + + result.total++ + if int(result.total) <= pageStart || len(result.items) >= limit { + continue + } + if int(result.total) > pageEnd { + continue + } + + result.items = append(result.items, rankedRecommendedTender{ + tender: tender, + rank: rec.Rank, + analysis: rec.Analysis, + tenderRef: tenderRef, + }) + } + } + + return result, nil +} diff --git a/internal/tender/recommendation_page_test.go b/internal/tender/recommendation_page_test.go new file mode 100644 index 0000000..ef7eb79 --- /dev/null +++ b/internal/tender/recommendation_page_test.go @@ -0,0 +1,86 @@ +package tender + +import ( + "context" + "fmt" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "tm/internal/company" +) + +type stubRecommendTenderRepository struct { + TenderRepository + batchCalls int +} + +func (s *stubRecommendTenderRepository) GetByProcedureReferences(_ context.Context, refs []ProcedureReference) (map[string]Tender, error) { + s.batchCalls++ + out := make(map[string]Tender, len(refs)) + for i, ref := range refs { + tender := Tender{ + ContractFolderID: ref.ContractFolderID, + NoticePublicationID: ref.NoticePublicationID, + Title: ref.Ref, + Status: TenderStatusActive, + } + tender.ID = bsonObjectIDFromCounter(i + s.batchCalls*1000) + out[ref.Ref] = tender + } + return out, nil +} + +func bsonObjectIDFromCounter(n int) bson.ObjectID { + var id [12]byte + id[11] = byte(n) + id[10] = byte(n >> 8) + return bson.ObjectID(id) +} + +func TestBuildRecommendationPageResolvesInBatchesAndPaginates(t *testing.T) { + repo := &stubRecommendTenderRepository{} + service := &tenderService{ + repository: repo, + logger: noopTenderTestLogger{}, + } + + recommendations := make([]company.RecommendedTenderResponse, 0, 120) + for i := 0; i < 120; i++ { + recommendations = append(recommendations, company.RecommendedTenderResponse{ + Rank: i + 1, + TenderID: fmt.Sprintf("PROC_folder/notice-%d", i), + Analysis: "fit", + }) + } + + form := &SearchForm{ + OnlyActiveDeadlines: false, + ExcludeRejectedTenders: false, + Status: []string{string(TenderStatusActive)}, + } + + got, err := service.buildRecommendationPage( + context.Background(), + recommendations, + form, + nil, + nil, + nil, + 0, + 10, + 10, + ) + if err != nil { + t.Fatalf("buildRecommendationPage() error = %v", err) + } + if repo.batchCalls != 2 { + t.Fatalf("batchCalls = %d, want 2", repo.batchCalls) + } + if got.total != 120 { + t.Fatalf("total = %d, want 120", got.total) + } + if len(got.items) != 10 { + t.Fatalf("len(items) = %d, want 10", len(got.items)) + } +} diff --git a/internal/tender/repository.go b/internal/tender/repository.go index 6176e42..8c5dca8 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -238,25 +238,42 @@ func (r *tenderRepository) GetByIDs(ctx context.Context, ids []string) ([]Tender return nil, nil } - objectIDs := make([]bson.ObjectID, 0, len(ids)) - for _, id := range ids { - objectID, err := bson.ObjectIDFromHex(id) + const batchSize = 500 + out := make([]Tender, 0, len(ids)) + for start := 0; start < len(ids); start += batchSize { + end := start + batchSize + if end > len(ids) { + end = len(ids) + } + batch := ids[start:end] + + objectIDs := make([]bson.ObjectID, 0, len(batch)) + for _, id := range batch { + objectID, err := bson.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + objectIDs = append(objectIDs, objectID) + } + + pagination := orm.Pagination{ + Limit: len(objectIDs), + SortField: "_id", + SortOrder: -1, + SkipCount: true, + } + result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, pagination) if err != nil { + r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{ + "count": len(batch), + "error": err.Error(), + }) return nil, err } - objectIDs = append(objectIDs, objectID) + out = append(out, result.Items...) } - result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, orm.NewPaginationBuilder().Build()) - if err != nil { - r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{ - "count": len(ids), - "error": err.Error(), - }) - return nil, err - } - - return result.Items, nil + return out, nil } // GetByContractNoticeID retrieves a tender by contract notice ID @@ -343,52 +360,75 @@ func (r *tenderRepository) GetByProcedureReference(ctx context.Context, contract return &result.Items[0], nil } -// GetByProcedureReferences loads tenders for multiple AI procedure refs in one query. +const procedureReferenceLookupBatchSize = 250 + +// GetByProcedureReferences loads tenders for multiple AI procedure refs using indexed notice lookups. func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error) { out := make(map[string]Tender) if len(refs) == 0 { return out, nil } - contractFolderIDs := make([]string, 0, len(refs)) + noticeIDs := make([]string, 0, len(refs)) for _, ref := range refs { folder := strings.TrimSpace(ref.ContractFolderID) notice := strings.TrimSpace(ref.NoticePublicationID) if folder == "" || notice == "" { continue } - contractFolderIDs = append(contractFolderIDs, folder) + noticeIDs = append(noticeIDs, notice) } - uniqueFolders := uniqueNonEmptyTrimmedStrings(contractFolderIDs) - if len(uniqueFolders) == 0 { + + uniqueNotices := uniqueNonEmptyTrimmedStrings(noticeIDs) + if len(uniqueNotices) == 0 { return out, nil } - findLimit := len(uniqueFolders) * 5 - if minLimit := len(refs) + len(uniqueFolders); findLimit < minLimit { - findLimit = minLimit - } - if findLimit > 100000 { - findLimit = 100000 + candidates := make([]Tender, 0, len(uniqueNotices)) + for start := 0; start < len(uniqueNotices); start += procedureReferenceLookupBatchSize { + end := start + procedureReferenceLookupBatchSize + if end > len(uniqueNotices) { + end = len(uniqueNotices) + } + + batch, err := r.findTendersByNoticePublicationIDs(ctx, uniqueNotices[start:end]) + if err != nil { + return nil, err + } + candidates = append(candidates, batch...) } - filter := bson.M{"contract_folder_id": bson.M{"$in": uniqueFolders}} + return mapProcedureReferencesToTenders(candidates, refs), nil +} + +func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context, noticeIDs []string) ([]Tender, error) { + if len(noticeIDs) == 0 { + return nil, nil + } + + filter := bson.M{ + "$or": []bson.M{ + {"notice_publication_id": bson.M{"$in": noticeIDs}}, + {"related_notice_publication_ids": bson.M{"$in": noticeIDs}}, + }, + } pagination := orm.Pagination{ - Limit: findLimit, + Limit: len(noticeIDs) * 2, SortField: "updated_at", SortOrder: -1, + SkipCount: true, } result, err := r.ormRepo.FindAll(ctx, filter, pagination) if err != nil { - r.logger.Error("Failed to get tenders by procedure references", map[string]interface{}{ - "count": len(uniqueFolders), + r.logger.Error("Failed to get tenders by notice publication IDs", map[string]interface{}{ + "count": len(noticeIDs), "error": err.Error(), }) return nil, err } - return mapProcedureReferencesToTenders(result.Items, refs), nil + return result.Items, nil } func mapProcedureReferencesToTenders(candidates []Tender, refs []ProcedureReference) map[string]Tender { diff --git a/internal/tender/service.go b/internal/tender/service.go index 5b362e0..6d2c17b 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -947,13 +947,8 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination nil } -const recommendRequestTimeout = 15 * time.Second - // Recommend retrieves AI-ranked tenders for the authenticated customer's company. func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) { - ctx, cancel := context.WithTimeout(ctx, recommendRequestTimeout) - defer cancel() - companyIDs := normalizeRecommendationCompanyScope(form.CompanyIDs) companyID := "" if form.CompanyID != nil { @@ -988,23 +983,8 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat return s.emptySearchResponse(pagination), nil } - var ( - tenderByRef map[string]Tender - excluded map[string]struct{} - excludedByCompany map[string]map[string]struct{} - ) - loadGroup, loadCtx := errgroup.WithContext(ctx) - loadGroup.Go(func() error { - var resolveErr error - tenderByRef, resolveErr = s.resolveRecommendedTenders(loadCtx, recommendations) - return resolveErr - }) - loadGroup.Go(func() error { - var exclusionErr error - excluded, excludedByCompany, exclusionErr = s.loadRecommendationExclusions(loadCtx, form, companyID, companyIDs) - return exclusionErr - }) - if err := loadGroup.Wait(); err != nil { + excluded, excludedByCompany, err := s.loadRecommendationExclusions(ctx, form, companyID, companyIDs) + if err != nil { if form.ExcludeRejectedTenders { s.logger.Error("Failed to build recommendation exclusions", map[string]interface{}{ "company_id": companyID, @@ -1017,54 +997,26 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat lang := s.pickResponseLanguage(form.Language) now := time.Now().Unix() - ordered := make([]rankedRecommendedTender, 0, len(recommendations)) - seenTenderIDs := make(map[string]struct{}, len(recommendations)) - for _, rec := range recommendations { - tenderRef := strings.TrimSpace(rec.TenderID) - tender, ok := tenderByRef[tenderRef] - if !ok { - continue - } - if len(companyIDs) > 0 { - if isRecommendedTenderExcludedForAllCompanies(tender, tenderRef, excludedByCompany, companyIDs) { - continue - } - } else if isRecommendedTenderExcluded(tender, tenderRef, excluded) { - continue - } - if form.OnlyActiveDeadlines && !tender.IsRecommendable(now) { - continue - } - if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) { - continue - } - if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) { - continue - } - - ordered = append(ordered, rankedRecommendedTender{ - tender: tender, - rank: rec.Rank, - analysis: rec.Analysis, - tenderRef: tenderRef, - }) + pageResult, err := s.buildRecommendationPage( + ctx, + recommendations, + form, + companyIDs, + excluded, + excludedByCompany, + now, + pagination.Offset, + pagination.Limit, + ) + if err != nil { + return nil, err } - total := int64(len(ordered)) - start := pagination.Offset - if start > len(ordered) { - start = len(ordered) - } - end := start + pagination.Limit - if end > len(ordered) { - end = len(ordered) - } - - paged := s.buildRecommendedTenderListResponses(ordered[start:end], lang) + paged := s.buildRecommendedTenderListResponses(pageResult.items, lang) return &SearchResponse{ Tenders: paged, - Metadata: pagination.ListMeta(total, "", end < len(ordered), start), + Metadata: pagination.ListMeta(pageResult.total, "", int(pagination.Offset)+len(paged) < int(pageResult.total), pagination.Offset), }, nil }