Enhance tender recommendation service with improved caching and pagination

- Introduced a context with a timeout for Redis cache retrieval in the `getCachedAIRecommendations` method, ensuring more robust handling of potential delays.
- Added a new file `recommendation_page.go` to encapsulate the logic for building recommendation pages, improving code organization and readability.
- Implemented a `buildRecommendationPage` method to handle pagination and batch processing of recommendations, enhancing performance and user experience.
- Created a new test file `recommendation_page_test.go` to validate the functionality of the recommendation page building process, ensuring correct pagination and batch resolution.

This update significantly improves the efficiency and maintainability of the tender recommendation service by optimizing caching and implementing structured pagination handling.

internal/company/recommendation.go

@@ -407,7 +407,10 @@ func (s *companyService) getCachedAIRecommendations(ctx context.Context, company
 		return nil, false
 	}
 
-	raw, err := s.redisClient.Get(ctx, aiRecommendationCacheKey(companyID))
+	readCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	defer cancel()
+
+	raw, err := s.redisClient.Get(readCtx, aiRecommendationCacheKey(companyID))
 	if err != nil {
 		if err != redis.Nil {
 			s.logger.Warn("Failed to read AI recommendation cache", map[string]interface{}{

internal/tender/recommendation_page.go

@@ -0,0 +1,94 @@
+package tender
+
+import (
+	"context"
+	"strings"
+
+	"tm/internal/company"
+)
+
+const recommendationResolveBatchSize = 100
+
+type recommendationPageResult struct {
+	items []rankedRecommendedTender
+	total int64
+}
+
+func (s *tenderService) buildRecommendationPage(
+	ctx context.Context,
+	recommendations []company.RecommendedTenderResponse,
+	form *SearchForm,
+	companyIDs []string,
+	excluded map[string]struct{},
+	excludedByCompany map[string]map[string]struct{},
+	now int64,
+	offset int,
+	limit int,
+) (recommendationPageResult, error) {
+	seenTenderIDs := make(map[string]struct{}, len(recommendations))
+	pageStart := offset
+	pageEnd := offset + limit
+	if pageStart < 0 {
+		pageStart = 0
+	}
+	if pageEnd < pageStart {
+		pageEnd = pageStart
+	}
+
+	result := recommendationPageResult{
+		items: make([]rankedRecommendedTender, 0, limit),
+	}
+
+	for start := 0; start < len(recommendations); start += recommendationResolveBatchSize {
+		end := start + recommendationResolveBatchSize
+		if end > len(recommendations) {
+			end = len(recommendations)
+		}
+
+		tenderByRef, err := s.resolveRecommendedTenders(ctx, recommendations[start:end])
+		if err != nil {
+			return recommendationPageResult{}, err
+		}
+
+		for _, rec := range recommendations[start:end] {
+			tenderRef := strings.TrimSpace(rec.TenderID)
+			tender, ok := tenderByRef[tenderRef]
+			if !ok {
+				continue
+			}
+			if len(companyIDs) > 0 {
+				if isRecommendedTenderExcludedForAllCompanies(tender, tenderRef, excludedByCompany, companyIDs) {
+					continue
+				}
+			} else if isRecommendedTenderExcluded(tender, tenderRef, excluded) {
+				continue
+			}
+			if form.OnlyActiveDeadlines && !tender.IsRecommendable(now) {
+				continue
+			}
+			if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) {
+				continue
+			}
+			if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) {
+				continue
+			}
+
+			result.total++
+			if int(result.total) <= pageStart || len(result.items) >= limit {
+				continue
+			}
+			if int(result.total) > pageEnd {
+				continue
+			}
+
+			result.items = append(result.items, rankedRecommendedTender{
+				tender:    tender,
+				rank:      rec.Rank,
+				analysis:  rec.Analysis,
+				tenderRef: tenderRef,
+			})
+		}
+	}
+
+	return result, nil
+}

internal/tender/recommendation_page_test.go

@@ -0,0 +1,86 @@
+package tender
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"go.mongodb.org/mongo-driver/v2/bson"
+
+	"tm/internal/company"
+)
+
+type stubRecommendTenderRepository struct {
+	TenderRepository
+	batchCalls int
+}
+
+func (s *stubRecommendTenderRepository) GetByProcedureReferences(_ context.Context, refs []ProcedureReference) (map[string]Tender, error) {
+	s.batchCalls++
+	out := make(map[string]Tender, len(refs))
+	for i, ref := range refs {
+		tender := Tender{
+			ContractFolderID:    ref.ContractFolderID,
+			NoticePublicationID: ref.NoticePublicationID,
+			Title:               ref.Ref,
+			Status:              TenderStatusActive,
+		}
+		tender.ID = bsonObjectIDFromCounter(i + s.batchCalls*1000)
+		out[ref.Ref] = tender
+	}
+	return out, nil
+}
+
+func bsonObjectIDFromCounter(n int) bson.ObjectID {
+	var id [12]byte
+	id[11] = byte(n)
+	id[10] = byte(n >> 8)
+	return bson.ObjectID(id)
+}
+
+func TestBuildRecommendationPageResolvesInBatchesAndPaginates(t *testing.T) {
+	repo := &stubRecommendTenderRepository{}
+	service := &tenderService{
+		repository: repo,
+		logger:     noopTenderTestLogger{},
+	}
+
+	recommendations := make([]company.RecommendedTenderResponse, 0, 120)
+	for i := 0; i < 120; i++ {
+		recommendations = append(recommendations, company.RecommendedTenderResponse{
+			Rank:     i + 1,
+			TenderID: fmt.Sprintf("PROC_folder/notice-%d", i),
+			Analysis: "fit",
+		})
+	}
+
+	form := &SearchForm{
+		OnlyActiveDeadlines:    false,
+		ExcludeRejectedTenders: false,
+		Status:                 []string{string(TenderStatusActive)},
+	}
+
+	got, err := service.buildRecommendationPage(
+		context.Background(),
+		recommendations,
+		form,
+		nil,
+		nil,
+		nil,
+		0,
+		10,
+		10,
+	)
+	if err != nil {
+		t.Fatalf("buildRecommendationPage() error = %v", err)
+	}
+	if repo.batchCalls != 2 {
+		t.Fatalf("batchCalls = %d, want 2", repo.batchCalls)
+	}
+	if got.total != 120 {
+		t.Fatalf("total = %d, want 120", got.total)
+	}
+	if len(got.items) != 10 {
+		t.Fatalf("len(items) = %d, want 10", len(got.items))
+	}
+}

internal/tender/repository.go

@@ -238,8 +238,17 @@ func (r *tenderRepository) GetByIDs(ctx context.Context, ids []string) ([]Tender
 		return nil, nil
 	}
 
-	objectIDs := make([]bson.ObjectID, 0, len(ids))
-	for _, id := range ids {
+	const batchSize = 500
+	out := make([]Tender, 0, len(ids))
+	for start := 0; start < len(ids); start += batchSize {
+		end := start + batchSize
+		if end > len(ids) {
+			end = len(ids)
+		}
+		batch := ids[start:end]
+
+		objectIDs := make([]bson.ObjectID, 0, len(batch))
+		for _, id := range batch {
 			objectID, err := bson.ObjectIDFromHex(id)
 			if err != nil {
 				return nil, err
@@ -247,16 +256,24 @@ func (r *tenderRepository) GetByIDs(ctx context.Context, ids []string) ([]Tender
 			objectIDs = append(objectIDs, objectID)
 		}
 
-	result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, orm.NewPaginationBuilder().Build())
+		pagination := orm.Pagination{
+			Limit:     len(objectIDs),
+			SortField: "_id",
+			SortOrder: -1,
+			SkipCount: true,
+		}
+		result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, pagination)
 		if err != nil {
 			r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{
-			"count": len(ids),
+				"count": len(batch),
 				"error": err.Error(),
 			})
 			return nil, err
 		}
+		out = append(out, result.Items...)
+	}
 
-	return result.Items, nil
+	return out, nil
 }
 
 // GetByContractNoticeID retrieves a tender by contract notice ID
@@ -343,52 +360,75 @@ func (r *tenderRepository) GetByProcedureReference(ctx context.Context, contract
 	return &result.Items[0], nil
 }
 
-// GetByProcedureReferences loads tenders for multiple AI procedure refs in one query.
+const procedureReferenceLookupBatchSize = 250
+
+// GetByProcedureReferences loads tenders for multiple AI procedure refs using indexed notice lookups.
 func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error) {
 	out := make(map[string]Tender)
 	if len(refs) == 0 {
 		return out, nil
 	}
 
-	contractFolderIDs := make([]string, 0, len(refs))
+	noticeIDs := make([]string, 0, len(refs))
 	for _, ref := range refs {
 		folder := strings.TrimSpace(ref.ContractFolderID)
 		notice := strings.TrimSpace(ref.NoticePublicationID)
 		if folder == "" || notice == "" {
 			continue
 		}
-		contractFolderIDs = append(contractFolderIDs, folder)
+		noticeIDs = append(noticeIDs, notice)
 	}
-	uniqueFolders := uniqueNonEmptyTrimmedStrings(contractFolderIDs)
-	if len(uniqueFolders) == 0 {
+
+	uniqueNotices := uniqueNonEmptyTrimmedStrings(noticeIDs)
+	if len(uniqueNotices) == 0 {
 		return out, nil
 	}
 
-	findLimit := len(uniqueFolders) * 5
-	if minLimit := len(refs) + len(uniqueFolders); findLimit < minLimit {
-		findLimit = minLimit
-	}
-	if findLimit > 100000 {
-		findLimit = 100000
+	candidates := make([]Tender, 0, len(uniqueNotices))
+	for start := 0; start < len(uniqueNotices); start += procedureReferenceLookupBatchSize {
+		end := start + procedureReferenceLookupBatchSize
+		if end > len(uniqueNotices) {
+			end = len(uniqueNotices)
 		}
 
-	filter := bson.M{"contract_folder_id": bson.M{"$in": uniqueFolders}}
+		batch, err := r.findTendersByNoticePublicationIDs(ctx, uniqueNotices[start:end])
+		if err != nil {
+			return nil, err
+		}
+		candidates = append(candidates, batch...)
+	}
+
+	return mapProcedureReferencesToTenders(candidates, refs), nil
+}
+
+func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context, noticeIDs []string) ([]Tender, error) {
+	if len(noticeIDs) == 0 {
+		return nil, nil
+	}
+
+	filter := bson.M{
+		"$or": []bson.M{
+			{"notice_publication_id": bson.M{"$in": noticeIDs}},
+			{"related_notice_publication_ids": bson.M{"$in": noticeIDs}},
+		},
+	}
 	pagination := orm.Pagination{
-		Limit:     findLimit,
+		Limit:     len(noticeIDs) * 2,
 		SortField: "updated_at",
 		SortOrder: -1,
+		SkipCount: true,
 	}
 
 	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
 	if err != nil {
-		r.logger.Error("Failed to get tenders by procedure references", map[string]interface{}{
-			"count": len(uniqueFolders),
+		r.logger.Error("Failed to get tenders by notice publication IDs", map[string]interface{}{
+			"count": len(noticeIDs),
 			"error": err.Error(),
 		})
 		return nil, err
 	}
 
-	return mapProcedureReferencesToTenders(result.Items, refs), nil
+	return result.Items, nil
 }
 
 func mapProcedureReferencesToTenders(candidates []Tender, refs []ProcedureReference) map[string]Tender {

internal/tender/service.go

@@ -947,13 +947,8 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
 		nil
 }
 
-const recommendRequestTimeout = 15 * time.Second
-
 // Recommend retrieves AI-ranked tenders for the authenticated customer's company.
 func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) {
-	ctx, cancel := context.WithTimeout(ctx, recommendRequestTimeout)
-	defer cancel()
-
 	companyIDs := normalizeRecommendationCompanyScope(form.CompanyIDs)
 	companyID := ""
 	if form.CompanyID != nil {
@@ -988,23 +983,8 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat
 		return s.emptySearchResponse(pagination), nil
 	}
 
-	var (
-		tenderByRef       map[string]Tender
-		excluded          map[string]struct{}
-		excludedByCompany map[string]map[string]struct{}
-	)
-	loadGroup, loadCtx := errgroup.WithContext(ctx)
-	loadGroup.Go(func() error {
-		var resolveErr error
-		tenderByRef, resolveErr = s.resolveRecommendedTenders(loadCtx, recommendations)
-		return resolveErr
-	})
-	loadGroup.Go(func() error {
-		var exclusionErr error
-		excluded, excludedByCompany, exclusionErr = s.loadRecommendationExclusions(loadCtx, form, companyID, companyIDs)
-		return exclusionErr
-	})
-	if err := loadGroup.Wait(); err != nil {
+	excluded, excludedByCompany, err := s.loadRecommendationExclusions(ctx, form, companyID, companyIDs)
+	if err != nil {
 		if form.ExcludeRejectedTenders {
 			s.logger.Error("Failed to build recommendation exclusions", map[string]interface{}{
 				"company_id":  companyID,
@@ -1017,54 +997,26 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat
 
 	lang := s.pickResponseLanguage(form.Language)
 	now := time.Now().Unix()
-	ordered := make([]rankedRecommendedTender, 0, len(recommendations))
-	seenTenderIDs := make(map[string]struct{}, len(recommendations))
-	for _, rec := range recommendations {
-		tenderRef := strings.TrimSpace(rec.TenderID)
-		tender, ok := tenderByRef[tenderRef]
-		if !ok {
-			continue
-		}
-		if len(companyIDs) > 0 {
-			if isRecommendedTenderExcludedForAllCompanies(tender, tenderRef, excludedByCompany, companyIDs) {
-				continue
-			}
-		} else if isRecommendedTenderExcluded(tender, tenderRef, excluded) {
-			continue
-		}
-		if form.OnlyActiveDeadlines && !tender.IsRecommendable(now) {
-			continue
-		}
-		if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) {
-			continue
-		}
-		if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) {
-			continue
+	pageResult, err := s.buildRecommendationPage(
+		ctx,
+		recommendations,
+		form,
+		companyIDs,
+		excluded,
+		excludedByCompany,
+		now,
+		pagination.Offset,
+		pagination.Limit,
+	)
+	if err != nil {
+		return nil, err
 	}
 
-		ordered = append(ordered, rankedRecommendedTender{
-			tender:    tender,
-			rank:      rec.Rank,
-			analysis:  rec.Analysis,
-			tenderRef: tenderRef,
-		})
-	}
-
-	total := int64(len(ordered))
-	start := pagination.Offset
-	if start > len(ordered) {
-		start = len(ordered)
-	}
-	end := start + pagination.Limit
-	if end > len(ordered) {
-		end = len(ordered)
-	}
-
-	paged := s.buildRecommendedTenderListResponses(ordered[start:end], lang)
+	paged := s.buildRecommendedTenderListResponses(pageResult.items, lang)
 
 	return &SearchResponse{
 		Tenders:  paged,
-		Metadata: pagination.ListMeta(total, "", end < len(ordered), start),
+		Metadata: pagination.ListMeta(pageResult.total, "", int(pagination.Offset)+len(paged) < int(pageResult.total), pagination.Offset),
 	}, nil
 }