Enhance tender recommendation service with improved caching and pagination

- Introduced a context with a timeout for Redis cache retrieval in the `getCachedAIRecommendations` method, ensuring more robust handling of potential delays.
- Added a new file `recommendation_page.go` to encapsulate the logic for building recommendation pages, improving code organization and readability.
- Implemented a `buildRecommendationPage` method to handle pagination and batch processing of recommendations, enhancing performance and user experience.
- Created a new test file `recommendation_page_test.go` to validate the functionality of the recommendation page building process, ensuring correct pagination and batch resolution.

This update significantly improves the efficiency and maintainability of the tender recommendation service by optimizing caching and implementing structured pagination handling.

internal/tender/repository.go

@@ -238,25 +238,42 @@ func (r *tenderRepository) GetByIDs(ctx context.Context, ids []string) ([]Tender
 		return nil, nil
 	}
 
-	objectIDs := make([]bson.ObjectID, 0, len(ids))
-	for _, id := range ids {
-		objectID, err := bson.ObjectIDFromHex(id)
+	const batchSize = 500
+	out := make([]Tender, 0, len(ids))
+	for start := 0; start < len(ids); start += batchSize {
+		end := start + batchSize
+		if end > len(ids) {
+			end = len(ids)
+		}
+		batch := ids[start:end]
+
+		objectIDs := make([]bson.ObjectID, 0, len(batch))
+		for _, id := range batch {
+			objectID, err := bson.ObjectIDFromHex(id)
+			if err != nil {
+				return nil, err
+			}
+			objectIDs = append(objectIDs, objectID)
+		}
+
+		pagination := orm.Pagination{
+			Limit:     len(objectIDs),
+			SortField: "_id",
+			SortOrder: -1,
+			SkipCount: true,
+		}
+		result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, pagination)
 		if err != nil {
+			r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{
+				"count": len(batch),
+				"error": err.Error(),
+			})
 			return nil, err
 		}
-		objectIDs = append(objectIDs, objectID)
+		out = append(out, result.Items...)
 	}
 
-	result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, orm.NewPaginationBuilder().Build())
-	if err != nil {
-		r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{
-			"count": len(ids),
-			"error": err.Error(),
-		})
-		return nil, err
-	}
-
-	return result.Items, nil
+	return out, nil
 }
 
 // GetByContractNoticeID retrieves a tender by contract notice ID
@@ -343,52 +360,75 @@ func (r *tenderRepository) GetByProcedureReference(ctx context.Context, contract
 	return &result.Items[0], nil
 }
 
-// GetByProcedureReferences loads tenders for multiple AI procedure refs in one query.
+const procedureReferenceLookupBatchSize = 250
+
+// GetByProcedureReferences loads tenders for multiple AI procedure refs using indexed notice lookups.
 func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error) {
 	out := make(map[string]Tender)
 	if len(refs) == 0 {
 		return out, nil
 	}
 
-	contractFolderIDs := make([]string, 0, len(refs))
+	noticeIDs := make([]string, 0, len(refs))
 	for _, ref := range refs {
 		folder := strings.TrimSpace(ref.ContractFolderID)
 		notice := strings.TrimSpace(ref.NoticePublicationID)
 		if folder == "" || notice == "" {
 			continue
 		}
-		contractFolderIDs = append(contractFolderIDs, folder)
+		noticeIDs = append(noticeIDs, notice)
 	}
-	uniqueFolders := uniqueNonEmptyTrimmedStrings(contractFolderIDs)
-	if len(uniqueFolders) == 0 {
+
+	uniqueNotices := uniqueNonEmptyTrimmedStrings(noticeIDs)
+	if len(uniqueNotices) == 0 {
 		return out, nil
 	}
 
-	findLimit := len(uniqueFolders) * 5
-	if minLimit := len(refs) + len(uniqueFolders); findLimit < minLimit {
-		findLimit = minLimit
-	}
-	if findLimit > 100000 {
-		findLimit = 100000
+	candidates := make([]Tender, 0, len(uniqueNotices))
+	for start := 0; start < len(uniqueNotices); start += procedureReferenceLookupBatchSize {
+		end := start + procedureReferenceLookupBatchSize
+		if end > len(uniqueNotices) {
+			end = len(uniqueNotices)
+		}
+
+		batch, err := r.findTendersByNoticePublicationIDs(ctx, uniqueNotices[start:end])
+		if err != nil {
+			return nil, err
+		}
+		candidates = append(candidates, batch...)
 	}
 
-	filter := bson.M{"contract_folder_id": bson.M{"$in": uniqueFolders}}
+	return mapProcedureReferencesToTenders(candidates, refs), nil
+}
+
+func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context, noticeIDs []string) ([]Tender, error) {
+	if len(noticeIDs) == 0 {
+		return nil, nil
+	}
+
+	filter := bson.M{
+		"$or": []bson.M{
+			{"notice_publication_id": bson.M{"$in": noticeIDs}},
+			{"related_notice_publication_ids": bson.M{"$in": noticeIDs}},
+		},
+	}
 	pagination := orm.Pagination{
-		Limit:     findLimit,
+		Limit:     len(noticeIDs) * 2,
 		SortField: "updated_at",
 		SortOrder: -1,
+		SkipCount: true,
 	}
 
 	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
 	if err != nil {
-		r.logger.Error("Failed to get tenders by procedure references", map[string]interface{}{
-			"count": len(uniqueFolders),
+		r.logger.Error("Failed to get tenders by notice publication IDs", map[string]interface{}{
+			"count": len(noticeIDs),
 			"error": err.Error(),
 		})
 		return nil, err
 	}
 
-	return mapProcedureReferencesToTenders(result.Items, refs), nil
+	return result.Items, nil
 }
 
 func mapProcedureReferencesToTenders(candidates []Tender, refs []ProcedureReference) map[string]Tender {