diff --git a/internal/company/recommendation.go b/internal/company/recommendation.go index 5bf06ec..3420715 100644 --- a/internal/company/recommendation.go +++ b/internal/company/recommendation.go @@ -407,7 +407,10 @@ func (s *companyService) getCachedAIRecommendations(ctx context.Context, company return nil, false } - raw, err := s.redisClient.Get(ctx, aiRecommendationCacheKey(companyID)) + readCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + + raw, err := s.redisClient.Get(readCtx, aiRecommendationCacheKey(companyID)) if err != nil { if err != redis.Nil { s.logger.Warn("Failed to read AI recommendation cache", map[string]interface{}{ diff --git a/internal/tender/recommendation_page.go b/internal/tender/recommendation_page.go new file mode 100644 index 0000000..2695e41 --- /dev/null +++ b/internal/tender/recommendation_page.go @@ -0,0 +1,94 @@ +package tender + +import ( + "context" + "strings" + + "tm/internal/company" +) + +const recommendationResolveBatchSize = 100 + +type recommendationPageResult struct { + items []rankedRecommendedTender + total int64 +} + +func (s *tenderService) buildRecommendationPage( + ctx context.Context, + recommendations []company.RecommendedTenderResponse, + form *SearchForm, + companyIDs []string, + excluded map[string]struct{}, + excludedByCompany map[string]map[string]struct{}, + now int64, + offset int, + limit int, +) (recommendationPageResult, error) { + seenTenderIDs := make(map[string]struct{}, len(recommendations)) + pageStart := offset + pageEnd := offset + limit + if pageStart < 0 { + pageStart = 0 + } + if pageEnd < pageStart { + pageEnd = pageStart + } + + result := recommendationPageResult{ + items: make([]rankedRecommendedTender, 0, limit), + } + + for start := 0; start < len(recommendations); start += recommendationResolveBatchSize { + end := start + recommendationResolveBatchSize + if end > len(recommendations) { + end = len(recommendations) + } + + tenderByRef, err := s.resolveRecommendedTenders(ctx, recommendations[start:end]) + if err != nil { + return recommendationPageResult{}, err + } + + for _, rec := range recommendations[start:end] { + tenderRef := strings.TrimSpace(rec.TenderID) + tender, ok := tenderByRef[tenderRef] + if !ok { + continue + } + if len(companyIDs) > 0 { + if isRecommendedTenderExcludedForAllCompanies(tender, tenderRef, excludedByCompany, companyIDs) { + continue + } + } else if isRecommendedTenderExcluded(tender, tenderRef, excluded) { + continue + } + if form.OnlyActiveDeadlines && !tender.IsRecommendable(now) { + continue + } + if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) { + continue + } + if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) { + continue + } + + result.total++ + if int(result.total) <= pageStart || len(result.items) >= limit { + continue + } + if int(result.total) > pageEnd { + continue + } + + result.items = append(result.items, rankedRecommendedTender{ + tender: tender, + rank: rec.Rank, + analysis: rec.Analysis, + tenderRef: tenderRef, + }) + } + } + + return result, nil +} diff --git a/internal/tender/recommendation_page_test.go b/internal/tender/recommendation_page_test.go new file mode 100644 index 0000000..ef7eb79 --- /dev/null +++ b/internal/tender/recommendation_page_test.go @@ -0,0 +1,86 @@ +package tender + +import ( + "context" + "fmt" + "testing" + + "go.mongodb.org/mongo-driver/v2/bson" + + "tm/internal/company" +) + +type stubRecommendTenderRepository struct { + TenderRepository + batchCalls int +} + +func (s *stubRecommendTenderRepository) GetByProcedureReferences(_ context.Context, refs []ProcedureReference) (map[string]Tender, error) { + s.batchCalls++ + out := make(map[string]Tender, len(refs)) + for i, ref := range refs { + tender := Tender{ + ContractFolderID: ref.ContractFolderID, + NoticePublicationID: ref.NoticePublicationID, + Title: ref.Ref, + Status: TenderStatusActive, + } + tender.ID = bsonObjectIDFromCounter(i + s.batchCalls*1000) + out[ref.Ref] = tender + } + return out, nil +} + +func bsonObjectIDFromCounter(n int) bson.ObjectID { + var id [12]byte + id[11] = byte(n) + id[10] = byte(n >> 8) + return bson.ObjectID(id) +} + +func TestBuildRecommendationPageResolvesInBatchesAndPaginates(t *testing.T) { + repo := &stubRecommendTenderRepository{} + service := &tenderService{ + repository: repo, + logger: noopTenderTestLogger{}, + } + + recommendations := make([]company.RecommendedTenderResponse, 0, 120) + for i := 0; i < 120; i++ { + recommendations = append(recommendations, company.RecommendedTenderResponse{ + Rank: i + 1, + TenderID: fmt.Sprintf("PROC_folder/notice-%d", i), + Analysis: "fit", + }) + } + + form := &SearchForm{ + OnlyActiveDeadlines: false, + ExcludeRejectedTenders: false, + Status: []string{string(TenderStatusActive)}, + } + + got, err := service.buildRecommendationPage( + context.Background(), + recommendations, + form, + nil, + nil, + nil, + 0, + 10, + 10, + ) + if err != nil { + t.Fatalf("buildRecommendationPage() error = %v", err) + } + if repo.batchCalls != 2 { + t.Fatalf("batchCalls = %d, want 2", repo.batchCalls) + } + if got.total != 120 { + t.Fatalf("total = %d, want 120", got.total) + } + if len(got.items) != 10 { + t.Fatalf("len(items) = %d, want 10", len(got.items)) + } +} diff --git a/internal/tender/repository.go b/internal/tender/repository.go index 6176e42..8c5dca8 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -238,25 +238,42 @@ func (r *tenderRepository) GetByIDs(ctx context.Context, ids []string) ([]Tender return nil, nil } - objectIDs := make([]bson.ObjectID, 0, len(ids)) - for _, id := range ids { - objectID, err := bson.ObjectIDFromHex(id) + const batchSize = 500 + out := make([]Tender, 0, len(ids)) + for start := 0; start < len(ids); start += batchSize { + end := start + batchSize + if end > len(ids) { + end = len(ids) + } + batch := ids[start:end] + + objectIDs := make([]bson.ObjectID, 0, len(batch)) + for _, id := range batch { + objectID, err := bson.ObjectIDFromHex(id) + if err != nil { + return nil, err + } + objectIDs = append(objectIDs, objectID) + } + + pagination := orm.Pagination{ + Limit: len(objectIDs), + SortField: "_id", + SortOrder: -1, + SkipCount: true, + } + result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, pagination) if err != nil { + r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{ + "count": len(batch), + "error": err.Error(), + }) return nil, err } - objectIDs = append(objectIDs, objectID) + out = append(out, result.Items...) } - result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, orm.NewPaginationBuilder().Build()) - if err != nil { - r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{ - "count": len(ids), - "error": err.Error(), - }) - return nil, err - } - - return result.Items, nil + return out, nil } // GetByContractNoticeID retrieves a tender by contract notice ID @@ -343,52 +360,75 @@ func (r *tenderRepository) GetByProcedureReference(ctx context.Context, contract return &result.Items[0], nil } -// GetByProcedureReferences loads tenders for multiple AI procedure refs in one query. +const procedureReferenceLookupBatchSize = 250 + +// GetByProcedureReferences loads tenders for multiple AI procedure refs using indexed notice lookups. func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error) { out := make(map[string]Tender) if len(refs) == 0 { return out, nil } - contractFolderIDs := make([]string, 0, len(refs)) + noticeIDs := make([]string, 0, len(refs)) for _, ref := range refs { folder := strings.TrimSpace(ref.ContractFolderID) notice := strings.TrimSpace(ref.NoticePublicationID) if folder == "" || notice == "" { continue } - contractFolderIDs = append(contractFolderIDs, folder) + noticeIDs = append(noticeIDs, notice) } - uniqueFolders := uniqueNonEmptyTrimmedStrings(contractFolderIDs) - if len(uniqueFolders) == 0 { + + uniqueNotices := uniqueNonEmptyTrimmedStrings(noticeIDs) + if len(uniqueNotices) == 0 { return out, nil } - findLimit := len(uniqueFolders) * 5 - if minLimit := len(refs) + len(uniqueFolders); findLimit < minLimit { - findLimit = minLimit - } - if findLimit > 100000 { - findLimit = 100000 + candidates := make([]Tender, 0, len(uniqueNotices)) + for start := 0; start < len(uniqueNotices); start += procedureReferenceLookupBatchSize { + end := start + procedureReferenceLookupBatchSize + if end > len(uniqueNotices) { + end = len(uniqueNotices) + } + + batch, err := r.findTendersByNoticePublicationIDs(ctx, uniqueNotices[start:end]) + if err != nil { + return nil, err + } + candidates = append(candidates, batch...) } - filter := bson.M{"contract_folder_id": bson.M{"$in": uniqueFolders}} + return mapProcedureReferencesToTenders(candidates, refs), nil +} + +func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context, noticeIDs []string) ([]Tender, error) { + if len(noticeIDs) == 0 { + return nil, nil + } + + filter := bson.M{ + "$or": []bson.M{ + {"notice_publication_id": bson.M{"$in": noticeIDs}}, + {"related_notice_publication_ids": bson.M{"$in": noticeIDs}}, + }, + } pagination := orm.Pagination{ - Limit: findLimit, + Limit: len(noticeIDs) * 2, SortField: "updated_at", SortOrder: -1, + SkipCount: true, } result, err := r.ormRepo.FindAll(ctx, filter, pagination) if err != nil { - r.logger.Error("Failed to get tenders by procedure references", map[string]interface{}{ - "count": len(uniqueFolders), + r.logger.Error("Failed to get tenders by notice publication IDs", map[string]interface{}{ + "count": len(noticeIDs), "error": err.Error(), }) return nil, err } - return mapProcedureReferencesToTenders(result.Items, refs), nil + return result.Items, nil } func mapProcedureReferencesToTenders(candidates []Tender, refs []ProcedureReference) map[string]Tender { diff --git a/internal/tender/service.go b/internal/tender/service.go index 5b362e0..6d2c17b 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -947,13 +947,8 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination nil } -const recommendRequestTimeout = 15 * time.Second - // Recommend retrieves AI-ranked tenders for the authenticated customer's company. func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) { - ctx, cancel := context.WithTimeout(ctx, recommendRequestTimeout) - defer cancel() - companyIDs := normalizeRecommendationCompanyScope(form.CompanyIDs) companyID := "" if form.CompanyID != nil { @@ -988,23 +983,8 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat return s.emptySearchResponse(pagination), nil } - var ( - tenderByRef map[string]Tender - excluded map[string]struct{} - excludedByCompany map[string]map[string]struct{} - ) - loadGroup, loadCtx := errgroup.WithContext(ctx) - loadGroup.Go(func() error { - var resolveErr error - tenderByRef, resolveErr = s.resolveRecommendedTenders(loadCtx, recommendations) - return resolveErr - }) - loadGroup.Go(func() error { - var exclusionErr error - excluded, excludedByCompany, exclusionErr = s.loadRecommendationExclusions(loadCtx, form, companyID, companyIDs) - return exclusionErr - }) - if err := loadGroup.Wait(); err != nil { + excluded, excludedByCompany, err := s.loadRecommendationExclusions(ctx, form, companyID, companyIDs) + if err != nil { if form.ExcludeRejectedTenders { s.logger.Error("Failed to build recommendation exclusions", map[string]interface{}{ "company_id": companyID, @@ -1017,54 +997,26 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat lang := s.pickResponseLanguage(form.Language) now := time.Now().Unix() - ordered := make([]rankedRecommendedTender, 0, len(recommendations)) - seenTenderIDs := make(map[string]struct{}, len(recommendations)) - for _, rec := range recommendations { - tenderRef := strings.TrimSpace(rec.TenderID) - tender, ok := tenderByRef[tenderRef] - if !ok { - continue - } - if len(companyIDs) > 0 { - if isRecommendedTenderExcludedForAllCompanies(tender, tenderRef, excludedByCompany, companyIDs) { - continue - } - } else if isRecommendedTenderExcluded(tender, tenderRef, excluded) { - continue - } - if form.OnlyActiveDeadlines && !tender.IsRecommendable(now) { - continue - } - if len(form.Status) > 0 && !statusAllowed(tender.Status, form.Status) { - continue - } - if markResolvedRecommendedTenderSeen(seenTenderIDs, tender) { - continue - } - - ordered = append(ordered, rankedRecommendedTender{ - tender: tender, - rank: rec.Rank, - analysis: rec.Analysis, - tenderRef: tenderRef, - }) + pageResult, err := s.buildRecommendationPage( + ctx, + recommendations, + form, + companyIDs, + excluded, + excludedByCompany, + now, + pagination.Offset, + pagination.Limit, + ) + if err != nil { + return nil, err } - total := int64(len(ordered)) - start := pagination.Offset - if start > len(ordered) { - start = len(ordered) - } - end := start + pagination.Limit - if end > len(ordered) { - end = len(ordered) - } - - paged := s.buildRecommendedTenderListResponses(ordered[start:end], lang) + paged := s.buildRecommendedTenderListResponses(pageResult.items, lang) return &SearchResponse{ Tenders: paged, - Metadata: pagination.ListMeta(total, "", end < len(ordered), start), + Metadata: pagination.ListMeta(pageResult.total, "", int(pagination.Offset)+len(paged) < int(pageResult.total), pagination.Offset), }, nil }