tm_back/internal/tender/repository.go

package tender

import (
	"context"
	"regexp"
	"strings"
	"time"
	"tm/pkg/logger"
	orm "tm/pkg/mongo"
	"tm/pkg/response"

	"go.mongodb.org/mongo-driver/v2/bson"
	"go.mongodb.org/mongo-driver/v2/mongo"
)

// TenderRepository interface defines tender data access methods
type TenderRepository interface {
	// Tender CRUD operations
	Create(ctx context.Context, tender *Tender) error
	GetByID(ctx context.Context, id string) (*Tender, error)
	GetByIDs(ctx context.Context, ids []string) ([]Tender, error)
	GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
	GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
	GetByProcedureReference(ctx context.Context, contractFolderID, noticePublicationID string) (*Tender, error)
	GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error)
	GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error)
	// GetLatestByContractFolderIDs returns the most recently updated tender per distinct contract_folder_id.
	GetLatestByContractFolderIDs(ctx context.Context, contractFolderIDs []string) (map[string]*Tender, error)
	GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error)
	FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error)
	GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
	GetTenderCountByType(ctx context.Context) (map[string]int64, error)
	GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
	Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error)
	GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
	GetPendingDocumentScrapeTenders(ctx context.Context, filter PendingDocumentScrapeFilter) ([]Tender, bool, error)
	GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
	GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error)
	Update(ctx context.Context, tender *Tender) error
	Delete(ctx context.Context, id string) error
	GetStatistics(ctx context.Context) (*TenderStatistics, error)
}

func collectionName() string {
	return "tenders"
}

// SearchPageResult is the paginated outcome of a tender search/list query.
type SearchPageResult struct {
	Items      []Tender
	TotalCount int64 // -1 when count was skipped
	NextCursor string
	HasMore    bool
	PageOffset int
}

// tenderSearchListProjection excludes large fields not needed for list/search API payloads.
func tenderSearchListProjection() bson.M {
	return bson.M{
		"content_xml":        0,
		"document_summaries": 0,
		"scraped_documents":  0,
		"selection_criteria": 0,
		"modifications":      0,
		"source_file_url":    0,
		"source_file_name":   0,
		// Not used by list response mapping; can be large per tender.
		"translations":                          0,
		"organizations":                         0,
		"review_organization":                   0,
		"processing_metadata.translated_data":   0,
		"processing_metadata.parsing_errors":    0,
		"processing_metadata.validation_errors": 0,
	}
}

// documentScraperListProjection loads only fields required by the document scraper API.
func documentScraperListProjection() bson.M {
	return bson.M{
		"contract_folder_id":    1,
		"notice_publication_id": 1,
		"document_uri":          1,
		"tender_url":            1,
		"title":                 1,
		"description":           1,
		"publication_date":      1,
		"submission_deadline":   1,
	}
}

// tenderRepository implements TenderRepository interface using MongoDB ORM
type tenderRepository struct {
	ormRepo      orm.Repository[Tender]
	mongoManager *orm.ConnectionManager
	logger       logger.Logger
}

// NewTenderRepository creates a new tender repository
func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) TenderRepository {
	// Create indexes for tenders collection
	tenderIndexes := []orm.Index{
		*orm.CreateUniqueIndex("tender_id_idx", bson.D{{Key: "tender_id", Value: 1}}),
		*orm.NewIndex("project_name_idx", bson.D{{Key: "project_name", Value: 1}}),
		*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
		// One tender row per TED ContractNoticeID when present (parallel ingest / lookup gaps).
		*orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}).
			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_notice_id")),
		*orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}),
		// One tender row per TED publication / procedure when ids are present (prevents parallel worker races).
		*orm.CreateUniqueIndex("notice_publication_id_unique", bson.D{{Key: "notice_publication_id", Value: 1}}).
			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("notice_publication_id")),
		*orm.CreateUniqueIndex("contract_folder_id_unique", bson.D{{Key: "contract_folder_id", Value: 1}}).
			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_folder_id")),
		// One tender per UBL procurement project (merges per-lot notices that share ProcurementProject/ID).
		*orm.CreateUniqueIndex("procurement_project_id_unique", bson.D{{Key: "procurement_project_id", Value: 1}}).
			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("procurement_project_id")),
		*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
		*orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}),
		*orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}),
		*orm.NewIndex("notice_type_code_idx", bson.D{{Key: "notice_type_code", Value: 1}}),
		*orm.NewIndex("form_type_idx", bson.D{{Key: "form_type", Value: 1}}),
		*orm.NewIndex("procurement_type_code_idx", bson.D{{Key: "procurement_type_code", Value: 1}}),
		*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
		*orm.NewIndex("publication_date_idx", bson.D{{Key: "publication_date", Value: -1}}),
		*orm.NewIndex("tender_deadline_idx", bson.D{{Key: "tender_deadline", Value: 1}}),
		*orm.NewIndex("estimated_value_idx", bson.D{{Key: "estimated_value", Value: -1}}),
		*orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}),
		// Stable keyset pagination for default admin list (created_at desc).
		*orm.NewIndex("created_at_id_idx", bson.D{
			{Key: "created_at", Value: -1},
			{Key: "_id", Value: -1},
		}),
		*orm.NewIndex("status_created_at_id_idx", bson.D{
			{Key: "status", Value: 1},
			{Key: "created_at", Value: -1},
			{Key: "_id", Value: -1},
		}),
		*orm.NewIndex("country_code_created_at_id_idx", bson.D{
			{Key: "country_code", Value: 1},
			{Key: "created_at", Value: -1},
			{Key: "_id", Value: -1},
		}),
		*orm.NewIndex("notice_type_code_created_at_id_idx", bson.D{
			{Key: "notice_type_code", Value: 1},
			{Key: "created_at", Value: -1},
			{Key: "_id", Value: -1},
		}),
		*orm.NewIndex("updated_at_idx", bson.D{{Key: "updated_at", Value: -1}}),
		// Compound indexes for common queries
		*orm.NewIndex("status_country_idx", bson.D{
			{Key: "status", Value: 1},
			{Key: "country_code", Value: 1},
		}),
		*orm.NewIndex("notice_type_status_idx", bson.D{
			{Key: "notice_type_code", Value: 1},
			{Key: "status", Value: 1},
		}),
		*orm.NewIndex("doc_scrape_pending_idx", bson.D{
			{Key: "country_code", Value: 1},
			{Key: "processing_metadata.documents_scraped", Value: 1},
			{Key: "tender_deadline", Value: 1},
			{Key: "created_at", Value: 1},
		}),
		*orm.NewIndex("documents_scraped_at_idx", bson.D{
			{Key: "processing_metadata.documents_scraped", Value: 1},
			{Key: "processing_metadata.documents_scraped_at", Value: -1},
		}),
		*orm.NewIndex("scraped_documents_scraped_at_idx", bson.D{
			{Key: "scraped_documents.scraped_at", Value: 1},
		}),
		// Index for CPV matching queries
		*orm.NewIndex("cpv_matching_idx", bson.D{
			{Key: "status", Value: 1},
			{Key: "main_classification", Value: 1},
			{Key: "additional_classifications", Value: 1},
			{Key: "publication_date", Value: -1},
		}),
		// Text index (legacy; list search uses $regex across title, description, and AI summary fields for unified matching)
		*orm.CreateTextIndex("search_idx", "title", "description"),
	}

	// Create indexes
	if err := mongoManager.CreateIndexes(collectionName(), tenderIndexes); err != nil {
		logger.Warn("Failed to create tender indexes", map[string]interface{}{
			"error": err.Error(),
		})
	}

	// Create ORM repositories
	tenderOrmRepo := orm.NewRepository[Tender](mongoManager.GetCollection(collectionName()), logger)

	return &tenderRepository{
		ormRepo:      tenderOrmRepo,
		mongoManager: mongoManager,
		logger:       logger,
	}
}

// Create creates a new tender
func (r *tenderRepository) Create(ctx context.Context, tender *Tender) error {
	now := time.Now().Unix()
	tender.SetCreatedAt(now)

	err := r.ormRepo.Create(ctx, tender)
	if err != nil {
		r.logger.Error("Failed to create tender", map[string]interface{}{
			"error":                 err.Error(),
			"notice_publication_id": tender.NoticePublicationID,
		})
		return err
	}

	r.logger.Info("Tender created successfully", map[string]interface{}{
		"tender_id":             tender.ID,
		"notice_publication_id": tender.NoticePublicationID,
	})

	return nil
}

// GetByID retrieves a tender by ID
func (r *tenderRepository) GetByID(ctx context.Context, id string) (*Tender, error) {
	tender, err := r.ormRepo.FindByID(ctx, id)
	if err != nil {
		r.logger.Error("Failed to get tender by ID", map[string]interface{}{
			"tender_id": id,
			"error":     err.Error(),
		})
		return nil, err
	}

	return tender, nil
}

// GetByIDs retrieves tenders by IDs in a single query.
func (r *tenderRepository) GetByIDs(ctx context.Context, ids []string) ([]Tender, error) {
	if len(ids) == 0 {
		return nil, nil
	}

	const batchSize = 500
	out := make([]Tender, 0, len(ids))
	for start := 0; start < len(ids); start += batchSize {
		end := start + batchSize
		if end > len(ids) {
			end = len(ids)
		}
		batch := ids[start:end]

		objectIDs := make([]bson.ObjectID, 0, len(batch))
		for _, id := range batch {
			objectID, err := bson.ObjectIDFromHex(id)
			if err != nil {
				return nil, err
			}
			objectIDs = append(objectIDs, objectID)
		}

		pagination := orm.Pagination{
			Limit:     len(objectIDs),
			SortField: "_id",
			SortOrder: -1,
			SkipCount: true,
		}
		result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, pagination)
		if err != nil {
			r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{
				"count": len(batch),
				"error": err.Error(),
			})
			return nil, err
		}
		out = append(out, result.Items...)
	}

	return out, nil
}

// GetByContractNoticeID retrieves a tender by contract notice ID
func (r *tenderRepository) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error) {
	filter := bson.M{"contract_notice_id": contractNoticeID}
	pagination := orm.Pagination{Limit: 1}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to get tender by contract notice ID", map[string]interface{}{
			"contract_notice_id": contractNoticeID,
			"error":              err.Error(),
		})
		return nil, err
	}

	if len(result.Items) == 0 {
		return nil, orm.ErrDocumentNotFound
	}

	return &result.Items[0], nil
}

// GetByNoticePublicationID retrieves a tender by notice publication ID.
// Matches both the canonical NoticePublicationID and any id retained in RelatedNoticePublicationIDs
// so historical TED detail links keep resolving after a tender update from a follow-up notice.
func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error) {
	filter := bson.M{
		"$or": []bson.M{
			{"notice_publication_id": noticePublicationID},
			{"related_notice_publication_ids": noticePublicationID},
		},
	}
	pagination := orm.Pagination{Limit: 1}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to get tender by notice publication ID", map[string]interface{}{
			"notice_publication_id": noticePublicationID,
			"error":                 err.Error(),
		})
		return nil, err
	}

	if len(result.Items) == 0 {
		return nil, orm.ErrDocumentNotFound
	}

	return &result.Items[0], nil
}

// GetByProcedureReference retrieves a tender by contract folder and notice publication id.
// Matches canonical notice_publication_id and related_notice_publication_ids.
func (r *tenderRepository) GetByProcedureReference(ctx context.Context, contractFolderID, noticePublicationID string) (*Tender, error) {
	contractFolderID = strings.TrimSpace(contractFolderID)
	noticePublicationID = strings.TrimSpace(noticePublicationID)
	if contractFolderID == "" || noticePublicationID == "" {
		return nil, orm.ErrDocumentNotFound
	}

	filter := bson.M{
		"contract_folder_id": contractFolderID,
		"$or": []bson.M{
			{"notice_publication_id": noticePublicationID},
			{"related_notice_publication_ids": noticePublicationID},
		},
	}
	pagination := orm.Pagination{Limit: 1, SortField: "updated_at", SortOrder: -1}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to get tender by procedure reference", map[string]interface{}{
			"contract_folder_id":    contractFolderID,
			"notice_publication_id": noticePublicationID,
			"error":                 err.Error(),
		})
		return nil, err
	}

	if len(result.Items) == 0 {
		return nil, orm.ErrDocumentNotFound
	}

	return &result.Items[0], nil
}

const procedureReferenceLookupBatchSize = 250

// GetByProcedureReferences loads tenders for multiple AI procedure refs using indexed notice lookups.
func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error) {
	out := make(map[string]Tender)
	if len(refs) == 0 {
		return out, nil
	}

	noticeIDs := make([]string, 0, len(refs))
	for _, ref := range refs {
		folder := strings.TrimSpace(ref.ContractFolderID)
		notice := strings.TrimSpace(ref.NoticePublicationID)
		if folder == "" || notice == "" {
			continue
		}
		noticeIDs = append(noticeIDs, notice)
	}

	uniqueNotices := uniqueNonEmptyTrimmedStrings(noticeIDs)
	if len(uniqueNotices) == 0 {
		return out, nil
	}

	candidates := make([]Tender, 0, len(uniqueNotices))
	for start := 0; start < len(uniqueNotices); start += procedureReferenceLookupBatchSize {
		end := start + procedureReferenceLookupBatchSize
		if end > len(uniqueNotices) {
			end = len(uniqueNotices)
		}

		batch, err := r.findTendersByNoticePublicationIDs(ctx, uniqueNotices[start:end])
		if err != nil {
			return nil, err
		}
		candidates = append(candidates, batch...)
	}

	return mapProcedureReferencesToTenders(candidates, refs), nil
}

func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context, noticeIDs []string) ([]Tender, error) {
	if len(noticeIDs) == 0 {
		return nil, nil
	}

	filter := bson.M{
		"$or": []bson.M{
			{"notice_publication_id": bson.M{"$in": noticeIDs}},
			{"related_notice_publication_ids": bson.M{"$in": noticeIDs}},
		},
	}
	pagination := orm.Pagination{
		Limit:     len(noticeIDs) * 2,
		SortField: "updated_at",
		SortOrder: -1,
		SkipCount: true,
	}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to get tenders by notice publication IDs", map[string]interface{}{
			"count": len(noticeIDs),
			"error": err.Error(),
		})
		return nil, err
	}

	return result.Items, nil
}

func mapProcedureReferencesToTenders(candidates []Tender, refs []ProcedureReference) map[string]Tender {
	byFolder := make(map[string][]Tender)
	for i := range candidates {
		folder := strings.TrimSpace(candidates[i].ContractFolderID)
		if folder == "" {
			continue
		}
		byFolder[folder] = append(byFolder[folder], candidates[i])
	}

	out := make(map[string]Tender, len(refs))
	for _, ref := range refs {
		folder := strings.TrimSpace(ref.ContractFolderID)
		for _, candidate := range byFolder[folder] {
			if tenderMatchesProcedureRef(candidate, ref.ContractFolderID, ref.NoticePublicationID) {
				out[ref.Ref] = candidate
				break
			}
		}
	}
	return out
}

// GetByContractFolderID retrieves a tender by its procedure-level identifier (ContractFolderID).
// TED groups multiple notice publications (e.g. competition notice + result notice) under the same
// ContractFolderID, so this lookup is the authoritative way to find the existing tender for a procedure
// when a follow-up notice arrives. Returns the most recently updated match if there are several.
func (r *tenderRepository) GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error) {
	filter := bson.M{"contract_folder_id": contractFolderID}
	pagination := orm.Pagination{
		Limit:     1,
		SortField: "updated_at",
		SortOrder: -1,
	}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to get tender by contract folder ID", map[string]interface{}{
			"contract_folder_id": contractFolderID,
			"error":              err.Error(),
		})
		return nil, err
	}

	if len(result.Items) == 0 {
		return nil, orm.ErrDocumentNotFound
	}

	return &result.Items[0], nil
}

// GetLatestByContractFolderIDs returns one tender per distinct contract_folder_id (latest updated_at).
func (r *tenderRepository) GetLatestByContractFolderIDs(ctx context.Context, contractFolderIDs []string) (map[string]*Tender, error) {
	out := make(map[string]*Tender)
	if len(contractFolderIDs) == 0 {
		return out, nil
	}

	uniq := uniqueNonEmptyTrimmedStrings(contractFolderIDs)
	if len(uniq) == 0 {
		return out, nil
	}

	// Allow a few rows per folder in case duplicate folder rows exist despite partial uniques.
	findLimit := len(uniq) * 3
	if findLimit < len(uniq)+1 {
		findLimit = len(uniq) + 1
	}
	if findLimit > 100000 {
		findLimit = 100000
	}

	result, err := r.ormRepo.FindAll(ctx,
		bson.M{"contract_folder_id": bson.M{"$in": uniq}},
		orm.Pagination{
			Limit:     findLimit,
			Skip:      0,
			SortField: "updated_at",
			SortOrder: -1,
		},
	)
	if err != nil {
		r.logger.Error("Failed to load tenders by contract folder ids", map[string]interface{}{
			"folder_count": len(uniq),
			"error":        err.Error(),
		})
		return nil, err
	}

	for i := range result.Items {
		t := &result.Items[i]
		cid := strings.TrimSpace(t.ContractFolderID)
		if cid == "" {
			continue
		}
		prev, ok := out[cid]
		if !ok || t.UpdatedAt > prev.UpdatedAt {
			out[cid] = t
		}
	}

	return out, nil
}

func uniqueNonEmptyTrimmedStrings(ids []string) []string {
	seen := make(map[string]struct{}, len(ids))
	out := make([]string, 0, len(ids))
	for _, id := range ids {
		id = strings.TrimSpace(id)
		if id == "" {
			continue
		}
		if _, dup := seen[id]; dup {
			continue
		}
		seen[id] = struct{}{}
		out = append(out, id)
	}
	return out
}

// GetByProcurementProjectID finds a tender by UBL ProcurementProject ID (shared across per-lot notices).
func (r *tenderRepository) GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error) {
	filter := bson.M{"procurement_project_id": procurementProjectID}
	pagination := orm.Pagination{
		Limit:     1,
		SortField: "updated_at",
		SortOrder: -1,
	}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to get tender by procurement project ID", map[string]interface{}{
			"procurement_project_id": procurementProjectID,
			"error":                  err.Error(),
		})
		return nil, err
	}

	if len(result.Items) == 0 {
		return nil, orm.ErrDocumentNotFound
	}

	return &result.Items[0], nil
}

// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted).
func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) {
	filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
	pagination := orm.Pagination{
		Limit:     limit,
		Skip:      skip,
		SortField: "_id",
		SortOrder: 1,
	}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to list tenders with content_xml", map[string]interface{}{
			"error": err.Error(),
		})
		return nil, err
	}

	return result.Items, nil
}

// Update updates an existing tender
func (r *tenderRepository) Update(ctx context.Context, tender *Tender) error {
	tender.UpdatedAt = time.Now().Unix()

	err := r.ormRepo.Update(ctx, tender)
	if err != nil {
		r.logger.Error("Failed to update tender", map[string]interface{}{
			"tender_id": tender.ID,
			"error":     err.Error(),
		})
		return err
	}

	r.logger.Info("Tender updated successfully", map[string]interface{}{
		"tender_id": tender.ID,
	})

	return nil
}

// Delete deletes a tender by ID
func (r *tenderRepository) Delete(ctx context.Context, id string) error {
	err := r.ormRepo.Delete(ctx, id)
	if err != nil {
		r.logger.Error("Failed to delete tender", map[string]interface{}{
			"tender_id": id,
			"error":     err.Error(),
		})
		return err
	}

	r.logger.Info("Tender deleted successfully", map[string]interface{}{
		"tender_id": id,
	})

	return nil
}

// Search searches tenders based on criteria
func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) {
	filter := r.buildSearchFilter(form)

	sort := "created_at"
	if pagination.SortBy != "" {
		sort = pagination.SortBy
	}

	sortOrder := "desc"
	if pagination.SortOrder != "" {
		sortOrder = pagination.SortOrder
	}

	mongoPagination, err := orm.BuildListPagination(
		pagination.Limit,
		pagination.Offset,
		pagination.Cursor,
		sort,
		sortOrder,
		filter,
		orm.ListPaginationOptions{Projection: tenderSearchListProjection()},
	)
	if err != nil {
		return nil, err
	}

	result, err := r.ormRepo.FindAll(ctx, filter, mongoPagination)
	if err != nil {
		r.logger.Error("Failed to search tenders", map[string]interface{}{
			"error": err.Error(),
		})
		return nil, err
	}

	return &SearchPageResult{
		Items:      result.Items,
		TotalCount: result.TotalCount,
		NextCursor: result.NextCursor,
		HasMore:    result.HasMore,
		PageOffset: result.PageOffset,
	}, nil
}

// GetExpiredTenders retrieves tenders that are expired
func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error) {
	filter := bson.M{
		"$or": []bson.M{
			{"status": TenderStatusExpired},
			{
				"tender_deadline": bson.M{
					"$lt": beforeDate,
				},
				"status": bson.M{"$ne": TenderStatusExpired},
			},
		},
	}

	pagination := orm.Pagination{
		Limit:     1000, // Large limit for maintenance operations
		SortField: "tender_deadline",
		SortOrder: 1,
	}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to get expired tenders", map[string]interface{}{
			"before_date": beforeDate,
			"error":       err.Error(),
		})
		return nil, err
	}

	return result.Items, nil
}

// PendingDocumentScrapeFilter defines query criteria for pending document scrape tenders.
type PendingDocumentScrapeFilter struct {
	Portals       []string
	MinWindowEnd  *int64 // when set, only tenders with an active publication submission window
	CreatedAtFrom *int64
	CreatedAtTo   *int64
	Limit         int
	Skip          int
}

func scrapePortalURLFilter(portals []string) bson.M {
	conditions := make([]bson.M, 0, len(portals)*2)
	for _, portal := range portals {
		portal = strings.TrimSpace(portal)
		if portal == "" {
			continue
		}
		regex := bson.M{"$regex": regexp.QuoteMeta(portal), "$options": "i"}
		conditions = append(conditions,
			bson.M{"document_uri": regex},
			bson.M{"tender_url": regex},
		)
	}
	if len(conditions) == 0 {
		return nil
	}
	return bson.M{"$or": conditions}
}

func publicationWindowActiveBSON(now int64) bson.M {
	return bson.M{
		"$or": []bson.M{
			{"submission_deadline": bson.M{"$gt": now}},
			{
				"$and": []bson.M{
					{"$or": []bson.M{
						{"submission_deadline": bson.M{"$exists": false}},
						{"submission_deadline": 0},
					}},
					{"publication_date": bson.M{"$gt": 0}},
				},
			},
		},
	}
}

// GetPendingDocumentScrapeTenders returns unscraped tenders whose document URL matches a supported portal.
// When MinWindowEnd is set, only tenders whose publication-based submission window is still open are returned.
func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, filter PendingDocumentScrapeFilter) ([]Tender, bool, error) {
	portalFilter := scrapePortalURLFilter(filter.Portals)
	if portalFilter == nil {
		return nil, false, nil
	}

	andFilters := []bson.M{
		{"processing_metadata.documents_scraped": bson.M{"$ne": true}},
		portalFilter,
	}

	if filter.MinWindowEnd != nil {
		andFilters = append(andFilters, publicationWindowActiveBSON(*filter.MinWindowEnd))
	}

	if filter.CreatedAtFrom != nil || filter.CreatedAtTo != nil {
		createdFilter := bson.M{}
		if filter.CreatedAtFrom != nil {
			createdFilter["$gte"] = *filter.CreatedAtFrom
		}
		if filter.CreatedAtTo != nil {
			createdFilter["$lte"] = *filter.CreatedAtTo
		}
		andFilters = append(andFilters, bson.M{"created_at": createdFilter})
	}

	query := bson.M{"$and": andFilters}

	pagination := orm.Pagination{
		Limit:      filter.Limit,
		Skip:       filter.Skip,
		SortField:  "created_at",
		SortOrder:  1,
		SkipCount:  true,
		Projection: documentScraperListProjection(),
	}

	result, err := r.ormRepo.FindAll(ctx, query, pagination)
	if err != nil {
		r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{
			"portals":         filter.Portals,
			"min_window_end":  filter.MinWindowEnd,
			"created_at_from": filter.CreatedAtFrom,
			"created_at_to":   filter.CreatedAtTo,
			"limit":           filter.Limit,
			"skip":            filter.Skip,
			"error":           err.Error(),
		})
		return nil, false, err
	}

	return result.Items, result.HasMore, nil
}

// GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized
func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
	filter := bson.M{
		"processing_metadata.documents_scraped": true,
		"$or": []bson.M{
			{"processing_metadata.documents_summarized": bson.M{"$exists": false}},
			{"processing_metadata.documents_summarized": false},
		},
	}

	pagination := orm.Pagination{
		Limit:     limit,
		Skip:      skip,
		SortField: "processing_metadata.documents_scraped_at",
		SortOrder: 1, // Oldest first (process oldest scraped documents first)
	}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
			"limit": limit,
			"skip":  skip,
			"error": err.Error(),
		})
		return nil, 0, err
	}

	return result.Items, result.TotalCount, nil
}

// GetUnTranslatedTenders returns tenders eligible for translation backfill.
func (r *tenderRepository) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) {
	_ = language

	filter := bson.M{
		"notice_publication_id": bson.M{"$ne": ""},
		"contract_folder_id":    bson.M{"$ne": ""},
		"title":                 bson.M{"$ne": ""},
		"description":           bson.M{"$ne": ""},
	}

	pagination := orm.Pagination{
		Limit:     limit,
		Skip:      skip,
		SortField: "created_at",
		SortOrder: 1, // Oldest first
	}

	result, err := r.ormRepo.FindAll(ctx, filter, pagination)
	if err != nil {
		r.logger.Error("Failed to get untranslated tenders", map[string]interface{}{
			"language": language,
			"limit":    limit,
			"skip":     skip,
			"error":    err.Error(),
		})
		return nil, 0, err
	}

	return result.Items, result.TotalCount, nil
}

// GetStatistics calculates tender statistics using aggregation
func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
	stats := &TenderStatistics{
		LastUpdated: time.Now().Unix(),
	}

	// Get total count
	total, err := r.ormRepo.Count(ctx, bson.M{})
	if err != nil {
		return nil, err
	}
	stats.TotalTenders = total

	// Get active count
	active, err := r.ormRepo.Count(ctx, bson.M{"status": TenderStatusActive})
	if err != nil {
		return nil, err
	}
	stats.ActiveTenders = active

	// Get expired count
	expired, err := r.ormRepo.Count(ctx, bson.M{"status": TenderStatusExpired})
	if err != nil {
		return nil, err
	}
	stats.ExpiredTenders = expired

	// Get statistics by country, type, and classification
	stats.TendersByCountry, _ = r.GetTenderCountByCountry(ctx)
	stats.TendersByType, _ = r.GetTenderCountByType(ctx)
	stats.TendersByClassification, _ = r.GetTenderCountByClassification(ctx)

	// Calculate value statistics using aggregation
	pipeline := mongo.Pipeline{
		{
			{Key: "$group", Value: bson.M{
				"_id":                   nil,
				"total_estimated_value": bson.M{"$sum": "$estimated_value"},
				"count":                 bson.M{"$sum": 1},
			}},
		},
	}

	result, err := r.ormRepo.Aggregate(ctx, pipeline)
	if err == nil && len(result) > 0 {
		if totalValue, ok := result[0]["total_estimated_value"].(float64); ok {
			stats.TotalEstimatedValue = totalValue
			if count, ok := result[0]["count"].(int32); ok && count > 0 {
				stats.AverageEstimatedValue = totalValue / float64(count)
			}
		}
	}

	return stats, nil
}

// GetTenderCountByCountry gets tender count grouped by country
func (r *tenderRepository) GetTenderCountByCountry(ctx context.Context) (map[string]int64, error) {
	pipeline := mongo.Pipeline{
		{
			{Key: "$group", Value: bson.M{
				"_id":   "$country_code",
				"count": bson.M{"$sum": 1},
			}},
		},
	}

	results, err := r.ormRepo.Aggregate(ctx, pipeline)
	if err != nil {
		return nil, err
	}

	counts := make(map[string]int64)
	for _, result := range results {
		if country, ok := result["_id"].(string); ok {
			if count, ok := result["count"].(int32); ok {
				counts[country] = int64(count)
			}
		}
	}

	return counts, nil
}

// GetTenderCountByType gets tender count grouped by type
func (r *tenderRepository) GetTenderCountByType(ctx context.Context) (map[string]int64, error) {
	pipeline := mongo.Pipeline{
		{
			{Key: "$group", Value: bson.M{
				"_id":   "$notice_type_code",
				"count": bson.M{"$sum": 1},
			}},
		},
	}

	results, err := r.ormRepo.Aggregate(ctx, pipeline)
	if err != nil {
		return nil, err
	}

	counts := make(map[string]int64)
	for _, result := range results {
		if noticeType, ok := result["_id"].(string); ok {
			if count, ok := result["count"].(int32); ok {
				counts[noticeType] = int64(count)
			}
		}
	}

	return counts, nil
}

// GetTenderCountByClassification gets tender count grouped by classification
func (r *tenderRepository) GetTenderCountByClassification(ctx context.Context) (map[string]int64, error) {
	pipeline := mongo.Pipeline{
		{
			{Key: "$group", Value: bson.M{
				"_id":   "$main_classification",
				"count": bson.M{"$sum": 1},
			}},
		},
		{
			{Key: "$limit", Value: 20},
		},
	}

	results, err := r.ormRepo.Aggregate(ctx, pipeline)
	if err != nil {
		return nil, err
	}

	counts := make(map[string]int64)
	for _, result := range results {
		if classification, ok := result["_id"].(string); ok {
			if count, ok := result["count"].(int32); ok {
				counts[classification] = int64(count)
			}
		}
	}

	return counts, nil
}

func dedupeNonEmptyStrings(values []string) []string {
	seen := make(map[string]struct{}, len(values))
	out := make([]string, 0, len(values))
	for _, v := range values {
		v = strings.TrimSpace(v)
		if v == "" {
			continue
		}
		if _, ok := seen[v]; ok {
			continue
		}
		seen[v] = struct{}{}
		out = append(out, v)
	}
	return out
}

// normalizeUnixTimestamp accepts seconds or milliseconds and always returns seconds.
func normalizeUnixTimestamp(value int64) int64 {
	// 1e12 is safely above current Unix seconds and in millisecond range for modern dates.
	if value > 1_000_000_000_000 {
		return value / 1000
	}
	return value
}

func exactOrRange(exact, from, to *int64) (*int64, *int64) {
	if exact != nil {
		v := normalizeUnixTimestamp(*exact)
		return &v, &v
	}

	var normalizedFrom *int64
	if from != nil {
		v := normalizeUnixTimestamp(*from)
		normalizedFrom = &v
	}

	var normalizedTo *int64
	if to != nil {
		v := normalizeUnixTimestamp(*to)
		normalizedTo = &v
	}

	return normalizedFrom, normalizedTo
}

// buildSearchFilter builds MongoDB filter from search form
func (r *tenderRepository) buildSearchFilter(form *SearchForm) bson.M {
	filter := bson.M{}

	var keywordOr []bson.M
	if form.Search != nil {
		q := strings.TrimSpace(*form.Search)
		if q != "" {
			pattern := regexp.QuoteMeta(q)
			keywordOr = []bson.M{
				{"title": bson.M{"$regex": pattern, "$options": "i"}},
				{"description": bson.M{"$regex": pattern, "$options": "i"}},
				{"document_summaries.summary": bson.M{"$regex": pattern, "$options": "i"}},
				{"ai_overall_summary": bson.M{"$regex": pattern, "$options": "i"}},
			}
		}
	}
	if form.Title != nil {
		title := strings.TrimSpace(*form.Title)
		if title != "" {
			filter["title"] = bson.M{
				"$regex":   regexp.QuoteMeta(title),
				"$options": "i",
			}
		}
	}
	if form.Description != nil {
		description := strings.TrimSpace(*form.Description)
		if description != "" {
			filter["description"] = bson.M{
				"$regex":   regexp.QuoteMeta(description),
				"$options": "i",
			}
		}
	}

	var noticeCodes []string
	if form.NoticeType != nil {
		if t := strings.TrimSpace(*form.NoticeType); t != "" {
			noticeCodes = append(noticeCodes, t)
		}
	}
	noticeCodes = append(noticeCodes, form.NoticeTypes...)
	noticeCodes = dedupeNonEmptyStrings(noticeCodes)
	if len(noticeCodes) == 1 {
		filter["notice_type_code"] = noticeCodes[0]
	} else if len(noticeCodes) > 1 {
		filter["notice_type_code"] = bson.M{"$in": noticeCodes}
	}

	formTypes := dedupeNonEmptyStrings(form.FormTypes)
	if len(formTypes) == 1 {
		filter["form_type"] = formTypes[0]
	} else if len(formTypes) > 1 {
		filter["form_type"] = bson.M{"$in": formTypes}
	}

	if form.ProcurementType != nil {
		filter["procurement_type_code"] = form.ProcurementType
	}

	countryCodes := dedupeNonEmptyStrings(form.CountryCodes)
	if form.CountryCode != nil {
		if c := strings.TrimSpace(*form.CountryCode); c != "" {
			countryCodes = append(countryCodes, c)
			countryCodes = dedupeNonEmptyStrings(countryCodes)
		}
	}
	if form.Country != nil {
		if c := strings.TrimSpace(*form.Country); c != "" {
			countryCodes = append(countryCodes, c)
			countryCodes = dedupeNonEmptyStrings(countryCodes)
		}
	}
	if len(countryCodes) == 1 {
		filter["country_code"] = countryCodes[0]
	} else if len(countryCodes) > 1 {
		filter["country_code"] = bson.M{"$in": countryCodes}
	}

	if len(form.RegionCodes) > 0 {
		filter["region_code"] = bson.M{"$in": dedupeNonEmptyStrings(form.RegionCodes)}
	}

	if form.MainClassification != nil {
		if mainClassification := strings.TrimSpace(*form.MainClassification); mainClassification != "" {
			filter["main_classification"] = mainClassification
		}
	}

	cpvCodes := dedupeNonEmptyStrings(form.Classifications)
	cpvCodes = append(cpvCodes, dedupeNonEmptyStrings(form.CpvCodes)...)
	cpvCodes = dedupeNonEmptyStrings(cpvCodes)
	var cpvOr []bson.M
	if len(cpvCodes) > 0 {
		cpvOr = []bson.M{
			{"main_classification": bson.M{"$in": cpvCodes}},
			{"additional_classifications": bson.M{"$in": cpvCodes}},
			{
				"procurement_lots": bson.M{
					"$elemMatch": bson.M{
						"$or": []bson.M{
							{"main_classification": bson.M{"$in": cpvCodes}},
							{"additional_classifications": bson.M{"$in": cpvCodes}},
						},
					},
				},
			},
		}
	}

	switch {
	case len(keywordOr) > 0 && len(cpvOr) > 0:
		filter["$and"] = []bson.M{
			{"$or": keywordOr},
			{"$or": cpvOr},
		}
	case len(keywordOr) > 0:
		filter["$or"] = keywordOr
	case len(cpvOr) > 0:
		filter["$or"] = cpvOr
	}

	if form.MinEstimatedValue != nil || form.MaxEstimatedValue != nil {
		valueFilter := bson.M{}
		if form.MinEstimatedValue != nil {
			valueFilter["$gte"] = *form.MinEstimatedValue
		}
		if form.MaxEstimatedValue != nil {
			valueFilter["$lte"] = *form.MaxEstimatedValue
		}
		filter["estimated_value"] = valueFilter
	}

	if form.Currency != "" {
		filter["currency"] = form.Currency
	}

	createdAtFrom, createdAtTo := exactOrRange(form.CreatedAt, form.CreatedAtFrom, form.CreatedAtTo)
	if createdAtFrom != nil || createdAtTo != nil {
		createdFilter := bson.M{}
		if createdAtFrom != nil {
			createdFilter["$gte"] = *createdAtFrom
		}
		if createdAtTo != nil {
			createdFilter["$lte"] = *createdAtTo
		}
		filter["created_at"] = createdFilter
	}

	deadlineFrom := form.DeadlineFrom
	if form.TenderDeadlineFrom != nil {
		deadlineFrom = form.TenderDeadlineFrom
	}
	deadlineTo := form.DeadlineTo
	if form.TenderDeadlineTo != nil {
		deadlineTo = form.TenderDeadlineTo
	}
	tenderDeadlineFrom, tenderDeadlineTo := exactOrRange(form.TenderDeadline, deadlineFrom, deadlineTo)
	if tenderDeadlineFrom != nil || tenderDeadlineTo != nil || form.OnlyActiveDeadlines {
		deadlineFilter := bson.M{}
		if form.OnlyActiveDeadlines {
			deadlineFilter["$gt"] = time.Now().Unix()
		}
		if tenderDeadlineFrom != nil {
			deadlineFilter["$gte"] = *tenderDeadlineFrom
		}
		if tenderDeadlineTo != nil {
			deadlineFilter["$lte"] = *tenderDeadlineTo
		}
		filter["tender_deadline"] = deadlineFilter
	}

	publicationDateFrom, publicationDateTo := exactOrRange(form.PublicationDate, form.PublicationDateFrom, form.PublicationDateTo)
	if publicationDateFrom != nil || publicationDateTo != nil {
		pubDateFilter := bson.M{}
		if publicationDateFrom != nil {
			pubDateFilter["$gte"] = *publicationDateFrom
		}
		if publicationDateTo != nil {
			pubDateFilter["$lte"] = *publicationDateTo
		}
		filter["publication_date"] = pubDateFilter
	}

	submissionDateFrom := form.SubmissionDateFrom
	if form.SubmissionDateAliasFrom != nil {
		submissionDateFrom = form.SubmissionDateAliasFrom
	}
	submissionDateTo := form.SubmissionDateTo
	if form.SubmissionDateAliasTo != nil {
		submissionDateTo = form.SubmissionDateAliasTo
	}
	submissionDeadlineFrom, submissionDeadlineTo := exactOrRange(form.SubmissionDeadline, submissionDateFrom, submissionDateTo)
	if submissionDeadlineFrom != nil || submissionDeadlineTo != nil {
		subFilter := bson.M{}
		if submissionDeadlineFrom != nil {
			subFilter["$gte"] = *submissionDeadlineFrom
		}
		if submissionDeadlineTo != nil {
			subFilter["$lte"] = *submissionDeadlineTo
		}
		filter["submission_deadline"] = subFilter
	}

	if len(form.Status) > 0 {
		filter["status"] = bson.M{"$in": dedupeNonEmptyStrings(form.Status)}
	}

	if len(form.Source) > 0 {
		filter["source"] = bson.M{"$in": dedupeNonEmptyStrings(form.Source)}
	}

	if form.BuyerOrganizationID != nil {
		filter["buyer_organization.id"] = *form.BuyerOrganizationID
	}

	if form.NoticePublicationID != nil {
		if noticePublicationID := strings.TrimSpace(*form.NoticePublicationID); noticePublicationID != "" {
			filter["notice_publication_id"] = noticePublicationID
		}
	}

	if len(form.Languages) > 0 {
		filter["notice_language_code"] = bson.M{"$in": dedupeNonEmptyStrings(form.Languages)}
	}

	if form.DocumentsScraped {
		folderIDs := dedupeNonEmptyStrings(form.ContractFolderIDsWithDocuments)
		scrapedClause := bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}
		if len(folderIDs) == 0 {
			scrapedClause = bson.M{"_id": bson.M{"$exists": false}}
		}
		if len(filter) == 0 {
			return scrapedClause
		}
		if and, ok := filter["$and"].([]bson.M); ok {
			filter["$and"] = append(and, scrapedClause)
		} else {
			existing := bson.M{}
			for k, v := range filter {
				existing[k] = v
			}
			filter = bson.M{
				"$and": []bson.M{existing, scrapedClause},
			}
		}
	}

	return filter
}