m.nazemi
1402 lines
41 KiB
Go
1402 lines
41 KiB
Go
package tender
|
|
|
|
import (
|
|
"context"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
"tm/pkg/logger"
|
|
orm "tm/pkg/mongo"
|
|
"tm/pkg/response"
|
|
|
|
"go.mongodb.org/mongo-driver/v2/bson"
|
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
|
)
|
|
|
|
// TenderRepository interface defines tender data access methods
|
|
type TenderRepository interface {
|
|
// Tender CRUD operations
|
|
Create(ctx context.Context, tender *Tender) error
|
|
GetByID(ctx context.Context, id string) (*Tender, error)
|
|
GetByIDs(ctx context.Context, ids []string) ([]Tender, error)
|
|
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
|
|
GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
|
|
GetByProcedureReference(ctx context.Context, contractFolderID, noticePublicationID string) (*Tender, error)
|
|
GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error)
|
|
GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error)
|
|
// GetLatestByContractFolderIDs returns the most recently updated tender per distinct contract_folder_id.
|
|
GetLatestByContractFolderIDs(ctx context.Context, contractFolderIDs []string) (map[string]*Tender, error)
|
|
GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error)
|
|
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
|
|
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
|
|
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
|
|
Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error)
|
|
GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
|
|
GetPendingDocumentScrapeTenders(ctx context.Context, filter PendingDocumentScrapeFilter) ([]Tender, bool, error)
|
|
GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
|
|
GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error)
|
|
Update(ctx context.Context, tender *Tender) error
|
|
Delete(ctx context.Context, id string) error
|
|
GetStatistics(ctx context.Context) (*TenderStatistics, error)
|
|
}
|
|
|
|
func collectionName() string {
|
|
return "tenders"
|
|
}
|
|
|
|
// SearchPageResult is the paginated outcome of a tender search/list query.
|
|
type SearchPageResult struct {
|
|
Items []Tender
|
|
TotalCount int64 // -1 when count was skipped
|
|
NextCursor string
|
|
HasMore bool
|
|
PageOffset int
|
|
}
|
|
|
|
// tenderSearchListProjection excludes large fields not needed for list/search API payloads.
|
|
func tenderSearchListProjection() bson.M {
|
|
return bson.M{
|
|
"content_xml": 0,
|
|
"document_summaries": 0,
|
|
"scraped_documents": 0,
|
|
"selection_criteria": 0,
|
|
"modifications": 0,
|
|
"source_file_url": 0,
|
|
"source_file_name": 0,
|
|
// Detail-only or bulky fields; list rows use title and summary metadata only.
|
|
"description": 0,
|
|
"ai_overall_summary": 0,
|
|
"procurement_lots": 0,
|
|
"awarded": 0,
|
|
"cancellation_reason": 0,
|
|
"suspension_reason": 0,
|
|
"translations": 0,
|
|
"organizations": 0,
|
|
"review_organization": 0,
|
|
"processing_metadata.enrichment_data": 0,
|
|
"processing_metadata.translated_data": 0,
|
|
"processing_metadata.parsing_errors": 0,
|
|
"processing_metadata.validation_errors": 0,
|
|
}
|
|
}
|
|
|
|
// documentScraperListProjection loads only fields required by the document scraper API.
|
|
func documentScraperListProjection() bson.M {
|
|
return bson.M{
|
|
"contract_folder_id": 1,
|
|
"notice_publication_id": 1,
|
|
"document_uri": 1,
|
|
"tender_url": 1,
|
|
"title": 1,
|
|
"description": 1,
|
|
"publication_date": 1,
|
|
"submission_deadline": 1,
|
|
}
|
|
}
|
|
|
|
// tenderRepository implements TenderRepository interface using MongoDB ORM
|
|
type tenderRepository struct {
|
|
ormRepo orm.Repository[Tender]
|
|
mongoManager *orm.ConnectionManager
|
|
logger logger.Logger
|
|
}
|
|
|
|
// NewTenderRepository creates a new tender repository
|
|
func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) TenderRepository {
|
|
// Create indexes for tenders collection
|
|
tenderIndexes := []orm.Index{
|
|
*orm.CreateUniqueIndex("tender_id_idx", bson.D{{Key: "tender_id", Value: 1}}),
|
|
*orm.NewIndex("project_name_idx", bson.D{{Key: "project_name", Value: 1}}),
|
|
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
|
|
// One tender row per TED ContractNoticeID when present (parallel ingest / lookup gaps).
|
|
*orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}).
|
|
WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_notice_id")),
|
|
*orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}),
|
|
// One tender row per TED publication / procedure when ids are present (prevents parallel worker races).
|
|
*orm.CreateUniqueIndex("notice_publication_id_unique", bson.D{{Key: "notice_publication_id", Value: 1}}).
|
|
WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("notice_publication_id")),
|
|
*orm.CreateUniqueIndex("contract_folder_id_unique", bson.D{{Key: "contract_folder_id", Value: 1}}).
|
|
WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_folder_id")),
|
|
// One tender per UBL procurement project (merges per-lot notices that share ProcurementProject/ID).
|
|
*orm.CreateUniqueIndex("procurement_project_id_unique", bson.D{{Key: "procurement_project_id", Value: 1}}).
|
|
WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("procurement_project_id")),
|
|
*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
|
|
*orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}),
|
|
*orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}),
|
|
*orm.NewIndex("notice_type_code_idx", bson.D{{Key: "notice_type_code", Value: 1}}),
|
|
*orm.NewIndex("form_type_idx", bson.D{{Key: "form_type", Value: 1}}),
|
|
*orm.NewIndex("procurement_type_code_idx", bson.D{{Key: "procurement_type_code", Value: 1}}),
|
|
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
|
|
*orm.NewIndex("publication_date_idx", bson.D{{Key: "publication_date", Value: -1}}),
|
|
*orm.NewIndex("tender_deadline_idx", bson.D{{Key: "tender_deadline", Value: 1}}),
|
|
*orm.NewIndex("submission_deadline_idx", bson.D{{Key: "submission_deadline", Value: 1}}),
|
|
*orm.NewIndex("closing_soon_submission_idx", bson.D{
|
|
{Key: "submission_deadline", Value: 1},
|
|
{Key: "status", Value: 1},
|
|
}),
|
|
*orm.NewIndex("closing_soon_tender_deadline_idx", bson.D{
|
|
{Key: "tender_deadline", Value: 1},
|
|
{Key: "status", Value: 1},
|
|
}),
|
|
*orm.NewIndex("estimated_value_idx", bson.D{{Key: "estimated_value", Value: -1}}),
|
|
*orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}),
|
|
// Stable keyset pagination for default admin list (created_at desc).
|
|
*orm.NewIndex("created_at_id_idx", bson.D{
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("status_created_at_id_idx", bson.D{
|
|
{Key: "status", Value: 1},
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("country_code_created_at_id_idx", bson.D{
|
|
{Key: "country_code", Value: 1},
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("notice_type_code_created_at_id_idx", bson.D{
|
|
{Key: "notice_type_code", Value: 1},
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("updated_at_idx", bson.D{{Key: "updated_at", Value: -1}}),
|
|
// Compound indexes for common queries
|
|
*orm.NewIndex("status_country_idx", bson.D{
|
|
{Key: "status", Value: 1},
|
|
{Key: "country_code", Value: 1},
|
|
}),
|
|
*orm.NewIndex("notice_type_status_idx", bson.D{
|
|
{Key: "notice_type_code", Value: 1},
|
|
{Key: "status", Value: 1},
|
|
}),
|
|
*orm.NewIndex("doc_scrape_pending_idx", bson.D{
|
|
{Key: "country_code", Value: 1},
|
|
{Key: "processing_metadata.documents_scraped", Value: 1},
|
|
{Key: "tender_deadline", Value: 1},
|
|
{Key: "created_at", Value: 1},
|
|
}),
|
|
*orm.NewIndex("documents_scraped_at_idx", bson.D{
|
|
{Key: "processing_metadata.documents_scraped", Value: 1},
|
|
{Key: "processing_metadata.documents_scraped_at", Value: -1},
|
|
}),
|
|
// Panel list: documents_scraped=true sorted by created_at desc (keyset pagination).
|
|
*orm.NewIndex("documents_scraped_created_at_id_idx", bson.D{
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}).WithPartialFilterExpression(bson.M{
|
|
"processing_metadata.documents_scraped": true,
|
|
}),
|
|
*orm.NewIndex("scraped_documents_scraped_at_idx", bson.D{
|
|
{Key: "scraped_documents.scraped_at", Value: 1},
|
|
}),
|
|
// Index for CPV matching queries
|
|
*orm.NewIndex("cpv_matching_idx", bson.D{
|
|
{Key: "status", Value: 1},
|
|
{Key: "main_classification", Value: 1},
|
|
{Key: "additional_classifications", Value: 1},
|
|
{Key: "publication_date", Value: -1},
|
|
}),
|
|
// Text index (legacy; list search uses $regex across title, description, and AI summary fields for unified matching)
|
|
*orm.CreateTextIndex("search_idx", "title", "description"),
|
|
}
|
|
|
|
// Create indexes
|
|
if err := mongoManager.CreateIndexes(collectionName(), tenderIndexes); err != nil {
|
|
logger.Warn("Failed to create tender indexes", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
}
|
|
|
|
// Create ORM repositories
|
|
tenderOrmRepo := orm.NewRepository[Tender](mongoManager.GetCollection(collectionName()), logger)
|
|
|
|
return &tenderRepository{
|
|
ormRepo: tenderOrmRepo,
|
|
mongoManager: mongoManager,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
// Create creates a new tender
|
|
func (r *tenderRepository) Create(ctx context.Context, tender *Tender) error {
|
|
now := time.Now().Unix()
|
|
tender.SetCreatedAt(now)
|
|
|
|
err := r.ormRepo.Create(ctx, tender)
|
|
if err != nil {
|
|
r.logger.Error("Failed to create tender", map[string]interface{}{
|
|
"error": err.Error(),
|
|
"notice_publication_id": tender.NoticePublicationID,
|
|
})
|
|
return err
|
|
}
|
|
|
|
r.logger.Info("Tender created successfully", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
"notice_publication_id": tender.NoticePublicationID,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetByID retrieves a tender by ID
|
|
func (r *tenderRepository) GetByID(ctx context.Context, id string) (*Tender, error) {
|
|
tender, err := r.ormRepo.FindByID(ctx, id)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by ID", map[string]interface{}{
|
|
"tender_id": id,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return tender, nil
|
|
}
|
|
|
|
// GetByIDs retrieves tenders by IDs in a single query.
|
|
func (r *tenderRepository) GetByIDs(ctx context.Context, ids []string) ([]Tender, error) {
|
|
if len(ids) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
const batchSize = 500
|
|
out := make([]Tender, 0, len(ids))
|
|
for start := 0; start < len(ids); start += batchSize {
|
|
end := start + batchSize
|
|
if end > len(ids) {
|
|
end = len(ids)
|
|
}
|
|
batch := ids[start:end]
|
|
|
|
objectIDs := make([]bson.ObjectID, 0, len(batch))
|
|
for _, id := range batch {
|
|
objectID, err := bson.ObjectIDFromHex(strings.TrimSpace(id))
|
|
if err != nil {
|
|
r.logger.Warn("Skipping invalid tender ID in batch lookup", map[string]interface{}{
|
|
"tender_id": id,
|
|
"error": err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
objectIDs = append(objectIDs, objectID)
|
|
}
|
|
if len(objectIDs) == 0 {
|
|
continue
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: len(objectIDs),
|
|
SortField: "_id",
|
|
SortOrder: -1,
|
|
SkipCount: true,
|
|
}
|
|
result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{
|
|
"count": len(batch),
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
out = append(out, result.Items...)
|
|
}
|
|
|
|
return out, nil
|
|
}
|
|
|
|
// GetByContractNoticeID retrieves a tender by contract notice ID
|
|
func (r *tenderRepository) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error) {
|
|
filter := bson.M{"contract_notice_id": contractNoticeID}
|
|
pagination := orm.Pagination{Limit: 1}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by contract notice ID", map[string]interface{}{
|
|
"contract_notice_id": contractNoticeID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// GetByNoticePublicationID retrieves a tender by notice publication ID.
|
|
// Matches both the canonical NoticePublicationID and any id retained in RelatedNoticePublicationIDs
|
|
// so historical TED detail links keep resolving after a tender update from a follow-up notice.
|
|
func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error) {
|
|
filter := bson.M{
|
|
"$or": []bson.M{
|
|
{"notice_publication_id": noticePublicationID},
|
|
{"related_notice_publication_ids": noticePublicationID},
|
|
},
|
|
}
|
|
pagination := orm.Pagination{Limit: 1}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by notice publication ID", map[string]interface{}{
|
|
"notice_publication_id": noticePublicationID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// GetByProcedureReference retrieves a tender by contract folder and notice publication id.
|
|
// Matches canonical notice_publication_id and related_notice_publication_ids.
|
|
func (r *tenderRepository) GetByProcedureReference(ctx context.Context, contractFolderID, noticePublicationID string) (*Tender, error) {
|
|
contractFolderID = strings.TrimSpace(contractFolderID)
|
|
noticePublicationID = strings.TrimSpace(noticePublicationID)
|
|
if contractFolderID == "" || noticePublicationID == "" {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
filter := bson.M{
|
|
"contract_folder_id": contractFolderID,
|
|
"$or": []bson.M{
|
|
{"notice_publication_id": noticePublicationID},
|
|
{"related_notice_publication_ids": noticePublicationID},
|
|
},
|
|
}
|
|
pagination := orm.Pagination{Limit: 1, SortField: "updated_at", SortOrder: -1}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by procedure reference", map[string]interface{}{
|
|
"contract_folder_id": contractFolderID,
|
|
"notice_publication_id": noticePublicationID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
const procedureReferenceLookupBatchSize = 250
|
|
|
|
const contractFolderLookupBatchSize = 250
|
|
|
|
// GetByProcedureReferences loads tenders for multiple AI procedure refs.
|
|
// It combines indexed notice lookups with contract-folder candidates so older notice
|
|
// references still resolve after tender merges (same behavior as before the perf refactor).
|
|
func (r *tenderRepository) GetByProcedureReferences(ctx context.Context, refs []ProcedureReference) (map[string]Tender, error) {
|
|
out := make(map[string]Tender)
|
|
if len(refs) == 0 {
|
|
return out, nil
|
|
}
|
|
|
|
folderIDs := make([]string, 0, len(refs))
|
|
noticeIDs := make([]string, 0, len(refs))
|
|
for _, ref := range refs {
|
|
folder := strings.TrimSpace(ref.ContractFolderID)
|
|
notice := strings.TrimSpace(ref.NoticePublicationID)
|
|
if folder == "" || notice == "" {
|
|
continue
|
|
}
|
|
folderIDs = append(folderIDs, folder)
|
|
noticeIDs = append(noticeIDs, notice)
|
|
}
|
|
|
|
uniqueNotices := uniqueNonEmptyTrimmedStrings(noticeIDs)
|
|
noticeCandidates := make([]Tender, 0, len(uniqueNotices))
|
|
for start := 0; start < len(uniqueNotices); start += procedureReferenceLookupBatchSize {
|
|
end := start + procedureReferenceLookupBatchSize
|
|
if end > len(uniqueNotices) {
|
|
end = len(uniqueNotices)
|
|
}
|
|
|
|
batch, err := r.findTendersByNoticePublicationIDs(ctx, uniqueNotices[start:end])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
noticeCandidates = append(noticeCandidates, batch...)
|
|
}
|
|
|
|
folderCandidates, err := r.findTendersByContractFolderIDs(ctx, folderIDs)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
candidates := mergeTenderCandidates(noticeCandidates, folderCandidates)
|
|
return mapProcedureReferencesToTenders(candidates, refs), nil
|
|
}
|
|
|
|
func (r *tenderRepository) findTendersByContractFolderIDs(ctx context.Context, folderIDs []string) ([]Tender, error) {
|
|
uniqueFolders := uniqueNonEmptyTrimmedStrings(folderIDs)
|
|
if len(uniqueFolders) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
out := make([]Tender, 0, len(uniqueFolders))
|
|
for start := 0; start < len(uniqueFolders); start += contractFolderLookupBatchSize {
|
|
end := start + contractFolderLookupBatchSize
|
|
if end > len(uniqueFolders) {
|
|
end = len(uniqueFolders)
|
|
}
|
|
batch := uniqueFolders[start:end]
|
|
|
|
findLimit := len(batch) * 5
|
|
if minLimit := len(batch) + 1; findLimit < minLimit {
|
|
findLimit = minLimit
|
|
}
|
|
|
|
filter := bson.M{"contract_folder_id": bson.M{"$in": batch}}
|
|
pagination := orm.Pagination{
|
|
Limit: findLimit,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
SkipCount: true,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tenders by contract folder IDs", map[string]interface{}{
|
|
"count": len(batch),
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
out = append(out, result.Items...)
|
|
}
|
|
|
|
return out, nil
|
|
}
|
|
|
|
func mergeTenderCandidates(groups ...[]Tender) []Tender {
|
|
seen := make(map[string]struct{})
|
|
out := make([]Tender, 0)
|
|
for _, group := range groups {
|
|
for i := range group {
|
|
tenderID := strings.TrimSpace(group[i].GetID())
|
|
if tenderID == "" {
|
|
out = append(out, group[i])
|
|
continue
|
|
}
|
|
if _, exists := seen[tenderID]; exists {
|
|
continue
|
|
}
|
|
seen[tenderID] = struct{}{}
|
|
out = append(out, group[i])
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (r *tenderRepository) findTendersByNoticePublicationIDs(ctx context.Context, noticeIDs []string) ([]Tender, error) {
|
|
if len(noticeIDs) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
filter := bson.M{
|
|
"$or": []bson.M{
|
|
{"notice_publication_id": bson.M{"$in": noticeIDs}},
|
|
{"related_notice_publication_ids": bson.M{"$in": noticeIDs}},
|
|
},
|
|
}
|
|
pagination := orm.Pagination{
|
|
Limit: len(noticeIDs) * 3,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
SkipCount: true,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tenders by notice publication IDs", map[string]interface{}{
|
|
"count": len(noticeIDs),
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return result.Items, nil
|
|
}
|
|
|
|
func mapProcedureReferencesToTenders(candidates []Tender, refs []ProcedureReference) map[string]Tender {
|
|
byFolder := make(map[string][]Tender)
|
|
for i := range candidates {
|
|
folder := strings.TrimSpace(candidates[i].ContractFolderID)
|
|
if folder == "" {
|
|
continue
|
|
}
|
|
byFolder[folder] = append(byFolder[folder], candidates[i])
|
|
}
|
|
|
|
out := make(map[string]Tender, len(refs))
|
|
for _, ref := range refs {
|
|
folder := strings.TrimSpace(ref.ContractFolderID)
|
|
for _, candidate := range byFolder[folder] {
|
|
if tenderMatchesProcedureRef(candidate, ref.ContractFolderID, ref.NoticePublicationID) {
|
|
out[ref.Ref] = candidate
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// GetByContractFolderID retrieves a tender by its procedure-level identifier (ContractFolderID).
|
|
// TED groups multiple notice publications (e.g. competition notice + result notice) under the same
|
|
// ContractFolderID, so this lookup is the authoritative way to find the existing tender for a procedure
|
|
// when a follow-up notice arrives. Returns the most recently updated match if there are several.
|
|
func (r *tenderRepository) GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error) {
|
|
filter := bson.M{"contract_folder_id": contractFolderID}
|
|
pagination := orm.Pagination{
|
|
Limit: 1,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by contract folder ID", map[string]interface{}{
|
|
"contract_folder_id": contractFolderID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// GetLatestByContractFolderIDs returns one tender per distinct contract_folder_id (latest updated_at).
|
|
func (r *tenderRepository) GetLatestByContractFolderIDs(ctx context.Context, contractFolderIDs []string) (map[string]*Tender, error) {
|
|
out := make(map[string]*Tender)
|
|
if len(contractFolderIDs) == 0 {
|
|
return out, nil
|
|
}
|
|
|
|
uniq := uniqueNonEmptyTrimmedStrings(contractFolderIDs)
|
|
if len(uniq) == 0 {
|
|
return out, nil
|
|
}
|
|
|
|
// Allow a few rows per folder in case duplicate folder rows exist despite partial uniques.
|
|
findLimit := len(uniq) * 3
|
|
if findLimit < len(uniq)+1 {
|
|
findLimit = len(uniq) + 1
|
|
}
|
|
if findLimit > 100000 {
|
|
findLimit = 100000
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx,
|
|
bson.M{"contract_folder_id": bson.M{"$in": uniq}},
|
|
orm.Pagination{
|
|
Limit: findLimit,
|
|
Skip: 0,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
},
|
|
)
|
|
if err != nil {
|
|
r.logger.Error("Failed to load tenders by contract folder ids", map[string]interface{}{
|
|
"folder_count": len(uniq),
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
for i := range result.Items {
|
|
t := &result.Items[i]
|
|
cid := strings.TrimSpace(t.ContractFolderID)
|
|
if cid == "" {
|
|
continue
|
|
}
|
|
prev, ok := out[cid]
|
|
if !ok || t.UpdatedAt > prev.UpdatedAt {
|
|
out[cid] = t
|
|
}
|
|
}
|
|
|
|
return out, nil
|
|
}
|
|
|
|
func uniqueNonEmptyTrimmedStrings(ids []string) []string {
|
|
seen := make(map[string]struct{}, len(ids))
|
|
out := make([]string, 0, len(ids))
|
|
for _, id := range ids {
|
|
id = strings.TrimSpace(id)
|
|
if id == "" {
|
|
continue
|
|
}
|
|
if _, dup := seen[id]; dup {
|
|
continue
|
|
}
|
|
seen[id] = struct{}{}
|
|
out = append(out, id)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// GetByProcurementProjectID finds a tender by UBL ProcurementProject ID (shared across per-lot notices).
|
|
func (r *tenderRepository) GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error) {
|
|
filter := bson.M{"procurement_project_id": procurementProjectID}
|
|
pagination := orm.Pagination{
|
|
Limit: 1,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by procurement project ID", map[string]interface{}{
|
|
"procurement_project_id": procurementProjectID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// Update updates an existing tender
|
|
func (r *tenderRepository) Update(ctx context.Context, tender *Tender) error {
|
|
tender.UpdatedAt = time.Now().Unix()
|
|
|
|
err := r.ormRepo.Update(ctx, tender)
|
|
if err != nil {
|
|
r.logger.Error("Failed to update tender", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
"error": err.Error(),
|
|
})
|
|
return err
|
|
}
|
|
|
|
r.logger.Info("Tender updated successfully", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// Delete deletes a tender by ID
|
|
func (r *tenderRepository) Delete(ctx context.Context, id string) error {
|
|
err := r.ormRepo.Delete(ctx, id)
|
|
if err != nil {
|
|
r.logger.Error("Failed to delete tender", map[string]interface{}{
|
|
"tender_id": id,
|
|
"error": err.Error(),
|
|
})
|
|
return err
|
|
}
|
|
|
|
r.logger.Info("Tender deleted successfully", map[string]interface{}{
|
|
"tender_id": id,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// Search searches tenders based on criteria
|
|
func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) {
|
|
filter := r.buildSearchFilter(form)
|
|
|
|
sort := "created_at"
|
|
if pagination.SortBy != "" {
|
|
sort = pagination.SortBy
|
|
}
|
|
|
|
sortOrder := "desc"
|
|
if pagination.SortOrder != "" {
|
|
sortOrder = pagination.SortOrder
|
|
}
|
|
|
|
listOpts := orm.ListPaginationOptions{Projection: tenderSearchListProjection()}
|
|
if pagination.IncludeTotal {
|
|
if pagination.Cursor != "" {
|
|
listOpts.IncludeCount = true
|
|
}
|
|
} else {
|
|
listOpts.SkipCount = true
|
|
}
|
|
|
|
mongoPagination, err := orm.BuildListPagination(
|
|
pagination.Limit,
|
|
pagination.Offset,
|
|
pagination.Cursor,
|
|
sort,
|
|
sortOrder,
|
|
filter,
|
|
listOpts,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, mongoPagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to search tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return &SearchPageResult{
|
|
Items: result.Items,
|
|
TotalCount: result.TotalCount,
|
|
NextCursor: result.NextCursor,
|
|
HasMore: result.HasMore,
|
|
PageOffset: result.PageOffset,
|
|
}, nil
|
|
}
|
|
|
|
// GetExpiredTenders retrieves tenders that are expired
|
|
func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error) {
|
|
filter := bson.M{
|
|
"$or": []bson.M{
|
|
{"status": TenderStatusExpired},
|
|
{
|
|
"tender_deadline": bson.M{
|
|
"$lt": beforeDate,
|
|
},
|
|
"status": bson.M{"$ne": TenderStatusExpired},
|
|
},
|
|
},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: 1000, // Large limit for maintenance operations
|
|
SortField: "tender_deadline",
|
|
SortOrder: 1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get expired tenders", map[string]interface{}{
|
|
"before_date": beforeDate,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return result.Items, nil
|
|
}
|
|
|
|
// PendingDocumentScrapeFilter defines query criteria for pending document scrape tenders.
|
|
type PendingDocumentScrapeFilter struct {
|
|
Portals []string
|
|
MinWindowEnd *int64 // when set, only tenders with an active publication submission window
|
|
CreatedAtFrom *int64
|
|
CreatedAtTo *int64
|
|
Limit int
|
|
Skip int
|
|
}
|
|
|
|
func scrapePortalURLFilter(portals []string) bson.M {
|
|
conditions := make([]bson.M, 0, len(portals)*2)
|
|
for _, portal := range portals {
|
|
portal = strings.TrimSpace(portal)
|
|
if portal == "" {
|
|
continue
|
|
}
|
|
regex := bson.M{"$regex": regexp.QuoteMeta(portal), "$options": "i"}
|
|
conditions = append(conditions,
|
|
bson.M{"document_uri": regex},
|
|
bson.M{"tender_url": regex},
|
|
)
|
|
}
|
|
if len(conditions) == 0 {
|
|
return nil
|
|
}
|
|
return bson.M{"$or": conditions}
|
|
}
|
|
|
|
func publicationWindowActiveBSON(now int64) bson.M {
|
|
return bson.M{
|
|
"$or": []bson.M{
|
|
{"submission_deadline": bson.M{"$gt": now}},
|
|
{
|
|
"$and": []bson.M{
|
|
{"$or": []bson.M{
|
|
{"submission_deadline": bson.M{"$exists": false}},
|
|
{"submission_deadline": 0},
|
|
}},
|
|
{"publication_date": bson.M{"$gt": 0}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// GetPendingDocumentScrapeTenders returns unscraped tenders whose document URL matches a supported portal.
|
|
// When MinWindowEnd is set, only tenders whose publication-based submission window is still open are returned.
|
|
func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, filter PendingDocumentScrapeFilter) ([]Tender, bool, error) {
|
|
portalFilter := scrapePortalURLFilter(filter.Portals)
|
|
if portalFilter == nil {
|
|
return nil, false, nil
|
|
}
|
|
|
|
andFilters := []bson.M{
|
|
{"processing_metadata.documents_scraped": bson.M{"$ne": true}},
|
|
portalFilter,
|
|
}
|
|
|
|
if filter.MinWindowEnd != nil {
|
|
andFilters = append(andFilters, publicationWindowActiveBSON(*filter.MinWindowEnd))
|
|
}
|
|
|
|
if filter.CreatedAtFrom != nil || filter.CreatedAtTo != nil {
|
|
createdFilter := bson.M{}
|
|
if filter.CreatedAtFrom != nil {
|
|
createdFilter["$gte"] = *filter.CreatedAtFrom
|
|
}
|
|
if filter.CreatedAtTo != nil {
|
|
createdFilter["$lte"] = *filter.CreatedAtTo
|
|
}
|
|
andFilters = append(andFilters, bson.M{"created_at": createdFilter})
|
|
}
|
|
|
|
query := bson.M{"$and": andFilters}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: filter.Limit,
|
|
Skip: filter.Skip,
|
|
SortField: "created_at",
|
|
SortOrder: 1,
|
|
SkipCount: true,
|
|
Projection: documentScraperListProjection(),
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, query, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{
|
|
"portals": filter.Portals,
|
|
"min_window_end": filter.MinWindowEnd,
|
|
"created_at_from": filter.CreatedAtFrom,
|
|
"created_at_to": filter.CreatedAtTo,
|
|
"limit": filter.Limit,
|
|
"skip": filter.Skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, false, err
|
|
}
|
|
|
|
return result.Items, result.HasMore, nil
|
|
}
|
|
|
|
// GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized
|
|
func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
|
|
filter := bson.M{
|
|
"processing_metadata.documents_scraped": true,
|
|
"$or": []bson.M{
|
|
{"processing_metadata.documents_summarized": bson.M{"$exists": false}},
|
|
{"processing_metadata.documents_summarized": false},
|
|
},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "processing_metadata.documents_scraped_at",
|
|
SortOrder: 1, // Oldest first (process oldest scraped documents first)
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
|
|
"limit": limit,
|
|
"skip": skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, err
|
|
}
|
|
|
|
return result.Items, result.TotalCount, nil
|
|
}
|
|
|
|
// GetUnTranslatedTenders returns tenders eligible for translation backfill.
|
|
func (r *tenderRepository) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) {
|
|
_ = language
|
|
|
|
filter := bson.M{
|
|
"notice_publication_id": bson.M{"$ne": ""},
|
|
"contract_folder_id": bson.M{"$ne": ""},
|
|
"title": bson.M{"$ne": ""},
|
|
"description": bson.M{"$ne": ""},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "created_at",
|
|
SortOrder: 1, // Oldest first
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get untranslated tenders", map[string]interface{}{
|
|
"language": language,
|
|
"limit": limit,
|
|
"skip": skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, err
|
|
}
|
|
|
|
return result.Items, result.TotalCount, nil
|
|
}
|
|
|
|
// GetStatistics calculates tender statistics using aggregation
|
|
func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
|
|
stats := &TenderStatistics{
|
|
LastUpdated: time.Now().Unix(),
|
|
}
|
|
|
|
// Get total count
|
|
total, err := r.ormRepo.Count(ctx, bson.M{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stats.TotalTenders = total
|
|
|
|
// Get active count
|
|
active, err := r.ormRepo.Count(ctx, bson.M{"status": TenderStatusActive})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stats.ActiveTenders = active
|
|
|
|
// Get expired count
|
|
expired, err := r.ormRepo.Count(ctx, bson.M{"status": TenderStatusExpired})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stats.ExpiredTenders = expired
|
|
|
|
// Get statistics by country, type, and classification
|
|
stats.TendersByCountry, _ = r.GetTenderCountByCountry(ctx)
|
|
stats.TendersByType, _ = r.GetTenderCountByType(ctx)
|
|
stats.TendersByClassification, _ = r.GetTenderCountByClassification(ctx)
|
|
|
|
// Calculate value statistics using aggregation
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": nil,
|
|
"total_estimated_value": bson.M{"$sum": "$estimated_value"},
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
}
|
|
|
|
result, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err == nil && len(result) > 0 {
|
|
if totalValue, ok := result[0]["total_estimated_value"].(float64); ok {
|
|
stats.TotalEstimatedValue = totalValue
|
|
if count, ok := result[0]["count"].(int32); ok && count > 0 {
|
|
stats.AverageEstimatedValue = totalValue / float64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return stats, nil
|
|
}
|
|
|
|
// GetTenderCountByCountry gets tender count grouped by country
|
|
func (r *tenderRepository) GetTenderCountByCountry(ctx context.Context) (map[string]int64, error) {
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": "$country_code",
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
}
|
|
|
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
counts := make(map[string]int64)
|
|
for _, result := range results {
|
|
if country, ok := result["_id"].(string); ok {
|
|
if count, ok := result["count"].(int32); ok {
|
|
counts[country] = int64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return counts, nil
|
|
}
|
|
|
|
// GetTenderCountByType gets tender count grouped by type
|
|
func (r *tenderRepository) GetTenderCountByType(ctx context.Context) (map[string]int64, error) {
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": "$notice_type_code",
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
}
|
|
|
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
counts := make(map[string]int64)
|
|
for _, result := range results {
|
|
if noticeType, ok := result["_id"].(string); ok {
|
|
if count, ok := result["count"].(int32); ok {
|
|
counts[noticeType] = int64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return counts, nil
|
|
}
|
|
|
|
// GetTenderCountByClassification gets tender count grouped by classification
|
|
func (r *tenderRepository) GetTenderCountByClassification(ctx context.Context) (map[string]int64, error) {
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": "$main_classification",
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
{
|
|
{Key: "$limit", Value: 20},
|
|
},
|
|
}
|
|
|
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
counts := make(map[string]int64)
|
|
for _, result := range results {
|
|
if classification, ok := result["_id"].(string); ok {
|
|
if count, ok := result["count"].(int32); ok {
|
|
counts[classification] = int64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return counts, nil
|
|
}
|
|
|
|
func dedupeNonEmptyStrings(values []string) []string {
|
|
seen := make(map[string]struct{}, len(values))
|
|
out := make([]string, 0, len(values))
|
|
for _, v := range values {
|
|
v = strings.TrimSpace(v)
|
|
if v == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[v]; ok {
|
|
continue
|
|
}
|
|
seen[v] = struct{}{}
|
|
out = append(out, v)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// normalizeUnixTimestamp accepts seconds or milliseconds and always returns seconds.
|
|
func normalizeUnixTimestamp(value int64) int64 {
|
|
// 1e12 is safely above current Unix seconds and in millisecond range for modern dates.
|
|
if value > 1_000_000_000_000 {
|
|
return value / 1000
|
|
}
|
|
return value
|
|
}
|
|
|
|
func exactOrRange(exact, from, to *int64) (*int64, *int64) {
|
|
if exact != nil {
|
|
v := normalizeUnixTimestamp(*exact)
|
|
return &v, &v
|
|
}
|
|
|
|
var normalizedFrom *int64
|
|
if from != nil {
|
|
v := normalizeUnixTimestamp(*from)
|
|
normalizedFrom = &v
|
|
}
|
|
|
|
var normalizedTo *int64
|
|
if to != nil {
|
|
v := normalizeUnixTimestamp(*to)
|
|
normalizedTo = &v
|
|
}
|
|
|
|
return normalizedFrom, normalizedTo
|
|
}
|
|
|
|
// buildSearchFilter builds MongoDB filter from search form
|
|
func (r *tenderRepository) buildSearchFilter(form *SearchForm) bson.M {
|
|
filter := bson.M{}
|
|
|
|
var keywordOr []bson.M
|
|
if form.Search != nil {
|
|
q := strings.TrimSpace(*form.Search)
|
|
if q != "" {
|
|
pattern := regexp.QuoteMeta(q)
|
|
keywordOr = []bson.M{
|
|
{"title": bson.M{"$regex": pattern, "$options": "i"}},
|
|
{"description": bson.M{"$regex": pattern, "$options": "i"}},
|
|
{"document_summaries.summary": bson.M{"$regex": pattern, "$options": "i"}},
|
|
{"ai_overall_summary": bson.M{"$regex": pattern, "$options": "i"}},
|
|
}
|
|
}
|
|
}
|
|
if form.Title != nil {
|
|
title := strings.TrimSpace(*form.Title)
|
|
if title != "" {
|
|
filter["title"] = bson.M{
|
|
"$regex": regexp.QuoteMeta(title),
|
|
"$options": "i",
|
|
}
|
|
}
|
|
}
|
|
if form.Description != nil {
|
|
description := strings.TrimSpace(*form.Description)
|
|
if description != "" {
|
|
filter["description"] = bson.M{
|
|
"$regex": regexp.QuoteMeta(description),
|
|
"$options": "i",
|
|
}
|
|
}
|
|
}
|
|
|
|
var noticeCodes []string
|
|
if form.NoticeType != nil {
|
|
if t := strings.TrimSpace(*form.NoticeType); t != "" {
|
|
noticeCodes = append(noticeCodes, t)
|
|
}
|
|
}
|
|
noticeCodes = append(noticeCodes, form.NoticeTypes...)
|
|
noticeCodes = dedupeNonEmptyStrings(noticeCodes)
|
|
if len(noticeCodes) == 1 {
|
|
filter["notice_type_code"] = noticeCodes[0]
|
|
} else if len(noticeCodes) > 1 {
|
|
filter["notice_type_code"] = bson.M{"$in": noticeCodes}
|
|
}
|
|
|
|
formTypes := dedupeNonEmptyStrings(form.FormTypes)
|
|
if len(formTypes) == 1 {
|
|
filter["form_type"] = formTypes[0]
|
|
} else if len(formTypes) > 1 {
|
|
filter["form_type"] = bson.M{"$in": formTypes}
|
|
}
|
|
|
|
if form.ProcurementType != nil {
|
|
filter["procurement_type_code"] = form.ProcurementType
|
|
}
|
|
|
|
countryCodes := dedupeNonEmptyStrings(form.CountryCodes)
|
|
if form.CountryCode != nil {
|
|
if c := strings.TrimSpace(*form.CountryCode); c != "" {
|
|
countryCodes = append(countryCodes, c)
|
|
countryCodes = dedupeNonEmptyStrings(countryCodes)
|
|
}
|
|
}
|
|
if form.Country != nil {
|
|
if c := strings.TrimSpace(*form.Country); c != "" {
|
|
countryCodes = append(countryCodes, c)
|
|
countryCodes = dedupeNonEmptyStrings(countryCodes)
|
|
}
|
|
}
|
|
if len(countryCodes) == 1 {
|
|
filter["country_code"] = countryCodes[0]
|
|
} else if len(countryCodes) > 1 {
|
|
filter["country_code"] = bson.M{"$in": countryCodes}
|
|
}
|
|
|
|
if len(form.RegionCodes) > 0 {
|
|
filter["region_code"] = bson.M{"$in": dedupeNonEmptyStrings(form.RegionCodes)}
|
|
}
|
|
|
|
if form.MainClassification != nil {
|
|
if mainClassification := strings.TrimSpace(*form.MainClassification); mainClassification != "" {
|
|
filter["main_classification"] = mainClassification
|
|
}
|
|
}
|
|
|
|
cpvCodes := dedupeNonEmptyStrings(form.Classifications)
|
|
cpvCodes = append(cpvCodes, dedupeNonEmptyStrings(form.CpvCodes)...)
|
|
cpvCodes = dedupeNonEmptyStrings(cpvCodes)
|
|
var cpvOr []bson.M
|
|
if len(cpvCodes) > 0 {
|
|
cpvOr = []bson.M{
|
|
{"main_classification": bson.M{"$in": cpvCodes}},
|
|
{"additional_classifications": bson.M{"$in": cpvCodes}},
|
|
{
|
|
"procurement_lots": bson.M{
|
|
"$elemMatch": bson.M{
|
|
"$or": []bson.M{
|
|
{"main_classification": bson.M{"$in": cpvCodes}},
|
|
{"additional_classifications": bson.M{"$in": cpvCodes}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
switch {
|
|
case len(keywordOr) > 0 && len(cpvOr) > 0:
|
|
filter["$and"] = []bson.M{
|
|
{"$or": keywordOr},
|
|
{"$or": cpvOr},
|
|
}
|
|
case len(keywordOr) > 0:
|
|
filter["$or"] = keywordOr
|
|
case len(cpvOr) > 0:
|
|
filter["$or"] = cpvOr
|
|
}
|
|
|
|
if form.MinEstimatedValue != nil || form.MaxEstimatedValue != nil {
|
|
valueFilter := bson.M{}
|
|
if form.MinEstimatedValue != nil {
|
|
valueFilter["$gte"] = *form.MinEstimatedValue
|
|
}
|
|
if form.MaxEstimatedValue != nil {
|
|
valueFilter["$lte"] = *form.MaxEstimatedValue
|
|
}
|
|
filter["estimated_value"] = valueFilter
|
|
}
|
|
|
|
if form.Currency != "" {
|
|
filter["currency"] = form.Currency
|
|
}
|
|
|
|
createdAtFrom, createdAtTo := exactOrRange(form.CreatedAt, form.CreatedAtFrom, form.CreatedAtTo)
|
|
if createdAtFrom != nil || createdAtTo != nil {
|
|
createdFilter := bson.M{}
|
|
if createdAtFrom != nil {
|
|
createdFilter["$gte"] = *createdAtFrom
|
|
}
|
|
if createdAtTo != nil {
|
|
createdFilter["$lte"] = *createdAtTo
|
|
}
|
|
filter["created_at"] = createdFilter
|
|
}
|
|
|
|
deadlineFrom := form.DeadlineFrom
|
|
if form.TenderDeadlineFrom != nil {
|
|
deadlineFrom = form.TenderDeadlineFrom
|
|
}
|
|
deadlineTo := form.DeadlineTo
|
|
if form.TenderDeadlineTo != nil {
|
|
deadlineTo = form.TenderDeadlineTo
|
|
}
|
|
tenderDeadlineFrom, tenderDeadlineTo := exactOrRange(form.TenderDeadline, deadlineFrom, deadlineTo)
|
|
if tenderDeadlineFrom != nil || tenderDeadlineTo != nil || form.OnlyActiveDeadlines {
|
|
deadlineFilter := bson.M{}
|
|
if form.OnlyActiveDeadlines {
|
|
deadlineFilter["$gt"] = time.Now().Unix()
|
|
}
|
|
if tenderDeadlineFrom != nil {
|
|
deadlineFilter["$gte"] = *tenderDeadlineFrom
|
|
}
|
|
if tenderDeadlineTo != nil {
|
|
deadlineFilter["$lte"] = *tenderDeadlineTo
|
|
}
|
|
filter["tender_deadline"] = deadlineFilter
|
|
}
|
|
|
|
publicationDateFrom, publicationDateTo := exactOrRange(form.PublicationDate, form.PublicationDateFrom, form.PublicationDateTo)
|
|
if publicationDateFrom != nil || publicationDateTo != nil {
|
|
pubDateFilter := bson.M{}
|
|
if publicationDateFrom != nil {
|
|
pubDateFilter["$gte"] = *publicationDateFrom
|
|
}
|
|
if publicationDateTo != nil {
|
|
pubDateFilter["$lte"] = *publicationDateTo
|
|
}
|
|
filter["publication_date"] = pubDateFilter
|
|
}
|
|
|
|
submissionDateFrom := form.SubmissionDateFrom
|
|
if form.SubmissionDateAliasFrom != nil {
|
|
submissionDateFrom = form.SubmissionDateAliasFrom
|
|
}
|
|
submissionDateTo := form.SubmissionDateTo
|
|
if form.SubmissionDateAliasTo != nil {
|
|
submissionDateTo = form.SubmissionDateAliasTo
|
|
}
|
|
submissionDeadlineFrom, submissionDeadlineTo := exactOrRange(form.SubmissionDeadline, submissionDateFrom, submissionDateTo)
|
|
if submissionDeadlineFrom != nil || submissionDeadlineTo != nil {
|
|
subFilter := bson.M{}
|
|
if submissionDeadlineFrom != nil {
|
|
subFilter["$gte"] = *submissionDeadlineFrom
|
|
}
|
|
if submissionDeadlineTo != nil {
|
|
subFilter["$lte"] = *submissionDeadlineTo
|
|
}
|
|
filter["submission_deadline"] = subFilter
|
|
}
|
|
|
|
if len(form.Status) > 0 {
|
|
filter["status"] = bson.M{"$in": dedupeNonEmptyStrings(form.Status)}
|
|
}
|
|
|
|
if len(form.Source) > 0 {
|
|
filter["source"] = bson.M{"$in": dedupeNonEmptyStrings(form.Source)}
|
|
}
|
|
|
|
if form.BuyerOrganizationID != nil {
|
|
filter["buyer_organization.id"] = *form.BuyerOrganizationID
|
|
}
|
|
|
|
if form.NoticePublicationID != nil {
|
|
if noticePublicationID := strings.TrimSpace(*form.NoticePublicationID); noticePublicationID != "" {
|
|
filter["notice_publication_id"] = noticePublicationID
|
|
}
|
|
}
|
|
|
|
if len(form.Languages) > 0 {
|
|
filter["notice_language_code"] = bson.M{"$in": dedupeNonEmptyStrings(form.Languages)}
|
|
}
|
|
|
|
if form.DocumentsScraped {
|
|
scrapedClause := bson.M{"processing_metadata.documents_scraped": true}
|
|
if len(filter) == 0 {
|
|
return scrapedClause
|
|
}
|
|
if and, ok := filter["$and"].([]bson.M); ok {
|
|
filter["$and"] = append(and, scrapedClause)
|
|
} else {
|
|
existing := bson.M{}
|
|
for k, v := range filter {
|
|
existing[k] = v
|
|
}
|
|
filter = bson.M{
|
|
"$and": []bson.M{existing, scrapedClause},
|
|
}
|
|
}
|
|
}
|
|
|
|
return filter
|
|
}
|