Mazyar
1006 lines
30 KiB
Go
1006 lines
30 KiB
Go
package tender
|
|
|
|
import (
|
|
"context"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
"tm/pkg/logger"
|
|
orm "tm/pkg/mongo"
|
|
"tm/pkg/response"
|
|
|
|
"go.mongodb.org/mongo-driver/v2/bson"
|
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
|
)
|
|
|
|
// TenderRepository interface defines tender data access methods
|
|
type TenderRepository interface {
|
|
// Tender CRUD operations
|
|
Create(ctx context.Context, tender *Tender) error
|
|
GetByID(ctx context.Context, id string) (*Tender, error)
|
|
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
|
|
GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
|
|
GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error)
|
|
GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error)
|
|
FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error)
|
|
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
|
|
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
|
|
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
|
|
Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error)
|
|
GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
|
|
GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
|
|
GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error)
|
|
GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
|
|
GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error)
|
|
Update(ctx context.Context, tender *Tender) error
|
|
Delete(ctx context.Context, id string) error
|
|
GetStatistics(ctx context.Context) (*TenderStatistics, error)
|
|
}
|
|
|
|
func collectionName() string {
|
|
return "tenders"
|
|
}
|
|
|
|
|
|
// SearchPageResult is the paginated outcome of a tender search/list query.
|
|
type SearchPageResult struct {
|
|
Items []Tender
|
|
TotalCount int64 // -1 when count was skipped
|
|
NextCursor string
|
|
HasMore bool
|
|
PageOffset int
|
|
}
|
|
|
|
// tenderSearchListProjection excludes large fields not needed for list/search API payloads.
|
|
func tenderSearchListProjection() bson.M {
|
|
return bson.M{
|
|
"content_xml": 0,
|
|
"document_summaries": 0,
|
|
"scraped_documents": 0,
|
|
"selection_criteria": 0,
|
|
"modifications": 0,
|
|
"source_file_url": 0,
|
|
"source_file_name": 0,
|
|
}
|
|
}
|
|
|
|
// documentScraperListProjection loads only fields required by the document scraper API.
|
|
func documentScraperListProjection() bson.M {
|
|
return bson.M{
|
|
"contract_folder_id": 1,
|
|
"notice_publication_id": 1,
|
|
"document_uri": 1,
|
|
"tender_url": 1,
|
|
"title": 1,
|
|
"description": 1,
|
|
}
|
|
}
|
|
|
|
// tenderRepository implements TenderRepository interface using MongoDB ORM
|
|
type tenderRepository struct {
|
|
ormRepo orm.Repository[Tender]
|
|
mongoManager *orm.ConnectionManager
|
|
logger logger.Logger
|
|
}
|
|
|
|
// NewTenderRepository creates a new tender repository
|
|
func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) TenderRepository {
|
|
// Create indexes for tenders collection
|
|
tenderIndexes := []orm.Index{
|
|
*orm.CreateUniqueIndex("tender_id_idx", bson.D{{Key: "tender_id", Value: 1}}),
|
|
*orm.NewIndex("project_name_idx", bson.D{{Key: "project_name", Value: 1}}),
|
|
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
|
|
// One tender row per TED ContractNoticeID when present (parallel ingest / lookup gaps).
|
|
*orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}).
|
|
WithPartialFilterExpression(bson.M{"contract_notice_id": bson.M{"$type": "string", "$ne": ""}}),
|
|
*orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}),
|
|
// One tender row per TED publication / procedure when ids are present (prevents parallel worker races).
|
|
*orm.CreateUniqueIndex("notice_publication_id_unique", bson.D{{Key: "notice_publication_id", Value: 1}}).
|
|
WithPartialFilterExpression(bson.M{"notice_publication_id": bson.M{"$type": "string", "$ne": ""}}),
|
|
*orm.CreateUniqueIndex("contract_folder_id_unique", bson.D{{Key: "contract_folder_id", Value: 1}}).
|
|
WithPartialFilterExpression(bson.M{"contract_folder_id": bson.M{"$type": "string", "$ne": ""}}),
|
|
// One tender per UBL procurement project (merges per-lot notices that share ProcurementProject/ID).
|
|
*orm.CreateUniqueIndex("procurement_project_id_unique", bson.D{{Key: "procurement_project_id", Value: 1}}).
|
|
WithPartialFilterExpression(bson.M{"procurement_project_id": bson.M{"$type": "string", "$ne": ""}}),
|
|
*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
|
|
*orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}),
|
|
*orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}),
|
|
*orm.NewIndex("notice_type_code_idx", bson.D{{Key: "notice_type_code", Value: 1}}),
|
|
*orm.NewIndex("form_type_idx", bson.D{{Key: "form_type", Value: 1}}),
|
|
*orm.NewIndex("procurement_type_code_idx", bson.D{{Key: "procurement_type_code", Value: 1}}),
|
|
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
|
|
*orm.NewIndex("publication_date_idx", bson.D{{Key: "publication_date", Value: -1}}),
|
|
*orm.NewIndex("tender_deadline_idx", bson.D{{Key: "tender_deadline", Value: 1}}),
|
|
*orm.NewIndex("estimated_value_idx", bson.D{{Key: "estimated_value", Value: -1}}),
|
|
*orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}),
|
|
// Stable keyset pagination for default admin list (created_at desc).
|
|
*orm.NewIndex("created_at_id_idx", bson.D{
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("status_created_at_id_idx", bson.D{
|
|
{Key: "status", Value: 1},
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("country_code_created_at_id_idx", bson.D{
|
|
{Key: "country_code", Value: 1},
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("notice_type_code_created_at_id_idx", bson.D{
|
|
{Key: "notice_type_code", Value: 1},
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("updated_at_idx", bson.D{{Key: "updated_at", Value: -1}}),
|
|
// Compound indexes for common queries
|
|
*orm.NewIndex("status_country_idx", bson.D{
|
|
{Key: "status", Value: 1},
|
|
{Key: "country_code", Value: 1},
|
|
}),
|
|
*orm.NewIndex("notice_type_status_idx", bson.D{
|
|
{Key: "notice_type_code", Value: 1},
|
|
{Key: "status", Value: 1},
|
|
}),
|
|
*orm.NewIndex("doc_scrape_pending_idx", bson.D{
|
|
{Key: "country_code", Value: 1},
|
|
{Key: "processing_metadata.documents_scraped", Value: 1},
|
|
{Key: "tender_deadline", Value: 1},
|
|
{Key: "created_at", Value: 1},
|
|
}),
|
|
// Index for CPV matching queries
|
|
*orm.NewIndex("cpv_matching_idx", bson.D{
|
|
{Key: "status", Value: 1},
|
|
{Key: "main_classification", Value: 1},
|
|
{Key: "additional_classifications", Value: 1},
|
|
{Key: "publication_date", Value: -1},
|
|
}),
|
|
// Text index (legacy; list search uses $regex across title, description, and AI summary fields for unified matching)
|
|
*orm.CreateTextIndex("search_idx", "title", "description"),
|
|
}
|
|
|
|
// Create indexes
|
|
if err := mongoManager.CreateIndexes(collectionName(), tenderIndexes); err != nil {
|
|
logger.Warn("Failed to create tender indexes", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
}
|
|
|
|
// Create ORM repositories
|
|
tenderOrmRepo := orm.NewRepository[Tender](mongoManager.GetCollection(collectionName()), logger)
|
|
|
|
return &tenderRepository{
|
|
ormRepo: tenderOrmRepo,
|
|
mongoManager: mongoManager,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
// Create creates a new tender
|
|
func (r *tenderRepository) Create(ctx context.Context, tender *Tender) error {
|
|
now := time.Now().Unix()
|
|
tender.SetCreatedAt(now)
|
|
|
|
err := r.ormRepo.Create(ctx, tender)
|
|
if err != nil {
|
|
r.logger.Error("Failed to create tender", map[string]interface{}{
|
|
"error": err.Error(),
|
|
"notice_publication_id": tender.NoticePublicationID,
|
|
})
|
|
return err
|
|
}
|
|
|
|
r.logger.Info("Tender created successfully", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
"notice_publication_id": tender.NoticePublicationID,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetByID retrieves a tender by ID
|
|
func (r *tenderRepository) GetByID(ctx context.Context, id string) (*Tender, error) {
|
|
tender, err := r.ormRepo.FindByID(ctx, id)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by ID", map[string]interface{}{
|
|
"tender_id": id,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return tender, nil
|
|
}
|
|
|
|
// GetByContractNoticeID retrieves a tender by contract notice ID
|
|
func (r *tenderRepository) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error) {
|
|
filter := bson.M{"contract_notice_id": contractNoticeID}
|
|
pagination := orm.Pagination{Limit: 1}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by contract notice ID", map[string]interface{}{
|
|
"contract_notice_id": contractNoticeID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// GetByNoticePublicationID retrieves a tender by notice publication ID.
|
|
// Matches both the canonical NoticePublicationID and any id retained in RelatedNoticePublicationIDs
|
|
// so historical TED detail links keep resolving after a tender update from a follow-up notice.
|
|
func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error) {
|
|
filter := bson.M{
|
|
"$or": []bson.M{
|
|
{"notice_publication_id": noticePublicationID},
|
|
{"related_notice_publication_ids": noticePublicationID},
|
|
},
|
|
}
|
|
pagination := orm.Pagination{Limit: 1}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by notice publication ID", map[string]interface{}{
|
|
"notice_publication_id": noticePublicationID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// GetByContractFolderID retrieves a tender by its procedure-level identifier (ContractFolderID).
|
|
// TED groups multiple notice publications (e.g. competition notice + result notice) under the same
|
|
// ContractFolderID, so this lookup is the authoritative way to find the existing tender for a procedure
|
|
// when a follow-up notice arrives. Returns the most recently updated match if there are several.
|
|
func (r *tenderRepository) GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error) {
|
|
filter := bson.M{"contract_folder_id": contractFolderID}
|
|
pagination := orm.Pagination{
|
|
Limit: 1,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by contract folder ID", map[string]interface{}{
|
|
"contract_folder_id": contractFolderID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// GetByProcurementProjectID finds a tender by UBL ProcurementProject ID (shared across per-lot notices).
|
|
func (r *tenderRepository) GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error) {
|
|
filter := bson.M{"procurement_project_id": procurementProjectID}
|
|
pagination := orm.Pagination{
|
|
Limit: 1,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by procurement project ID", map[string]interface{}{
|
|
"procurement_project_id": procurementProjectID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted).
|
|
func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) {
|
|
filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "_id",
|
|
SortOrder: 1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to list tenders with content_xml", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return result.Items, nil
|
|
}
|
|
|
|
// Update updates an existing tender
|
|
func (r *tenderRepository) Update(ctx context.Context, tender *Tender) error {
|
|
tender.UpdatedAt = time.Now().Unix()
|
|
|
|
err := r.ormRepo.Update(ctx, tender)
|
|
if err != nil {
|
|
r.logger.Error("Failed to update tender", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
"error": err.Error(),
|
|
})
|
|
return err
|
|
}
|
|
|
|
r.logger.Info("Tender updated successfully", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// Delete deletes a tender by ID
|
|
func (r *tenderRepository) Delete(ctx context.Context, id string) error {
|
|
err := r.ormRepo.Delete(ctx, id)
|
|
if err != nil {
|
|
r.logger.Error("Failed to delete tender", map[string]interface{}{
|
|
"tender_id": id,
|
|
"error": err.Error(),
|
|
})
|
|
return err
|
|
}
|
|
|
|
r.logger.Info("Tender deleted successfully", map[string]interface{}{
|
|
"tender_id": id,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// Search searches tenders based on criteria
|
|
func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) {
|
|
filter := r.buildSearchFilter(form)
|
|
|
|
sort := "created_at"
|
|
if pagination.SortBy != "" {
|
|
sort = pagination.SortBy
|
|
}
|
|
|
|
sortOrder := "desc"
|
|
if pagination.SortOrder != "" {
|
|
sortOrder = pagination.SortOrder
|
|
}
|
|
|
|
mongoPagination, err := orm.BuildListPagination(
|
|
pagination.Limit,
|
|
pagination.Offset,
|
|
pagination.Cursor,
|
|
sort,
|
|
sortOrder,
|
|
filter,
|
|
orm.ListPaginationOptions{Projection: tenderSearchListProjection()},
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, mongoPagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to search tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return &SearchPageResult{
|
|
Items: result.Items,
|
|
TotalCount: result.TotalCount,
|
|
NextCursor: result.NextCursor,
|
|
HasMore: result.HasMore,
|
|
PageOffset: result.PageOffset,
|
|
}, nil
|
|
}
|
|
|
|
// GetExpiredTenders retrieves tenders that are expired
|
|
func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error) {
|
|
filter := bson.M{
|
|
"$or": []bson.M{
|
|
{"status": TenderStatusExpired},
|
|
{
|
|
"tender_deadline": bson.M{
|
|
"$lt": beforeDate,
|
|
},
|
|
"status": bson.M{"$ne": TenderStatusExpired},
|
|
},
|
|
},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: 1000, // Large limit for maintenance operations
|
|
SortField: "tender_deadline",
|
|
SortOrder: 1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get expired tenders", map[string]interface{}{
|
|
"before_date": beforeDate,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return result.Items, nil
|
|
}
|
|
|
|
// GetUnScrapedTenders retrieves tenders that haven't been processed for document scraping
|
|
func (r *tenderRepository) GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
|
|
filter := bson.M{
|
|
"$or": []bson.M{
|
|
{"processing_metadata.documents_scraped": bson.M{"$exists": false}},
|
|
{"processing_metadata.documents_scraped": false},
|
|
{"processing_metadata.documents_scraped": nil},
|
|
},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "created_at",
|
|
SortOrder: 1, // Oldest first
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get un-scraped tenders", map[string]interface{}{
|
|
"limit": limit,
|
|
"skip": skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, err
|
|
}
|
|
|
|
return result.Items, result.TotalCount, nil
|
|
}
|
|
|
|
// GetPendingDocumentScrapeTenders returns unscraped tenders for one country with an active deadline.
|
|
func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, countryCode string, minDeadline int64, limit, skip int) ([]Tender, bool, error) {
|
|
filter := bson.M{
|
|
"country_code": countryCode,
|
|
"tender_deadline": bson.M{
|
|
"$gt": minDeadline,
|
|
},
|
|
"processing_metadata.documents_scraped": bson.M{"$ne": true},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "created_at",
|
|
SortOrder: 1,
|
|
SkipCount: true,
|
|
Projection: documentScraperListProjection(),
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{
|
|
"country_code": countryCode,
|
|
"min_deadline": minDeadline,
|
|
"limit": limit,
|
|
"skip": skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, false, err
|
|
}
|
|
|
|
return result.Items, result.HasMore, nil
|
|
}
|
|
|
|
// GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized
|
|
func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
|
|
filter := bson.M{
|
|
"processing_metadata.documents_scraped": true,
|
|
"$or": []bson.M{
|
|
{"processing_metadata.documents_summarized": bson.M{"$exists": false}},
|
|
{"processing_metadata.documents_summarized": false},
|
|
},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "processing_metadata.documents_scraped_at",
|
|
SortOrder: 1, // Oldest first (process oldest scraped documents first)
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
|
|
"limit": limit,
|
|
"skip": skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, err
|
|
}
|
|
|
|
return result.Items, result.TotalCount, nil
|
|
}
|
|
|
|
// GetUnTranslatedTenders retrieves tenders that don't have a translation for the given language yet.
|
|
func (r *tenderRepository) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) {
|
|
translationKey := "translations." + language
|
|
metadataKey := "processing_metadata.translated_data." + language
|
|
|
|
filter := bson.M{
|
|
"notice_publication_id": bson.M{"$ne": ""},
|
|
"title": bson.M{"$ne": ""},
|
|
"description": bson.M{"$ne": ""},
|
|
"$and": []bson.M{
|
|
{
|
|
"$or": []bson.M{
|
|
{translationKey: bson.M{"$exists": false}},
|
|
{translationKey: nil},
|
|
},
|
|
},
|
|
{
|
|
"$or": []bson.M{
|
|
{metadataKey: bson.M{"$exists": false}},
|
|
{metadataKey: bson.M{"$ne": "done"}},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "created_at",
|
|
SortOrder: 1, // Oldest first
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get untranslated tenders", map[string]interface{}{
|
|
"language": language,
|
|
"limit": limit,
|
|
"skip": skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, err
|
|
}
|
|
|
|
return result.Items, result.TotalCount, nil
|
|
}
|
|
|
|
// GetStatistics calculates tender statistics using aggregation
|
|
func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
|
|
stats := &TenderStatistics{
|
|
LastUpdated: time.Now().Unix(),
|
|
}
|
|
|
|
// Get total count
|
|
total, err := r.ormRepo.Count(ctx, bson.M{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stats.TotalTenders = total
|
|
|
|
// Get active count
|
|
active, err := r.ormRepo.Count(ctx, bson.M{"status": TenderStatusActive})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stats.ActiveTenders = active
|
|
|
|
// Get expired count
|
|
expired, err := r.ormRepo.Count(ctx, bson.M{"status": TenderStatusExpired})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stats.ExpiredTenders = expired
|
|
|
|
// Get statistics by country, type, and classification
|
|
stats.TendersByCountry, _ = r.GetTenderCountByCountry(ctx)
|
|
stats.TendersByType, _ = r.GetTenderCountByType(ctx)
|
|
stats.TendersByClassification, _ = r.GetTenderCountByClassification(ctx)
|
|
|
|
// Calculate value statistics using aggregation
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": nil,
|
|
"total_estimated_value": bson.M{"$sum": "$estimated_value"},
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
}
|
|
|
|
result, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err == nil && len(result) > 0 {
|
|
if totalValue, ok := result[0]["total_estimated_value"].(float64); ok {
|
|
stats.TotalEstimatedValue = totalValue
|
|
if count, ok := result[0]["count"].(int32); ok && count > 0 {
|
|
stats.AverageEstimatedValue = totalValue / float64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return stats, nil
|
|
}
|
|
|
|
// GetTenderCountByCountry gets tender count grouped by country
|
|
func (r *tenderRepository) GetTenderCountByCountry(ctx context.Context) (map[string]int64, error) {
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": "$country_code",
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
}
|
|
|
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
counts := make(map[string]int64)
|
|
for _, result := range results {
|
|
if country, ok := result["_id"].(string); ok {
|
|
if count, ok := result["count"].(int32); ok {
|
|
counts[country] = int64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return counts, nil
|
|
}
|
|
|
|
// GetTenderCountByType gets tender count grouped by type
|
|
func (r *tenderRepository) GetTenderCountByType(ctx context.Context) (map[string]int64, error) {
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": "$notice_type_code",
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
}
|
|
|
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
counts := make(map[string]int64)
|
|
for _, result := range results {
|
|
if noticeType, ok := result["_id"].(string); ok {
|
|
if count, ok := result["count"].(int32); ok {
|
|
counts[noticeType] = int64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return counts, nil
|
|
}
|
|
|
|
// GetTenderCountByClassification gets tender count grouped by classification
|
|
func (r *tenderRepository) GetTenderCountByClassification(ctx context.Context) (map[string]int64, error) {
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": "$main_classification",
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
{
|
|
{Key: "$limit", Value: 20},
|
|
},
|
|
}
|
|
|
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
counts := make(map[string]int64)
|
|
for _, result := range results {
|
|
if classification, ok := result["_id"].(string); ok {
|
|
if count, ok := result["count"].(int32); ok {
|
|
counts[classification] = int64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return counts, nil
|
|
}
|
|
|
|
func dedupeNonEmptyStrings(values []string) []string {
|
|
seen := make(map[string]struct{}, len(values))
|
|
out := make([]string, 0, len(values))
|
|
for _, v := range values {
|
|
v = strings.TrimSpace(v)
|
|
if v == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[v]; ok {
|
|
continue
|
|
}
|
|
seen[v] = struct{}{}
|
|
out = append(out, v)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// normalizeUnixTimestamp accepts seconds or milliseconds and always returns seconds.
|
|
func normalizeUnixTimestamp(value int64) int64 {
|
|
// 1e12 is safely above current Unix seconds and in millisecond range for modern dates.
|
|
if value > 1_000_000_000_000 {
|
|
return value / 1000
|
|
}
|
|
return value
|
|
}
|
|
|
|
func exactOrRange(exact, from, to *int64) (*int64, *int64) {
|
|
if exact != nil {
|
|
v := normalizeUnixTimestamp(*exact)
|
|
return &v, &v
|
|
}
|
|
|
|
var normalizedFrom *int64
|
|
if from != nil {
|
|
v := normalizeUnixTimestamp(*from)
|
|
normalizedFrom = &v
|
|
}
|
|
|
|
var normalizedTo *int64
|
|
if to != nil {
|
|
v := normalizeUnixTimestamp(*to)
|
|
normalizedTo = &v
|
|
}
|
|
|
|
return normalizedFrom, normalizedTo
|
|
}
|
|
|
|
// buildSearchFilter builds MongoDB filter from search form
|
|
func (r *tenderRepository) buildSearchFilter(form *SearchForm) bson.M {
|
|
filter := bson.M{}
|
|
|
|
var keywordOr []bson.M
|
|
if form.Search != nil {
|
|
q := strings.TrimSpace(*form.Search)
|
|
if q != "" {
|
|
pattern := regexp.QuoteMeta(q)
|
|
keywordOr = []bson.M{
|
|
{"title": bson.M{"$regex": pattern, "$options": "i"}},
|
|
{"description": bson.M{"$regex": pattern, "$options": "i"}},
|
|
{"document_summaries.summary": bson.M{"$regex": pattern, "$options": "i"}},
|
|
{"ai_overall_summary": bson.M{"$regex": pattern, "$options": "i"}},
|
|
}
|
|
}
|
|
}
|
|
if form.Title != nil {
|
|
title := strings.TrimSpace(*form.Title)
|
|
if title != "" {
|
|
filter["title"] = bson.M{
|
|
"$regex": regexp.QuoteMeta(title),
|
|
"$options": "i",
|
|
}
|
|
}
|
|
}
|
|
if form.Description != nil {
|
|
description := strings.TrimSpace(*form.Description)
|
|
if description != "" {
|
|
filter["description"] = bson.M{
|
|
"$regex": regexp.QuoteMeta(description),
|
|
"$options": "i",
|
|
}
|
|
}
|
|
}
|
|
|
|
var noticeCodes []string
|
|
if form.NoticeType != nil {
|
|
if t := strings.TrimSpace(*form.NoticeType); t != "" {
|
|
noticeCodes = append(noticeCodes, t)
|
|
}
|
|
}
|
|
noticeCodes = append(noticeCodes, form.NoticeTypes...)
|
|
noticeCodes = dedupeNonEmptyStrings(noticeCodes)
|
|
if len(noticeCodes) == 1 {
|
|
filter["notice_type_code"] = noticeCodes[0]
|
|
} else if len(noticeCodes) > 1 {
|
|
filter["notice_type_code"] = bson.M{"$in": noticeCodes}
|
|
}
|
|
|
|
formTypes := dedupeNonEmptyStrings(form.FormTypes)
|
|
if len(formTypes) == 1 {
|
|
filter["form_type"] = formTypes[0]
|
|
} else if len(formTypes) > 1 {
|
|
filter["form_type"] = bson.M{"$in": formTypes}
|
|
}
|
|
|
|
if form.ProcurementType != nil {
|
|
filter["procurement_type_code"] = form.ProcurementType
|
|
}
|
|
|
|
countryCodes := dedupeNonEmptyStrings(form.CountryCodes)
|
|
if form.CountryCode != nil {
|
|
if c := strings.TrimSpace(*form.CountryCode); c != "" {
|
|
countryCodes = append(countryCodes, c)
|
|
countryCodes = dedupeNonEmptyStrings(countryCodes)
|
|
}
|
|
}
|
|
if form.Country != nil {
|
|
if c := strings.TrimSpace(*form.Country); c != "" {
|
|
countryCodes = append(countryCodes, c)
|
|
countryCodes = dedupeNonEmptyStrings(countryCodes)
|
|
}
|
|
}
|
|
if len(countryCodes) == 1 {
|
|
filter["country_code"] = countryCodes[0]
|
|
} else if len(countryCodes) > 1 {
|
|
filter["country_code"] = bson.M{"$in": countryCodes}
|
|
}
|
|
|
|
if len(form.RegionCodes) > 0 {
|
|
filter["region_code"] = bson.M{"$in": dedupeNonEmptyStrings(form.RegionCodes)}
|
|
}
|
|
|
|
if form.MainClassification != nil {
|
|
if mainClassification := strings.TrimSpace(*form.MainClassification); mainClassification != "" {
|
|
filter["main_classification"] = mainClassification
|
|
}
|
|
}
|
|
|
|
cpvCodes := dedupeNonEmptyStrings(form.Classifications)
|
|
cpvCodes = append(cpvCodes, dedupeNonEmptyStrings(form.CpvCodes)...)
|
|
cpvCodes = dedupeNonEmptyStrings(cpvCodes)
|
|
var cpvOr []bson.M
|
|
if len(cpvCodes) > 0 {
|
|
cpvOr = []bson.M{
|
|
{"main_classification": bson.M{"$in": cpvCodes}},
|
|
{"additional_classifications": bson.M{"$in": cpvCodes}},
|
|
{
|
|
"procurement_lots": bson.M{
|
|
"$elemMatch": bson.M{
|
|
"$or": []bson.M{
|
|
{"main_classification": bson.M{"$in": cpvCodes}},
|
|
{"additional_classifications": bson.M{"$in": cpvCodes}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
switch {
|
|
case len(keywordOr) > 0 && len(cpvOr) > 0:
|
|
filter["$and"] = []bson.M{
|
|
{"$or": keywordOr},
|
|
{"$or": cpvOr},
|
|
}
|
|
case len(keywordOr) > 0:
|
|
filter["$or"] = keywordOr
|
|
case len(cpvOr) > 0:
|
|
filter["$or"] = cpvOr
|
|
}
|
|
|
|
if form.MinEstimatedValue != nil || form.MaxEstimatedValue != nil {
|
|
valueFilter := bson.M{}
|
|
if form.MinEstimatedValue != nil {
|
|
valueFilter["$gte"] = *form.MinEstimatedValue
|
|
}
|
|
if form.MaxEstimatedValue != nil {
|
|
valueFilter["$lte"] = *form.MaxEstimatedValue
|
|
}
|
|
filter["estimated_value"] = valueFilter
|
|
}
|
|
|
|
if form.Currency != "" {
|
|
filter["currency"] = form.Currency
|
|
}
|
|
|
|
createdAtFrom, createdAtTo := exactOrRange(form.CreatedAt, form.CreatedAtFrom, form.CreatedAtTo)
|
|
if createdAtFrom != nil || createdAtTo != nil {
|
|
createdFilter := bson.M{}
|
|
if createdAtFrom != nil {
|
|
createdFilter["$gte"] = *createdAtFrom
|
|
}
|
|
if createdAtTo != nil {
|
|
createdFilter["$lte"] = *createdAtTo
|
|
}
|
|
filter["created_at"] = createdFilter
|
|
}
|
|
|
|
deadlineFrom := form.DeadlineFrom
|
|
if form.TenderDeadlineFrom != nil {
|
|
deadlineFrom = form.TenderDeadlineFrom
|
|
}
|
|
deadlineTo := form.DeadlineTo
|
|
if form.TenderDeadlineTo != nil {
|
|
deadlineTo = form.TenderDeadlineTo
|
|
}
|
|
tenderDeadlineFrom, tenderDeadlineTo := exactOrRange(form.TenderDeadline, deadlineFrom, deadlineTo)
|
|
if tenderDeadlineFrom != nil || tenderDeadlineTo != nil || form.OnlyActiveDeadlines {
|
|
deadlineFilter := bson.M{}
|
|
if form.OnlyActiveDeadlines {
|
|
deadlineFilter["$gt"] = time.Now().Unix()
|
|
}
|
|
if tenderDeadlineFrom != nil {
|
|
deadlineFilter["$gte"] = *tenderDeadlineFrom
|
|
}
|
|
if tenderDeadlineTo != nil {
|
|
deadlineFilter["$lte"] = *tenderDeadlineTo
|
|
}
|
|
filter["tender_deadline"] = deadlineFilter
|
|
}
|
|
|
|
publicationDateFrom, publicationDateTo := exactOrRange(form.PublicationDate, form.PublicationDateFrom, form.PublicationDateTo)
|
|
if publicationDateFrom != nil || publicationDateTo != nil {
|
|
pubDateFilter := bson.M{}
|
|
if publicationDateFrom != nil {
|
|
pubDateFilter["$gte"] = *publicationDateFrom
|
|
}
|
|
if publicationDateTo != nil {
|
|
pubDateFilter["$lte"] = *publicationDateTo
|
|
}
|
|
filter["publication_date"] = pubDateFilter
|
|
}
|
|
|
|
submissionDateFrom := form.SubmissionDateFrom
|
|
if form.SubmissionDateAliasFrom != nil {
|
|
submissionDateFrom = form.SubmissionDateAliasFrom
|
|
}
|
|
submissionDateTo := form.SubmissionDateTo
|
|
if form.SubmissionDateAliasTo != nil {
|
|
submissionDateTo = form.SubmissionDateAliasTo
|
|
}
|
|
submissionDeadlineFrom, submissionDeadlineTo := exactOrRange(form.SubmissionDeadline, submissionDateFrom, submissionDateTo)
|
|
if submissionDeadlineFrom != nil || submissionDeadlineTo != nil {
|
|
subFilter := bson.M{}
|
|
if submissionDeadlineFrom != nil {
|
|
subFilter["$gte"] = *submissionDeadlineFrom
|
|
}
|
|
if submissionDeadlineTo != nil {
|
|
subFilter["$lte"] = *submissionDeadlineTo
|
|
}
|
|
filter["submission_deadline"] = subFilter
|
|
}
|
|
|
|
if len(form.Status) > 0 {
|
|
filter["status"] = bson.M{"$in": dedupeNonEmptyStrings(form.Status)}
|
|
}
|
|
|
|
if len(form.Source) > 0 {
|
|
filter["source"] = bson.M{"$in": dedupeNonEmptyStrings(form.Source)}
|
|
}
|
|
|
|
if form.BuyerOrganizationID != nil {
|
|
filter["buyer_organization.id"] = *form.BuyerOrganizationID
|
|
}
|
|
|
|
if len(form.Languages) > 0 {
|
|
filter["notice_language_code"] = bson.M{"$in": dedupeNonEmptyStrings(form.Languages)}
|
|
}
|
|
|
|
return filter
|
|
}
|