Mazyar
31d8efef0b
continuous-integration/drone/push Build is passing
- Added `NoticePublicationID` field to the `SearchForm` for filtering tenders by TED notice publication ID. - Updated Swagger documentation in the handler methods to include the new `notice_publication_id` parameter for relevant endpoints. - Modified the repository's search filter to incorporate the new `NoticePublicationID` field, allowing for more precise search queries. This update improves the search capabilities within the tender management system, enhancing user experience and data retrieval accuracy.
1099 lines
32 KiB
Go
1099 lines
32 KiB
Go
package tender
|
|
|
|
import (
|
|
"context"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
"tm/pkg/logger"
|
|
orm "tm/pkg/mongo"
|
|
"tm/pkg/response"
|
|
|
|
"go.mongodb.org/mongo-driver/v2/bson"
|
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
|
)
|
|
|
|
// TenderRepository interface defines tender data access methods
|
|
type TenderRepository interface {
|
|
// Tender CRUD operations
|
|
Create(ctx context.Context, tender *Tender) error
|
|
GetByID(ctx context.Context, id string) (*Tender, error)
|
|
GetByIDs(ctx context.Context, ids []string) ([]Tender, error)
|
|
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
|
|
GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
|
|
GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error)
|
|
// GetLatestByContractFolderIDs returns the most recently updated tender per distinct contract_folder_id.
|
|
GetLatestByContractFolderIDs(ctx context.Context, contractFolderIDs []string) (map[string]*Tender, error)
|
|
GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error)
|
|
FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error)
|
|
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
|
|
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
|
|
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
|
|
Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error)
|
|
GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
|
|
GetPendingDocumentScrapeTenders(ctx context.Context, countryCodes []string, minDeadline int64, limit, skip int) ([]Tender, bool, error)
|
|
GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
|
|
GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error)
|
|
Update(ctx context.Context, tender *Tender) error
|
|
Delete(ctx context.Context, id string) error
|
|
GetStatistics(ctx context.Context) (*TenderStatistics, error)
|
|
}
|
|
|
|
func collectionName() string {
|
|
return "tenders"
|
|
}
|
|
|
|
// SearchPageResult is the paginated outcome of a tender search/list query.
|
|
type SearchPageResult struct {
|
|
Items []Tender
|
|
TotalCount int64 // -1 when count was skipped
|
|
NextCursor string
|
|
HasMore bool
|
|
PageOffset int
|
|
}
|
|
|
|
// tenderSearchListProjection excludes large fields not needed for list/search API payloads.
|
|
func tenderSearchListProjection() bson.M {
|
|
return bson.M{
|
|
"content_xml": 0,
|
|
"document_summaries": 0,
|
|
"scraped_documents": 0,
|
|
"selection_criteria": 0,
|
|
"modifications": 0,
|
|
"source_file_url": 0,
|
|
"source_file_name": 0,
|
|
}
|
|
}
|
|
|
|
// documentScraperListProjection loads only fields required by the document scraper API.
|
|
func documentScraperListProjection() bson.M {
|
|
return bson.M{
|
|
"contract_folder_id": 1,
|
|
"notice_publication_id": 1,
|
|
"document_uri": 1,
|
|
"tender_url": 1,
|
|
"title": 1,
|
|
"description": 1,
|
|
}
|
|
}
|
|
|
|
// tenderRepository implements TenderRepository interface using MongoDB ORM
|
|
type tenderRepository struct {
|
|
ormRepo orm.Repository[Tender]
|
|
mongoManager *orm.ConnectionManager
|
|
logger logger.Logger
|
|
}
|
|
|
|
// NewTenderRepository creates a new tender repository
|
|
func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) TenderRepository {
|
|
// Create indexes for tenders collection
|
|
tenderIndexes := []orm.Index{
|
|
*orm.CreateUniqueIndex("tender_id_idx", bson.D{{Key: "tender_id", Value: 1}}),
|
|
*orm.NewIndex("project_name_idx", bson.D{{Key: "project_name", Value: 1}}),
|
|
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
|
|
// One tender row per TED ContractNoticeID when present (parallel ingest / lookup gaps).
|
|
*orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}).
|
|
WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_notice_id")),
|
|
*orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}),
|
|
// One tender row per TED publication / procedure when ids are present (prevents parallel worker races).
|
|
*orm.CreateUniqueIndex("notice_publication_id_unique", bson.D{{Key: "notice_publication_id", Value: 1}}).
|
|
WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("notice_publication_id")),
|
|
*orm.CreateUniqueIndex("contract_folder_id_unique", bson.D{{Key: "contract_folder_id", Value: 1}}).
|
|
WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_folder_id")),
|
|
// One tender per UBL procurement project (merges per-lot notices that share ProcurementProject/ID).
|
|
*orm.CreateUniqueIndex("procurement_project_id_unique", bson.D{{Key: "procurement_project_id", Value: 1}}).
|
|
WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("procurement_project_id")),
|
|
*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
|
|
*orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}),
|
|
*orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}),
|
|
*orm.NewIndex("notice_type_code_idx", bson.D{{Key: "notice_type_code", Value: 1}}),
|
|
*orm.NewIndex("form_type_idx", bson.D{{Key: "form_type", Value: 1}}),
|
|
*orm.NewIndex("procurement_type_code_idx", bson.D{{Key: "procurement_type_code", Value: 1}}),
|
|
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
|
|
*orm.NewIndex("publication_date_idx", bson.D{{Key: "publication_date", Value: -1}}),
|
|
*orm.NewIndex("tender_deadline_idx", bson.D{{Key: "tender_deadline", Value: 1}}),
|
|
*orm.NewIndex("estimated_value_idx", bson.D{{Key: "estimated_value", Value: -1}}),
|
|
*orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}),
|
|
// Stable keyset pagination for default admin list (created_at desc).
|
|
*orm.NewIndex("created_at_id_idx", bson.D{
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("status_created_at_id_idx", bson.D{
|
|
{Key: "status", Value: 1},
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("country_code_created_at_id_idx", bson.D{
|
|
{Key: "country_code", Value: 1},
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("notice_type_code_created_at_id_idx", bson.D{
|
|
{Key: "notice_type_code", Value: 1},
|
|
{Key: "created_at", Value: -1},
|
|
{Key: "_id", Value: -1},
|
|
}),
|
|
*orm.NewIndex("updated_at_idx", bson.D{{Key: "updated_at", Value: -1}}),
|
|
// Compound indexes for common queries
|
|
*orm.NewIndex("status_country_idx", bson.D{
|
|
{Key: "status", Value: 1},
|
|
{Key: "country_code", Value: 1},
|
|
}),
|
|
*orm.NewIndex("notice_type_status_idx", bson.D{
|
|
{Key: "notice_type_code", Value: 1},
|
|
{Key: "status", Value: 1},
|
|
}),
|
|
*orm.NewIndex("doc_scrape_pending_idx", bson.D{
|
|
{Key: "country_code", Value: 1},
|
|
{Key: "processing_metadata.documents_scraped", Value: 1},
|
|
{Key: "tender_deadline", Value: 1},
|
|
{Key: "created_at", Value: 1},
|
|
}),
|
|
*orm.NewIndex("documents_scraped_at_idx", bson.D{
|
|
{Key: "processing_metadata.documents_scraped", Value: 1},
|
|
{Key: "processing_metadata.documents_scraped_at", Value: -1},
|
|
}),
|
|
*orm.NewIndex("scraped_documents_scraped_at_idx", bson.D{
|
|
{Key: "scraped_documents.scraped_at", Value: 1},
|
|
}),
|
|
// Index for CPV matching queries
|
|
*orm.NewIndex("cpv_matching_idx", bson.D{
|
|
{Key: "status", Value: 1},
|
|
{Key: "main_classification", Value: 1},
|
|
{Key: "additional_classifications", Value: 1},
|
|
{Key: "publication_date", Value: -1},
|
|
}),
|
|
// Text index (legacy; list search uses $regex across title, description, and AI summary fields for unified matching)
|
|
*orm.CreateTextIndex("search_idx", "title", "description"),
|
|
}
|
|
|
|
// Create indexes
|
|
if err := mongoManager.CreateIndexes(collectionName(), tenderIndexes); err != nil {
|
|
logger.Warn("Failed to create tender indexes", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
}
|
|
|
|
// Create ORM repositories
|
|
tenderOrmRepo := orm.NewRepository[Tender](mongoManager.GetCollection(collectionName()), logger)
|
|
|
|
return &tenderRepository{
|
|
ormRepo: tenderOrmRepo,
|
|
mongoManager: mongoManager,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
// Create creates a new tender
|
|
func (r *tenderRepository) Create(ctx context.Context, tender *Tender) error {
|
|
now := time.Now().Unix()
|
|
tender.SetCreatedAt(now)
|
|
|
|
err := r.ormRepo.Create(ctx, tender)
|
|
if err != nil {
|
|
r.logger.Error("Failed to create tender", map[string]interface{}{
|
|
"error": err.Error(),
|
|
"notice_publication_id": tender.NoticePublicationID,
|
|
})
|
|
return err
|
|
}
|
|
|
|
r.logger.Info("Tender created successfully", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
"notice_publication_id": tender.NoticePublicationID,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// GetByID retrieves a tender by ID
|
|
func (r *tenderRepository) GetByID(ctx context.Context, id string) (*Tender, error) {
|
|
tender, err := r.ormRepo.FindByID(ctx, id)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by ID", map[string]interface{}{
|
|
"tender_id": id,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return tender, nil
|
|
}
|
|
|
|
// GetByIDs retrieves tenders by IDs in a single query.
|
|
func (r *tenderRepository) GetByIDs(ctx context.Context, ids []string) ([]Tender, error) {
|
|
if len(ids) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
objectIDs := make([]bson.ObjectID, 0, len(ids))
|
|
for _, id := range ids {
|
|
objectID, err := bson.ObjectIDFromHex(id)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
objectIDs = append(objectIDs, objectID)
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, bson.M{"_id": bson.M{"$in": objectIDs}}, orm.NewPaginationBuilder().Build())
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tenders by IDs", map[string]interface{}{
|
|
"count": len(ids),
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return result.Items, nil
|
|
}
|
|
|
|
// GetByContractNoticeID retrieves a tender by contract notice ID
|
|
func (r *tenderRepository) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error) {
|
|
filter := bson.M{"contract_notice_id": contractNoticeID}
|
|
pagination := orm.Pagination{Limit: 1}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by contract notice ID", map[string]interface{}{
|
|
"contract_notice_id": contractNoticeID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// GetByNoticePublicationID retrieves a tender by notice publication ID.
|
|
// Matches both the canonical NoticePublicationID and any id retained in RelatedNoticePublicationIDs
|
|
// so historical TED detail links keep resolving after a tender update from a follow-up notice.
|
|
func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error) {
|
|
filter := bson.M{
|
|
"$or": []bson.M{
|
|
{"notice_publication_id": noticePublicationID},
|
|
{"related_notice_publication_ids": noticePublicationID},
|
|
},
|
|
}
|
|
pagination := orm.Pagination{Limit: 1}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by notice publication ID", map[string]interface{}{
|
|
"notice_publication_id": noticePublicationID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// GetByContractFolderID retrieves a tender by its procedure-level identifier (ContractFolderID).
|
|
// TED groups multiple notice publications (e.g. competition notice + result notice) under the same
|
|
// ContractFolderID, so this lookup is the authoritative way to find the existing tender for a procedure
|
|
// when a follow-up notice arrives. Returns the most recently updated match if there are several.
|
|
func (r *tenderRepository) GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error) {
|
|
filter := bson.M{"contract_folder_id": contractFolderID}
|
|
pagination := orm.Pagination{
|
|
Limit: 1,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by contract folder ID", map[string]interface{}{
|
|
"contract_folder_id": contractFolderID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// GetLatestByContractFolderIDs returns one tender per distinct contract_folder_id (latest updated_at).
|
|
func (r *tenderRepository) GetLatestByContractFolderIDs(ctx context.Context, contractFolderIDs []string) (map[string]*Tender, error) {
|
|
out := make(map[string]*Tender)
|
|
if len(contractFolderIDs) == 0 {
|
|
return out, nil
|
|
}
|
|
|
|
uniq := uniqueNonEmptyTrimmedStrings(contractFolderIDs)
|
|
if len(uniq) == 0 {
|
|
return out, nil
|
|
}
|
|
|
|
// Allow a few rows per folder in case duplicate folder rows exist despite partial uniques.
|
|
findLimit := len(uniq) * 3
|
|
if findLimit < len(uniq)+1 {
|
|
findLimit = len(uniq) + 1
|
|
}
|
|
if findLimit > 100000 {
|
|
findLimit = 100000
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx,
|
|
bson.M{"contract_folder_id": bson.M{"$in": uniq}},
|
|
orm.Pagination{
|
|
Limit: findLimit,
|
|
Skip: 0,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
},
|
|
)
|
|
if err != nil {
|
|
r.logger.Error("Failed to load tenders by contract folder ids", map[string]interface{}{
|
|
"folder_count": len(uniq),
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
for i := range result.Items {
|
|
t := &result.Items[i]
|
|
cid := strings.TrimSpace(t.ContractFolderID)
|
|
if cid == "" {
|
|
continue
|
|
}
|
|
prev, ok := out[cid]
|
|
if !ok || t.UpdatedAt > prev.UpdatedAt {
|
|
out[cid] = t
|
|
}
|
|
}
|
|
|
|
return out, nil
|
|
}
|
|
|
|
func uniqueNonEmptyTrimmedStrings(ids []string) []string {
|
|
seen := make(map[string]struct{}, len(ids))
|
|
out := make([]string, 0, len(ids))
|
|
for _, id := range ids {
|
|
id = strings.TrimSpace(id)
|
|
if id == "" {
|
|
continue
|
|
}
|
|
if _, dup := seen[id]; dup {
|
|
continue
|
|
}
|
|
seen[id] = struct{}{}
|
|
out = append(out, id)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// GetByProcurementProjectID finds a tender by UBL ProcurementProject ID (shared across per-lot notices).
|
|
func (r *tenderRepository) GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error) {
|
|
filter := bson.M{"procurement_project_id": procurementProjectID}
|
|
pagination := orm.Pagination{
|
|
Limit: 1,
|
|
SortField: "updated_at",
|
|
SortOrder: -1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get tender by procurement project ID", map[string]interface{}{
|
|
"procurement_project_id": procurementProjectID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
if len(result.Items) == 0 {
|
|
return nil, orm.ErrDocumentNotFound
|
|
}
|
|
|
|
return &result.Items[0], nil
|
|
}
|
|
|
|
// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted).
|
|
func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) {
|
|
filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "_id",
|
|
SortOrder: 1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to list tenders with content_xml", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return result.Items, nil
|
|
}
|
|
|
|
// Update updates an existing tender
|
|
func (r *tenderRepository) Update(ctx context.Context, tender *Tender) error {
|
|
tender.UpdatedAt = time.Now().Unix()
|
|
|
|
err := r.ormRepo.Update(ctx, tender)
|
|
if err != nil {
|
|
r.logger.Error("Failed to update tender", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
"error": err.Error(),
|
|
})
|
|
return err
|
|
}
|
|
|
|
r.logger.Info("Tender updated successfully", map[string]interface{}{
|
|
"tender_id": tender.ID,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// Delete deletes a tender by ID
|
|
func (r *tenderRepository) Delete(ctx context.Context, id string) error {
|
|
err := r.ormRepo.Delete(ctx, id)
|
|
if err != nil {
|
|
r.logger.Error("Failed to delete tender", map[string]interface{}{
|
|
"tender_id": id,
|
|
"error": err.Error(),
|
|
})
|
|
return err
|
|
}
|
|
|
|
r.logger.Info("Tender deleted successfully", map[string]interface{}{
|
|
"tender_id": id,
|
|
})
|
|
|
|
return nil
|
|
}
|
|
|
|
// Search searches tenders based on criteria
|
|
func (r *tenderRepository) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchPageResult, error) {
|
|
filter := r.buildSearchFilter(form)
|
|
|
|
sort := "created_at"
|
|
if pagination.SortBy != "" {
|
|
sort = pagination.SortBy
|
|
}
|
|
|
|
sortOrder := "desc"
|
|
if pagination.SortOrder != "" {
|
|
sortOrder = pagination.SortOrder
|
|
}
|
|
|
|
mongoPagination, err := orm.BuildListPagination(
|
|
pagination.Limit,
|
|
pagination.Offset,
|
|
pagination.Cursor,
|
|
sort,
|
|
sortOrder,
|
|
filter,
|
|
orm.ListPaginationOptions{Projection: tenderSearchListProjection()},
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, mongoPagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to search tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return &SearchPageResult{
|
|
Items: result.Items,
|
|
TotalCount: result.TotalCount,
|
|
NextCursor: result.NextCursor,
|
|
HasMore: result.HasMore,
|
|
PageOffset: result.PageOffset,
|
|
}, nil
|
|
}
|
|
|
|
// GetExpiredTenders retrieves tenders that are expired
|
|
func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error) {
|
|
filter := bson.M{
|
|
"$or": []bson.M{
|
|
{"status": TenderStatusExpired},
|
|
{
|
|
"tender_deadline": bson.M{
|
|
"$lt": beforeDate,
|
|
},
|
|
"status": bson.M{"$ne": TenderStatusExpired},
|
|
},
|
|
},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: 1000, // Large limit for maintenance operations
|
|
SortField: "tender_deadline",
|
|
SortOrder: 1,
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get expired tenders", map[string]interface{}{
|
|
"before_date": beforeDate,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
return result.Items, nil
|
|
}
|
|
|
|
// GetPendingDocumentScrapeTenders returns unscraped tenders for the given countries with an active deadline.
|
|
func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, countryCodes []string, minDeadline int64, limit, skip int) ([]Tender, bool, error) {
|
|
if len(countryCodes) == 0 {
|
|
return nil, false, nil
|
|
}
|
|
|
|
filter := bson.M{
|
|
"country_code": bson.M{"$in": countryCodes},
|
|
"tender_deadline": bson.M{
|
|
"$gt": minDeadline,
|
|
},
|
|
"processing_metadata.documents_scraped": bson.M{"$ne": true},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "created_at",
|
|
SortOrder: 1,
|
|
SkipCount: true,
|
|
Projection: documentScraperListProjection(),
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{
|
|
"country_codes": countryCodes,
|
|
"min_deadline": minDeadline,
|
|
"limit": limit,
|
|
"skip": skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, false, err
|
|
}
|
|
|
|
return result.Items, result.HasMore, nil
|
|
}
|
|
|
|
// GetUnSummarizedTenders retrieves tenders that have documents scraped but not summarized
|
|
func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) {
|
|
filter := bson.M{
|
|
"processing_metadata.documents_scraped": true,
|
|
"$or": []bson.M{
|
|
{"processing_metadata.documents_summarized": bson.M{"$exists": false}},
|
|
{"processing_metadata.documents_summarized": false},
|
|
},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "processing_metadata.documents_scraped_at",
|
|
SortOrder: 1, // Oldest first (process oldest scraped documents first)
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
|
|
"limit": limit,
|
|
"skip": skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, err
|
|
}
|
|
|
|
return result.Items, result.TotalCount, nil
|
|
}
|
|
|
|
// GetUnTranslatedTenders returns tenders eligible for translation backfill.
|
|
func (r *tenderRepository) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) {
|
|
_ = language
|
|
|
|
filter := bson.M{
|
|
"notice_publication_id": bson.M{"$ne": ""},
|
|
"contract_folder_id": bson.M{"$ne": ""},
|
|
"title": bson.M{"$ne": ""},
|
|
"description": bson.M{"$ne": ""},
|
|
}
|
|
|
|
pagination := orm.Pagination{
|
|
Limit: limit,
|
|
Skip: skip,
|
|
SortField: "created_at",
|
|
SortOrder: 1, // Oldest first
|
|
}
|
|
|
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
|
if err != nil {
|
|
r.logger.Error("Failed to get untranslated tenders", map[string]interface{}{
|
|
"language": language,
|
|
"limit": limit,
|
|
"skip": skip,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, 0, err
|
|
}
|
|
|
|
return result.Items, result.TotalCount, nil
|
|
}
|
|
|
|
// GetStatistics calculates tender statistics using aggregation
|
|
func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
|
|
stats := &TenderStatistics{
|
|
LastUpdated: time.Now().Unix(),
|
|
}
|
|
|
|
// Get total count
|
|
total, err := r.ormRepo.Count(ctx, bson.M{})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stats.TotalTenders = total
|
|
|
|
// Get active count
|
|
active, err := r.ormRepo.Count(ctx, bson.M{"status": TenderStatusActive})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stats.ActiveTenders = active
|
|
|
|
// Get expired count
|
|
expired, err := r.ormRepo.Count(ctx, bson.M{"status": TenderStatusExpired})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
stats.ExpiredTenders = expired
|
|
|
|
// Get statistics by country, type, and classification
|
|
stats.TendersByCountry, _ = r.GetTenderCountByCountry(ctx)
|
|
stats.TendersByType, _ = r.GetTenderCountByType(ctx)
|
|
stats.TendersByClassification, _ = r.GetTenderCountByClassification(ctx)
|
|
|
|
// Calculate value statistics using aggregation
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": nil,
|
|
"total_estimated_value": bson.M{"$sum": "$estimated_value"},
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
}
|
|
|
|
result, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err == nil && len(result) > 0 {
|
|
if totalValue, ok := result[0]["total_estimated_value"].(float64); ok {
|
|
stats.TotalEstimatedValue = totalValue
|
|
if count, ok := result[0]["count"].(int32); ok && count > 0 {
|
|
stats.AverageEstimatedValue = totalValue / float64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return stats, nil
|
|
}
|
|
|
|
// GetTenderCountByCountry gets tender count grouped by country
|
|
func (r *tenderRepository) GetTenderCountByCountry(ctx context.Context) (map[string]int64, error) {
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": "$country_code",
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
}
|
|
|
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
counts := make(map[string]int64)
|
|
for _, result := range results {
|
|
if country, ok := result["_id"].(string); ok {
|
|
if count, ok := result["count"].(int32); ok {
|
|
counts[country] = int64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return counts, nil
|
|
}
|
|
|
|
// GetTenderCountByType gets tender count grouped by type
|
|
func (r *tenderRepository) GetTenderCountByType(ctx context.Context) (map[string]int64, error) {
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": "$notice_type_code",
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
}
|
|
|
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
counts := make(map[string]int64)
|
|
for _, result := range results {
|
|
if noticeType, ok := result["_id"].(string); ok {
|
|
if count, ok := result["count"].(int32); ok {
|
|
counts[noticeType] = int64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return counts, nil
|
|
}
|
|
|
|
// GetTenderCountByClassification gets tender count grouped by classification
|
|
func (r *tenderRepository) GetTenderCountByClassification(ctx context.Context) (map[string]int64, error) {
|
|
pipeline := mongo.Pipeline{
|
|
{
|
|
{Key: "$group", Value: bson.M{
|
|
"_id": "$main_classification",
|
|
"count": bson.M{"$sum": 1},
|
|
}},
|
|
},
|
|
{
|
|
{Key: "$limit", Value: 20},
|
|
},
|
|
}
|
|
|
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
counts := make(map[string]int64)
|
|
for _, result := range results {
|
|
if classification, ok := result["_id"].(string); ok {
|
|
if count, ok := result["count"].(int32); ok {
|
|
counts[classification] = int64(count)
|
|
}
|
|
}
|
|
}
|
|
|
|
return counts, nil
|
|
}
|
|
|
|
func dedupeNonEmptyStrings(values []string) []string {
|
|
seen := make(map[string]struct{}, len(values))
|
|
out := make([]string, 0, len(values))
|
|
for _, v := range values {
|
|
v = strings.TrimSpace(v)
|
|
if v == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[v]; ok {
|
|
continue
|
|
}
|
|
seen[v] = struct{}{}
|
|
out = append(out, v)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// normalizeUnixTimestamp accepts seconds or milliseconds and always returns seconds.
|
|
func normalizeUnixTimestamp(value int64) int64 {
|
|
// 1e12 is safely above current Unix seconds and in millisecond range for modern dates.
|
|
if value > 1_000_000_000_000 {
|
|
return value / 1000
|
|
}
|
|
return value
|
|
}
|
|
|
|
func exactOrRange(exact, from, to *int64) (*int64, *int64) {
|
|
if exact != nil {
|
|
v := normalizeUnixTimestamp(*exact)
|
|
return &v, &v
|
|
}
|
|
|
|
var normalizedFrom *int64
|
|
if from != nil {
|
|
v := normalizeUnixTimestamp(*from)
|
|
normalizedFrom = &v
|
|
}
|
|
|
|
var normalizedTo *int64
|
|
if to != nil {
|
|
v := normalizeUnixTimestamp(*to)
|
|
normalizedTo = &v
|
|
}
|
|
|
|
return normalizedFrom, normalizedTo
|
|
}
|
|
|
|
// buildSearchFilter builds MongoDB filter from search form
|
|
func (r *tenderRepository) buildSearchFilter(form *SearchForm) bson.M {
|
|
filter := bson.M{}
|
|
|
|
var keywordOr []bson.M
|
|
if form.Search != nil {
|
|
q := strings.TrimSpace(*form.Search)
|
|
if q != "" {
|
|
pattern := regexp.QuoteMeta(q)
|
|
keywordOr = []bson.M{
|
|
{"title": bson.M{"$regex": pattern, "$options": "i"}},
|
|
{"description": bson.M{"$regex": pattern, "$options": "i"}},
|
|
{"document_summaries.summary": bson.M{"$regex": pattern, "$options": "i"}},
|
|
{"ai_overall_summary": bson.M{"$regex": pattern, "$options": "i"}},
|
|
}
|
|
}
|
|
}
|
|
if form.Title != nil {
|
|
title := strings.TrimSpace(*form.Title)
|
|
if title != "" {
|
|
filter["title"] = bson.M{
|
|
"$regex": regexp.QuoteMeta(title),
|
|
"$options": "i",
|
|
}
|
|
}
|
|
}
|
|
if form.Description != nil {
|
|
description := strings.TrimSpace(*form.Description)
|
|
if description != "" {
|
|
filter["description"] = bson.M{
|
|
"$regex": regexp.QuoteMeta(description),
|
|
"$options": "i",
|
|
}
|
|
}
|
|
}
|
|
|
|
var noticeCodes []string
|
|
if form.NoticeType != nil {
|
|
if t := strings.TrimSpace(*form.NoticeType); t != "" {
|
|
noticeCodes = append(noticeCodes, t)
|
|
}
|
|
}
|
|
noticeCodes = append(noticeCodes, form.NoticeTypes...)
|
|
noticeCodes = dedupeNonEmptyStrings(noticeCodes)
|
|
if len(noticeCodes) == 1 {
|
|
filter["notice_type_code"] = noticeCodes[0]
|
|
} else if len(noticeCodes) > 1 {
|
|
filter["notice_type_code"] = bson.M{"$in": noticeCodes}
|
|
}
|
|
|
|
formTypes := dedupeNonEmptyStrings(form.FormTypes)
|
|
if len(formTypes) == 1 {
|
|
filter["form_type"] = formTypes[0]
|
|
} else if len(formTypes) > 1 {
|
|
filter["form_type"] = bson.M{"$in": formTypes}
|
|
}
|
|
|
|
if form.ProcurementType != nil {
|
|
filter["procurement_type_code"] = form.ProcurementType
|
|
}
|
|
|
|
countryCodes := dedupeNonEmptyStrings(form.CountryCodes)
|
|
if form.CountryCode != nil {
|
|
if c := strings.TrimSpace(*form.CountryCode); c != "" {
|
|
countryCodes = append(countryCodes, c)
|
|
countryCodes = dedupeNonEmptyStrings(countryCodes)
|
|
}
|
|
}
|
|
if form.Country != nil {
|
|
if c := strings.TrimSpace(*form.Country); c != "" {
|
|
countryCodes = append(countryCodes, c)
|
|
countryCodes = dedupeNonEmptyStrings(countryCodes)
|
|
}
|
|
}
|
|
if len(countryCodes) == 1 {
|
|
filter["country_code"] = countryCodes[0]
|
|
} else if len(countryCodes) > 1 {
|
|
filter["country_code"] = bson.M{"$in": countryCodes}
|
|
}
|
|
|
|
if len(form.RegionCodes) > 0 {
|
|
filter["region_code"] = bson.M{"$in": dedupeNonEmptyStrings(form.RegionCodes)}
|
|
}
|
|
|
|
if form.MainClassification != nil {
|
|
if mainClassification := strings.TrimSpace(*form.MainClassification); mainClassification != "" {
|
|
filter["main_classification"] = mainClassification
|
|
}
|
|
}
|
|
|
|
cpvCodes := dedupeNonEmptyStrings(form.Classifications)
|
|
cpvCodes = append(cpvCodes, dedupeNonEmptyStrings(form.CpvCodes)...)
|
|
cpvCodes = dedupeNonEmptyStrings(cpvCodes)
|
|
var cpvOr []bson.M
|
|
if len(cpvCodes) > 0 {
|
|
cpvOr = []bson.M{
|
|
{"main_classification": bson.M{"$in": cpvCodes}},
|
|
{"additional_classifications": bson.M{"$in": cpvCodes}},
|
|
{
|
|
"procurement_lots": bson.M{
|
|
"$elemMatch": bson.M{
|
|
"$or": []bson.M{
|
|
{"main_classification": bson.M{"$in": cpvCodes}},
|
|
{"additional_classifications": bson.M{"$in": cpvCodes}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
switch {
|
|
case len(keywordOr) > 0 && len(cpvOr) > 0:
|
|
filter["$and"] = []bson.M{
|
|
{"$or": keywordOr},
|
|
{"$or": cpvOr},
|
|
}
|
|
case len(keywordOr) > 0:
|
|
filter["$or"] = keywordOr
|
|
case len(cpvOr) > 0:
|
|
filter["$or"] = cpvOr
|
|
}
|
|
|
|
if form.MinEstimatedValue != nil || form.MaxEstimatedValue != nil {
|
|
valueFilter := bson.M{}
|
|
if form.MinEstimatedValue != nil {
|
|
valueFilter["$gte"] = *form.MinEstimatedValue
|
|
}
|
|
if form.MaxEstimatedValue != nil {
|
|
valueFilter["$lte"] = *form.MaxEstimatedValue
|
|
}
|
|
filter["estimated_value"] = valueFilter
|
|
}
|
|
|
|
if form.Currency != "" {
|
|
filter["currency"] = form.Currency
|
|
}
|
|
|
|
createdAtFrom, createdAtTo := exactOrRange(form.CreatedAt, form.CreatedAtFrom, form.CreatedAtTo)
|
|
if createdAtFrom != nil || createdAtTo != nil {
|
|
createdFilter := bson.M{}
|
|
if createdAtFrom != nil {
|
|
createdFilter["$gte"] = *createdAtFrom
|
|
}
|
|
if createdAtTo != nil {
|
|
createdFilter["$lte"] = *createdAtTo
|
|
}
|
|
filter["created_at"] = createdFilter
|
|
}
|
|
|
|
deadlineFrom := form.DeadlineFrom
|
|
if form.TenderDeadlineFrom != nil {
|
|
deadlineFrom = form.TenderDeadlineFrom
|
|
}
|
|
deadlineTo := form.DeadlineTo
|
|
if form.TenderDeadlineTo != nil {
|
|
deadlineTo = form.TenderDeadlineTo
|
|
}
|
|
tenderDeadlineFrom, tenderDeadlineTo := exactOrRange(form.TenderDeadline, deadlineFrom, deadlineTo)
|
|
if tenderDeadlineFrom != nil || tenderDeadlineTo != nil || form.OnlyActiveDeadlines {
|
|
deadlineFilter := bson.M{}
|
|
if form.OnlyActiveDeadlines {
|
|
deadlineFilter["$gt"] = time.Now().Unix()
|
|
}
|
|
if tenderDeadlineFrom != nil {
|
|
deadlineFilter["$gte"] = *tenderDeadlineFrom
|
|
}
|
|
if tenderDeadlineTo != nil {
|
|
deadlineFilter["$lte"] = *tenderDeadlineTo
|
|
}
|
|
filter["tender_deadline"] = deadlineFilter
|
|
}
|
|
|
|
publicationDateFrom, publicationDateTo := exactOrRange(form.PublicationDate, form.PublicationDateFrom, form.PublicationDateTo)
|
|
if publicationDateFrom != nil || publicationDateTo != nil {
|
|
pubDateFilter := bson.M{}
|
|
if publicationDateFrom != nil {
|
|
pubDateFilter["$gte"] = *publicationDateFrom
|
|
}
|
|
if publicationDateTo != nil {
|
|
pubDateFilter["$lte"] = *publicationDateTo
|
|
}
|
|
filter["publication_date"] = pubDateFilter
|
|
}
|
|
|
|
submissionDateFrom := form.SubmissionDateFrom
|
|
if form.SubmissionDateAliasFrom != nil {
|
|
submissionDateFrom = form.SubmissionDateAliasFrom
|
|
}
|
|
submissionDateTo := form.SubmissionDateTo
|
|
if form.SubmissionDateAliasTo != nil {
|
|
submissionDateTo = form.SubmissionDateAliasTo
|
|
}
|
|
submissionDeadlineFrom, submissionDeadlineTo := exactOrRange(form.SubmissionDeadline, submissionDateFrom, submissionDateTo)
|
|
if submissionDeadlineFrom != nil || submissionDeadlineTo != nil {
|
|
subFilter := bson.M{}
|
|
if submissionDeadlineFrom != nil {
|
|
subFilter["$gte"] = *submissionDeadlineFrom
|
|
}
|
|
if submissionDeadlineTo != nil {
|
|
subFilter["$lte"] = *submissionDeadlineTo
|
|
}
|
|
filter["submission_deadline"] = subFilter
|
|
}
|
|
|
|
if len(form.Status) > 0 {
|
|
filter["status"] = bson.M{"$in": dedupeNonEmptyStrings(form.Status)}
|
|
}
|
|
|
|
if len(form.Source) > 0 {
|
|
filter["source"] = bson.M{"$in": dedupeNonEmptyStrings(form.Source)}
|
|
}
|
|
|
|
if form.BuyerOrganizationID != nil {
|
|
filter["buyer_organization.id"] = *form.BuyerOrganizationID
|
|
}
|
|
|
|
if form.NoticePublicationID != nil {
|
|
if noticePublicationID := strings.TrimSpace(*form.NoticePublicationID); noticePublicationID != "" {
|
|
filter["notice_publication_id"] = noticePublicationID
|
|
}
|
|
}
|
|
|
|
if len(form.Languages) > 0 {
|
|
filter["notice_language_code"] = bson.M{"$in": dedupeNonEmptyStrings(form.Languages)}
|
|
}
|
|
|
|
if form.DocumentsScraped {
|
|
folderIDs := dedupeNonEmptyStrings(form.ContractFolderIDsWithDocuments)
|
|
scrapedClause := bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}
|
|
if len(folderIDs) == 0 {
|
|
scrapedClause = bson.M{"_id": bson.M{"$exists": false}}
|
|
}
|
|
if len(filter) == 0 {
|
|
return scrapedClause
|
|
}
|
|
if and, ok := filter["$and"].([]bson.M); ok {
|
|
filter["$and"] = append(and, scrapedClause)
|
|
} else {
|
|
existing := bson.M{}
|
|
for k, v := range filter {
|
|
existing[k] = v
|
|
}
|
|
filter = bson.M{
|
|
"$and": []bson.M{existing, scrapedClause},
|
|
}
|
|
}
|
|
}
|
|
|
|
return filter
|
|
}
|