mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tender duplication fix

Browse Source
This commit is contained in:
Mazyar
2026-05-12 17:02:39 +03:30
parent 29b4aa50bd
commit 5e8d4f67b2
7 changed files with 290 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/notice/entity.go
+2
View File
@@ -14,6 +14,8 @@ type Notice struct {
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
// ProcurementProjectID is UBL ProcurementProject/ID — stable across per-lot notices for one procedure.
ProcurementProjectID string `bson:"procurement_project_id,omitempty" json:"procurement_project_id,omitempty"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
internal/notice/repository.go
+3
View File
@@ -41,6 +41,9 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
*orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}),
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
// One row per TED ContractNoticeID when present (parallel scrape races).
*orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}).
WithPartialFilterExpression(bson.M{"contract_notice_id": bson.M{"$type": "string", "$ne": ""}}),
}
// Create indexes
internal/tender/entity.go
+2
View File
@@ -39,6 +39,8 @@ type Tender struct {
// (same procedure / contract_folder_id). Ordered oldest-first; the latest is also reflected in NoticePublicationID.
RelatedNoticePublicationIDs []string `bson:"related_notice_publication_ids,omitempty" json:"related_notice_publication_ids,omitempty"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
// ProcurementProjectID is UBL ProcurementProject/ID — merges per-lot CAN/CN notices into one tender.
ProcurementProjectID string `bson:"procurement_project_id,omitempty" json:"procurement_project_id,omitempty"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
internal/tender/repository.go
+34
View File
@@ -21,6 +21,7 @@ type TenderRepository interface {
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error)
GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error)
FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error)
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
@@ -54,6 +55,14 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
*orm.NewIndex("project_name_idx", bson.D{{Key: "project_name", Value: 1}}),
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
*orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}),
// One tender row per TED publication / procedure when ids are present (prevents parallel worker races).
*orm.CreateUniqueIndex("notice_publication_id_unique", bson.D{{Key: "notice_publication_id", Value: 1}}).
WithPartialFilterExpression(bson.M{"notice_publication_id": bson.M{"$type": "string", "$ne": ""}}),
*orm.CreateUniqueIndex("contract_folder_id_unique", bson.D{{Key: "contract_folder_id", Value: 1}}).
WithPartialFilterExpression(bson.M{"contract_folder_id": bson.M{"$type": "string", "$ne": ""}}),
// One tender per UBL procurement project (merges per-lot notices that share ProcurementProject/ID).
*orm.CreateUniqueIndex("procurement_project_id_unique", bson.D{{Key: "procurement_project_id", Value: 1}}).
WithPartialFilterExpression(bson.M{"procurement_project_id": bson.M{"$type": "string", "$ne": ""}}),
*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
*orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}),
*orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}),
@@ -216,6 +225,31 @@ func (r *tenderRepository) GetByContractFolderID(ctx context.Context, contractFo
return &result.Items[0], nil
}
// GetByProcurementProjectID finds a tender by UBL ProcurementProject ID (shared across per-lot notices).
func (r *tenderRepository) GetByProcurementProjectID(ctx context.Context, procurementProjectID string) (*Tender, error) {
filter := bson.M{"procurement_project_id": procurementProjectID}
pagination := orm.Pagination{
Limit: 1,
SortField: "updated_at",
SortOrder: -1,
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to get tender by procurement project ID", map[string]interface{}{
"procurement_project_id": procurementProjectID,
"error": err.Error(),
})
return nil, err
}
if len(result.Items) == 0 {
return nil, orm.ErrDocumentNotFound
}
return &result.Items[0], nil
}
// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted).
func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) {
filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}