mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge tenders which have the same procedure id - worker performance fix

Browse Source
This commit is contained in:
Mazyar
2026-05-11 15:10:01 +03:30
parent 8ba667e4f4
commit b4efb97f47
22 changed files with 661 additions and 2234 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/tender/entity.go
+53 -50
View File
@@ -32,56 +32,59 @@ type ScrapedDocument struct {
// Tender represents a tender/contract notice entity
type Tender struct {
mongo.Model `bson:",inline"`
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"`
PublicationDate int64 `bson:"publication_date" json:"publication_date"`
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"`
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"`
GazetteID string `bson:"gazette_id" json:"gazette_id"`
Title string `bson:"title" json:"title"`
Description string `bson:"description" json:"description"`
ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"`
ProcedureCode string `bson:"procedure_code" json:"procedure_code"`
MainClassification string `bson:"main_classification" json:"main_classification"`
AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"`
EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"`
Currency string `bson:"currency" json:"currency"`
Duration string `bson:"duration" json:"duration"`
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
CountryCode string `bson:"country_code" json:"country_code"`
RegionCode string `bson:"region_code" json:"region_code"`
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
Organizations []Organization `bson:"organizations" json:"organizations"`
SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"`
OfficialLanguages []string `bson:"official_languages" json:"official_languages"`
Status TenderStatus `bson:"status" json:"status"`
Source TenderSource `bson:"source" json:"source"`
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"`
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
mongo.Model `bson:",inline"`
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
// RelatedNoticePublicationIDs lists every TED notice publication id that has contributed to this tender
// (same procedure / contract_folder_id). Ordered oldest-first; the latest is also reflected in NoticePublicationID.
RelatedNoticePublicationIDs []string `bson:"related_notice_publication_ids,omitempty" json:"related_notice_publication_ids,omitempty"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"`
PublicationDate int64 `bson:"publication_date" json:"publication_date"`
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"`
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"`
GazetteID string `bson:"gazette_id" json:"gazette_id"`
Title string `bson:"title" json:"title"`
Description string `bson:"description" json:"description"`
ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"`
ProcedureCode string `bson:"procedure_code" json:"procedure_code"`
MainClassification string `bson:"main_classification" json:"main_classification"`
AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"`
EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"`
Currency string `bson:"currency" json:"currency"`
Duration string `bson:"duration" json:"duration"`
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
CountryCode string `bson:"country_code" json:"country_code"`
RegionCode string `bson:"region_code" json:"region_code"`
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
Organizations []Organization `bson:"organizations" json:"organizations"`
SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"`
OfficialLanguages []string `bson:"official_languages" json:"official_languages"`
Status TenderStatus `bson:"status" json:"status"`
Source TenderSource `bson:"source" json:"source"`
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"`
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
// AIOverallSummary optional denormalized overall summary for search/list (e.g. synced from AI pipeline storage).
AIOverallSummary string `bson:"ai_overall_summary,omitempty" json:"ai_overall_summary,omitempty"`
TenderID string `bson:"tender_id" json:"tender_id"` // PBL Project Number (SCDYYNNN format)
internal/tender/repository.go
+39 -2
View File
@@ -20,6 +20,7 @@ type TenderRepository interface {
GetByID(ctx context.Context, id string) (*Tender, error)
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error)
FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error)
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
@@ -52,6 +53,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
*orm.CreateUniqueIndex("tender_id_idx", bson.D{{Key: "tender_id", Value: 1}}),
*orm.NewIndex("project_name_idx", bson.D{{Key: "project_name", Value: 1}}),
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
*orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}),
*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
*orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}),
*orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}),
@@ -158,9 +160,16 @@ func (r *tenderRepository) GetByContractNoticeID(ctx context.Context, contractNo
return &result.Items[0], nil
}
// GetByNoticePublicationID retrieves a tender by notice publication ID
// GetByNoticePublicationID retrieves a tender by notice publication ID.
// Matches both the canonical NoticePublicationID and any id retained in RelatedNoticePublicationIDs
// so historical TED detail links keep resolving after a tender update from a follow-up notice.
func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error) {
filter := bson.M{"notice_publication_id": noticePublicationID}
filter := bson.M{
"$or": []bson.M{
{"notice_publication_id": noticePublicationID},
{"related_notice_publication_ids": noticePublicationID},
},
}
pagination := orm.Pagination{Limit: 1}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
@@ -179,6 +188,34 @@ func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticeP
return &result.Items[0], nil
}
// GetByContractFolderID retrieves a tender by its procedure-level identifier (ContractFolderID).
// TED groups multiple notice publications (e.g. competition notice + result notice) under the same
// ContractFolderID, so this lookup is the authoritative way to find the existing tender for a procedure
// when a follow-up notice arrives. Returns the most recently updated match if there are several.
func (r *tenderRepository) GetByContractFolderID(ctx context.Context, contractFolderID string) (*Tender, error) {
filter := bson.M{"contract_folder_id": contractFolderID}
pagination := orm.Pagination{
Limit: 1,
SortField: "updated_at",
SortOrder: -1,
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to get tender by contract folder ID", map[string]interface{}{
"contract_folder_id": contractFolderID,
"error": err.Error(),
})
return nil, err
}
if len(result.Items) == 0 {
return nil, orm.ErrDocumentNotFound
}
return &result.Items[0], nil
}
// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted).
func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) {
filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}