mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added missing fields - scraper and worker fixes

Browse Source
This commit is contained in:
Mazyar
2026-05-11 00:09:37 +03:30
parent e136f0eaa7
commit eb6706e061
12 changed files with 667 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/notice/entity.go
+20
View File
@@ -15,7 +15,12 @@ type Notice struct {
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
@@ -40,6 +45,8 @@ type Notice struct {
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
@@ -99,6 +106,19 @@ type SelectionCriterion struct {
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
}
// ProcurementLot captures per-lot procurement fields from TED notices.
type ProcurementLot struct {
LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"`
Title string `bson:"title,omitempty" json:"title,omitempty"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"`
MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"`
AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"`
Duration string `bson:"duration,omitempty" json:"duration,omitempty"`
EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"`
Currency string `bson:"currency,omitempty" json:"currency,omitempty"`
}
// ProcessingMetadata contains metadata about how the tender was processed
type ProcessingMetadata struct {
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"`
internal/notice/repository.go
+23
View File
@@ -16,6 +16,7 @@ type Repository interface {
BulkImport(ctx context.Context, notices []Notice) error
GetByID(ctx context.Context, id string) (*Notice, error)
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error)
FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error)
GetUnProcessedNotices(ctx context.Context, ProcessingLimit int, skip int) ([]Notice, int64, error)
GetProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error)
Delete(ctx context.Context, id string) error
@@ -39,6 +40,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
// processing_metadata.processed
*orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}),
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
}
// Create indexes
@@ -144,6 +146,27 @@ func (r *noticeRepository) GetByContractNoticeID(ctx context.Context, contractNo
return &result.Items[0], nil
}
// FindNoticesWithContentXML returns notices that have non-empty stored TED XML (for backfill / remapping).
func (r *noticeRepository) FindNoticesWithContentXML(ctx context.Context, limit, skip int) ([]Notice, error) {
filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
pagination := orm.Pagination{
Limit: limit,
Skip: skip,
SortField: "_id",
SortOrder: 1,
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to list notices with content_xml", map[string]interface{}{
"error": err.Error(),
})
return nil, err
}
return result.Items, nil
}
// GetUnProcessedNotices retrieves unprocessed notices
func (r *noticeRepository) GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) {
internal/tender/entity.go
+21
View File
@@ -37,7 +37,12 @@ type Tender struct {
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
FormType string `bson:"form_type,omitempty" json:"form_type,omitempty"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeIdentifier string `bson:"notice_identifier,omitempty" json:"notice_identifier,omitempty"`
NoticeVersion string `bson:"notice_version,omitempty" json:"notice_version,omitempty"`
NoticeDispatchAt int64 `bson:"notice_dispatch_at,omitempty" json:"notice_dispatch_at,omitempty"`
ESenderDispatchAt int64 `bson:"esender_dispatch_at,omitempty" json:"esender_dispatch_at,omitempty"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
@@ -62,6 +67,8 @@ type Tender struct {
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
BuyerProfileURL string `bson:"buyer_profile_url,omitempty" json:"buyer_profile_url,omitempty"`
ProcurementLots []ProcurementLot `bson:"procurement_lots,omitempty" json:"procurement_lots,omitempty"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
@@ -73,6 +80,7 @@ type Tender struct {
Source TenderSource `bson:"source" json:"source"`
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"`
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
TenderID string `bson:"tender_id" json:"tender_id"` // PBL Project Number (SCDYYNNN format)
ProjectName string `bson:"project_name" json:"project_name"` // PBL Project Name (<InternalProjectNumber>_<Client-OpportunityName>_<PartnerCode>)
@@ -133,6 +141,19 @@ type SelectionCriterion struct {
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
}
// ProcurementLot captures per-lot procurement fields from TED notices.
type ProcurementLot struct {
LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"`
Title string `bson:"title,omitempty" json:"title,omitempty"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
MainNatureOfContract string `bson:"main_nature_of_contract,omitempty" json:"main_nature_of_contract,omitempty"`
MainClassification string `bson:"main_classification,omitempty" json:"main_classification,omitempty"`
AdditionalClassifications []string `bson:"additional_classifications,omitempty" json:"additional_classifications,omitempty"`
Duration string `bson:"duration,omitempty" json:"duration,omitempty"`
EstimatedValue float64 `bson:"estimated_value,omitempty" json:"estimated_value,omitempty"`
Currency string `bson:"currency,omitempty" json:"currency,omitempty"`
}
// ProcessingMetadata contains metadata about how the tender was processed
type ProcessingMetadata struct {
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` // Unix milliseconds
internal/tender/repository.go
+22
View File
@@ -18,6 +18,7 @@ type TenderRepository interface {
GetByID(ctx context.Context, id string) (*Tender, error)
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Tender, error)
GetByNoticePublicationID(ctx context.Context, noticePublicationID string) (*Tender, error)
FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error)
GetTenderCountByCountry(ctx context.Context) (map[string]int64, error)
GetTenderCountByType(ctx context.Context) (map[string]int64, error)
GetTenderCountByClassification(ctx context.Context) (map[string]int64, error)
@@ -175,6 +176,27 @@ func (r *tenderRepository) GetByNoticePublicationID(ctx context.Context, noticeP
return &result.Items[0], nil
}
// FindTendersWithContentXML returns tenders that still have TED XML (for backfill after notices were deleted).
func (r *tenderRepository) FindTendersWithContentXML(ctx context.Context, limit, skip int) ([]Tender, error) {
filter := bson.M{"content_xml": bson.M{"$exists": true, "$ne": ""}}
pagination := orm.Pagination{
Limit: limit,
Skip: skip,
SortField: "_id",
SortOrder: 1,
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to list tenders with content_xml", map[string]interface{}{
"error": err.Error(),
})
return nil, err
}
return result.Items, nil
}
// Update updates an existing tender
func (r *tenderRepository) Update(ctx context.Context, tender *Tender) error {
tender.UpdatedAt = time.Now().Unix()