Update .gitignore, adjust TED scraper configuration, and enhance Tender entity structure
- Added 'ted_samples/' to .gitignore to exclude TED sample files from version control. - Updated the TED scraper configuration to extend the cleanup duration from 24 hours to 72 hours, allowing for better resource management. - Enhanced the Tender entity by adding new fields: TenderDeadline, SubmissionDeadline, ApplicationDeadline, and SubmissionURL, improving the data model for tender management. - Updated the TenderResponse structure to include the new fields, ensuring consistency in API responses. - Implemented logic in the TED scraper to calculate submission and application deadlines based on tender deadlines, enhancing the scraping functionality.
This commit is contained in:
n.nakhostin
@@ -17,9 +17,12 @@ type Tender struct {
|
||||
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
|
||||
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
|
||||
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
|
||||
IssueDate int64 `bson:"issue_date" json:"issue_date"` // Unix milliseconds
|
||||
IssueTime int64 `bson:"issue_time" json:"issue_time"` // Unix milliseconds
|
||||
PublicationDate int64 `bson:"publication_date" json:"publication_date"` // Unix milliseconds
|
||||
IssueDate int64 `bson:"issue_date" json:"issue_date"` // Unix milliseconds
|
||||
IssueTime int64 `bson:"issue_time" json:"issue_time"` // Unix milliseconds
|
||||
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds
|
||||
PublicationDate int64 `bson:"publication_date" json:"publication_date"` // Unix milliseconds
|
||||
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds
|
||||
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` // Unix milliseconds
|
||||
GazetteID string `bson:"gazette_id" json:"gazette_id"`
|
||||
Title string `bson:"title" json:"title"`
|
||||
Description string `bson:"description" json:"description"`
|
||||
@@ -31,7 +34,6 @@ type Tender struct {
|
||||
Currency string `bson:"currency" json:"currency"`
|
||||
Duration string `bson:"duration" json:"duration"`
|
||||
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
|
||||
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds
|
||||
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
|
||||
CountryCode string `bson:"country_code" json:"country_code"`
|
||||
RegionCode string `bson:"region_code" json:"region_code"`
|
||||
@@ -39,6 +41,7 @@ type Tender struct {
|
||||
PostalCode string `bson:"postal_code" json:"postal_code"`
|
||||
DocumentURI string `bson:"document_uri" json:"document_uri"`
|
||||
TenderURL string `bson:"tender_url" json:"tender_url"`
|
||||
SubmissionURL string `bson:"submission_url" json:"submission_url"`
|
||||
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
|
||||
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
|
||||
Organizations []Organization `bson:"organizations" json:"organizations"`
|
||||
@@ -49,8 +52,7 @@ type Tender struct {
|
||||
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
|
||||
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
|
||||
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
|
||||
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds
|
||||
TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
|
||||
TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ type ListTendersResponse struct {
|
||||
type TenderResponse struct {
|
||||
ID string `json:"id"`
|
||||
NoticePublicationID string `json:"notice_publication_id"`
|
||||
PublicationDate int64 `json:"publication_date"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
ProcurementTypeCode string `json:"procurement_type_code"`
|
||||
@@ -40,8 +39,11 @@ type TenderResponse struct {
|
||||
Currency string `json:"currency"`
|
||||
Duration string `json:"duration"`
|
||||
DurationUnit string `json:"duration_unit"`
|
||||
PublicationDate int64 `json:"publication_date"`
|
||||
TenderDeadline int64 `json:"tender_deadline"`
|
||||
SubmissionDeadline int64 `json:"submission_deadline"`
|
||||
ApplicationDeadline int64 `json:"application_deadline"`
|
||||
SubmissionURL string `json:"submission_url"`
|
||||
CountryCode string `json:"country_code"`
|
||||
BuyerOrganization *OrganizationResponse `json:"buyer_organization"`
|
||||
Status TenderStatus `json:"status"`
|
||||
@@ -56,7 +58,6 @@ func (t *Tender) ToTenderResponse() *TenderResponse {
|
||||
response := &TenderResponse{
|
||||
ID: t.ID.Hex(),
|
||||
NoticePublicationID: t.NoticePublicationID,
|
||||
PublicationDate: t.PublicationDate,
|
||||
Title: t.Title,
|
||||
Description: t.Description,
|
||||
ProcurementTypeCode: t.ProcurementTypeCode,
|
||||
@@ -65,8 +66,11 @@ func (t *Tender) ToTenderResponse() *TenderResponse {
|
||||
Currency: t.Currency,
|
||||
Duration: t.Duration,
|
||||
DurationUnit: t.DurationUnit,
|
||||
PublicationDate: t.PublicationDate,
|
||||
TenderDeadline: t.TenderDeadline,
|
||||
SubmissionDeadline: t.SubmissionDeadline,
|
||||
ApplicationDeadline: t.ApplicationDeadline,
|
||||
SubmissionURL: t.SubmissionURL,
|
||||
CountryCode: t.CountryCode,
|
||||
BuyerOrganization: &OrganizationResponse{Name: t.BuyerOrganization.Name},
|
||||
Status: t.Status,
|
||||
|
||||
Reference in New Issue
Block a user