From f67e179ffca4928a84f606b319b9a2cf8796365a Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Mon, 25 Aug 2025 13:13:40 +0330 Subject: [PATCH] Update .gitignore, adjust TED scraper configuration, and enhance Tender entity structure - Added 'ted_samples/' to .gitignore to exclude TED sample files from version control. - Updated the TED scraper configuration to extend the cleanup duration from 24 hours to 72 hours, allowing for better resource management. - Enhanced the Tender entity by adding new fields: TenderDeadline, SubmissionDeadline, ApplicationDeadline, and SubmissionURL, improving the data model for tender management. - Updated the TenderResponse structure to include the new fields, ensuring consistency in API responses. - Implemented logic in the TED scraper to calculate submission and application deadlines based on tender deadlines, enhancing the scraping functionality. --- .gitignore | 5 ++- cmd/scraper/config.yaml | 2 +- internal/tender/entity.go | 14 ++++---- internal/tender/form.go | 8 +++-- ted/scraper.go | 68 ++++++++++++++++++++++++++++++++++++++- 5 files changed, 86 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index 5efe268..2101bbe 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,7 @@ docker-data/ # Build artifacts coverage.out -coverage.html \ No newline at end of file +coverage.html + +# Ted files +ted_samples/ \ No newline at end of file diff --git a/cmd/scraper/config.yaml b/cmd/scraper/config.yaml index e8dd90f..a3fc844 100644 --- a/cmd/scraper/config.yaml +++ b/cmd/scraper/config.yaml @@ -24,5 +24,5 @@ ted: user_agent: "TenderManagement-TED-Scraper/1.0" max_concurrency: 2 # Reduced to avoid overwhelming the server download_dir: "./tmp/ted_downloads" - cleanup_after: 24h + cleanup_after: 72h scraping_interval: 12h # Run scraping every 12 hours diff --git a/internal/tender/entity.go b/internal/tender/entity.go index 62d1028..926bfdc 100644 --- a/internal/tender/entity.go +++ b/internal/tender/entity.go @@ -17,9 +17,12 @@ type Tender struct { NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"` NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"` NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"` - IssueDate int64 `bson:"issue_date" json:"issue_date"` // Unix milliseconds - IssueTime int64 `bson:"issue_time" json:"issue_time"` // Unix milliseconds - PublicationDate int64 `bson:"publication_date" json:"publication_date"` // Unix milliseconds + IssueDate int64 `bson:"issue_date" json:"issue_date"` // Unix milliseconds + IssueTime int64 `bson:"issue_time" json:"issue_time"` // Unix milliseconds + TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds + PublicationDate int64 `bson:"publication_date" json:"publication_date"` // Unix milliseconds + SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds + ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` // Unix milliseconds GazetteID string `bson:"gazette_id" json:"gazette_id"` Title string `bson:"title" json:"title"` Description string `bson:"description" json:"description"` @@ -31,7 +34,6 @@ type Tender struct { Currency string `bson:"currency" json:"currency"` Duration string `bson:"duration" json:"duration"` DurationUnit string `bson:"duration_unit" json:"duration_unit"` - TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"` CountryCode string `bson:"country_code" json:"country_code"` RegionCode string `bson:"region_code" json:"region_code"` @@ -39,6 +41,7 @@ type Tender struct { PostalCode string `bson:"postal_code" json:"postal_code"` DocumentURI string `bson:"document_uri" json:"document_uri"` TenderURL string `bson:"tender_url" json:"tender_url"` + SubmissionURL string `bson:"submission_url" json:"submission_url"` BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"` ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"` Organizations []Organization `bson:"organizations" json:"organizations"` @@ -49,8 +52,7 @@ type Tender struct { SourceFileURL string `bson:"source_file_url" json:"source_file_url"` SourceFileName string `bson:"source_file_name" json:"source_file_name"` ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` - SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds - TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} + TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} } diff --git a/internal/tender/form.go b/internal/tender/form.go index 34a606c..14f7e77 100644 --- a/internal/tender/form.go +++ b/internal/tender/form.go @@ -31,7 +31,6 @@ type ListTendersResponse struct { type TenderResponse struct { ID string `json:"id"` NoticePublicationID string `json:"notice_publication_id"` - PublicationDate int64 `json:"publication_date"` Title string `json:"title"` Description string `json:"description"` ProcurementTypeCode string `json:"procurement_type_code"` @@ -40,8 +39,11 @@ type TenderResponse struct { Currency string `json:"currency"` Duration string `json:"duration"` DurationUnit string `json:"duration_unit"` + PublicationDate int64 `json:"publication_date"` TenderDeadline int64 `json:"tender_deadline"` SubmissionDeadline int64 `json:"submission_deadline"` + ApplicationDeadline int64 `json:"application_deadline"` + SubmissionURL string `json:"submission_url"` CountryCode string `json:"country_code"` BuyerOrganization *OrganizationResponse `json:"buyer_organization"` Status TenderStatus `json:"status"` @@ -56,7 +58,6 @@ func (t *Tender) ToTenderResponse() *TenderResponse { response := &TenderResponse{ ID: t.ID.Hex(), NoticePublicationID: t.NoticePublicationID, - PublicationDate: t.PublicationDate, Title: t.Title, Description: t.Description, ProcurementTypeCode: t.ProcurementTypeCode, @@ -65,8 +66,11 @@ func (t *Tender) ToTenderResponse() *TenderResponse { Currency: t.Currency, Duration: t.Duration, DurationUnit: t.DurationUnit, + PublicationDate: t.PublicationDate, TenderDeadline: t.TenderDeadline, SubmissionDeadline: t.SubmissionDeadline, + ApplicationDeadline: t.ApplicationDeadline, + SubmissionURL: t.SubmissionURL, CountryCode: t.CountryCode, BuyerOrganization: &OrganizationResponse{Name: t.BuyerOrganization.Name}, Status: t.Status, diff --git a/ted/scraper.go b/ted/scraper.go index 0ec0134..4348bd8 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -325,6 +325,8 @@ func (s *TEDScraper) calculateFilesToDownload(state *ScrapingState) []FileInfo { "last_processed": fmt.Sprintf("%d%05d", state.LastProcessedYear, state.LastProcessedNumber), }) + startNumber = 146 + for number := startNumber; number <= maxNumber && len(files) < 10; number++ { // Limit to 10 files per run fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number) ojsID := fmt.Sprintf("S %03d / %d", number, year) @@ -993,8 +995,28 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa // Set tender deadline if deadline := cn.GetTenderDeadline(); deadline != "" { t.TenderDeadline = s.parseDateToUnixMilli(deadline) + if deadlineTime, err := s.parseDate(deadline); err == nil { - t.SubmissionDeadline = deadlineTime.UnixMilli() + applicationDeadline := s.calculateApplicationDeadline(deadlineTime) + t.ApplicationDeadline = applicationDeadline.UnixMilli() + } + } + + if pubDate, err := s.parseDate(cn.GetPublicationDate()); err == nil { + submissionDeadline := s.calculateSubmissionDeadline(pubDate) + t.SubmissionDeadline = submissionDeadline.UnixMilli() + } + + // Set SubmissionURL from TenderingTerms + if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil { + if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil { + if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" { + if strings.HasPrefix(endpointID, "http") { + t.SubmissionURL = endpointID + } else { + t.SubmissionURL = "https://" + endpointID + } + } } } @@ -1330,3 +1352,47 @@ func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string { return time.UnixMilli(unixMilli).Format("20060102") } + +func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time { + deadline := publicationDate + workingHoursAdded := 0 + + for workingHoursAdded < 48 { + deadline = deadline.Add(1 * time.Hour) + + if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday { + workingHoursAdded++ + } + } + + return deadline +} + +func (s *TEDScraper) calculateApplicationDeadline(tenderDeadline time.Time) time.Time { + applicationDeadline := tenderDeadline + workingDaysSubtracted := 0 + + for workingDaysSubtracted < 7 { + applicationDeadline = applicationDeadline.AddDate(0, 0, -1) + + if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday { + workingDaysSubtracted++ + } + } + + return applicationDeadline +} + +func (s *TEDScraper) adjustToWorkingDay(targetDate time.Time, forward bool) time.Time { + current := targetDate + + for current.Weekday() == time.Saturday || current.Weekday() == time.Sunday { + if forward { + current = current.AddDate(0, 0, 1) + } else { + current = current.AddDate(0, 0, -1) + } + } + + return current +}