mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update .gitignore, adjust TED scraper configuration, and enhance Tender entity structure

Browse Source 
- Added 'ted_samples/' to .gitignore to exclude TED sample files from version control.
- Updated the TED scraper configuration to extend the cleanup duration from 24 hours to 72 hours, allowing for better resource management.
- Enhanced the Tender entity by adding new fields: TenderDeadline, SubmissionDeadline, ApplicationDeadline, and SubmissionURL, improving the data model for tender management.
- Updated the TenderResponse structure to include the new fields, ensuring consistency in API responses.
- Implemented logic in the TED scraper to calculate submission and application deadlines based on tender deadlines, enhancing the scraping functionality.
This commit is contained in:
n.nakhostin
2025-08-25 13:13:40 +03:30
parent b26d77b4cf
commit f67e179ffc
5 changed files with 86 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3
View File
@@ -63,3 +63,6 @@ docker-data/
# Build artifacts
coverage.out
coverage.html
# Ted files
ted_samples/
cmd/scraper/config.yaml
+1 -1
View File
@@ -24,5 +24,5 @@ ted:
user_agent: "TenderManagement-TED-Scraper/1.0"
max_concurrency: 2 # Reduced to avoid overwhelming the server
download_dir: "./tmp/ted_downloads"
cleanup_after: 24h
cleanup_after: 72h
scraping_interval: 12h # Run scraping every 12 hours
internal/tender/entity.go
+8 -6
View File
@@ -17,9 +17,12 @@ type Tender struct {
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"` // Unix milliseconds
IssueTime int64 `bson:"issue_time" json:"issue_time"` // Unix milliseconds
PublicationDate int64 `bson:"publication_date" json:"publication_date"` // Unix milliseconds
IssueDate int64 `bson:"issue_date" json:"issue_date"` // Unix milliseconds
IssueTime int64 `bson:"issue_time" json:"issue_time"` // Unix milliseconds
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds
PublicationDate int64 `bson:"publication_date" json:"publication_date"` // Unix milliseconds
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` // Unix milliseconds
GazetteID string `bson:"gazette_id" json:"gazette_id"`
Title string `bson:"title" json:"title"`
Description string `bson:"description" json:"description"`
@@ -31,7 +34,6 @@ type Tender struct {
Currency string `bson:"currency" json:"currency"`
Duration string `bson:"duration" json:"duration"`
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
CountryCode string `bson:"country_code" json:"country_code"`
RegionCode string `bson:"region_code" json:"region_code"`
@@ -39,6 +41,7 @@ type Tender struct {
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
Organizations []Organization `bson:"organizations" json:"organizations"`
@@ -49,8 +52,7 @@ type Tender struct {
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds
TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
}
internal/tender/form.go
+6 -2
View File
@@ -31,7 +31,6 @@ type ListTendersResponse struct {
type TenderResponse struct {
ID string `json:"id"`
NoticePublicationID string `json:"notice_publication_id"`
PublicationDate int64 `json:"publication_date"`
Title string `json:"title"`
Description string `json:"description"`
ProcurementTypeCode string `json:"procurement_type_code"`
@@ -40,8 +39,11 @@ type TenderResponse struct {
Currency string `json:"currency"`
Duration string `json:"duration"`
DurationUnit string `json:"duration_unit"`
PublicationDate int64 `json:"publication_date"`
TenderDeadline int64 `json:"tender_deadline"`
SubmissionDeadline int64 `json:"submission_deadline"`
ApplicationDeadline int64 `json:"application_deadline"`
SubmissionURL string `json:"submission_url"`
CountryCode string `json:"country_code"`
BuyerOrganization *OrganizationResponse `json:"buyer_organization"`
Status TenderStatus `json:"status"`
@@ -56,7 +58,6 @@ func (t *Tender) ToTenderResponse() *TenderResponse {
response := &TenderResponse{
ID: t.ID.Hex(),
NoticePublicationID: t.NoticePublicationID,
PublicationDate: t.PublicationDate,
Title: t.Title,
Description: t.Description,
ProcurementTypeCode: t.ProcurementTypeCode,
@@ -65,8 +66,11 @@ func (t *Tender) ToTenderResponse() *TenderResponse {
Currency: t.Currency,
Duration: t.Duration,
DurationUnit: t.DurationUnit,
PublicationDate: t.PublicationDate,
TenderDeadline: t.TenderDeadline,
SubmissionDeadline: t.SubmissionDeadline,
ApplicationDeadline: t.ApplicationDeadline,
SubmissionURL: t.SubmissionURL,
CountryCode: t.CountryCode,
BuyerOrganization: &OrganizationResponse{Name: t.BuyerOrganization.Name},
Status: t.Status,
ted/scraper.go
+67 -1
View File
@@ -325,6 +325,8 @@ func (s *TEDScraper) calculateFilesToDownload(state *ScrapingState) []FileInfo {
"last_processed": fmt.Sprintf("%d%05d", state.LastProcessedYear, state.LastProcessedNumber),
})
startNumber = 146
for number := startNumber; number <= maxNumber && len(files) < 10; number++ { // Limit to 10 files per run
fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number)
ojsID := fmt.Sprintf("S %03d / %d", number, year)
@@ -993,8 +995,28 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
// Set tender deadline
if deadline := cn.GetTenderDeadline(); deadline != "" {
t.TenderDeadline = s.parseDateToUnixMilli(deadline)
if deadlineTime, err := s.parseDate(deadline); err == nil {
t.SubmissionDeadline = deadlineTime.UnixMilli()
applicationDeadline := s.calculateApplicationDeadline(deadlineTime)
t.ApplicationDeadline = applicationDeadline.UnixMilli()
}
}
if pubDate, err := s.parseDate(cn.GetPublicationDate()); err == nil {
submissionDeadline := s.calculateSubmissionDeadline(pubDate)
t.SubmissionDeadline = submissionDeadline.UnixMilli()
}
// Set SubmissionURL from TenderingTerms
if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil {
if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
if strings.HasPrefix(endpointID, "http") {
t.SubmissionURL = endpointID
} else {
t.SubmissionURL = "https://" + endpointID
}
}
}
}
@@ -1330,3 +1352,47 @@ func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string {
return time.UnixMilli(unixMilli).Format("20060102")
}
func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time {
deadline := publicationDate
workingHoursAdded := 0
for workingHoursAdded < 48 {
deadline = deadline.Add(1 * time.Hour)
if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday {
workingHoursAdded++
}
}
return deadline
}
func (s *TEDScraper) calculateApplicationDeadline(tenderDeadline time.Time) time.Time {
applicationDeadline := tenderDeadline
workingDaysSubtracted := 0
for workingDaysSubtracted < 7 {
applicationDeadline = applicationDeadline.AddDate(0, 0, -1)
if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday {
workingDaysSubtracted++
}
}
return applicationDeadline
}
func (s *TEDScraper) adjustToWorkingDay(targetDate time.Time, forward bool) time.Time {
current := targetDate
for current.Weekday() == time.Saturday || current.Weekday() == time.Sunday {
if forward {
current = current.AddDate(0, 0, 1)
} else {
current = current.AddDate(0, 0, -1)
}
}
return current
}