Update .gitignore, adjust TED scraper configuration, and enhance Tender entity structure

- Added 'ted_samples/' to .gitignore to exclude TED sample files from version control.
- Updated the TED scraper configuration to extend the cleanup duration from 24 hours to 72 hours, allowing for better resource management.
- Enhanced the Tender entity by adding new fields: TenderDeadline, SubmissionDeadline, ApplicationDeadline, and SubmissionURL, improving the data model for tender management.
- Updated the TenderResponse structure to include the new fields, ensuring consistency in API responses.
- Implemented logic in the TED scraper to calculate submission and application deadlines based on tender deadlines, enhancing the scraping functionality.

.gitignore

@@ -62,4 +62,7 @@ docker-data/
 
 # Build artifacts
 coverage.out
-coverage.html 
+coverage.html 
+
+# Ted files
+ted_samples/

cmd/scraper/config.yaml

@@ -24,5 +24,5 @@ ted:
   user_agent: "TenderManagement-TED-Scraper/1.0"
   max_concurrency: 2  # Reduced to avoid overwhelming the server
   download_dir: "./tmp/ted_downloads"
-  cleanup_after: 24h
+  cleanup_after: 72h
   scraping_interval: 12h  # Run scraping every 12 hours

internal/tender/entity.go

@@ -17,9 +17,12 @@ type Tender struct {
 	NoticeTypeCode            string               `bson:"notice_type_code" json:"notice_type_code"`
 	NoticeSubTypeCode         string               `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
 	NoticeLanguageCode        string               `bson:"notice_language_code" json:"notice_language_code"`
-	IssueDate                 int64                `bson:"issue_date" json:"issue_date"`             // Unix milliseconds
+	IssueDate                 int64                `bson:"issue_date" json:"issue_date"`                     // Unix milliseconds
-	IssueTime                 int64                `bson:"issue_time" json:"issue_time"`             // Unix milliseconds
+	IssueTime                 int64                `bson:"issue_time" json:"issue_time"`                     // Unix milliseconds
-	PublicationDate           int64                `bson:"publication_date" json:"publication_date"` // Unix milliseconds
+	TenderDeadline            int64                `bson:"tender_deadline" json:"tender_deadline"`           // Unix milliseconds
+	PublicationDate           int64                `bson:"publication_date" json:"publication_date"`         // Unix milliseconds
+	SubmissionDeadline        int64                `bson:"submission_deadline" json:"submission_deadline"`   // Unix milliseconds
+	ApplicationDeadline       int64                `bson:"application_deadline" json:"application_deadline"` // Unix milliseconds
 	GazetteID                 string               `bson:"gazette_id" json:"gazette_id"`
 	Title                     string               `bson:"title" json:"title"`
 	Description               string               `bson:"description" json:"description"`
@@ -31,7 +34,6 @@ type Tender struct {
 	Currency                  string               `bson:"currency" json:"currency"`
 	Duration                  string               `bson:"duration" json:"duration"`
 	DurationUnit              string               `bson:"duration_unit" json:"duration_unit"`
-	TenderDeadline            int64                `bson:"tender_deadline" json:"tender_deadline"` // Unix milliseconds
 	PlaceOfPerformance        string               `bson:"place_of_performance" json:"place_of_performance"`
 	CountryCode               string               `bson:"country_code" json:"country_code"`
 	RegionCode                string               `bson:"region_code" json:"region_code"`
@@ -39,6 +41,7 @@ type Tender struct {
 	PostalCode                string               `bson:"postal_code" json:"postal_code"`
 	DocumentURI               string               `bson:"document_uri" json:"document_uri"`
 	TenderURL                 string               `bson:"tender_url" json:"tender_url"`
+	SubmissionURL             string               `bson:"submission_url" json:"submission_url"`
 	BuyerOrganization         *Organization        `bson:"buyer_organization" json:"buyer_organization"`
 	ReviewOrganization        *Organization        `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
 	Organizations             []Organization       `bson:"organizations" json:"organizations"`
@@ -49,8 +52,7 @@ type Tender struct {
 	SourceFileURL             string               `bson:"source_file_url" json:"source_file_url"`
 	SourceFileName            string               `bson:"source_file_name" json:"source_file_name"`
 	ProcessingMetadata        ProcessingMetadata   `bson:"processing_metadata" json:"processing_metadata"`
-	SubmissionDeadline        int64                `bson:"submission_deadline" json:"submission_deadline"` // Unix milliseconds
+	TenderID                  string               `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
-	TenderID                  string               `bson:"tender_id" json:"tender_id"`                     // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
 
 }
 

internal/tender/form.go

@@ -31,7 +31,6 @@ type ListTendersResponse struct {
 type TenderResponse struct {
 	ID                  string                `json:"id"`
 	NoticePublicationID string                `json:"notice_publication_id"`
-	PublicationDate     int64                 `json:"publication_date"`
 	Title               string                `json:"title"`
 	Description         string                `json:"description"`
 	ProcurementTypeCode string                `json:"procurement_type_code"`
@@ -40,8 +39,11 @@ type TenderResponse struct {
 	Currency            string                `json:"currency"`
 	Duration            string                `json:"duration"`
 	DurationUnit        string                `json:"duration_unit"`
+	PublicationDate     int64                 `json:"publication_date"`
 	TenderDeadline      int64                 `json:"tender_deadline"`
 	SubmissionDeadline  int64                 `json:"submission_deadline"`
+	ApplicationDeadline int64                 `json:"application_deadline"`
+	SubmissionURL       string                `json:"submission_url"`
 	CountryCode         string                `json:"country_code"`
 	BuyerOrganization   *OrganizationResponse `json:"buyer_organization"`
 	Status              TenderStatus          `json:"status"`
@@ -56,7 +58,6 @@ func (t *Tender) ToTenderResponse() *TenderResponse {
 	response := &TenderResponse{
 		ID:                  t.ID.Hex(),
 		NoticePublicationID: t.NoticePublicationID,
-		PublicationDate:     t.PublicationDate,
 		Title:               t.Title,
 		Description:         t.Description,
 		ProcurementTypeCode: t.ProcurementTypeCode,
@@ -65,8 +66,11 @@ func (t *Tender) ToTenderResponse() *TenderResponse {
 		Currency:            t.Currency,
 		Duration:            t.Duration,
 		DurationUnit:        t.DurationUnit,
+		PublicationDate:     t.PublicationDate,
 		TenderDeadline:      t.TenderDeadline,
 		SubmissionDeadline:  t.SubmissionDeadline,
+		ApplicationDeadline: t.ApplicationDeadline,
+		SubmissionURL:       t.SubmissionURL,
 		CountryCode:         t.CountryCode,
 		BuyerOrganization:   &OrganizationResponse{Name: t.BuyerOrganization.Name},
 		Status:              t.Status,

ted/scraper.go

@@ -325,6 +325,8 @@ func (s *TEDScraper) calculateFilesToDownload(state *ScrapingState) []FileInfo {
 		"last_processed": fmt.Sprintf("%d%05d", state.LastProcessedYear, state.LastProcessedNumber),
 	})
 
+	startNumber = 146
+
 	for number := startNumber; number <= maxNumber && len(files) < 10; number++ { // Limit to 10 files per run
 		fileURL := fmt.Sprintf("%s/%d%05d", s.config.BaseURL, year, number)
 		ojsID := fmt.Sprintf("S %03d / %d", number, year)
@@ -993,8 +995,28 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
 	// Set tender deadline
 	if deadline := cn.GetTenderDeadline(); deadline != "" {
 		t.TenderDeadline = s.parseDateToUnixMilli(deadline)
+
 		if deadlineTime, err := s.parseDate(deadline); err == nil {
-			t.SubmissionDeadline = deadlineTime.UnixMilli()
+			applicationDeadline := s.calculateApplicationDeadline(deadlineTime)
+			t.ApplicationDeadline = applicationDeadline.UnixMilli()
+		}
+	}
+
+	if pubDate, err := s.parseDate(cn.GetPublicationDate()); err == nil {
+		submissionDeadline := s.calculateSubmissionDeadline(pubDate)
+		t.SubmissionDeadline = submissionDeadline.UnixMilli()
+	}
+
+	// Set SubmissionURL from TenderingTerms
+	if cn.ProcurementProjectLot != nil && cn.ProcurementProjectLot.TenderingTerms != nil {
+		if tenderingTerms := cn.ProcurementProjectLot.TenderingTerms; tenderingTerms.TenderRecipientParty != nil {
+			if endpointID := tenderingTerms.TenderRecipientParty.EndpointID; endpointID != "" {
+				if strings.HasPrefix(endpointID, "http") {
+					t.SubmissionURL = endpointID
+				} else {
+					t.SubmissionURL = "https://" + endpointID
+				}
+			}
 		}
 	}
 
@@ -1330,3 +1352,47 @@ func (s *TEDScraper) unixMilliToDateString(unixMilli int64) string {
 
 	return time.UnixMilli(unixMilli).Format("20060102")
 }
+
+func (s *TEDScraper) calculateSubmissionDeadline(publicationDate time.Time) time.Time {
+	deadline := publicationDate
+	workingHoursAdded := 0
+
+	for workingHoursAdded < 48 {
+		deadline = deadline.Add(1 * time.Hour)
+
+		if deadline.Weekday() != time.Saturday && deadline.Weekday() != time.Sunday {
+			workingHoursAdded++
+		}
+	}
+
+	return deadline
+}
+
+func (s *TEDScraper) calculateApplicationDeadline(tenderDeadline time.Time) time.Time {
+	applicationDeadline := tenderDeadline
+	workingDaysSubtracted := 0
+
+	for workingDaysSubtracted < 7 {
+		applicationDeadline = applicationDeadline.AddDate(0, 0, -1)
+
+		if applicationDeadline.Weekday() != time.Saturday && applicationDeadline.Weekday() != time.Sunday {
+			workingDaysSubtracted++
+		}
+	}
+
+	return applicationDeadline
+}
+
+func (s *TEDScraper) adjustToWorkingDay(targetDate time.Time, forward bool) time.Time {
+	current := targetDate
+
+	for current.Weekday() == time.Saturday || current.Weekday() == time.Sunday {
+		if forward {
+			current = current.AddDate(0, 0, 1)
+		} else {
+			current = current.AddDate(0, 0, -1)
+		}
+	}
+
+	return current
+}