Refactor document scraper to utilize active publication windows for tender retrieval

- Updated the ListPendingTenders method to filter tenders based on their active publication submission window instead of just deadlines, enhancing the accuracy of tender listings.
- Introduced a new HasActivePublicationWindow method in the Tender entity to determine if the publication-based submission window is still open.
- Modified the GetTenderByNoticeID method to reflect the new logic for filtering tenders based on their publication status.
- Enhanced the documentation for relevant API endpoints to clarify the changes in tender retrieval criteria.

This update improves the precision of tender scraping by ensuring only relevant tenders with active submission windows are processed, contributing to better data management and scraping efficiency.

internal/tender/entity.go

@@ -280,6 +280,12 @@ func (t *Tender) PublicationSubmissionDeadline() int64 {
 	return ted.CalculateSubmissionDeadline(time.Unix(t.PublicationDate, 0)).Unix()
 }
 
+// HasActivePublicationWindow reports whether the publication-based submission window is still open.
+func (t *Tender) HasActivePublicationWindow(now int64) bool {
+	deadline := t.PublicationSubmissionDeadline()
+	return deadline > 0 && deadline > now
+}
+
 // IsRecommendable reports whether a tender should appear in customer AI recommendation results.
 // Recommendations use the publication-based submission window, not tender_deadline.
 func (t *Tender) IsRecommendable(now int64) bool {
@@ -287,11 +293,7 @@ func (t *Tender) IsRecommendable(now int64) bool {
 	case TenderStatusExpired, TenderStatusCancelled, TenderStatusAwarded, TenderStatusClosed:
 		return false
 	}
-	deadline := t.PublicationSubmissionDeadline()
-	if deadline <= 0 || deadline <= now {
-		return false
-	}
-	return true
+	return t.HasActivePublicationWindow(now)
 }
 
 // GetTenderURL returns the TED tender URL

internal/tender/entity_test.go

@@ -88,6 +88,50 @@ func TestTenderPublicationSubmissionDeadline(t *testing.T) {
 	}
 }
 
+func TestTenderHasActivePublicationWindow(t *testing.T) {
+	now := int64(1_700_000_000)
+
+	tests := []struct {
+		name   string
+		tender Tender
+		want   bool
+	}{
+		{
+			name:   "future submission deadline",
+			tender: Tender{SubmissionDeadline: now + 3600},
+			want:   true,
+		},
+		{
+			name:   "past submission deadline",
+			tender: Tender{SubmissionDeadline: now - 3600},
+			want:   false,
+		},
+		{
+			name:   "future tender deadline only",
+			tender: Tender{TenderDeadline: now + 86400},
+			want:   false,
+		},
+		{
+			name:   "publication date only with open window",
+			tender: Tender{PublicationDate: now - 3600},
+			want:   true,
+		},
+		{
+			name:   "no publication or submission deadline",
+			tender: Tender{},
+			want:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.tender.HasActivePublicationWindow(now); got != tt.want {
+				t.Fatalf("HasActivePublicationWindow() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestTenderIsRecommendable(t *testing.T) {
 	now := int64(1_700_000_000)
 

internal/tender/repository.go

@@ -83,6 +83,8 @@ func documentScraperListProjection() bson.M {
 		"tender_url":            1,
 		"title":                 1,
 		"description":           1,
+		"publication_date":      1,
+		"submission_deadline":   1,
 	}
 }
 
@@ -656,7 +658,7 @@ func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int
 // PendingDocumentScrapeFilter defines query criteria for pending document scrape tenders.
 type PendingDocumentScrapeFilter struct {
 	Portals       []string
-	MinDeadline   *int64 // when set, tender_deadline must be greater than this value
+	MinWindowEnd  *int64 // when set, only tenders with an active publication submission window
 	CreatedAtFrom *int64
 	CreatedAtTo   *int64
 	Limit         int
@@ -682,25 +684,38 @@ func scrapePortalURLFilter(portals []string) bson.M {
 	return bson.M{"$or": conditions}
 }
 
+func publicationWindowActiveBSON(now int64) bson.M {
+	return bson.M{
+		"$or": []bson.M{
+			{"submission_deadline": bson.M{"$gt": now}},
+			{
+				"$and": []bson.M{
+					{"$or": []bson.M{
+						{"submission_deadline": bson.M{"$exists": false}},
+						{"submission_deadline": 0},
+					}},
+					{"publication_date": bson.M{"$gt": 0}},
+				},
+			},
+		},
+	}
+}
+
 // GetPendingDocumentScrapeTenders returns unscraped tenders whose document URL matches a supported portal.
-// When MinDeadline is set, only tenders with tender_deadline greater than that value are returned.
+// When MinWindowEnd is set, only tenders whose publication-based submission window is still open are returned.
 func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, filter PendingDocumentScrapeFilter) ([]Tender, bool, error) {
 	portalFilter := scrapePortalURLFilter(filter.Portals)
 	if portalFilter == nil {
 		return nil, false, nil
 	}
 
-	query := bson.M{
-		"processing_metadata.documents_scraped": bson.M{"$ne": true},
-	}
-	for key, value := range portalFilter {
-		query[key] = value
+	andFilters := []bson.M{
+		{"processing_metadata.documents_scraped": bson.M{"$ne": true}},
+		portalFilter,
 	}
 
-	if filter.MinDeadline != nil {
-		query["tender_deadline"] = bson.M{
-			"$gt": *filter.MinDeadline,
-		}
+	if filter.MinWindowEnd != nil {
+		andFilters = append(andFilters, publicationWindowActiveBSON(*filter.MinWindowEnd))
 	}
 
 	if filter.CreatedAtFrom != nil || filter.CreatedAtTo != nil {
@@ -711,9 +726,11 @@ func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context,
 		if filter.CreatedAtTo != nil {
 			createdFilter["$lte"] = *filter.CreatedAtTo
 		}
-		query["created_at"] = createdFilter
+		andFilters = append(andFilters, bson.M{"created_at": createdFilter})
 	}
 
+	query := bson.M{"$and": andFilters}
+
 	pagination := orm.Pagination{
 		Limit:      filter.Limit,
 		Skip:       filter.Skip,
@@ -727,7 +744,7 @@ func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context,
 	if err != nil {
 		r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{
 			"portals":         filter.Portals,
-			"min_deadline":    filter.MinDeadline,
+			"min_window_end":  filter.MinWindowEnd,
 			"created_at_from": filter.CreatedAtFrom,
 			"created_at_to":   filter.CreatedAtTo,
 			"limit":           filter.Limit,