Refactor document scraper to utilize active publication windows for tender retrieval

- Updated the ListPendingTenders method to filter tenders based on their active publication submission window instead of just deadlines, enhancing the accuracy of tender listings.
- Introduced a new HasActivePublicationWindow method in the Tender entity to determine if the publication-based submission window is still open.
- Modified the GetTenderByNoticeID method to reflect the new logic for filtering tenders based on their publication status.
- Enhanced the documentation for relevant API endpoints to clarify the changes in tender retrieval criteria.

This update improves the precision of tender scraping by ensuring only relevant tenders with active submission windows are processed, contributing to better data management and scraping efficiency.

internal/document_scraper/service.go

@@ -67,7 +67,7 @@ func (s *service) fetchScrapePortals(ctx context.Context) ([]string, error) {
 
 // ListPendingTenders retrieves tenders that have not yet been scraped for documents.
 // Only returns tenders whose document URL matches a portal supported by the AI service.
-// By default only tenders with active deadlines are included; set IncludeExpired to include expired tenders.
+// By default only tenders with an active publication submission window are included; set IncludeExpired to include expired tenders.
 func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) {
 	req.Normalize()
 
@@ -106,7 +106,7 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis
 	}
 	if !req.IncludeExpired {
 		now := time.Now().Unix()
-		filter.MinDeadline = &now
+		filter.MinWindowEnd = &now
 	}
 
 	tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, filter)
@@ -117,6 +117,11 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis
 		return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
 	}
 
+	if !req.IncludeExpired {
+		now := time.Now().Unix()
+		tenders = filterActivePublicationWindow(tenders, now)
+	}
+
 	result := &DocumentScraperListResponse{
 		Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
 	}
@@ -144,7 +149,7 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis
 }
 
 // GetTenderByNoticeID retrieves a specific tender by its notice publication ID.
-// Only returns unexpired tenders whose document URL matches a supported scrape portal.
+// Only returns tenders with an active publication submission window whose document URL matches a supported scrape portal.
 func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
 	s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
 		"notice_id": noticeID,
@@ -169,11 +174,13 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do
 	}
 
 	now := time.Now().Unix()
-	if t.TenderDeadline <= now || !matchesScrapePortals(documentURL(t), portals) {
-		s.logger.Info("Tender filtered out: unsupported portal or deadline passed", map[string]interface{}{
-			"notice_id":       noticeID,
-			"tender_deadline": t.TenderDeadline,
-			"now":             now,
+	if !t.HasActivePublicationWindow(now) || !matchesScrapePortals(documentURL(t), portals) {
+		s.logger.Info("Tender filtered out: unsupported portal or publication window closed", map[string]interface{}{
+			"notice_id":            noticeID,
+			"publication_date":     t.PublicationDate,
+			"submission_deadline":  t.SubmissionDeadline,
+			"publication_deadline": t.PublicationSubmissionDeadline(),
+			"now":                  now,
 		})
 		return nil, fmt.Errorf("tender not found")
 	}
@@ -187,3 +194,16 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do
 
 	return result, nil
 }
+
+func filterActivePublicationWindow(tenders []tender.Tender, now int64) []tender.Tender {
+	if len(tenders) == 0 {
+		return tenders
+	}
+	active := make([]tender.Tender, 0, len(tenders))
+	for i := range tenders {
+		if tenders[i].HasActivePublicationWindow(now) {
+			active = append(active, tenders[i])
+		}
+	}
+	return active
+}