diff --git a/internal/document_scraper/handler.go b/internal/document_scraper/handler.go index b50744b..f1e5fd1 100644 --- a/internal/document_scraper/handler.go +++ b/internal/document_scraper/handler.go @@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler { // ListPendingTenders retrieves tenders that need document scraping // @Summary List pending tenders for document scraping -// @Description Retrieve tenders that have not yet been scraped for documents and whose document URL matches a portal supported by the AI scraping service. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with active deadlines are returned; set include_expired=true to include expired tenders. +// @Description Retrieve tenders that have not yet been scraped for documents and whose document URL matches a portal supported by the AI scraping service. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with an active publication submission window are returned (submission_deadline, or 48 working hours after publication_date); set include_expired=true to include expired tenders. // @Tags Document-Scraper // @Produce json // @Param limit query int false "Number of items per page (default: 20, max: 100)" @@ -61,7 +61,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error { // GetTenderByNoticeID retrieves a specific tender by its notice publication ID // @Summary Get tender by notice ID for document scraping -// @Description Retrieve a specific unexpired tender whose document URL matches a supported scrape portal. +// @Description Retrieve a specific tender whose publication submission window is still open and whose document URL matches a supported scrape portal. // @Tags Document-Scraper // @Produce json // @Param notice_id path string true "Notice Publication ID" diff --git a/internal/document_scraper/service.go b/internal/document_scraper/service.go index 312c8c8..570a908 100644 --- a/internal/document_scraper/service.go +++ b/internal/document_scraper/service.go @@ -67,7 +67,7 @@ func (s *service) fetchScrapePortals(ctx context.Context) ([]string, error) { // ListPendingTenders retrieves tenders that have not yet been scraped for documents. // Only returns tenders whose document URL matches a portal supported by the AI service. -// By default only tenders with active deadlines are included; set IncludeExpired to include expired tenders. +// By default only tenders with an active publication submission window are included; set IncludeExpired to include expired tenders. func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) { req.Normalize() @@ -106,7 +106,7 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis } if !req.IncludeExpired { now := time.Now().Unix() - filter.MinDeadline = &now + filter.MinWindowEnd = &now } tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, filter) @@ -117,6 +117,11 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err) } + if !req.IncludeExpired { + now := time.Now().Unix() + tenders = filterActivePublicationWindow(tenders, now) + } + result := &DocumentScraperListResponse{ Tenders: make([]DocumentScraperTenderResponse, len(tenders)), } @@ -144,7 +149,7 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis } // GetTenderByNoticeID retrieves a specific tender by its notice publication ID. -// Only returns unexpired tenders whose document URL matches a supported scrape portal. +// Only returns tenders with an active publication submission window whose document URL matches a supported scrape portal. func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) { s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{ "notice_id": noticeID, @@ -169,11 +174,13 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do } now := time.Now().Unix() - if t.TenderDeadline <= now || !matchesScrapePortals(documentURL(t), portals) { - s.logger.Info("Tender filtered out: unsupported portal or deadline passed", map[string]interface{}{ - "notice_id": noticeID, - "tender_deadline": t.TenderDeadline, - "now": now, + if !t.HasActivePublicationWindow(now) || !matchesScrapePortals(documentURL(t), portals) { + s.logger.Info("Tender filtered out: unsupported portal or publication window closed", map[string]interface{}{ + "notice_id": noticeID, + "publication_date": t.PublicationDate, + "submission_deadline": t.SubmissionDeadline, + "publication_deadline": t.PublicationSubmissionDeadline(), + "now": now, }) return nil, fmt.Errorf("tender not found") } @@ -187,3 +194,16 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do return result, nil } + +func filterActivePublicationWindow(tenders []tender.Tender, now int64) []tender.Tender { + if len(tenders) == 0 { + return tenders + } + active := make([]tender.Tender, 0, len(tenders)) + for i := range tenders { + if tenders[i].HasActivePublicationWindow(now) { + active = append(active, tenders[i]) + } + } + return active +} diff --git a/internal/tender/entity.go b/internal/tender/entity.go index a344bc4..9a47d3c 100644 --- a/internal/tender/entity.go +++ b/internal/tender/entity.go @@ -280,6 +280,12 @@ func (t *Tender) PublicationSubmissionDeadline() int64 { return ted.CalculateSubmissionDeadline(time.Unix(t.PublicationDate, 0)).Unix() } +// HasActivePublicationWindow reports whether the publication-based submission window is still open. +func (t *Tender) HasActivePublicationWindow(now int64) bool { + deadline := t.PublicationSubmissionDeadline() + return deadline > 0 && deadline > now +} + // IsRecommendable reports whether a tender should appear in customer AI recommendation results. // Recommendations use the publication-based submission window, not tender_deadline. func (t *Tender) IsRecommendable(now int64) bool { @@ -287,11 +293,7 @@ func (t *Tender) IsRecommendable(now int64) bool { case TenderStatusExpired, TenderStatusCancelled, TenderStatusAwarded, TenderStatusClosed: return false } - deadline := t.PublicationSubmissionDeadline() - if deadline <= 0 || deadline <= now { - return false - } - return true + return t.HasActivePublicationWindow(now) } // GetTenderURL returns the TED tender URL diff --git a/internal/tender/entity_test.go b/internal/tender/entity_test.go index a9996f1..5a6c826 100644 --- a/internal/tender/entity_test.go +++ b/internal/tender/entity_test.go @@ -88,6 +88,50 @@ func TestTenderPublicationSubmissionDeadline(t *testing.T) { } } +func TestTenderHasActivePublicationWindow(t *testing.T) { + now := int64(1_700_000_000) + + tests := []struct { + name string + tender Tender + want bool + }{ + { + name: "future submission deadline", + tender: Tender{SubmissionDeadline: now + 3600}, + want: true, + }, + { + name: "past submission deadline", + tender: Tender{SubmissionDeadline: now - 3600}, + want: false, + }, + { + name: "future tender deadline only", + tender: Tender{TenderDeadline: now + 86400}, + want: false, + }, + { + name: "publication date only with open window", + tender: Tender{PublicationDate: now - 3600}, + want: true, + }, + { + name: "no publication or submission deadline", + tender: Tender{}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.tender.HasActivePublicationWindow(now); got != tt.want { + t.Fatalf("HasActivePublicationWindow() = %v, want %v", got, tt.want) + } + }) + } +} + func TestTenderIsRecommendable(t *testing.T) { now := int64(1_700_000_000) diff --git a/internal/tender/repository.go b/internal/tender/repository.go index e7b3094..80d11cd 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -83,6 +83,8 @@ func documentScraperListProjection() bson.M { "tender_url": 1, "title": 1, "description": 1, + "publication_date": 1, + "submission_deadline": 1, } } @@ -656,7 +658,7 @@ func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int // PendingDocumentScrapeFilter defines query criteria for pending document scrape tenders. type PendingDocumentScrapeFilter struct { Portals []string - MinDeadline *int64 // when set, tender_deadline must be greater than this value + MinWindowEnd *int64 // when set, only tenders with an active publication submission window CreatedAtFrom *int64 CreatedAtTo *int64 Limit int @@ -682,25 +684,38 @@ func scrapePortalURLFilter(portals []string) bson.M { return bson.M{"$or": conditions} } +func publicationWindowActiveBSON(now int64) bson.M { + return bson.M{ + "$or": []bson.M{ + {"submission_deadline": bson.M{"$gt": now}}, + { + "$and": []bson.M{ + {"$or": []bson.M{ + {"submission_deadline": bson.M{"$exists": false}}, + {"submission_deadline": 0}, + }}, + {"publication_date": bson.M{"$gt": 0}}, + }, + }, + }, + } +} + // GetPendingDocumentScrapeTenders returns unscraped tenders whose document URL matches a supported portal. -// When MinDeadline is set, only tenders with tender_deadline greater than that value are returned. +// When MinWindowEnd is set, only tenders whose publication-based submission window is still open are returned. func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, filter PendingDocumentScrapeFilter) ([]Tender, bool, error) { portalFilter := scrapePortalURLFilter(filter.Portals) if portalFilter == nil { return nil, false, nil } - query := bson.M{ - "processing_metadata.documents_scraped": bson.M{"$ne": true}, - } - for key, value := range portalFilter { - query[key] = value + andFilters := []bson.M{ + {"processing_metadata.documents_scraped": bson.M{"$ne": true}}, + portalFilter, } - if filter.MinDeadline != nil { - query["tender_deadline"] = bson.M{ - "$gt": *filter.MinDeadline, - } + if filter.MinWindowEnd != nil { + andFilters = append(andFilters, publicationWindowActiveBSON(*filter.MinWindowEnd)) } if filter.CreatedAtFrom != nil || filter.CreatedAtTo != nil { @@ -711,9 +726,11 @@ func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, if filter.CreatedAtTo != nil { createdFilter["$lte"] = *filter.CreatedAtTo } - query["created_at"] = createdFilter + andFilters = append(andFilters, bson.M{"created_at": createdFilter}) } + query := bson.M{"$and": andFilters} + pagination := orm.Pagination{ Limit: filter.Limit, Skip: filter.Skip, @@ -727,7 +744,7 @@ func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, if err != nil { r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{ "portals": filter.Portals, - "min_deadline": filter.MinDeadline, + "min_window_end": filter.MinWindowEnd, "created_at_from": filter.CreatedAtFrom, "created_at_to": filter.CreatedAtTo, "limit": filter.Limit,