From c46a8d54f43e4e0ab8ef5312e788e9b9002f7ac2 Mon Sep 17 00:00:00 2001 From: Mazyar Date: Wed, 1 Jul 2026 19:22:45 +0330 Subject: [PATCH] Refactor document scraper to utilize active publication windows for tender retrieval - Updated the ListPendingTenders method to filter tenders based on their active publication submission window instead of just deadlines, enhancing the accuracy of tender listings. - Introduced a new HasActivePublicationWindow method in the Tender entity to determine if the publication-based submission window is still open. - Modified the GetTenderByNoticeID method to reflect the new logic for filtering tenders based on their publication status. - Enhanced the documentation for relevant API endpoints to clarify the changes in tender retrieval criteria. This update improves the precision of tender scraping by ensuring only relevant tenders with active submission windows are processed, contributing to better data management and scraping efficiency. --- internal/document_scraper/handler.go | 4 +-- internal/document_scraper/service.go | 36 ++++++++++++++++++----- internal/tender/entity.go | 12 ++++---- internal/tender/entity_test.go | 44 ++++++++++++++++++++++++++++ internal/tender/repository.go | 43 +++++++++++++++++++-------- 5 files changed, 111 insertions(+), 28 deletions(-) diff --git a/internal/document_scraper/handler.go b/internal/document_scraper/handler.go index b50744b..f1e5fd1 100644 --- a/internal/document_scraper/handler.go +++ b/internal/document_scraper/handler.go @@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler { // ListPendingTenders retrieves tenders that need document scraping // @Summary List pending tenders for document scraping -// @Description Retrieve tenders that have not yet been scraped for documents and whose document URL matches a portal supported by the AI scraping service. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with active deadlines are returned; set include_expired=true to include expired tenders. +// @Description Retrieve tenders that have not yet been scraped for documents and whose document URL matches a portal supported by the AI scraping service. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with an active publication submission window are returned (submission_deadline, or 48 working hours after publication_date); set include_expired=true to include expired tenders. // @Tags Document-Scraper // @Produce json // @Param limit query int false "Number of items per page (default: 20, max: 100)" @@ -61,7 +61,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error { // GetTenderByNoticeID retrieves a specific tender by its notice publication ID // @Summary Get tender by notice ID for document scraping -// @Description Retrieve a specific unexpired tender whose document URL matches a supported scrape portal. +// @Description Retrieve a specific tender whose publication submission window is still open and whose document URL matches a supported scrape portal. // @Tags Document-Scraper // @Produce json // @Param notice_id path string true "Notice Publication ID" diff --git a/internal/document_scraper/service.go b/internal/document_scraper/service.go index 312c8c8..570a908 100644 --- a/internal/document_scraper/service.go +++ b/internal/document_scraper/service.go @@ -67,7 +67,7 @@ func (s *service) fetchScrapePortals(ctx context.Context) ([]string, error) { // ListPendingTenders retrieves tenders that have not yet been scraped for documents. // Only returns tenders whose document URL matches a portal supported by the AI service. -// By default only tenders with active deadlines are included; set IncludeExpired to include expired tenders. +// By default only tenders with an active publication submission window are included; set IncludeExpired to include expired tenders. func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) { req.Normalize() @@ -106,7 +106,7 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis } if !req.IncludeExpired { now := time.Now().Unix() - filter.MinDeadline = &now + filter.MinWindowEnd = &now } tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, filter) @@ -117,6 +117,11 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err) } + if !req.IncludeExpired { + now := time.Now().Unix() + tenders = filterActivePublicationWindow(tenders, now) + } + result := &DocumentScraperListResponse{ Tenders: make([]DocumentScraperTenderResponse, len(tenders)), } @@ -144,7 +149,7 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis } // GetTenderByNoticeID retrieves a specific tender by its notice publication ID. -// Only returns unexpired tenders whose document URL matches a supported scrape portal. +// Only returns tenders with an active publication submission window whose document URL matches a supported scrape portal. func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) { s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{ "notice_id": noticeID, @@ -169,11 +174,13 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do } now := time.Now().Unix() - if t.TenderDeadline <= now || !matchesScrapePortals(documentURL(t), portals) { - s.logger.Info("Tender filtered out: unsupported portal or deadline passed", map[string]interface{}{ - "notice_id": noticeID, - "tender_deadline": t.TenderDeadline, - "now": now, + if !t.HasActivePublicationWindow(now) || !matchesScrapePortals(documentURL(t), portals) { + s.logger.Info("Tender filtered out: unsupported portal or publication window closed", map[string]interface{}{ + "notice_id": noticeID, + "publication_date": t.PublicationDate, + "submission_deadline": t.SubmissionDeadline, + "publication_deadline": t.PublicationSubmissionDeadline(), + "now": now, }) return nil, fmt.Errorf("tender not found") } @@ -187,3 +194,16 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do return result, nil } + +func filterActivePublicationWindow(tenders []tender.Tender, now int64) []tender.Tender { + if len(tenders) == 0 { + return tenders + } + active := make([]tender.Tender, 0, len(tenders)) + for i := range tenders { + if tenders[i].HasActivePublicationWindow(now) { + active = append(active, tenders[i]) + } + } + return active +} diff --git a/internal/tender/entity.go b/internal/tender/entity.go index a344bc4..9a47d3c 100644 --- a/internal/tender/entity.go +++ b/internal/tender/entity.go @@ -280,6 +280,12 @@ func (t *Tender) PublicationSubmissionDeadline() int64 { return ted.CalculateSubmissionDeadline(time.Unix(t.PublicationDate, 0)).Unix() } +// HasActivePublicationWindow reports whether the publication-based submission window is still open. +func (t *Tender) HasActivePublicationWindow(now int64) bool { + deadline := t.PublicationSubmissionDeadline() + return deadline > 0 && deadline > now +} + // IsRecommendable reports whether a tender should appear in customer AI recommendation results. // Recommendations use the publication-based submission window, not tender_deadline. func (t *Tender) IsRecommendable(now int64) bool { @@ -287,11 +293,7 @@ func (t *Tender) IsRecommendable(now int64) bool { case TenderStatusExpired, TenderStatusCancelled, TenderStatusAwarded, TenderStatusClosed: return false } - deadline := t.PublicationSubmissionDeadline() - if deadline <= 0 || deadline <= now { - return false - } - return true + return t.HasActivePublicationWindow(now) } // GetTenderURL returns the TED tender URL diff --git a/internal/tender/entity_test.go b/internal/tender/entity_test.go index a9996f1..5a6c826 100644 --- a/internal/tender/entity_test.go +++ b/internal/tender/entity_test.go @@ -88,6 +88,50 @@ func TestTenderPublicationSubmissionDeadline(t *testing.T) { } } +func TestTenderHasActivePublicationWindow(t *testing.T) { + now := int64(1_700_000_000) + + tests := []struct { + name string + tender Tender + want bool + }{ + { + name: "future submission deadline", + tender: Tender{SubmissionDeadline: now + 3600}, + want: true, + }, + { + name: "past submission deadline", + tender: Tender{SubmissionDeadline: now - 3600}, + want: false, + }, + { + name: "future tender deadline only", + tender: Tender{TenderDeadline: now + 86400}, + want: false, + }, + { + name: "publication date only with open window", + tender: Tender{PublicationDate: now - 3600}, + want: true, + }, + { + name: "no publication or submission deadline", + tender: Tender{}, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.tender.HasActivePublicationWindow(now); got != tt.want { + t.Fatalf("HasActivePublicationWindow() = %v, want %v", got, tt.want) + } + }) + } +} + func TestTenderIsRecommendable(t *testing.T) { now := int64(1_700_000_000) diff --git a/internal/tender/repository.go b/internal/tender/repository.go index e7b3094..80d11cd 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -83,6 +83,8 @@ func documentScraperListProjection() bson.M { "tender_url": 1, "title": 1, "description": 1, + "publication_date": 1, + "submission_deadline": 1, } } @@ -656,7 +658,7 @@ func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int // PendingDocumentScrapeFilter defines query criteria for pending document scrape tenders. type PendingDocumentScrapeFilter struct { Portals []string - MinDeadline *int64 // when set, tender_deadline must be greater than this value + MinWindowEnd *int64 // when set, only tenders with an active publication submission window CreatedAtFrom *int64 CreatedAtTo *int64 Limit int @@ -682,25 +684,38 @@ func scrapePortalURLFilter(portals []string) bson.M { return bson.M{"$or": conditions} } +func publicationWindowActiveBSON(now int64) bson.M { + return bson.M{ + "$or": []bson.M{ + {"submission_deadline": bson.M{"$gt": now}}, + { + "$and": []bson.M{ + {"$or": []bson.M{ + {"submission_deadline": bson.M{"$exists": false}}, + {"submission_deadline": 0}, + }}, + {"publication_date": bson.M{"$gt": 0}}, + }, + }, + }, + } +} + // GetPendingDocumentScrapeTenders returns unscraped tenders whose document URL matches a supported portal. -// When MinDeadline is set, only tenders with tender_deadline greater than that value are returned. +// When MinWindowEnd is set, only tenders whose publication-based submission window is still open are returned. func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, filter PendingDocumentScrapeFilter) ([]Tender, bool, error) { portalFilter := scrapePortalURLFilter(filter.Portals) if portalFilter == nil { return nil, false, nil } - query := bson.M{ - "processing_metadata.documents_scraped": bson.M{"$ne": true}, - } - for key, value := range portalFilter { - query[key] = value + andFilters := []bson.M{ + {"processing_metadata.documents_scraped": bson.M{"$ne": true}}, + portalFilter, } - if filter.MinDeadline != nil { - query["tender_deadline"] = bson.M{ - "$gt": *filter.MinDeadline, - } + if filter.MinWindowEnd != nil { + andFilters = append(andFilters, publicationWindowActiveBSON(*filter.MinWindowEnd)) } if filter.CreatedAtFrom != nil || filter.CreatedAtTo != nil { @@ -711,9 +726,11 @@ func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, if filter.CreatedAtTo != nil { createdFilter["$lte"] = *filter.CreatedAtTo } - query["created_at"] = createdFilter + andFilters = append(andFilters, bson.M{"created_at": createdFilter}) } + query := bson.M{"$and": andFilters} + pagination := orm.Pagination{ Limit: filter.Limit, Skip: filter.Skip, @@ -727,7 +744,7 @@ func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, if err != nil { r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{ "portals": filter.Portals, - "min_deadline": filter.MinDeadline, + "min_window_end": filter.MinWindowEnd, "created_at_from": filter.CreatedAtFrom, "created_at_to": filter.CreatedAtTo, "limit": filter.Limit,