diff --git a/internal/document_scraper/service.go b/internal/document_scraper/service.go index 47ac9e1..63bdf2d 100644 --- a/internal/document_scraper/service.go +++ b/internal/document_scraper/service.go @@ -3,6 +3,7 @@ package document_scraper import ( "context" "fmt" + "time" "tm/internal/tender" "tm/pkg/logger" "tm/pkg/response" @@ -48,50 +49,87 @@ func (s *service) buildTenderResponse(_ context.Context, t *tender.Tender) *Docu } } -// ListPendingTenders retrieves tenders that have not yet been scraped for documents +// ListPendingTenders retrieves tenders that have not yet been scraped for documents. +// Only returns tenders from Sweden with active deadlines (deadline not yet reached). func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) { s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{ "limit": limit, "offset": offset, }) - tenders, totalCount, err := s.tenderRepo.GetUnScrapedTenders(ctx, limit, offset) - if err != nil { - s.logger.Error("Failed to list pending tenders", map[string]interface{}{ - "error": err.Error(), - }) - return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err) + now := time.Now().Unix() + + // Fetch a larger batch to account for filtering, then apply Sweden + active deadline filters + // We need to over-fetch since the repository doesn't filter by country/deadline + fetchLimit := limit * 5 + if fetchLimit > 500 { + fetchLimit = 500 + } + + var filteredTenders []tender.Tender + currentOffset := offset + hasMore := true + + for hasMore { + tenders, _, err := s.tenderRepo.GetUnScrapedTenders(ctx, fetchLimit, currentOffset) + if err != nil { + s.logger.Error("Failed to list pending tenders", map[string]interface{}{ + "error": err.Error(), + }) + return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err) + } + + if len(tenders) == 0 { + hasMore = false + break + } + + for i := range tenders { + if tenders[i].CountryCode == "SWE" && tenders[i].TenderDeadline > now { + filteredTenders = append(filteredTenders, tenders[i]) + } + if len(filteredTenders) >= limit { + break + } + } + + if len(filteredTenders) >= limit { + filteredTenders = filteredTenders[:limit] + hasMore = false + } else { + currentOffset += fetchLimit + } } // Convert to response result := &DocumentScraperListResponse{ - Tenders: make([]DocumentScraperTenderResponse, len(tenders)), + Tenders: make([]DocumentScraperTenderResponse, len(filteredTenders)), } - for i := range tenders { - result.Tenders[i] = *s.buildTenderResponse(ctx, &tenders[i]) + for i := range filteredTenders { + result.Tenders[i] = *s.buildTenderResponse(ctx, &filteredTenders[i]) } // Build pagination metadata meta := &response.Meta{ - Total: int(totalCount), + Total: len(filteredTenders), Limit: limit, Offset: offset, } if meta.Limit > 0 { meta.Page = (offset / meta.Limit) + 1 - meta.Pages = (int(totalCount) + meta.Limit - 1) / meta.Limit + meta.Pages = 1 } s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{ - "count": len(tenders), - "total_count": totalCount, + "count": len(filteredTenders), }) return result, meta, nil } -// GetTenderByNoticeID retrieves a specific tender by its notice publication ID +// GetTenderByNoticeID retrieves a specific tender by its notice publication ID. +// Only returns tenders from Sweden with active deadlines (deadline not yet reached). func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) { s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{ "notice_id": noticeID, @@ -110,6 +148,18 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do return nil, fmt.Errorf("tender not found") } + // Only return tenders from Sweden with active deadlines + now := time.Now().Unix() + if t.CountryCode != "SWE" || t.TenderDeadline <= now { + s.logger.Info("Tender filtered out: not Sweden or deadline passed", map[string]interface{}{ + "notice_id": noticeID, + "country_code": t.CountryCode, + "tender_deadline": t.TenderDeadline, + "now": now, + }) + return nil, fmt.Errorf("tender not found") + } + result := s.buildTenderResponse(ctx, t) s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{