get pending tenders slow request fix

internal/document_scraper/service.go

@@ -9,6 +9,8 @@ import (
 	"tm/pkg/response"
 )
 
+const documentScraperCountryCode = "SWE"
+
 // Service defines business logic for the document scraper API
 type Service interface {
 	// ListPendingTenders retrieves tenders that need document scraping
@@ -40,70 +42,35 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D
 
 	now := time.Now().Unix()
 
-	// Fetch a larger batch to account for filtering, then apply Sweden + active deadline filters
-	// We need to over-fetch since the repository doesn't filter by country/deadline
-	fetchLimit := limit * 5
-	if fetchLimit > 500 {
-		fetchLimit = 500
+	tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, documentScraperCountryCode, now, limit, offset)
+	if err != nil {
+		s.logger.Error("Failed to list pending tenders", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
 	}
 
-	var filteredTenders []tender.Tender
-	currentOffset := offset
-	hasMore := true
-
-	for hasMore {
-		tenders, _, err := s.tenderRepo.GetUnScrapedTenders(ctx, fetchLimit, currentOffset)
-		if err != nil {
-			s.logger.Error("Failed to list pending tenders", map[string]interface{}{
-				"error": err.Error(),
-			})
-			return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
-		}
-
-		if len(tenders) == 0 {
-			hasMore = false
-			break
-		}
-
-		for i := range tenders {
-			if tenders[i].CountryCode == "SWE" && tenders[i].TenderDeadline > now {
-				filteredTenders = append(filteredTenders, tenders[i])
-			}
-			if len(filteredTenders) >= limit {
-				break
-			}
-		}
-
-		if len(filteredTenders) >= limit {
-			filteredTenders = filteredTenders[:limit]
-			hasMore = false
-		} else {
-			currentOffset += fetchLimit
-		}
-	}
-
-	// Convert to response
 	result := &DocumentScraperListResponse{
-		Tenders: make([]DocumentScraperTenderResponse, len(filteredTenders)),
+		Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
 	}
 
-	for i := range filteredTenders {
-		result.Tenders[i] = *BuildDocumentScraperTenderResponse(&filteredTenders[i])
+	for i := range tenders {
+		result.Tenders[i] = *BuildDocumentScraperTenderResponse(&tenders[i])
 	}
 
-	// Build pagination metadata
 	meta := &response.Meta{
-		Total:  len(filteredTenders),
-		Limit:  limit,
-		Offset: offset,
+		Total:   len(tenders),
+		Limit:   limit,
+		Offset:  offset,
+		HasMore: hasMore,
 	}
 	if meta.Limit > 0 {
 		meta.Page = (offset / meta.Limit) + 1
-		meta.Pages = 1
 	}
 
 	s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
-		"count": len(filteredTenders),
+		"count":    len(tenders),
+		"has_more": hasMore,
 	})
 
 	return result, meta, nil
@@ -129,9 +96,8 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do
 		return nil, fmt.Errorf("tender not found")
 	}
 
-	// Only return tenders from Sweden with active deadlines
 	now := time.Now().Unix()
-	if t.CountryCode != "SWE" || t.TenderDeadline <= now {
+	if t.CountryCode != documentScraperCountryCode || t.TenderDeadline <= now {
 		s.logger.Info("Tender filtered out: not Sweden or deadline passed", map[string]interface{}{
 			"notice_id":       noticeID,
 			"country_code":    t.CountryCode,