package document_scraper import ( "context" "fmt" "strings" "time" "tm/internal/tender" "tm/pkg/ai_summarizer" "tm/pkg/logger" "tm/pkg/response" ) // ScrapePortalsProvider fetches supported document scraping portal identifiers. type ScrapePortalsProvider interface { GetScrapePortals(ctx context.Context) (*ai_summarizer.ScrapePortalsResponse, error) } // Service defines business logic for the document scraper API type Service interface { // ListPendingTenders retrieves tenders that need document scraping ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) // GetTenderByNoticeID retrieves a specific tender by its notice publication ID GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) } type service struct { tenderRepo tender.TenderRepository portalsProvider ScrapePortalsProvider logger logger.Logger } // NewService creates a new document scraper service func NewService(tenderRepo tender.TenderRepository, portalsProvider ScrapePortalsProvider, logger logger.Logger) Service { return &service{ tenderRepo: tenderRepo, portalsProvider: portalsProvider, logger: logger, } } func (s *service) fetchScrapePortals(ctx context.Context) ([]string, error) { if s.portalsProvider == nil { s.logger.Warn("Scrape portals provider not configured", map[string]interface{}{}) return nil, ErrScrapePortalsUnavailable } resp, err := s.portalsProvider.GetScrapePortals(ctx) if err != nil { s.logger.Error("Failed to fetch scrape portals", map[string]interface{}{ "error": err.Error(), }) return nil, fmt.Errorf("fetch scrape portals: %w", err) } if resp == nil { return nil, nil } portals := make([]string, 0, len(*resp)) for _, portal := range *resp { if p := strings.TrimSpace(portal); p != "" { portals = append(portals, p) } } return portals, nil } // ListPendingTenders retrieves tenders that have not yet been scraped for documents. // Only returns tenders whose document URL matches a portal supported by the AI service. // By default only tenders with an active publication submission window are included; set IncludeExpired to include expired tenders. func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) { req.Normalize() if req.CreatedAtFrom != nil && req.CreatedAtTo != nil && *req.CreatedAtFrom > *req.CreatedAtTo { return nil, nil, ErrInvalidDateRange } portals, err := s.fetchScrapePortals(ctx) if err != nil { return nil, nil, err } if len(portals) == 0 { s.logger.Info("No scrape portals configured; returning empty pending tender list", map[string]interface{}{}) meta := &response.Meta{Total: 0, Limit: req.Limit, Offset: req.Offset, HasMore: false} if meta.Limit > 0 { meta.Page = (req.Offset / meta.Limit) + 1 } return &DocumentScraperListResponse{Tenders: []DocumentScraperTenderResponse{}}, meta, nil } s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{ "limit": req.Limit, "offset": req.Offset, "created_at_from": req.CreatedAtFrom, "created_at_to": req.CreatedAtTo, "include_expired": req.IncludeExpired, "portal_count": len(portals), }) filter := tender.PendingDocumentScrapeFilter{ Portals: portals, CreatedAtFrom: req.CreatedAtFrom, CreatedAtTo: req.CreatedAtTo, Limit: req.Limit, Skip: req.Offset, } if !req.IncludeExpired { now := time.Now().Unix() filter.MinWindowEnd = &now } tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, filter) if err != nil { s.logger.Error("Failed to list pending tenders", map[string]interface{}{ "error": err.Error(), }) return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err) } if !req.IncludeExpired { now := time.Now().Unix() tenders = filterActivePublicationWindow(tenders, now) } result := &DocumentScraperListResponse{ Tenders: make([]DocumentScraperTenderResponse, len(tenders)), } for i := range tenders { result.Tenders[i] = *BuildDocumentScraperTenderResponse(&tenders[i]) } meta := &response.Meta{ Total: len(tenders), Limit: req.Limit, Offset: req.Offset, HasMore: hasMore, } if meta.Limit > 0 { meta.Page = (req.Offset / meta.Limit) + 1 } s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{ "count": len(tenders), "has_more": hasMore, }) return result, meta, nil } // GetTenderByNoticeID retrieves a specific tender by its notice publication ID. // Only returns tenders with an active publication submission window whose document URL matches a supported scrape portal. func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) { s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{ "notice_id": noticeID, }) portals, err := s.fetchScrapePortals(ctx) if err != nil { return nil, err } t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID) if err != nil { s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{ "notice_id": noticeID, "error": err.Error(), }) return nil, fmt.Errorf("failed to get tender: %w", err) } if t == nil { return nil, fmt.Errorf("tender not found") } now := time.Now().Unix() if !t.HasActivePublicationWindow(now) || !matchesScrapePortals(documentURL(t), portals) { s.logger.Info("Tender filtered out: unsupported portal or publication window closed", map[string]interface{}{ "notice_id": noticeID, "publication_date": t.PublicationDate, "submission_deadline": t.SubmissionDeadline, "publication_deadline": t.PublicationSubmissionDeadline(), "now": now, }) return nil, fmt.Errorf("tender not found") } result := BuildDocumentScraperTenderResponse(t) s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{ "notice_id": noticeID, "tender_id": t.ID.Hex(), }) return result, nil } func filterActivePublicationWindow(tenders []tender.Tender, now int64) []tender.Tender { if len(tenders) == 0 { return tenders } active := make([]tender.Tender, 0, len(tenders)) for i := range tenders { if tenders[i].HasActivePublicationWindow(now) { active = append(active, tenders[i]) } } return active }