Enhance document scraper service with AI portal integration and error handling

- Updated the document scraper service to include a new ScrapePortalsProvider interface, allowing for dynamic retrieval of supported scraping portals.
- Modified the ListPendingTenders and GetTenderByNoticeID methods to filter tenders based on document URLs that match the configured portals.
- Introduced new error handling for cases when the scrape portals provider is not configured, returning appropriate service unavailable responses.
- Enhanced API documentation to reflect changes in tender retrieval logic and added error response details for unsupported portal scenarios.

This update improves the document scraping functionality by integrating AI portal support, enhancing the overall reliability and flexibility of the tender management system.

cmd/web/main.go

@@ -231,7 +231,7 @@ func main() {
 	contactService := contact.NewService(contactRepository, logger, notificationSDK)
 	cmsService := cms.NewService(cmsRepository, logger)
 	kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, goRulesClient, conf.GoRules.RuleID, logger)
-	documentScraperService := document_scraper.NewService(tenderRepository, logger)
+	documentScraperService := document_scraper.NewService(tenderRepository, aiPipelineClient, logger)
 	dashboardRepository := dashboard.NewRepository(mongoManager, logger, procedureDocumentsLister)
 	dashboardService := dashboard.NewService(dashboardRepository, logger)
 	aiPipelineService := ai_pipeline.NewService(aiPipelineClient, logger, tenderService)

internal/document_scraper/errors.go

@@ -5,4 +5,6 @@ import "errors"
 var (
 	// ErrInvalidDateRange is returned when created_at_from is after created_at_to.
 	ErrInvalidDateRange = errors.New("invalid date range: created_at_from must be before or equal to created_at_to")
+	// ErrScrapePortalsUnavailable is returned when the scrape portals provider is not configured.
+	ErrScrapePortalsUnavailable = errors.New("scrape portals provider is not configured")
 )

internal/document_scraper/handler.go

@@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler {
 
 // ListPendingTenders retrieves tenders that need document scraping
 // @Summary List pending tenders for document scraping
-// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with active deadlines are returned; set include_expired=true to include expired tenders.
+// @Description Retrieve tenders that have not yet been scraped for documents and whose document URL matches a portal supported by the AI scraping service. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with active deadlines are returned; set include_expired=true to include expired tenders.
 // @Tags Document-Scraper
 // @Produce json
 // @Param limit query int false "Number of items per page (default: 20, max: 100)"
@@ -35,6 +35,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler {
 // @Success 200 {object} response.APIResponse{data=DocumentScraperListResponse} "List of pending tenders"
 // @Failure 400 {object} response.APIResponse "Invalid query parameters"
 // @Failure 401 {object} response.APIResponse "Invalid or missing API key"
+// @Failure 503 {object} response.APIResponse "Scrape portals service is not configured"
 // @Failure 500 {object} response.APIResponse "Internal server error"
 // @Security ApiKeyAuth
 // @Router /api/v1/scraper/tenders [get]
@@ -49,6 +50,9 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
 		if errors.Is(err, ErrInvalidDateRange) {
 			return response.BadRequest(c, "Invalid date range", err.Error())
 		}
+		if errors.Is(err, ErrScrapePortalsUnavailable) {
+			return response.ServiceUnavailable(c, "Scrape portals service is not configured")
+		}
 		return response.InternalServerError(c, "Failed to list pending tenders")
 	}
 
@@ -57,7 +61,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
 
 // GetTenderByNoticeID retrieves a specific tender by its notice publication ID
 // @Summary Get tender by notice ID for document scraping
-// @Description Retrieve a specific tender by its notice publication ID (contract_folder_id, notice_publication_id, document_url, title, description).
+// @Description Retrieve a specific unexpired tender whose document URL matches a supported scrape portal.
 // @Tags Document-Scraper
 // @Produce json
 // @Param notice_id path string true "Notice Publication ID"
@@ -65,6 +69,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
 // @Failure 400 {object} response.APIResponse "Invalid notice ID"
 // @Failure 401 {object} response.APIResponse "Invalid or missing API key"
 // @Failure 404 {object} response.APIResponse "Tender not found"
+// @Failure 503 {object} response.APIResponse "Scrape portals service is not configured"
 // @Failure 500 {object} response.APIResponse "Internal server error"
 // @Security ApiKeyAuth
 // @Router /api/v1/scraper/tenders/{notice_id} [get]
@@ -76,6 +81,9 @@ func (h *Handler) GetTenderByNoticeID(c echo.Context) error {
 
 	tender, err := h.service.GetTenderByNoticeID(c.Request().Context(), noticeID)
 	if err != nil {
+		if errors.Is(err, ErrScrapePortalsUnavailable) {
+			return response.ServiceUnavailable(c, "Scrape portals service is not configured")
+		}
 		if err.Error() == "tender not found" {
 			return response.NotFound(c, "Tender not found")
 		}

internal/document_scraper/portal.go

@@ -0,0 +1,33 @@
+package document_scraper
+
+import (
+	"strings"
+
+	"tm/internal/tender"
+)
+
+// documentURL returns the URL used for portal matching and the scraper API payload.
+func documentURL(t *tender.Tender) string {
+	if t == nil {
+		return ""
+	}
+	if u := strings.TrimSpace(t.DocumentURI); u != "" {
+		return u
+	}
+	return strings.TrimSpace(t.TenderURL)
+}
+
+// matchesScrapePortals reports whether url contains any supported portal identifier.
+func matchesScrapePortals(url string, portals []string) bool {
+	if url == "" || len(portals) == 0 {
+		return false
+	}
+	urlLower := strings.ToLower(url)
+	for _, portal := range portals {
+		p := strings.TrimSpace(portal)
+		if p != "" && strings.Contains(urlLower, strings.ToLower(p)) {
+			return true
+		}
+	}
+	return false
+}

internal/document_scraper/service.go

@@ -3,21 +3,17 @@ package document_scraper
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 	"tm/internal/tender"
+	"tm/pkg/ai_summarizer"
 	"tm/pkg/logger"
 	"tm/pkg/response"
 )
 
-var documentScraperCountryCodes = []string{"SWE", "FIN", "NOR"}
-
-func isDocumentScraperCountry(countryCode string) bool {
-	for _, code := range documentScraperCountryCodes {
-		if countryCode == code {
-			return true
-		}
-	}
-	return false
+// ScrapePortalsProvider fetches supported document scraping portal identifiers.
+type ScrapePortalsProvider interface {
+	GetScrapePortals(ctx context.Context) (*ai_summarizer.ScrapePortalsResponse, error)
 }
 
 // Service defines business logic for the document scraper API
@@ -30,20 +26,48 @@ type Service interface {
 
 type service struct {
 	tenderRepo      tender.TenderRepository
+	portalsProvider ScrapePortalsProvider
 	logger          logger.Logger
 }
 
 // NewService creates a new document scraper service
-func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Service {
+func NewService(tenderRepo tender.TenderRepository, portalsProvider ScrapePortalsProvider, logger logger.Logger) Service {
 	return &service{
 		tenderRepo:      tenderRepo,
+		portalsProvider: portalsProvider,
 		logger:          logger,
 	}
 }
 
+func (s *service) fetchScrapePortals(ctx context.Context) ([]string, error) {
+	if s.portalsProvider == nil {
+		s.logger.Warn("Scrape portals provider not configured", map[string]interface{}{})
+		return nil, ErrScrapePortalsUnavailable
+	}
+
+	resp, err := s.portalsProvider.GetScrapePortals(ctx)
+	if err != nil {
+		s.logger.Error("Failed to fetch scrape portals", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("fetch scrape portals: %w", err)
+	}
+	if resp == nil {
+		return nil, nil
+	}
+
+	portals := make([]string, 0, len(*resp))
+	for _, portal := range *resp {
+		if p := strings.TrimSpace(portal); p != "" {
+			portals = append(portals, p)
+		}
+	}
+	return portals, nil
+}
+
 // ListPendingTenders retrieves tenders that have not yet been scraped for documents.
-// Only returns tenders from Sweden, Finland, and Norway. By default only tenders with active
-// deadlines are included; set IncludeExpired to include tenders whose deadline has passed.
+// Only returns tenders whose document URL matches a portal supported by the AI service.
+// By default only tenders with active deadlines are included; set IncludeExpired to include expired tenders.
 func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) {
 	req.Normalize()
 
@@ -51,16 +75,30 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis
 		return nil, nil, ErrInvalidDateRange
 	}
 
+	portals, err := s.fetchScrapePortals(ctx)
+	if err != nil {
+		return nil, nil, err
+	}
+	if len(portals) == 0 {
+		s.logger.Info("No scrape portals configured; returning empty pending tender list", map[string]interface{}{})
+		meta := &response.Meta{Total: 0, Limit: req.Limit, Offset: req.Offset, HasMore: false}
+		if meta.Limit > 0 {
+			meta.Page = (req.Offset / meta.Limit) + 1
+		}
+		return &DocumentScraperListResponse{Tenders: []DocumentScraperTenderResponse{}}, meta, nil
+	}
+
 	s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{
 		"limit":           req.Limit,
 		"offset":          req.Offset,
 		"created_at_from": req.CreatedAtFrom,
 		"created_at_to":   req.CreatedAtTo,
 		"include_expired": req.IncludeExpired,
+		"portal_count":    len(portals),
 	})
 
 	filter := tender.PendingDocumentScrapeFilter{
-		CountryCodes:  documentScraperCountryCodes,
+		Portals:       portals,
 		CreatedAtFrom: req.CreatedAtFrom,
 		CreatedAtTo:   req.CreatedAtTo,
 		Limit:         req.Limit,
@@ -106,12 +144,17 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis
 }
 
 // GetTenderByNoticeID retrieves a specific tender by its notice publication ID.
-// Only returns tenders from Sweden, Finland, and Norway with active deadlines (deadline not yet reached).
+// Only returns unexpired tenders whose document URL matches a supported scrape portal.
 func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
 	s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
 		"notice_id": noticeID,
 	})
 
+	portals, err := s.fetchScrapePortals(ctx)
+	if err != nil {
+		return nil, err
+	}
+
 	t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID)
 	if err != nil {
 		s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
@@ -126,10 +169,9 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do
 	}
 
 	now := time.Now().Unix()
-	if !isDocumentScraperCountry(t.CountryCode) || t.TenderDeadline <= now {
-		s.logger.Info("Tender filtered out: unsupported country or deadline passed", map[string]interface{}{
+	if t.TenderDeadline <= now || !matchesScrapePortals(documentURL(t), portals) {
+		s.logger.Info("Tender filtered out: unsupported portal or deadline passed", map[string]interface{}{
 			"notice_id":       noticeID,
-			"country_code":    t.CountryCode,
 			"tender_deadline": t.TenderDeadline,
 			"now":             now,
 		})

internal/document_scraper/tender_response.go

@@ -1,10 +1,6 @@
 package document_scraper
 
-import (
-	"strings"
-
-	"tm/internal/tender"
-)
+import "tm/internal/tender"
 
 // BuildDocumentScraperTenderResponse maps a stored tender to the scraper API contract.
 func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTenderResponse {
@@ -12,15 +8,10 @@ func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTender
 		return &DocumentScraperTenderResponse{}
 	}
 
-	documentURL := strings.TrimSpace(t.DocumentURI)
-	if documentURL == "" {
-		documentURL = strings.TrimSpace(t.TenderURL)
-	}
-
 	return &DocumentScraperTenderResponse{
 		ContractFolderID:    t.ContractFolderID,
 		NoticePublicationID: t.NoticePublicationID,
-		DocumentURL:         documentURL,
+		DocumentURL:         documentURL(t),
 		Title:               t.Title,
 		Description:         t.Description,
 	}

internal/tender/repository.go

@@ -557,7 +557,7 @@ func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int
 
 // PendingDocumentScrapeFilter defines query criteria for pending document scrape tenders.
 type PendingDocumentScrapeFilter struct {
-	CountryCodes  []string
+	Portals       []string
 	MinDeadline   *int64 // when set, tender_deadline must be greater than this value
 	CreatedAtFrom *int64
 	CreatedAtTo   *int64
@@ -565,17 +565,39 @@ type PendingDocumentScrapeFilter struct {
 	Skip          int
 }
 
-// GetPendingDocumentScrapeTenders returns unscraped tenders for the given countries.
+func scrapePortalURLFilter(portals []string) bson.M {
+	conditions := make([]bson.M, 0, len(portals)*2)
+	for _, portal := range portals {
+		portal = strings.TrimSpace(portal)
+		if portal == "" {
+			continue
+		}
+		regex := bson.M{"$regex": regexp.QuoteMeta(portal), "$options": "i"}
+		conditions = append(conditions,
+			bson.M{"document_uri": regex},
+			bson.M{"tender_url": regex},
+		)
+	}
+	if len(conditions) == 0 {
+		return nil
+	}
+	return bson.M{"$or": conditions}
+}
+
+// GetPendingDocumentScrapeTenders returns unscraped tenders whose document URL matches a supported portal.
 // When MinDeadline is set, only tenders with tender_deadline greater than that value are returned.
 func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, filter PendingDocumentScrapeFilter) ([]Tender, bool, error) {
-	if len(filter.CountryCodes) == 0 {
+	portalFilter := scrapePortalURLFilter(filter.Portals)
+	if portalFilter == nil {
 		return nil, false, nil
 	}
 
 	query := bson.M{
-		"country_code": bson.M{"$in": filter.CountryCodes},
 		"processing_metadata.documents_scraped": bson.M{"$ne": true},
 	}
+	for key, value := range portalFilter {
+		query[key] = value
+	}
 
 	if filter.MinDeadline != nil {
 		query["tender_deadline"] = bson.M{
@@ -606,7 +628,7 @@ func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context,
 	result, err := r.ormRepo.FindAll(ctx, query, pagination)
 	if err != nil {
 		r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{
-			"country_codes":   filter.CountryCodes,
+			"portals":         filter.Portals,
 			"min_deadline":    filter.MinDeadline,
 			"created_at_from": filter.CreatedAtFrom,
 			"created_at_to":   filter.CreatedAtTo,