diff --git a/cmd/web/main.go b/cmd/web/main.go index a81ab52..b4017bb 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -231,7 +231,7 @@ func main() { contactService := contact.NewService(contactRepository, logger, notificationSDK) cmsService := cms.NewService(cmsRepository, logger) kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, goRulesClient, conf.GoRules.RuleID, logger) - documentScraperService := document_scraper.NewService(tenderRepository, logger) + documentScraperService := document_scraper.NewService(tenderRepository, aiPipelineClient, logger) dashboardRepository := dashboard.NewRepository(mongoManager, logger, procedureDocumentsLister) dashboardService := dashboard.NewService(dashboardRepository, logger) aiPipelineService := ai_pipeline.NewService(aiPipelineClient, logger, tenderService) diff --git a/internal/document_scraper/errors.go b/internal/document_scraper/errors.go index 58e3a7a..49b7506 100644 --- a/internal/document_scraper/errors.go +++ b/internal/document_scraper/errors.go @@ -5,4 +5,6 @@ import "errors" var ( // ErrInvalidDateRange is returned when created_at_from is after created_at_to. ErrInvalidDateRange = errors.New("invalid date range: created_at_from must be before or equal to created_at_to") + // ErrScrapePortalsUnavailable is returned when the scrape portals provider is not configured. + ErrScrapePortalsUnavailable = errors.New("scrape portals provider is not configured") ) diff --git a/internal/document_scraper/handler.go b/internal/document_scraper/handler.go index 1dfa30f..b50744b 100644 --- a/internal/document_scraper/handler.go +++ b/internal/document_scraper/handler.go @@ -24,7 +24,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler { // ListPendingTenders retrieves tenders that need document scraping // @Summary List pending tenders for document scraping -// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with active deadlines are returned; set include_expired=true to include expired tenders. +// @Description Retrieve tenders that have not yet been scraped for documents and whose document URL matches a portal supported by the AI scraping service. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with active deadlines are returned; set include_expired=true to include expired tenders. // @Tags Document-Scraper // @Produce json // @Param limit query int false "Number of items per page (default: 20, max: 100)" @@ -35,6 +35,7 @@ func NewHandler(service Service, logger logger.Logger) *Handler { // @Success 200 {object} response.APIResponse{data=DocumentScraperListResponse} "List of pending tenders" // @Failure 400 {object} response.APIResponse "Invalid query parameters" // @Failure 401 {object} response.APIResponse "Invalid or missing API key" +// @Failure 503 {object} response.APIResponse "Scrape portals service is not configured" // @Failure 500 {object} response.APIResponse "Internal server error" // @Security ApiKeyAuth // @Router /api/v1/scraper/tenders [get] @@ -49,6 +50,9 @@ func (h *Handler) ListPendingTenders(c echo.Context) error { if errors.Is(err, ErrInvalidDateRange) { return response.BadRequest(c, "Invalid date range", err.Error()) } + if errors.Is(err, ErrScrapePortalsUnavailable) { + return response.ServiceUnavailable(c, "Scrape portals service is not configured") + } return response.InternalServerError(c, "Failed to list pending tenders") } @@ -57,7 +61,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error { // GetTenderByNoticeID retrieves a specific tender by its notice publication ID // @Summary Get tender by notice ID for document scraping -// @Description Retrieve a specific tender by its notice publication ID (contract_folder_id, notice_publication_id, document_url, title, description). +// @Description Retrieve a specific unexpired tender whose document URL matches a supported scrape portal. // @Tags Document-Scraper // @Produce json // @Param notice_id path string true "Notice Publication ID" @@ -65,6 +69,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error { // @Failure 400 {object} response.APIResponse "Invalid notice ID" // @Failure 401 {object} response.APIResponse "Invalid or missing API key" // @Failure 404 {object} response.APIResponse "Tender not found" +// @Failure 503 {object} response.APIResponse "Scrape portals service is not configured" // @Failure 500 {object} response.APIResponse "Internal server error" // @Security ApiKeyAuth // @Router /api/v1/scraper/tenders/{notice_id} [get] @@ -76,6 +81,9 @@ func (h *Handler) GetTenderByNoticeID(c echo.Context) error { tender, err := h.service.GetTenderByNoticeID(c.Request().Context(), noticeID) if err != nil { + if errors.Is(err, ErrScrapePortalsUnavailable) { + return response.ServiceUnavailable(c, "Scrape portals service is not configured") + } if err.Error() == "tender not found" { return response.NotFound(c, "Tender not found") } diff --git a/internal/document_scraper/portal.go b/internal/document_scraper/portal.go new file mode 100644 index 0000000..229ce92 --- /dev/null +++ b/internal/document_scraper/portal.go @@ -0,0 +1,33 @@ +package document_scraper + +import ( + "strings" + + "tm/internal/tender" +) + +// documentURL returns the URL used for portal matching and the scraper API payload. +func documentURL(t *tender.Tender) string { + if t == nil { + return "" + } + if u := strings.TrimSpace(t.DocumentURI); u != "" { + return u + } + return strings.TrimSpace(t.TenderURL) +} + +// matchesScrapePortals reports whether url contains any supported portal identifier. +func matchesScrapePortals(url string, portals []string) bool { + if url == "" || len(portals) == 0 { + return false + } + urlLower := strings.ToLower(url) + for _, portal := range portals { + p := strings.TrimSpace(portal) + if p != "" && strings.Contains(urlLower, strings.ToLower(p)) { + return true + } + } + return false +} diff --git a/internal/document_scraper/service.go b/internal/document_scraper/service.go index 94f5fc3..312c8c8 100644 --- a/internal/document_scraper/service.go +++ b/internal/document_scraper/service.go @@ -3,21 +3,17 @@ package document_scraper import ( "context" "fmt" + "strings" "time" "tm/internal/tender" + "tm/pkg/ai_summarizer" "tm/pkg/logger" "tm/pkg/response" ) -var documentScraperCountryCodes = []string{"SWE", "FIN", "NOR"} - -func isDocumentScraperCountry(countryCode string) bool { - for _, code := range documentScraperCountryCodes { - if countryCode == code { - return true - } - } - return false +// ScrapePortalsProvider fetches supported document scraping portal identifiers. +type ScrapePortalsProvider interface { + GetScrapePortals(ctx context.Context) (*ai_summarizer.ScrapePortalsResponse, error) } // Service defines business logic for the document scraper API @@ -29,21 +25,49 @@ type Service interface { } type service struct { - tenderRepo tender.TenderRepository - logger logger.Logger + tenderRepo tender.TenderRepository + portalsProvider ScrapePortalsProvider + logger logger.Logger } // NewService creates a new document scraper service -func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Service { +func NewService(tenderRepo tender.TenderRepository, portalsProvider ScrapePortalsProvider, logger logger.Logger) Service { return &service{ - tenderRepo: tenderRepo, - logger: logger, + tenderRepo: tenderRepo, + portalsProvider: portalsProvider, + logger: logger, } } +func (s *service) fetchScrapePortals(ctx context.Context) ([]string, error) { + if s.portalsProvider == nil { + s.logger.Warn("Scrape portals provider not configured", map[string]interface{}{}) + return nil, ErrScrapePortalsUnavailable + } + + resp, err := s.portalsProvider.GetScrapePortals(ctx) + if err != nil { + s.logger.Error("Failed to fetch scrape portals", map[string]interface{}{ + "error": err.Error(), + }) + return nil, fmt.Errorf("fetch scrape portals: %w", err) + } + if resp == nil { + return nil, nil + } + + portals := make([]string, 0, len(*resp)) + for _, portal := range *resp { + if p := strings.TrimSpace(portal); p != "" { + portals = append(portals, p) + } + } + return portals, nil +} + // ListPendingTenders retrieves tenders that have not yet been scraped for documents. -// Only returns tenders from Sweden, Finland, and Norway. By default only tenders with active -// deadlines are included; set IncludeExpired to include tenders whose deadline has passed. +// Only returns tenders whose document URL matches a portal supported by the AI service. +// By default only tenders with active deadlines are included; set IncludeExpired to include expired tenders. func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) { req.Normalize() @@ -51,16 +75,30 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis return nil, nil, ErrInvalidDateRange } + portals, err := s.fetchScrapePortals(ctx) + if err != nil { + return nil, nil, err + } + if len(portals) == 0 { + s.logger.Info("No scrape portals configured; returning empty pending tender list", map[string]interface{}{}) + meta := &response.Meta{Total: 0, Limit: req.Limit, Offset: req.Offset, HasMore: false} + if meta.Limit > 0 { + meta.Page = (req.Offset / meta.Limit) + 1 + } + return &DocumentScraperListResponse{Tenders: []DocumentScraperTenderResponse{}}, meta, nil + } + s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{ - "limit": req.Limit, - "offset": req.Offset, - "created_at_from": req.CreatedAtFrom, - "created_at_to": req.CreatedAtTo, - "include_expired": req.IncludeExpired, + "limit": req.Limit, + "offset": req.Offset, + "created_at_from": req.CreatedAtFrom, + "created_at_to": req.CreatedAtTo, + "include_expired": req.IncludeExpired, + "portal_count": len(portals), }) filter := tender.PendingDocumentScrapeFilter{ - CountryCodes: documentScraperCountryCodes, + Portals: portals, CreatedAtFrom: req.CreatedAtFrom, CreatedAtTo: req.CreatedAtTo, Limit: req.Limit, @@ -106,12 +144,17 @@ func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperLis } // GetTenderByNoticeID retrieves a specific tender by its notice publication ID. -// Only returns tenders from Sweden, Finland, and Norway with active deadlines (deadline not yet reached). +// Only returns unexpired tenders whose document URL matches a supported scrape portal. func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) { s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{ "notice_id": noticeID, }) + portals, err := s.fetchScrapePortals(ctx) + if err != nil { + return nil, err + } + t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID) if err != nil { s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{ @@ -126,10 +169,9 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do } now := time.Now().Unix() - if !isDocumentScraperCountry(t.CountryCode) || t.TenderDeadline <= now { - s.logger.Info("Tender filtered out: unsupported country or deadline passed", map[string]interface{}{ + if t.TenderDeadline <= now || !matchesScrapePortals(documentURL(t), portals) { + s.logger.Info("Tender filtered out: unsupported portal or deadline passed", map[string]interface{}{ "notice_id": noticeID, - "country_code": t.CountryCode, "tender_deadline": t.TenderDeadline, "now": now, }) diff --git a/internal/document_scraper/tender_response.go b/internal/document_scraper/tender_response.go index c44e19d..dc9444e 100644 --- a/internal/document_scraper/tender_response.go +++ b/internal/document_scraper/tender_response.go @@ -1,10 +1,6 @@ package document_scraper -import ( - "strings" - - "tm/internal/tender" -) +import "tm/internal/tender" // BuildDocumentScraperTenderResponse maps a stored tender to the scraper API contract. func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTenderResponse { @@ -12,15 +8,10 @@ func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTender return &DocumentScraperTenderResponse{} } - documentURL := strings.TrimSpace(t.DocumentURI) - if documentURL == "" { - documentURL = strings.TrimSpace(t.TenderURL) - } - return &DocumentScraperTenderResponse{ ContractFolderID: t.ContractFolderID, NoticePublicationID: t.NoticePublicationID, - DocumentURL: documentURL, + DocumentURL: documentURL(t), Title: t.Title, Description: t.Description, } diff --git a/internal/tender/repository.go b/internal/tender/repository.go index 3339166..42e0413 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -557,7 +557,7 @@ func (r *tenderRepository) GetExpiredTenders(ctx context.Context, beforeDate int // PendingDocumentScrapeFilter defines query criteria for pending document scrape tenders. type PendingDocumentScrapeFilter struct { - CountryCodes []string + Portals []string MinDeadline *int64 // when set, tender_deadline must be greater than this value CreatedAtFrom *int64 CreatedAtTo *int64 @@ -565,17 +565,39 @@ type PendingDocumentScrapeFilter struct { Skip int } -// GetPendingDocumentScrapeTenders returns unscraped tenders for the given countries. +func scrapePortalURLFilter(portals []string) bson.M { + conditions := make([]bson.M, 0, len(portals)*2) + for _, portal := range portals { + portal = strings.TrimSpace(portal) + if portal == "" { + continue + } + regex := bson.M{"$regex": regexp.QuoteMeta(portal), "$options": "i"} + conditions = append(conditions, + bson.M{"document_uri": regex}, + bson.M{"tender_url": regex}, + ) + } + if len(conditions) == 0 { + return nil + } + return bson.M{"$or": conditions} +} + +// GetPendingDocumentScrapeTenders returns unscraped tenders whose document URL matches a supported portal. // When MinDeadline is set, only tenders with tender_deadline greater than that value are returned. func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, filter PendingDocumentScrapeFilter) ([]Tender, bool, error) { - if len(filter.CountryCodes) == 0 { + portalFilter := scrapePortalURLFilter(filter.Portals) + if portalFilter == nil { return nil, false, nil } query := bson.M{ - "country_code": bson.M{"$in": filter.CountryCodes}, "processing_metadata.documents_scraped": bson.M{"$ne": true}, } + for key, value := range portalFilter { + query[key] = value + } if filter.MinDeadline != nil { query["tender_deadline"] = bson.M{ @@ -606,7 +628,7 @@ func (r *tenderRepository) GetPendingDocumentScrapeTenders(ctx context.Context, result, err := r.ormRepo.FindAll(ctx, query, pagination) if err != nil { r.logger.Error("Failed to get pending document scrape tenders", map[string]interface{}{ - "country_codes": filter.CountryCodes, + "portals": filter.Portals, "min_deadline": filter.MinDeadline, "created_at_from": filter.CreatedAtFrom, "created_at_to": filter.CreatedAtTo,