e5fa0dfe47
continuous-integration/drone/push Build is passing
- Introduced the GetScrapePortals method in the AI pipeline handler to list document scraping portals supported by the Opplens AI service. - Updated the service layer to include GetScrapePortals, which retrieves the portals from the client and handles errors appropriately. - Enhanced the routes to register the new endpoint for retrieving scrape portals. - Added a new error type for invalid date ranges in the document scraper, improving validation and error handling. This update expands the AI pipeline capabilities, allowing for better management of document scraping portals within the tender management system.
148 lines
4.4 KiB
Go
148 lines
4.4 KiB
Go
package document_scraper
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
"tm/internal/tender"
|
|
"tm/pkg/logger"
|
|
"tm/pkg/response"
|
|
)
|
|
|
|
var documentScraperCountryCodes = []string{"SWE", "FIN", "NOR"}
|
|
|
|
func isDocumentScraperCountry(countryCode string) bool {
|
|
for _, code := range documentScraperCountryCodes {
|
|
if countryCode == code {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// Service defines business logic for the document scraper API
|
|
type Service interface {
|
|
// ListPendingTenders retrieves tenders that need document scraping
|
|
ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error)
|
|
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
|
|
GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error)
|
|
}
|
|
|
|
type service struct {
|
|
tenderRepo tender.TenderRepository
|
|
logger logger.Logger
|
|
}
|
|
|
|
// NewService creates a new document scraper service
|
|
func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Service {
|
|
return &service{
|
|
tenderRepo: tenderRepo,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
// ListPendingTenders retrieves tenders that have not yet been scraped for documents.
|
|
// Only returns tenders from Sweden, Finland, and Norway. By default only tenders with active
|
|
// deadlines are included; set IncludeExpired to include tenders whose deadline has passed.
|
|
func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) {
|
|
req.Normalize()
|
|
|
|
if req.CreatedAtFrom != nil && req.CreatedAtTo != nil && *req.CreatedAtFrom > *req.CreatedAtTo {
|
|
return nil, nil, ErrInvalidDateRange
|
|
}
|
|
|
|
s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{
|
|
"limit": req.Limit,
|
|
"offset": req.Offset,
|
|
"created_at_from": req.CreatedAtFrom,
|
|
"created_at_to": req.CreatedAtTo,
|
|
"include_expired": req.IncludeExpired,
|
|
})
|
|
|
|
filter := tender.PendingDocumentScrapeFilter{
|
|
CountryCodes: documentScraperCountryCodes,
|
|
CreatedAtFrom: req.CreatedAtFrom,
|
|
CreatedAtTo: req.CreatedAtTo,
|
|
Limit: req.Limit,
|
|
Skip: req.Offset,
|
|
}
|
|
if !req.IncludeExpired {
|
|
now := time.Now().Unix()
|
|
filter.MinDeadline = &now
|
|
}
|
|
|
|
tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, filter)
|
|
if err != nil {
|
|
s.logger.Error("Failed to list pending tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
|
|
}
|
|
|
|
result := &DocumentScraperListResponse{
|
|
Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
|
|
}
|
|
|
|
for i := range tenders {
|
|
result.Tenders[i] = *BuildDocumentScraperTenderResponse(&tenders[i])
|
|
}
|
|
|
|
meta := &response.Meta{
|
|
Total: len(tenders),
|
|
Limit: req.Limit,
|
|
Offset: req.Offset,
|
|
HasMore: hasMore,
|
|
}
|
|
if meta.Limit > 0 {
|
|
meta.Page = (req.Offset / meta.Limit) + 1
|
|
}
|
|
|
|
s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
|
|
"count": len(tenders),
|
|
"has_more": hasMore,
|
|
})
|
|
|
|
return result, meta, nil
|
|
}
|
|
|
|
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID.
|
|
// Only returns tenders from Sweden, Finland, and Norway with active deadlines (deadline not yet reached).
|
|
func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
|
|
s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
|
|
"notice_id": noticeID,
|
|
})
|
|
|
|
t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID)
|
|
if err != nil {
|
|
s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
|
|
"notice_id": noticeID,
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to get tender: %w", err)
|
|
}
|
|
|
|
if t == nil {
|
|
return nil, fmt.Errorf("tender not found")
|
|
}
|
|
|
|
now := time.Now().Unix()
|
|
if !isDocumentScraperCountry(t.CountryCode) || t.TenderDeadline <= now {
|
|
s.logger.Info("Tender filtered out: unsupported country or deadline passed", map[string]interface{}{
|
|
"notice_id": noticeID,
|
|
"country_code": t.CountryCode,
|
|
"tender_deadline": t.TenderDeadline,
|
|
"now": now,
|
|
})
|
|
return nil, fmt.Errorf("tender not found")
|
|
}
|
|
|
|
result := BuildDocumentScraperTenderResponse(t)
|
|
|
|
s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{
|
|
"notice_id": noticeID,
|
|
"tender_id": t.ID.Hex(),
|
|
})
|
|
|
|
return result, nil
|
|
}
|