Files
tm_back/internal/document_scraper/service.go
T
Mazyar e5fa0dfe47
continuous-integration/drone/push Build is passing
Add GetScrapePortals endpoint and related service functionality
- Introduced the GetScrapePortals method in the AI pipeline handler to list document scraping portals supported by the Opplens AI service.
- Updated the service layer to include GetScrapePortals, which retrieves the portals from the client and handles errors appropriately.
- Enhanced the routes to register the new endpoint for retrieving scrape portals.
- Added a new error type for invalid date ranges in the document scraper, improving validation and error handling.

This update expands the AI pipeline capabilities, allowing for better management of document scraping portals within the tender management system.
2026-06-18 19:53:07 +03:30

148 lines
4.4 KiB
Go

package document_scraper
import (
"context"
"fmt"
"time"
"tm/internal/tender"
"tm/pkg/logger"
"tm/pkg/response"
)
var documentScraperCountryCodes = []string{"SWE", "FIN", "NOR"}
func isDocumentScraperCountry(countryCode string) bool {
for _, code := range documentScraperCountryCodes {
if countryCode == code {
return true
}
}
return false
}
// Service defines business logic for the document scraper API
type Service interface {
// ListPendingTenders retrieves tenders that need document scraping
ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error)
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error)
}
type service struct {
tenderRepo tender.TenderRepository
logger logger.Logger
}
// NewService creates a new document scraper service
func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Service {
return &service{
tenderRepo: tenderRepo,
logger: logger,
}
}
// ListPendingTenders retrieves tenders that have not yet been scraped for documents.
// Only returns tenders from Sweden, Finland, and Norway. By default only tenders with active
// deadlines are included; set IncludeExpired to include tenders whose deadline has passed.
func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) {
req.Normalize()
if req.CreatedAtFrom != nil && req.CreatedAtTo != nil && *req.CreatedAtFrom > *req.CreatedAtTo {
return nil, nil, ErrInvalidDateRange
}
s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{
"limit": req.Limit,
"offset": req.Offset,
"created_at_from": req.CreatedAtFrom,
"created_at_to": req.CreatedAtTo,
"include_expired": req.IncludeExpired,
})
filter := tender.PendingDocumentScrapeFilter{
CountryCodes: documentScraperCountryCodes,
CreatedAtFrom: req.CreatedAtFrom,
CreatedAtTo: req.CreatedAtTo,
Limit: req.Limit,
Skip: req.Offset,
}
if !req.IncludeExpired {
now := time.Now().Unix()
filter.MinDeadline = &now
}
tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, filter)
if err != nil {
s.logger.Error("Failed to list pending tenders", map[string]interface{}{
"error": err.Error(),
})
return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
}
result := &DocumentScraperListResponse{
Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
}
for i := range tenders {
result.Tenders[i] = *BuildDocumentScraperTenderResponse(&tenders[i])
}
meta := &response.Meta{
Total: len(tenders),
Limit: req.Limit,
Offset: req.Offset,
HasMore: hasMore,
}
if meta.Limit > 0 {
meta.Page = (req.Offset / meta.Limit) + 1
}
s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
"count": len(tenders),
"has_more": hasMore,
})
return result, meta, nil
}
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID.
// Only returns tenders from Sweden, Finland, and Norway with active deadlines (deadline not yet reached).
func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
"notice_id": noticeID,
})
t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID)
if err != nil {
s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
"notice_id": noticeID,
"error": err.Error(),
})
return nil, fmt.Errorf("failed to get tender: %w", err)
}
if t == nil {
return nil, fmt.Errorf("tender not found")
}
now := time.Now().Unix()
if !isDocumentScraperCountry(t.CountryCode) || t.TenderDeadline <= now {
s.logger.Info("Tender filtered out: unsupported country or deadline passed", map[string]interface{}{
"notice_id": noticeID,
"country_code": t.CountryCode,
"tender_deadline": t.TenderDeadline,
"now": now,
})
return nil, fmt.Errorf("tender not found")
}
result := BuildDocumentScraperTenderResponse(t)
s.logger.Info("Tender retrieved successfully for document scraper", map[string]interface{}{
"notice_id": noticeID,
"tender_id": t.ID.Hex(),
})
return result, nil
}