Mazyar
98 lines
2.6 KiB
Go
98 lines
2.6 KiB
Go
package scraper
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"tm/pkg/logger"
|
|
"tm/pkg/scraper"
|
|
)
|
|
|
|
// Service defines the interface for scraper business operations
|
|
type Service interface {
|
|
// ScrapeDocuments scrapes documents and uploads them to MinIO
|
|
ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
|
|
}
|
|
|
|
// scraperService implements Service interface
|
|
type scraperService struct {
|
|
sdk scraper.SDK
|
|
logger logger.Logger
|
|
}
|
|
|
|
// NewService creates a new scraper service with stored credentials
|
|
func NewService(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) Service {
|
|
sdk := scraper.NewClient(baseURL, timeout, logger, username, password, loginURL)
|
|
return &scraperService{
|
|
sdk: sdk,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
// ScrapeDocuments scrapes documents and uploads them to MinIO
|
|
func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
|
|
s.logger.Info("Starting document scraping", map[string]interface{}{
|
|
"notice_id": form.NoticeID,
|
|
})
|
|
|
|
// Convert form to SDK request (only dynamic fields)
|
|
req := &scraper.ScrapeRequest{
|
|
NoticeID: form.NoticeID,
|
|
NoticeURL: form.NoticeURL,
|
|
}
|
|
|
|
// Call SDK (credentials are already configured)
|
|
response, err := s.sdk.ScrapeDocuments(ctx, req)
|
|
if err != nil {
|
|
s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{
|
|
"error": err.Error(),
|
|
"notice_id": form.NoticeID,
|
|
})
|
|
return nil, err
|
|
}
|
|
|
|
// Convert SDK response to service response
|
|
serviceResp := &ScrapeResponse{
|
|
Success: response.Success,
|
|
NoticeID: response.NoticeID,
|
|
UploadedCount: response.UploadedCount,
|
|
UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)),
|
|
Errors: make([]FileError, len(response.Errors)),
|
|
Buckets: make([]BucketInfo, len(response.Buckets)),
|
|
}
|
|
|
|
// Convert uploaded files
|
|
for i, file := range response.UploadedFiles {
|
|
serviceResp.UploadedFiles[i] = UploadedFileInfo{
|
|
Filename: file.Filename,
|
|
ObjectName: file.ObjectName,
|
|
Size: file.Size,
|
|
}
|
|
}
|
|
|
|
// Convert errors
|
|
for i, err := range response.Errors {
|
|
serviceResp.Errors[i] = FileError{
|
|
Filename: err.Filename,
|
|
Error: err.Error,
|
|
}
|
|
}
|
|
|
|
// Convert buckets
|
|
for i, bucket := range response.Buckets {
|
|
serviceResp.Buckets[i] = BucketInfo{
|
|
Name: bucket.Name,
|
|
NoticeID: bucket.NoticeID,
|
|
Count: bucket.Count,
|
|
}
|
|
}
|
|
|
|
s.logger.Info("Document scraping completed", map[string]interface{}{
|
|
"notice_id": serviceResp.NoticeID,
|
|
"uploaded_count": serviceResp.UploadedCount,
|
|
"success": serviceResp.Success,
|
|
})
|
|
|
|
return serviceResp, nil
|
|
}
|