Mazyar
133 lines
3.7 KiB
Go
133 lines
3.7 KiB
Go
package scraper
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"time"
|
|
"tm/pkg/logger"
|
|
)
|
|
|
|
// Service defines the interface for scraper business operations
|
|
type Service interface {
|
|
// ScrapeDocuments scrapes documents and uploads them to MinIO
|
|
ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
|
|
}
|
|
|
|
// scraperService implements Service interface
|
|
type scraperService struct {
|
|
httpClient *http.Client
|
|
baseURL string
|
|
logger logger.Logger
|
|
}
|
|
|
|
// NewService creates a new scraper service
|
|
func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
|
|
return &scraperService{
|
|
httpClient: &http.Client{
|
|
Timeout: timeout,
|
|
},
|
|
baseURL: baseURL,
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
// ScrapeDocuments scrapes documents and uploads them to MinIO
|
|
func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
|
|
s.logger.Info("Starting document scraping", map[string]interface{}{
|
|
"notice_id": form.NoticeID,
|
|
"category": form.Category,
|
|
})
|
|
|
|
// Prepare request body
|
|
requestBody := map[string]interface{}{
|
|
"notice_id": form.NoticeID,
|
|
"username": form.Username,
|
|
"password": form.Password,
|
|
"login_url": form.LoginURL,
|
|
"notice_url": form.NoticeURL,
|
|
"category": form.Category,
|
|
"subcategory": form.SubCategory,
|
|
}
|
|
|
|
bodyBytes, err := json.Marshal(requestBody)
|
|
if err != nil {
|
|
s.logger.Error("Failed to marshal request body", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
|
}
|
|
|
|
// Create HTTP request
|
|
url := fmt.Sprintf("%s/download", s.baseURL)
|
|
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
|
|
if err != nil {
|
|
s.logger.Error("Failed to create HTTP request", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
// Send request
|
|
resp, err := s.httpClient.Do(req)
|
|
if err != nil {
|
|
s.logger.Error("Failed to send request to scraper service", map[string]interface{}{
|
|
"error": err.Error(),
|
|
"url": url,
|
|
})
|
|
return nil, fmt.Errorf("failed to send request: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// Read response body
|
|
respBody, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
s.logger.Error("Failed to read response body", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return nil, fmt.Errorf("failed to read response: %w", err)
|
|
}
|
|
|
|
// Handle error responses
|
|
if resp.StatusCode >= 400 {
|
|
var errorResp map[string]interface{}
|
|
if err := json.Unmarshal(respBody, &errorResp); err == nil {
|
|
if errorMsg, ok := errorResp["error"].(string); ok {
|
|
s.logger.Error("Scraper service returned error", map[string]interface{}{
|
|
"status_code": resp.StatusCode,
|
|
"error": errorMsg,
|
|
})
|
|
return nil, fmt.Errorf("scraper service error: %s", errorMsg)
|
|
}
|
|
}
|
|
s.logger.Error("Scraper service returned error", map[string]interface{}{
|
|
"status_code": resp.StatusCode,
|
|
"body": string(respBody),
|
|
})
|
|
return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode)
|
|
}
|
|
|
|
// Parse success response
|
|
var scrapeResp ScrapeResponse
|
|
if err := json.Unmarshal(respBody, &scrapeResp); err != nil {
|
|
s.logger.Error("Failed to unmarshal response", map[string]interface{}{
|
|
"error": err.Error(),
|
|
"body": string(respBody),
|
|
})
|
|
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
|
|
}
|
|
|
|
s.logger.Info("Document scraping completed", map[string]interface{}{
|
|
"notice_id": scrapeResp.NoticeID,
|
|
"uploaded_count": scrapeResp.UploadedCount,
|
|
"success": scrapeResp.Success,
|
|
})
|
|
|
|
return &scrapeResp, nil
|
|
}
|