package scraper import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "time" "tm/pkg/logger" ) // Service defines the interface for scraper business operations type Service interface { // ScrapeDocuments scrapes documents and uploads them to MinIO ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) } // scraperService implements Service interface type scraperService struct { httpClient *http.Client baseURL string logger logger.Logger } // NewService creates a new scraper service func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service { return &scraperService{ httpClient: &http.Client{ Timeout: timeout, }, baseURL: baseURL, logger: logger, } } // ScrapeDocuments scrapes documents and uploads them to MinIO func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) { s.logger.Info("Starting document scraping", map[string]interface{}{ "notice_id": form.NoticeID, "category": form.Category, }) // Prepare request body requestBody := map[string]interface{}{ "notice_id": form.NoticeID, "username": form.Username, "password": form.Password, "login_url": form.LoginURL, "notice_url": form.NoticeURL, "category": form.Category, "subcategory": form.SubCategory, } bodyBytes, err := json.Marshal(requestBody) if err != nil { s.logger.Error("Failed to marshal request body", map[string]interface{}{ "error": err.Error(), }) return nil, fmt.Errorf("failed to marshal request: %w", err) } // Create HTTP request url := fmt.Sprintf("%s/download", s.baseURL) req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) if err != nil { s.logger.Error("Failed to create HTTP request", map[string]interface{}{ "error": err.Error(), }) return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("Content-Type", "application/json") // Send request resp, err := s.httpClient.Do(req) if err != nil { s.logger.Error("Failed to send request to scraper service", map[string]interface{}{ "error": err.Error(), "url": url, }) return nil, fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() // Read response body respBody, err := io.ReadAll(resp.Body) if err != nil { s.logger.Error("Failed to read response body", map[string]interface{}{ "error": err.Error(), }) return nil, fmt.Errorf("failed to read response: %w", err) } // Handle error responses if resp.StatusCode >= 400 { var errorResp map[string]interface{} if err := json.Unmarshal(respBody, &errorResp); err == nil { if errorMsg, ok := errorResp["error"].(string); ok { s.logger.Error("Scraper service returned error", map[string]interface{}{ "status_code": resp.StatusCode, "error": errorMsg, }) return nil, fmt.Errorf("scraper service error: %s", errorMsg) } } s.logger.Error("Scraper service returned error", map[string]interface{}{ "status_code": resp.StatusCode, "body": string(respBody), }) return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode) } // Parse success response var scrapeResp ScrapeResponse if err := json.Unmarshal(respBody, &scrapeResp); err != nil { s.logger.Error("Failed to unmarshal response", map[string]interface{}{ "error": err.Error(), "body": string(respBody), }) return nil, fmt.Errorf("failed to unmarshal response: %w", err) } s.logger.Info("Document scraping completed", map[string]interface{}{ "notice_id": scrapeResp.NoticeID, "uploaded_count": scrapeResp.UploadedCount, "success": scrapeResp.Success, }) return &scrapeResp, nil }