tendsign scraper

internal/scraper/service.go

@@ -0,0 +1,132 @@
+package scraper
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+	"tm/pkg/logger"
+)
+
+// Service defines the interface for scraper business operations
+type Service interface {
+	// ScrapeDocuments scrapes documents and uploads them to MinIO
+	ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
+}
+
+// scraperService implements Service interface
+type scraperService struct {
+	httpClient *http.Client
+	baseURL    string
+	logger     logger.Logger
+}
+
+// NewService creates a new scraper service
+func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
+	return &scraperService{
+		httpClient: &http.Client{
+			Timeout: timeout,
+		},
+		baseURL: baseURL,
+		logger:  logger,
+	}
+}
+
+// ScrapeDocuments scrapes documents and uploads them to MinIO
+func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
+	s.logger.Info("Starting document scraping", map[string]interface{}{
+		"notice_id": form.NoticeID,
+		"category":  form.Category,
+	})
+
+	// Prepare request body
+	requestBody := map[string]interface{}{
+		"notice_id":   form.NoticeID,
+		"username":    form.Username,
+		"password":    form.Password,
+		"login_url":   form.LoginURL,
+		"notice_url":  form.NoticeURL,
+		"category":    form.Category,
+		"subcategory": form.SubCategory,
+	}
+
+	bodyBytes, err := json.Marshal(requestBody)
+	if err != nil {
+		s.logger.Error("Failed to marshal request body", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	// Create HTTP request
+	url := fmt.Sprintf("%s/download", s.baseURL)
+	req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
+	if err != nil {
+		s.logger.Error("Failed to create HTTP request", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	// Send request
+	resp, err := s.httpClient.Do(req)
+	if err != nil {
+		s.logger.Error("Failed to send request to scraper service", map[string]interface{}{
+			"error": err.Error(),
+			"url":   url,
+		})
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Read response body
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		s.logger.Error("Failed to read response body", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to read response: %w", err)
+	}
+
+	// Handle error responses
+	if resp.StatusCode >= 400 {
+		var errorResp map[string]interface{}
+		if err := json.Unmarshal(respBody, &errorResp); err == nil {
+			if errorMsg, ok := errorResp["error"].(string); ok {
+				s.logger.Error("Scraper service returned error", map[string]interface{}{
+					"status_code": resp.StatusCode,
+					"error":       errorMsg,
+				})
+				return nil, fmt.Errorf("scraper service error: %s", errorMsg)
+			}
+		}
+		s.logger.Error("Scraper service returned error", map[string]interface{}{
+			"status_code": resp.StatusCode,
+			"body":        string(respBody),
+		})
+		return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode)
+	}
+
+	// Parse success response
+	var scrapeResp ScrapeResponse
+	if err := json.Unmarshal(respBody, &scrapeResp); err != nil {
+		s.logger.Error("Failed to unmarshal response", map[string]interface{}{
+			"error": err.Error(),
+			"body":  string(respBody),
+		})
+		return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+	}
+
+	s.logger.Info("Document scraping completed", map[string]interface{}{
+		"notice_id":      scrapeResp.NoticeID,
+		"uploaded_count": scrapeResp.UploadedCount,
+		"success":        scrapeResp.Success,
+	})
+
+	return &scrapeResp, nil
+}