package scraper import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "time" "tm/pkg/logger" ) // SDK represents the scraper SDK interface type SDK interface { // ScrapeDocuments scrapes documents for a given notice ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) // ListBuckets lists all buckets with notice IDs ListBuckets(ctx context.Context) (*ListBucketsResponse, error) // HealthCheck checks if the scraper service is healthy HealthCheck(ctx context.Context) error } // ScrapeRequest represents a request to scrape documents (only dynamic fields) type ScrapeRequest struct { NoticeID string `json:"notice_id"` NoticeURL string `json:"notice_url"` } // ScrapeResponse represents the response from scraping operation type ScrapeResponse struct { Success bool `json:"success"` NoticeID string `json:"notice_id"` UploadedCount int `json:"uploaded_count"` UploadedFiles []UploadedFile `json:"uploaded_files"` Errors []FileError `json:"errors,omitempty"` Buckets []BucketInfo `json:"buckets,omitempty"` } // UploadedFile represents information about an uploaded file type UploadedFile struct { Filename string `json:"filename"` ObjectName string `json:"object_name"` Size int64 `json:"size"` } // FileError represents an error for a specific file type FileError struct { Filename string `json:"filename"` Error string `json:"error"` } // BucketInfo represents information about a bucket type BucketInfo struct { Name string `json:"name"` NoticeID string `json:"notice_id"` Count int `json:"count"` } // ListBucketsResponse represents the response from listing buckets type ListBucketsResponse struct { Buckets []BucketInfo `json:"buckets"` Total int `json:"total"` } // Client implements the SDK interface type Client struct { httpClient *http.Client baseURL string logger logger.Logger } // NewClient creates a new scraper SDK client with stored credentials func NewClient(baseURL string, timeout time.Duration, logger logger.Logger) SDK { return &Client{ httpClient: &http.Client{ Timeout: timeout, }, baseURL: baseURL, logger: logger, } } // ScrapeDocuments scrapes documents and uploads them to MinIO func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) { c.logger.Info("Starting document scraping via SDK", map[string]interface{}{ "notice_id": req.NoticeID, "base_url": c.baseURL, }) // Prepare request body with stored credentials + dynamic fields requestBody := map[string]interface{}{ "notice_id": req.NoticeID, "notice_url": req.NoticeURL, } // Use unified handler API var response ScrapeResponse err := c.makeRequest(ctx, "POST", "/download", requestBody, &response) if err != nil { return nil, err } // Get buckets list after successful scraping bucketsResp, err := c.ListBuckets(ctx) if err != nil { c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{ "error": err.Error(), }) } else { response.Buckets = bucketsResp.Buckets } c.logger.Info("Document scraping completed via SDK", map[string]interface{}{ "notice_id": response.NoticeID, "uploaded_count": response.UploadedCount, "success": response.Success, }) return &response, nil } // ListBuckets lists all buckets with notice IDs func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) { var response ListBucketsResponse err := c.makeRequest(ctx, "GET", "/buckets", nil, &response) return &response, err } // HealthCheck checks if the scraper service is healthy func (c *Client) HealthCheck(ctx context.Context) error { var response map[string]interface{} err := c.makeRequest(ctx, "GET", "/health", nil, &response) if err != nil { return err } // Health check doesn't need to return data, just check if request succeeded return nil } // Unified HTTP Handler API // makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error { // Build full URL url := fmt.Sprintf("%s%s", c.baseURL, endpoint) // Marshal request body to JSON if provided var bodyReader io.Reader if requestBody != nil { jsonData, err := json.Marshal(requestBody) if err != nil { c.logger.Error("Failed to marshal request body", map[string]interface{}{ "error": err.Error(), "endpoint": endpoint, }) return fmt.Errorf("failed to marshal request: %w", err) } bodyReader = bytes.NewReader(jsonData) } // Create HTTP request req, err := http.NewRequestWithContext(ctx, method, url, bodyReader) if err != nil { c.logger.Error("Failed to create HTTP request", map[string]interface{}{ "error": err.Error(), "method": method, "url": url, }) return fmt.Errorf("failed to create request: %w", err) } // Set headers if requestBody != nil { req.Header.Set("Content-Type", "application/json") } req.Header.Set("Accept", "application/json") // Send request resp, err := c.httpClient.Do(req) if err != nil { c.logger.Error("Failed to send HTTP request", map[string]interface{}{ "error": err.Error(), "method": method, "url": url, }) return fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() // Read response body respBody, err := io.ReadAll(resp.Body) if err != nil { c.logger.Error("Failed to read response body", map[string]interface{}{ "error": err.Error(), "url": url, }) return fmt.Errorf("failed to read response: %w", err) } // Handle error responses if resp.StatusCode >= 400 { // Try to parse error response var errorResp map[string]interface{} if err := json.Unmarshal(respBody, &errorResp); err == nil { if errorMsg, ok := errorResp["error"].(string); ok { c.logger.Error("Request failed with error response", map[string]interface{}{ "status_code": resp.StatusCode, "error": errorMsg, "url": url, }) return fmt.Errorf("request failed: %s", errorMsg) } } c.logger.Error("Request failed with status code", map[string]interface{}{ "status_code": resp.StatusCode, "url": url, "body": string(respBody), }) return fmt.Errorf("request failed with status %d", resp.StatusCode) } // Handle successful responses if responseBody != nil { if err := json.Unmarshal(respBody, responseBody); err != nil { c.logger.Error("Failed to unmarshal response", map[string]interface{}{ "error": err.Error(), "url": url, "body": string(respBody), }) return fmt.Errorf("failed to unmarshal response: %w", err) } } c.logger.Debug("Request completed successfully", map[string]interface{}{ "method": method, "url": url, "status_code": resp.StatusCode, }) return nil }