tm_back/pkg/scraper/sdk.go

package scraper

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"time"

	"tm/pkg/logger"
)

// SDK represents the scraper SDK interface
type SDK interface {
	// ScrapeDocuments scrapes documents for a given notice
	ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error)

	// ListBuckets lists all buckets with notice IDs
	ListBuckets(ctx context.Context) (*ListBucketsResponse, error)

	// HealthCheck checks if the scraper service is healthy
	HealthCheck(ctx context.Context) error
}

// ScrapeRequest represents a request to scrape documents (only dynamic fields)
type ScrapeRequest struct {
	NoticeID  string `json:"notice_id"`
	NoticeURL string `json:"notice_url"`
}

// ScrapeResponse represents the response from scraping operation
type ScrapeResponse struct {
	Success       bool           `json:"success"`
	NoticeID      string         `json:"notice_id"`
	UploadedCount int            `json:"uploaded_count"`
	UploadedFiles []UploadedFile `json:"uploaded_files"`
	Errors        []FileError    `json:"errors,omitempty"`
	Buckets       []BucketInfo   `json:"buckets,omitempty"`
}

// UploadedFile represents information about an uploaded file
type UploadedFile struct {
	Filename   string `json:"filename"`
	ObjectName string `json:"object_name"`
	Size       int64  `json:"size"`
}

// FileError represents an error for a specific file
type FileError struct {
	Filename string `json:"filename"`
	Error    string `json:"error"`
}

// BucketInfo represents information about a bucket
type BucketInfo struct {
	Name     string `json:"name"`
	NoticeID string `json:"notice_id"`
	Count    int    `json:"count"`
}

// ListBucketsResponse represents the response from listing buckets
type ListBucketsResponse struct {
	Buckets []BucketInfo `json:"buckets"`
	Total   int          `json:"total"`
}

// Client implements the SDK interface
type Client struct {
	httpClient *http.Client
	baseURL    string
	logger     logger.Logger
	// Stored credentials
	username string
	password string
	loginURL string
}

// NewClient creates a new scraper SDK client with stored credentials
func NewClient(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) SDK {
	return &Client{
		httpClient: &http.Client{
			Timeout: timeout,
		},
		baseURL:  baseURL,
		logger:   logger,
		username: username,
		password: password,
		loginURL: loginURL,
	}
}

// ScrapeDocuments scrapes documents and uploads them to MinIO
func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) {
	c.logger.Info("Starting document scraping via SDK", map[string]interface{}{
		"notice_id": req.NoticeID,
		"base_url":  c.baseURL,
	})

	// Prepare request body with stored credentials + dynamic fields
	requestBody := map[string]interface{}{
		"notice_id":  req.NoticeID,
		"username":   c.username,
		"password":   c.password,
		"login_url":  c.loginURL,
		"notice_url": req.NoticeURL,
	}

	// Use unified handler API
	var response ScrapeResponse
	err := c.makeRequest(ctx, "POST", "/download", requestBody, &response)
	if err != nil {
		return nil, err
	}

	// Get buckets list after successful scraping
	bucketsResp, err := c.ListBuckets(ctx)
	if err != nil {
		c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{
			"error": err.Error(),
		})
	} else {
		response.Buckets = bucketsResp.Buckets
	}

	c.logger.Info("Document scraping completed via SDK", map[string]interface{}{
		"notice_id":      response.NoticeID,
		"uploaded_count": response.UploadedCount,
		"success":        response.Success,
	})

	return &response, nil
}

// ListBuckets lists all buckets with notice IDs
func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) {
	var response ListBucketsResponse
	err := c.makeRequest(ctx, "GET", "/buckets", nil, &response)
	return &response, err
}

// HealthCheck checks if the scraper service is healthy
func (c *Client) HealthCheck(ctx context.Context) error {
	var response map[string]interface{}
	err := c.makeRequest(ctx, "GET", "/health", nil, &response)
	if err != nil {
		return err
	}

	// Health check doesn't need to return data, just check if request succeeded
	return nil
}

// Unified HTTP Handler API
// makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling
func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error {
	// Build full URL
	url := fmt.Sprintf("%s%s", c.baseURL, endpoint)

	// Marshal request body to JSON if provided
	var bodyReader io.Reader
	if requestBody != nil {
		jsonData, err := json.Marshal(requestBody)
		if err != nil {
			c.logger.Error("Failed to marshal request body", map[string]interface{}{
				"error":    err.Error(),
				"endpoint": endpoint,
			})
			return fmt.Errorf("failed to marshal request: %w", err)
		}
		bodyReader = bytes.NewReader(jsonData)
	}

	// Create HTTP request
	req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
	if err != nil {
		c.logger.Error("Failed to create HTTP request", map[string]interface{}{
			"error":  err.Error(),
			"method": method,
			"url":    url,
		})
		return fmt.Errorf("failed to create request: %w", err)
	}

	// Set headers
	if requestBody != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	req.Header.Set("Accept", "application/json")

	// Send request
	resp, err := c.httpClient.Do(req)
	if err != nil {
		c.logger.Error("Failed to send HTTP request", map[string]interface{}{
			"error":  err.Error(),
			"method": method,
			"url":    url,
		})
		return fmt.Errorf("failed to send request: %w", err)
	}
	defer resp.Body.Close()

	// Read response body
	respBody, err := io.ReadAll(resp.Body)
	if err != nil {
		c.logger.Error("Failed to read response body", map[string]interface{}{
			"error": err.Error(),
			"url":   url,
		})
		return fmt.Errorf("failed to read response: %w", err)
	}

	// Handle error responses
	if resp.StatusCode >= 400 {
		// Try to parse error response
		var errorResp map[string]interface{}
		if err := json.Unmarshal(respBody, &errorResp); err == nil {
			if errorMsg, ok := errorResp["error"].(string); ok {
				c.logger.Error("Request failed with error response", map[string]interface{}{
					"status_code": resp.StatusCode,
					"error":       errorMsg,
					"url":         url,
				})
				return fmt.Errorf("request failed: %s", errorMsg)
			}
		}

		c.logger.Error("Request failed with status code", map[string]interface{}{
			"status_code": resp.StatusCode,
			"url":         url,
			"body":        string(respBody),
		})
		return fmt.Errorf("request failed with status %d", resp.StatusCode)
	}

	// Handle successful responses
	if responseBody != nil {
		if err := json.Unmarshal(respBody, responseBody); err != nil {
			c.logger.Error("Failed to unmarshal response", map[string]interface{}{
				"error": err.Error(),
				"url":   url,
				"body":  string(respBody),
			})
			return fmt.Errorf("failed to unmarshal response: %w", err)
		}
	}

	c.logger.Debug("Request completed successfully", map[string]interface{}{
		"method":      method,
		"url":         url,
		"status_code": resp.StatusCode,
	})

	return nil
}