Removed old document scraper service
This commit is contained in:
m.nazemi
@@ -1,246 +0,0 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"tm/pkg/logger"
|
||||
)
|
||||
|
||||
// SDK represents the scraper SDK interface
|
||||
type SDK interface {
|
||||
// ScrapeDocuments scrapes documents for a given notice
|
||||
ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error)
|
||||
|
||||
// ListBuckets lists all buckets with notice IDs
|
||||
ListBuckets(ctx context.Context) (*ListBucketsResponse, error)
|
||||
|
||||
// HealthCheck checks if the scraper service is healthy
|
||||
HealthCheck(ctx context.Context) error
|
||||
}
|
||||
|
||||
// ScrapeRequest represents a request to scrape documents (only dynamic fields)
|
||||
type ScrapeRequest struct {
|
||||
NoticeID string `json:"notice_id"`
|
||||
NoticeURL string `json:"notice_url"`
|
||||
}
|
||||
|
||||
// ScrapeResponse represents the response from scraping operation
|
||||
type ScrapeResponse struct {
|
||||
Success bool `json:"success"`
|
||||
NoticeID string `json:"notice_id"`
|
||||
UploadedCount int `json:"uploaded_count"`
|
||||
UploadedFiles []UploadedFile `json:"uploaded_files"`
|
||||
Errors []FileError `json:"errors,omitempty"`
|
||||
Buckets []BucketInfo `json:"buckets,omitempty"`
|
||||
}
|
||||
|
||||
// UploadedFile represents information about an uploaded file
|
||||
type UploadedFile struct {
|
||||
Filename string `json:"filename"`
|
||||
ObjectName string `json:"object_name"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// FileError represents an error for a specific file
|
||||
type FileError struct {
|
||||
Filename string `json:"filename"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// BucketInfo represents information about a bucket
|
||||
type BucketInfo struct {
|
||||
Name string `json:"name"`
|
||||
NoticeID string `json:"notice_id"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// ListBucketsResponse represents the response from listing buckets
|
||||
type ListBucketsResponse struct {
|
||||
Buckets []BucketInfo `json:"buckets"`
|
||||
Total int `json:"total"`
|
||||
}
|
||||
|
||||
// Client implements the SDK interface
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
baseURL string
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
// NewClient creates a new scraper SDK client with stored credentials
|
||||
func NewClient(baseURL string, timeout time.Duration, logger logger.Logger) SDK {
|
||||
return &Client{
|
||||
httpClient: &http.Client{
|
||||
Timeout: timeout,
|
||||
},
|
||||
baseURL: baseURL,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// ScrapeDocuments scrapes documents and uploads them to MinIO
|
||||
func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) {
|
||||
c.logger.Info("Starting document scraping via SDK", map[string]interface{}{
|
||||
"notice_id": req.NoticeID,
|
||||
"base_url": c.baseURL,
|
||||
})
|
||||
|
||||
// Prepare request body with stored credentials + dynamic fields
|
||||
requestBody := map[string]interface{}{
|
||||
"notice_id": req.NoticeID,
|
||||
"notice_url": req.NoticeURL,
|
||||
}
|
||||
|
||||
// Use unified handler API
|
||||
var response ScrapeResponse
|
||||
err := c.makeRequest(ctx, "POST", "/download", requestBody, &response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get buckets list after successful scraping
|
||||
bucketsResp, err := c.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
} else {
|
||||
response.Buckets = bucketsResp.Buckets
|
||||
}
|
||||
|
||||
c.logger.Info("Document scraping completed via SDK", map[string]interface{}{
|
||||
"notice_id": response.NoticeID,
|
||||
"uploaded_count": response.UploadedCount,
|
||||
"success": response.Success,
|
||||
})
|
||||
|
||||
return &response, nil
|
||||
}
|
||||
|
||||
// ListBuckets lists all buckets with notice IDs
|
||||
func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) {
|
||||
var response ListBucketsResponse
|
||||
err := c.makeRequest(ctx, "GET", "/buckets", nil, &response)
|
||||
return &response, err
|
||||
}
|
||||
|
||||
// HealthCheck checks if the scraper service is healthy
|
||||
func (c *Client) HealthCheck(ctx context.Context) error {
|
||||
var response map[string]interface{}
|
||||
err := c.makeRequest(ctx, "GET", "/health", nil, &response)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Health check doesn't need to return data, just check if request succeeded
|
||||
return nil
|
||||
}
|
||||
|
||||
// Unified HTTP Handler API
|
||||
// makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling
|
||||
func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error {
|
||||
// Build full URL
|
||||
url := fmt.Sprintf("%s%s", c.baseURL, endpoint)
|
||||
|
||||
// Marshal request body to JSON if provided
|
||||
var bodyReader io.Reader
|
||||
if requestBody != nil {
|
||||
jsonData, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to marshal request body", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"endpoint": endpoint,
|
||||
})
|
||||
return fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
bodyReader = bytes.NewReader(jsonData)
|
||||
}
|
||||
|
||||
// Create HTTP request
|
||||
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to create HTTP request", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"method": method,
|
||||
"url": url,
|
||||
})
|
||||
return fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
// Set headers
|
||||
if requestBody != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
// Send request
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to send HTTP request", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"method": method,
|
||||
"url": url,
|
||||
})
|
||||
return fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read response body
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
c.logger.Error("Failed to read response body", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"url": url,
|
||||
})
|
||||
return fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
// Handle error responses
|
||||
if resp.StatusCode >= 400 {
|
||||
// Try to parse error response
|
||||
var errorResp map[string]interface{}
|
||||
if err := json.Unmarshal(respBody, &errorResp); err == nil {
|
||||
if errorMsg, ok := errorResp["error"].(string); ok {
|
||||
c.logger.Error("Request failed with error response", map[string]interface{}{
|
||||
"status_code": resp.StatusCode,
|
||||
"error": errorMsg,
|
||||
"url": url,
|
||||
})
|
||||
return fmt.Errorf("request failed: %s", errorMsg)
|
||||
}
|
||||
}
|
||||
|
||||
c.logger.Error("Request failed with status code", map[string]interface{}{
|
||||
"status_code": resp.StatusCode,
|
||||
"url": url,
|
||||
"body": string(respBody),
|
||||
})
|
||||
return fmt.Errorf("request failed with status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Handle successful responses
|
||||
if responseBody != nil {
|
||||
if err := json.Unmarshal(respBody, responseBody); err != nil {
|
||||
c.logger.Error("Failed to unmarshal response", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"url": url,
|
||||
"body": string(respBody),
|
||||
})
|
||||
return fmt.Errorf("failed to unmarshal response: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
c.logger.Debug("Request completed successfully", map[string]interface{}{
|
||||
"method": method,
|
||||
"url": url,
|
||||
"status_code": resp.StatusCode,
|
||||
})
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user