scraper sdk

cmd/web/bootstrap/config.go

@@ -21,8 +21,11 @@ type Config struct {
 }
 
 type ScraperConfig struct {
-	BaseURL string        `env:"SCRAPER_BASE_URL"`
+	BaseURL  string        `env:"SCRAPER_BASE_URL"`
-	Timeout time.Duration `env:"SCRAPER_TIMEOUT"`
+	Timeout  time.Duration `env:"SCRAPER_TIMEOUT"`
+	Username string        `env:"SCRAPER_USERNAME"`
+	Password string        `env:"SCRAPER_PASSWORD"`
+	LoginURL string        `env:"SCRAPER_LOGIN_URL"`
 }
 
 type AuthConfig struct {

cmd/web/main.go

@@ -180,7 +180,7 @@ func main() {
 	notificationService := notification.NewService(notificationSDK, userService, customerService, logger)
 	contactService := contact.NewService(contactRepository, logger)
 	cmsService := cms.NewService(cmsRepository, logger)
-	scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger)
+	scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, conf.Scraper.Username, conf.Scraper.Password, conf.Scraper.LoginURL)
 	logger.Info("Services initialized successfully", map[string]interface{}{
 		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
 	})

internal/scraper/form.go

@@ -2,13 +2,8 @@ package scraper
 
 // ScrapeRequestForm represents the request form for scraping documents
 type ScrapeRequestForm struct {
-	NoticeID    string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
+	NoticeID  string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
-	Username    string `json:"username" valid:"required,email" example:"feed@opplenz.com"`
+	NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
-	Password    string `json:"password" valid:"required,length(8|200)" example:"password123"`
-	LoginURL    string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"`
-	NoticeURL   string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
-	Category    string `json:"category" valid:"optional,length(1|100)" example:"tenders"`
-	SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"`
 }
 
 // ScrapeResponse represents the response from scraping operation
@@ -18,6 +13,7 @@ type ScrapeResponse struct {
 	UploadedCount int                `json:"uploaded_count"`
 	UploadedFiles []UploadedFileInfo `json:"uploaded_files"`
 	Errors        []FileError        `json:"errors,omitempty"`
+	Buckets       []BucketInfo       `json:"buckets,omitempty"`
 }
 
 // UploadedFileInfo represents information about an uploaded file
@@ -32,3 +28,10 @@ type FileError struct {
 	Filename string `json:"filename"`
 	Error    string `json:"error"`
 }
+
+// BucketInfo represents information about a bucket
+type BucketInfo struct {
+	Name     string `json:"name"`
+	NoticeID string `json:"notice_id"`
+	Count    int    `json:"count"`
+}

internal/scraper/service.go

@@ -1,14 +1,11 @@
 package scraper
 
 import (
-	"bytes"
 	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
 	"time"
+
 	"tm/pkg/logger"
+	"tm/pkg/scraper"
 )
 
 // Service defines the interface for scraper business operations
@@ -19,19 +16,16 @@ type Service interface {
 
 // scraperService implements Service interface
 type scraperService struct {
-	httpClient *http.Client
+	sdk    scraper.SDK
-	baseURL    string
+	logger logger.Logger
-	logger     logger.Logger
 }
 
-// NewService creates a new scraper service
+// NewService creates a new scraper service with stored credentials
-func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
+func NewService(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) Service {
+	sdk := scraper.NewClient(baseURL, timeout, logger, username, password, loginURL)
 	return &scraperService{
-		httpClient: &http.Client{
+		sdk:    sdk,
-			Timeout: timeout,
+		logger: logger,
-		},
-		baseURL: baseURL,
-		logger:  logger,
 	}
 }
 
@@ -39,94 +33,65 @@ func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Ser
 func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
 	s.logger.Info("Starting document scraping", map[string]interface{}{
 		"notice_id": form.NoticeID,
-		"category":  form.Category,
 	})
 
-	// Prepare request body
+	// Convert form to SDK request (only dynamic fields)
-	requestBody := map[string]interface{}{
+	req := &scraper.ScrapeRequest{
-		"notice_id":   form.NoticeID,
+		NoticeID:  form.NoticeID,
-		"username":    form.Username,
+		NoticeURL: form.NoticeURL,
-		"password":    form.Password,
-		"login_url":   form.LoginURL,
-		"notice_url":  form.NoticeURL,
-		"category":    form.Category,
-		"subcategory": form.SubCategory,
 	}
 
-	bodyBytes, err := json.Marshal(requestBody)
+	// Call SDK (credentials are already configured)
+	response, err := s.sdk.ScrapeDocuments(ctx, req)
 	if err != nil {
-		s.logger.Error("Failed to marshal request body", map[string]interface{}{
+		s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{
-			"error": err.Error(),
+			"error":     err.Error(),
+			"notice_id": form.NoticeID,
 		})
-		return nil, fmt.Errorf("failed to marshal request: %w", err)
+		return nil, err
 	}
 
-	// Create HTTP request
+	// Convert SDK response to service response
-	url := fmt.Sprintf("%s/download", s.baseURL)
+	serviceResp := &ScrapeResponse{
-	req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
+		Success:       response.Success,
-	if err != nil {
+		NoticeID:      response.NoticeID,
-		s.logger.Error("Failed to create HTTP request", map[string]interface{}{
+		UploadedCount: response.UploadedCount,
-			"error": err.Error(),
+		UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)),
-		})
+		Errors:        make([]FileError, len(response.Errors)),
-		return nil, fmt.Errorf("failed to create request: %w", err)
+		Buckets:       make([]BucketInfo, len(response.Buckets)),
 	}
 
-	req.Header.Set("Content-Type", "application/json")
+	// Convert uploaded files
-
+	for i, file := range response.UploadedFiles {
-	// Send request
+		serviceResp.UploadedFiles[i] = UploadedFileInfo{
-	resp, err := s.httpClient.Do(req)
+			Filename:   file.Filename,
-	if err != nil {
+			ObjectName: file.ObjectName,
-		s.logger.Error("Failed to send request to scraper service", map[string]interface{}{
+			Size:       file.Size,
-			"error": err.Error(),
-			"url":   url,
-		})
-		return nil, fmt.Errorf("failed to send request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	// Read response body
-	respBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		s.logger.Error("Failed to read response body", map[string]interface{}{
-			"error": err.Error(),
-		})
-		return nil, fmt.Errorf("failed to read response: %w", err)
-	}
-
-	// Handle error responses
-	if resp.StatusCode >= 400 {
-		var errorResp map[string]interface{}
-		if err := json.Unmarshal(respBody, &errorResp); err == nil {
-			if errorMsg, ok := errorResp["error"].(string); ok {
-				s.logger.Error("Scraper service returned error", map[string]interface{}{
-					"status_code": resp.StatusCode,
-					"error":       errorMsg,
-				})
-				return nil, fmt.Errorf("scraper service error: %s", errorMsg)
-			}
 		}
-		s.logger.Error("Scraper service returned error", map[string]interface{}{
-			"status_code": resp.StatusCode,
-			"body":        string(respBody),
-		})
-		return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode)
 	}
 
-	// Parse success response
+	// Convert errors
-	var scrapeResp ScrapeResponse
+	for i, err := range response.Errors {
-	if err := json.Unmarshal(respBody, &scrapeResp); err != nil {
+		serviceResp.Errors[i] = FileError{
-		s.logger.Error("Failed to unmarshal response", map[string]interface{}{
+			Filename: err.Filename,
-			"error": err.Error(),
+			Error:    err.Error,
-			"body":  string(respBody),
+		}
-		})
+	}
-		return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+
+	// Convert buckets
+	for i, bucket := range response.Buckets {
+		serviceResp.Buckets[i] = BucketInfo{
+			Name:     bucket.Name,
+			NoticeID: bucket.NoticeID,
+			Count:    bucket.Count,
+		}
 	}
 
 	s.logger.Info("Document scraping completed", map[string]interface{}{
-		"notice_id":      scrapeResp.NoticeID,
+		"notice_id":      serviceResp.NoticeID,
-		"uploaded_count": scrapeResp.UploadedCount,
+		"uploaded_count": serviceResp.UploadedCount,
-		"success":        scrapeResp.Success,
+		"success":        serviceResp.Success,
 	})
 
-	return &scrapeResp, nil
+	return serviceResp, nil
 }

pkg/scraper/Dockerfile

@@ -1,43 +0,0 @@
-FROM python:3.11-slim
-
-RUN apt-get update && apt-get install -y \
-    wget \
-    gnupg \
-    ca-certificates \
-    fonts-liberation \
-    libasound2 \
-    libatk-bridge2.0-0 \
-    libatk1.0-0 \
-    libatspi2.0-0 \
-    libcups2 \
-    libdbus-1-3 \
-    libdrm2 \
-    libgbm1 \
-    libgtk-3-0 \
-    libnspr4 \
-    libnss3 \
-    libx11-6 \
-    libx11-xcb1 \
-    libxcb1 \
-    libxcomposite1 \
-    libxdamage1 \
-    libxext6 \
-    libxfixes3 \
-    libxkbcommon0 \
-    libxrandr2 \
-    xdg-utils \
-    && rm -rf /var/lib/apt/lists/*
-
-WORKDIR /app
-
-COPY requirements.txt .
-RUN pip install --no-cache-dir -r requirements.txt
-
-RUN playwright install chromium
-
-COPY Scraper.py .
-COPY app.py .
-
-EXPOSE 8000
-
-CMD ["python", "app.py"]

pkg/scraper/Scraper.py

@@ -1,121 +0,0 @@
-from playwright.sync_api import sync_playwright
-import re
-import os
-import time
-
-def clean_filename(filename):
-    """Removes illegal characters from filenames."""
-    return re.sub(r'[<>:"/\\|?*]', '_', filename).strip()
-
-def download_documents(username, password, login_url, notice_url, download_dir=None):
-    """
-    Download documents from TendSign website.
-    
-    Args:
-        username: Login username
-        password: Login password
-        login_url: URL to login page
-        notice_url: URL to notice entry page
-        download_dir: Directory to save downloads (optional, creates temp dir if not provided)
-    
-    Returns:
-        List of downloaded file paths
-    """
-    if download_dir is None:
-        download_dir = f"TendSign_Downloads_{int(time.time())}"
-    
-    os.makedirs(download_dir, exist_ok=True)
-    print(f"[*] Downloads will be saved to: {download_dir}")
-    
-    downloaded_files = []
-    
-    with sync_playwright() as p:
-        print("[*] Starting browser...")
-        browser = p.chromium.launch(headless=True)
-        context = browser.new_context(
-            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
-            viewport={"width": 1920, "height": 1080},
-            locale="sv-SE",
-        )
-        context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});")
-        page = context.new_page()
-
-        try:
-            # === 1. LOGIN ===
-            print("[*] Logging in...")
-            page.goto(login_url, wait_until="networkidle")
-            page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000)
-            page.fill('input[name="UcomLogin$txt_UserName"]', username)
-            page.fill('input[name="UcomLogin$txt_Password"]', password)
-            page.click('input[name="UcomLogin$btn_Submit"]')
-            page.wait_for_url("**/supplier/**", timeout=30000)
-            print("[SUCCESS] Logged in")
-
-            # === 2. NAVIGATE TO DOCUMENTS TAB ===
-            print(f"[*] Navigating to notice page...")
-            page.goto(notice_url, wait_until="networkidle", timeout=60000)
-
-            print("[*] Switching to 'Dokument' tab...")
-            with page.expect_navigation(wait_until="networkidle", timeout=15000):
-                page.click('a[href*="s_view_RFQfiles.aspx"]')
-
-            # === 3. LOCATE ALL FILE LINKS ===
-            print("[*] Looking for file links starting with 'Bilaga'...")
-
-            FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]"
-
-            try:
-                page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000)
-            except Exception:
-                raise Exception("Failed to locate any file links starting with 'Bilaga'.")
-
-            file_elements = page.locator(FILE_LINK_SELECTOR).all()
-            
-            print(f"[*] Found {len(file_elements)} file links to download.")
-
-            if not file_elements:
-                print("[!] No file links found. Exiting.")
-                return downloaded_files
-
-            # === 4. DOWNLOAD LOOP ===
-            for i, element in enumerate(file_elements):
-                file_text = element.inner_text().strip()
-                print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...")
-
-                try:
-                    with page.expect_download(timeout=60000) as download_info:
-                        element.click()
-
-                    download = download_info.value
-                    
-                    suggested_name = download.suggested_filename
-                    clean_name = clean_filename(suggested_name)
-                    save_path = os.path.join(download_dir, clean_name)
-
-                    download.save_as(save_path)
-                    downloaded_files.append(save_path)
-                    print(f"   -> Successfully saved: {clean_name}")
-
-                except Exception as e:
-                    print(f"   -> [ERROR] Download failed for {file_text}: {e}")
-                
-                time.sleep(1)
-
-            print("\n[DONE] All downloads processed.")
-            return downloaded_files
-
-        except Exception as e:
-            print(f"[CRITICAL ERROR] {e}")
-            page.screenshot(path=os.path.join(download_dir, "error_debug.png"))
-            raise
-        finally:
-            browser.close()
-
-if __name__ == "__main__":
-    # Default values for backward compatibility
-    USERNAME = "feed@opplenz.com"
-    PASSWORD = "f38Dgma!rLn2VR3DZaBn"
-    LOGIN_URL = "https://tendsign.com/login.aspx"
-    NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"
-    
-    download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL)

pkg/scraper/app.py

@@ -1,151 +0,0 @@
-from flask import Flask, request, jsonify
-from Scraper import download_documents
-import os
-from minio import Minio
-from minio.error import S3Error
-import tempfile
-import shutil
-
-app = Flask(__name__)
-
-# MinIO configuration from environment variables
-MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000')
-MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
-MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
-MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
-MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents')
-
-def get_minio_client():
-    """Create and return MinIO client"""
-    return Minio(
-        MINIO_ENDPOINT,
-        access_key=MINIO_ACCESS_KEY,
-        secret_key=MINIO_SECRET_KEY,
-        secure=MINIO_USE_SSL
-    )
-
-def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET):
-    """Upload a file to MinIO"""
-    try:
-        client = get_minio_client()
-        
-        # Ensure bucket exists
-        if not client.bucket_exists(bucket):
-            client.make_bucket(bucket)
-        
-        # Upload file
-        client.fput_object(bucket, object_name, file_path)
-        return True, None
-    except S3Error as e:
-        return False, str(e)
-    except Exception as e:
-        return False, str(e)
-
-@app.route('/health', methods=['GET'])
-def health():
-    """Health check endpoint"""
-    return jsonify({"status": "healthy"}), 200
-
-@app.route('/download', methods=['POST'])
-def download():
-    """Download documents and upload to MinIO"""
-    try:
-        data = request.get_json()
-        
-        # Validate required fields
-        if not data:
-            return jsonify({"error": "Request body is required"}), 400
-        
-        notice_id = data.get('notice_id')
-        username = data.get('username')
-        password = data.get('password')
-        login_url = data.get('login_url')
-        notice_url = data.get('notice_url')
-        category = data.get('category', 'tenders')
-        subcategory = data.get('subcategory', '')
-        
-        if not all([notice_id, username, password, login_url, notice_url]):
-            return jsonify({
-                "error": "Missing required fields: notice_id, username, password, login_url, notice_url"
-            }), 400
-        
-        # Create temporary directory for downloads
-        temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_")
-        
-        try:
-            # Download documents using the scraper
-            downloaded_files = download_documents(
-                username=username,
-                password=password,
-                login_url=login_url,
-                notice_url=notice_url,
-                download_dir=temp_dir
-            )
-            
-            if not downloaded_files:
-                return jsonify({
-                    "error": "No files were downloaded",
-                    "notice_id": notice_id
-                }), 404
-            
-            # Upload each file to MinIO
-            uploaded_files = []
-            errors = []
-            
-            for file_path in downloaded_files:
-                try:
-                    filename = os.path.basename(file_path)
-                    # Create object name with hierarchical path
-                    if subcategory:
-                        object_name = f"{category}/{subcategory}/{notice_id}/{filename}"
-                    else:
-                        object_name = f"{category}/{notice_id}/{filename}"
-                    
-                    success, error = upload_to_minio(file_path, object_name)
-                    
-                    if success:
-                        uploaded_files.append({
-                            "filename": filename,
-                            "object_name": object_name,
-                            "size": os.path.getsize(file_path)
-                        })
-                    else:
-                        errors.append({
-                            "filename": filename,
-                            "error": error
-                        })
-                except Exception as e:
-                    errors.append({
-                        "filename": os.path.basename(file_path),
-                        "error": str(e)
-                    })
-            
-            # Clean up temporary directory
-            shutil.rmtree(temp_dir, ignore_errors=True)
-            
-            return jsonify({
-                "success": True,
-                "notice_id": notice_id,
-                "uploaded_count": len(uploaded_files),
-                "uploaded_files": uploaded_files,
-                "errors": errors if errors else None
-            }), 200
-            
-        except Exception as e:
-            # Clean up on error
-            shutil.rmtree(temp_dir, ignore_errors=True)
-            return jsonify({
-                "error": "Failed to download documents",
-                "message": str(e),
-                "notice_id": notice_id
-            }), 500
-            
-    except Exception as e:
-        return jsonify({
-            "error": "Internal server error",
-            "message": str(e)
-        }), 500
-
-if __name__ == '__main__':
-    port = int(os.getenv('PORT', 8000))
-    app.run(host='0.0.0.0', port=port, debug=False)

pkg/scraper/docker-compose.scraper.yml

@@ -1,45 +0,0 @@
-version: '3.8'
-
-services:
-  scraper:
-    build:
-      context: .
-      dockerfile: pkg/scraper/Dockerfile
-    container_name: tm-scraper
-    ports:
-      - "8000:8000"
-    environment:
-      - PORT=8000
-      - MINIO_ENDPOINT=minio:9000
-      - MINIO_ACCESS_KEY=minioadmin
-      - MINIO_SECRET_KEY=minioadmin
-      - MINIO_USE_SSL=false
-      - MINIO_BUCKET=documents
-    networks:
-      - tm-network
-    depends_on:
-      - minio
-    restart: unless-stopped
-
-  minio:
-    image: minio/minio:latest
-    container_name: tm-minio
-    ports:
-      - "9000:9000"
-      - "9001:9001"
-    environment:
-      - MINIO_ROOT_USER=minioadmin
-      - MINIO_ROOT_PASSWORD=minioadmin
-    command: server /data --console-address ":9001"
-    volumes:
-      - minio-data:/data
-    networks:
-      - tm-network
-    restart: unless-stopped
-
-networks:
-  tm-network:
-    driver: bridge
-
-volumes:
-  minio-data:

pkg/scraper/requirements.txt

@@ -1,5 +0,0 @@
-playwright==1.40.0
-flask==3.0.0
-minio==7.2.0
-requests==2.31.0
-python-dotenv==1.0.0

pkg/scraper/sdk.go

@@ -0,0 +1,256 @@
+package scraper
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"tm/pkg/logger"
+)
+
+// SDK represents the scraper SDK interface
+type SDK interface {
+	// ScrapeDocuments scrapes documents for a given notice
+	ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error)
+
+	// ListBuckets lists all buckets with notice IDs
+	ListBuckets(ctx context.Context) (*ListBucketsResponse, error)
+
+	// HealthCheck checks if the scraper service is healthy
+	HealthCheck(ctx context.Context) error
+}
+
+// ScrapeRequest represents a request to scrape documents (only dynamic fields)
+type ScrapeRequest struct {
+	NoticeID  string `json:"notice_id"`
+	NoticeURL string `json:"notice_url"`
+}
+
+// ScrapeResponse represents the response from scraping operation
+type ScrapeResponse struct {
+	Success       bool           `json:"success"`
+	NoticeID      string         `json:"notice_id"`
+	UploadedCount int            `json:"uploaded_count"`
+	UploadedFiles []UploadedFile `json:"uploaded_files"`
+	Errors        []FileError    `json:"errors,omitempty"`
+	Buckets       []BucketInfo   `json:"buckets,omitempty"`
+}
+
+// UploadedFile represents information about an uploaded file
+type UploadedFile struct {
+	Filename   string `json:"filename"`
+	ObjectName string `json:"object_name"`
+	Size       int64  `json:"size"`
+}
+
+// FileError represents an error for a specific file
+type FileError struct {
+	Filename string `json:"filename"`
+	Error    string `json:"error"`
+}
+
+// BucketInfo represents information about a bucket
+type BucketInfo struct {
+	Name     string `json:"name"`
+	NoticeID string `json:"notice_id"`
+	Count    int    `json:"count"`
+}
+
+// ListBucketsResponse represents the response from listing buckets
+type ListBucketsResponse struct {
+	Buckets []BucketInfo `json:"buckets"`
+	Total   int          `json:"total"`
+}
+
+// Client implements the SDK interface
+type Client struct {
+	httpClient *http.Client
+	baseURL    string
+	logger     logger.Logger
+	// Stored credentials
+	username string
+	password string
+	loginURL string
+}
+
+// NewClient creates a new scraper SDK client with stored credentials
+func NewClient(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) SDK {
+	return &Client{
+		httpClient: &http.Client{
+			Timeout: timeout,
+		},
+		baseURL:  baseURL,
+		logger:   logger,
+		username: username,
+		password: password,
+		loginURL: loginURL,
+	}
+}
+
+// ScrapeDocuments scrapes documents and uploads them to MinIO
+func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) {
+	c.logger.Info("Starting document scraping via SDK", map[string]interface{}{
+		"notice_id": req.NoticeID,
+		"base_url":  c.baseURL,
+	})
+
+	// Prepare request body with stored credentials + dynamic fields
+	requestBody := map[string]interface{}{
+		"notice_id":  req.NoticeID,
+		"username":   c.username,
+		"password":   c.password,
+		"login_url":  c.loginURL,
+		"notice_url": req.NoticeURL,
+	}
+
+	// Use unified handler API
+	var response ScrapeResponse
+	err := c.makeRequest(ctx, "POST", "/download", requestBody, &response)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get buckets list after successful scraping
+	bucketsResp, err := c.ListBuckets(ctx)
+	if err != nil {
+		c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{
+			"error": err.Error(),
+		})
+	} else {
+		response.Buckets = bucketsResp.Buckets
+	}
+
+	c.logger.Info("Document scraping completed via SDK", map[string]interface{}{
+		"notice_id":      response.NoticeID,
+		"uploaded_count": response.UploadedCount,
+		"success":        response.Success,
+	})
+
+	return &response, nil
+}
+
+// ListBuckets lists all buckets with notice IDs
+func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) {
+	var response ListBucketsResponse
+	err := c.makeRequest(ctx, "GET", "/buckets", nil, &response)
+	return &response, err
+}
+
+// HealthCheck checks if the scraper service is healthy
+func (c *Client) HealthCheck(ctx context.Context) error {
+	var response map[string]interface{}
+	err := c.makeRequest(ctx, "GET", "/health", nil, &response)
+	if err != nil {
+		return err
+	}
+
+	// Health check doesn't need to return data, just check if request succeeded
+	return nil
+}
+
+// Unified HTTP Handler API
+// makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling
+func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error {
+	// Build full URL
+	url := fmt.Sprintf("%s%s", c.baseURL, endpoint)
+
+	// Marshal request body to JSON if provided
+	var bodyReader io.Reader
+	if requestBody != nil {
+		jsonData, err := json.Marshal(requestBody)
+		if err != nil {
+			c.logger.Error("Failed to marshal request body", map[string]interface{}{
+				"error":    err.Error(),
+				"endpoint": endpoint,
+			})
+			return fmt.Errorf("failed to marshal request: %w", err)
+		}
+		bodyReader = bytes.NewReader(jsonData)
+	}
+
+	// Create HTTP request
+	req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
+	if err != nil {
+		c.logger.Error("Failed to create HTTP request", map[string]interface{}{
+			"error":  err.Error(),
+			"method": method,
+			"url":    url,
+		})
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	// Set headers
+	if requestBody != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	req.Header.Set("Accept", "application/json")
+
+	// Send request
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		c.logger.Error("Failed to send HTTP request", map[string]interface{}{
+			"error":  err.Error(),
+			"method": method,
+			"url":    url,
+		})
+		return fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Read response body
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		c.logger.Error("Failed to read response body", map[string]interface{}{
+			"error": err.Error(),
+			"url":   url,
+		})
+		return fmt.Errorf("failed to read response: %w", err)
+	}
+
+	// Handle error responses
+	if resp.StatusCode >= 400 {
+		// Try to parse error response
+		var errorResp map[string]interface{}
+		if err := json.Unmarshal(respBody, &errorResp); err == nil {
+			if errorMsg, ok := errorResp["error"].(string); ok {
+				c.logger.Error("Request failed with error response", map[string]interface{}{
+					"status_code": resp.StatusCode,
+					"error":       errorMsg,
+					"url":         url,
+				})
+				return fmt.Errorf("request failed: %s", errorMsg)
+			}
+		}
+
+		c.logger.Error("Request failed with status code", map[string]interface{}{
+			"status_code": resp.StatusCode,
+			"url":         url,
+			"body":        string(respBody),
+		})
+		return fmt.Errorf("request failed with status %d", resp.StatusCode)
+	}
+
+	// Handle successful responses
+	if responseBody != nil {
+		if err := json.Unmarshal(respBody, responseBody); err != nil {
+			c.logger.Error("Failed to unmarshal response", map[string]interface{}{
+				"error": err.Error(),
+				"url":   url,
+				"body":  string(respBody),
+			})
+			return fmt.Errorf("failed to unmarshal response: %w", err)
+		}
+	}
+
+	c.logger.Debug("Request completed successfully", map[string]interface{}{
+		"method":      method,
+		"url":         url,
+		"status_code": resp.StatusCode,
+	})
+
+	return nil
+}