diff --git a/cmd/web/bootstrap/config.go b/cmd/web/bootstrap/config.go index f473da6..3bdef7f 100644 --- a/cmd/web/bootstrap/config.go +++ b/cmd/web/bootstrap/config.go @@ -21,8 +21,11 @@ type Config struct { } type ScraperConfig struct { - BaseURL string `env:"SCRAPER_BASE_URL"` - Timeout time.Duration `env:"SCRAPER_TIMEOUT"` + BaseURL string `env:"SCRAPER_BASE_URL"` + Timeout time.Duration `env:"SCRAPER_TIMEOUT"` + Username string `env:"SCRAPER_USERNAME"` + Password string `env:"SCRAPER_PASSWORD"` + LoginURL string `env:"SCRAPER_LOGIN_URL"` } type AuthConfig struct { diff --git a/cmd/web/main.go b/cmd/web/main.go index 902cc93..f5b7db2 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -180,7 +180,7 @@ func main() { notificationService := notification.NewService(notificationSDK, userService, customerService, logger) contactService := contact.NewService(contactRepository, logger) cmsService := cms.NewService(cmsRepository, logger) - scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger) + scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, conf.Scraper.Username, conf.Scraper.Password, conf.Scraper.LoginURL) logger.Info("Services initialized successfully", map[string]interface{}{ "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"}, }) diff --git a/internal/scraper/form.go b/internal/scraper/form.go index 25c0452..3c0a5b7 100644 --- a/internal/scraper/form.go +++ b/internal/scraper/form.go @@ -2,13 +2,8 @@ package scraper // ScrapeRequestForm represents the request form for scraping documents type ScrapeRequestForm struct { - NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"` - Username string `json:"username" valid:"required,email" example:"feed@opplenz.com"` - Password string `json:"password" valid:"required,length(8|200)" example:"password123"` - LoginURL string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"` - NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"` - Category string `json:"category" valid:"optional,length(1|100)" example:"tenders"` - SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"` + NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"` + NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"` } // ScrapeResponse represents the response from scraping operation @@ -18,6 +13,7 @@ type ScrapeResponse struct { UploadedCount int `json:"uploaded_count"` UploadedFiles []UploadedFileInfo `json:"uploaded_files"` Errors []FileError `json:"errors,omitempty"` + Buckets []BucketInfo `json:"buckets,omitempty"` } // UploadedFileInfo represents information about an uploaded file @@ -32,3 +28,10 @@ type FileError struct { Filename string `json:"filename"` Error string `json:"error"` } + +// BucketInfo represents information about a bucket +type BucketInfo struct { + Name string `json:"name"` + NoticeID string `json:"notice_id"` + Count int `json:"count"` +} diff --git a/internal/scraper/service.go b/internal/scraper/service.go index 9896f11..26f925e 100644 --- a/internal/scraper/service.go +++ b/internal/scraper/service.go @@ -1,14 +1,11 @@ package scraper import ( - "bytes" "context" - "encoding/json" - "fmt" - "io" - "net/http" "time" + "tm/pkg/logger" + "tm/pkg/scraper" ) // Service defines the interface for scraper business operations @@ -19,19 +16,16 @@ type Service interface { // scraperService implements Service interface type scraperService struct { - httpClient *http.Client - baseURL string - logger logger.Logger + sdk scraper.SDK + logger logger.Logger } -// NewService creates a new scraper service -func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service { +// NewService creates a new scraper service with stored credentials +func NewService(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) Service { + sdk := scraper.NewClient(baseURL, timeout, logger, username, password, loginURL) return &scraperService{ - httpClient: &http.Client{ - Timeout: timeout, - }, - baseURL: baseURL, - logger: logger, + sdk: sdk, + logger: logger, } } @@ -39,94 +33,65 @@ func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Ser func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) { s.logger.Info("Starting document scraping", map[string]interface{}{ "notice_id": form.NoticeID, - "category": form.Category, }) - // Prepare request body - requestBody := map[string]interface{}{ - "notice_id": form.NoticeID, - "username": form.Username, - "password": form.Password, - "login_url": form.LoginURL, - "notice_url": form.NoticeURL, - "category": form.Category, - "subcategory": form.SubCategory, + // Convert form to SDK request (only dynamic fields) + req := &scraper.ScrapeRequest{ + NoticeID: form.NoticeID, + NoticeURL: form.NoticeURL, } - bodyBytes, err := json.Marshal(requestBody) + // Call SDK (credentials are already configured) + response, err := s.sdk.ScrapeDocuments(ctx, req) if err != nil { - s.logger.Error("Failed to marshal request body", map[string]interface{}{ - "error": err.Error(), + s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{ + "error": err.Error(), + "notice_id": form.NoticeID, }) - return nil, fmt.Errorf("failed to marshal request: %w", err) + return nil, err } - // Create HTTP request - url := fmt.Sprintf("%s/download", s.baseURL) - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) - if err != nil { - s.logger.Error("Failed to create HTTP request", map[string]interface{}{ - "error": err.Error(), - }) - return nil, fmt.Errorf("failed to create request: %w", err) + // Convert SDK response to service response + serviceResp := &ScrapeResponse{ + Success: response.Success, + NoticeID: response.NoticeID, + UploadedCount: response.UploadedCount, + UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)), + Errors: make([]FileError, len(response.Errors)), + Buckets: make([]BucketInfo, len(response.Buckets)), } - req.Header.Set("Content-Type", "application/json") - - // Send request - resp, err := s.httpClient.Do(req) - if err != nil { - s.logger.Error("Failed to send request to scraper service", map[string]interface{}{ - "error": err.Error(), - "url": url, - }) - return nil, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - // Read response body - respBody, err := io.ReadAll(resp.Body) - if err != nil { - s.logger.Error("Failed to read response body", map[string]interface{}{ - "error": err.Error(), - }) - return nil, fmt.Errorf("failed to read response: %w", err) - } - - // Handle error responses - if resp.StatusCode >= 400 { - var errorResp map[string]interface{} - if err := json.Unmarshal(respBody, &errorResp); err == nil { - if errorMsg, ok := errorResp["error"].(string); ok { - s.logger.Error("Scraper service returned error", map[string]interface{}{ - "status_code": resp.StatusCode, - "error": errorMsg, - }) - return nil, fmt.Errorf("scraper service error: %s", errorMsg) - } + // Convert uploaded files + for i, file := range response.UploadedFiles { + serviceResp.UploadedFiles[i] = UploadedFileInfo{ + Filename: file.Filename, + ObjectName: file.ObjectName, + Size: file.Size, } - s.logger.Error("Scraper service returned error", map[string]interface{}{ - "status_code": resp.StatusCode, - "body": string(respBody), - }) - return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode) } - // Parse success response - var scrapeResp ScrapeResponse - if err := json.Unmarshal(respBody, &scrapeResp); err != nil { - s.logger.Error("Failed to unmarshal response", map[string]interface{}{ - "error": err.Error(), - "body": string(respBody), - }) - return nil, fmt.Errorf("failed to unmarshal response: %w", err) + // Convert errors + for i, err := range response.Errors { + serviceResp.Errors[i] = FileError{ + Filename: err.Filename, + Error: err.Error, + } + } + + // Convert buckets + for i, bucket := range response.Buckets { + serviceResp.Buckets[i] = BucketInfo{ + Name: bucket.Name, + NoticeID: bucket.NoticeID, + Count: bucket.Count, + } } s.logger.Info("Document scraping completed", map[string]interface{}{ - "notice_id": scrapeResp.NoticeID, - "uploaded_count": scrapeResp.UploadedCount, - "success": scrapeResp.Success, + "notice_id": serviceResp.NoticeID, + "uploaded_count": serviceResp.UploadedCount, + "success": serviceResp.Success, }) - return &scrapeResp, nil + return serviceResp, nil } diff --git a/pkg/scraper/Dockerfile b/pkg/scraper/Dockerfile deleted file mode 100644 index 42e6b3a..0000000 --- a/pkg/scraper/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM python:3.11-slim - -RUN apt-get update && apt-get install -y \ - wget \ - gnupg \ - ca-certificates \ - fonts-liberation \ - libasound2 \ - libatk-bridge2.0-0 \ - libatk1.0-0 \ - libatspi2.0-0 \ - libcups2 \ - libdbus-1-3 \ - libdrm2 \ - libgbm1 \ - libgtk-3-0 \ - libnspr4 \ - libnss3 \ - libx11-6 \ - libx11-xcb1 \ - libxcb1 \ - libxcomposite1 \ - libxdamage1 \ - libxext6 \ - libxfixes3 \ - libxkbcommon0 \ - libxrandr2 \ - xdg-utils \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -RUN playwright install chromium - -COPY Scraper.py . -COPY app.py . - -EXPOSE 8000 - -CMD ["python", "app.py"] \ No newline at end of file diff --git a/pkg/scraper/Scraper.py b/pkg/scraper/Scraper.py deleted file mode 100644 index 78075d6..0000000 --- a/pkg/scraper/Scraper.py +++ /dev/null @@ -1,121 +0,0 @@ -from playwright.sync_api import sync_playwright -import re -import os -import time - -def clean_filename(filename): - """Removes illegal characters from filenames.""" - return re.sub(r'[<>:"/\\|?*]', '_', filename).strip() - -def download_documents(username, password, login_url, notice_url, download_dir=None): - """ - Download documents from TendSign website. - - Args: - username: Login username - password: Login password - login_url: URL to login page - notice_url: URL to notice entry page - download_dir: Directory to save downloads (optional, creates temp dir if not provided) - - Returns: - List of downloaded file paths - """ - if download_dir is None: - download_dir = f"TendSign_Downloads_{int(time.time())}" - - os.makedirs(download_dir, exist_ok=True) - print(f"[*] Downloads will be saved to: {download_dir}") - - downloaded_files = [] - - with sync_playwright() as p: - print("[*] Starting browser...") - browser = p.chromium.launch(headless=True) - context = browser.new_context( - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - viewport={"width": 1920, "height": 1080}, - locale="sv-SE", - ) - context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});") - page = context.new_page() - - try: - # === 1. LOGIN === - print("[*] Logging in...") - page.goto(login_url, wait_until="networkidle") - page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000) - page.fill('input[name="UcomLogin$txt_UserName"]', username) - page.fill('input[name="UcomLogin$txt_Password"]', password) - page.click('input[name="UcomLogin$btn_Submit"]') - page.wait_for_url("**/supplier/**", timeout=30000) - print("[SUCCESS] Logged in") - - # === 2. NAVIGATE TO DOCUMENTS TAB === - print(f"[*] Navigating to notice page...") - page.goto(notice_url, wait_until="networkidle", timeout=60000) - - print("[*] Switching to 'Dokument' tab...") - with page.expect_navigation(wait_until="networkidle", timeout=15000): - page.click('a[href*="s_view_RFQfiles.aspx"]') - - # === 3. LOCATE ALL FILE LINKS === - print("[*] Looking for file links starting with 'Bilaga'...") - - FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]" - - try: - page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000) - except Exception: - raise Exception("Failed to locate any file links starting with 'Bilaga'.") - - file_elements = page.locator(FILE_LINK_SELECTOR).all() - - print(f"[*] Found {len(file_elements)} file links to download.") - - if not file_elements: - print("[!] No file links found. Exiting.") - return downloaded_files - - # === 4. DOWNLOAD LOOP === - for i, element in enumerate(file_elements): - file_text = element.inner_text().strip() - print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...") - - try: - with page.expect_download(timeout=60000) as download_info: - element.click() - - download = download_info.value - - suggested_name = download.suggested_filename - clean_name = clean_filename(suggested_name) - save_path = os.path.join(download_dir, clean_name) - - download.save_as(save_path) - downloaded_files.append(save_path) - print(f" -> Successfully saved: {clean_name}") - - except Exception as e: - print(f" -> [ERROR] Download failed for {file_text}: {e}") - - time.sleep(1) - - print("\n[DONE] All downloads processed.") - return downloaded_files - - except Exception as e: - print(f"[CRITICAL ERROR] {e}") - page.screenshot(path=os.path.join(download_dir, "error_debug.png")) - raise - finally: - browser.close() - -if __name__ == "__main__": - # Default values for backward compatibility - USERNAME = "feed@opplenz.com" - PASSWORD = "f38Dgma!rLn2VR3DZaBn" - LOGIN_URL = "https://tendsign.com/login.aspx" - NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896" - - download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL) \ No newline at end of file diff --git a/pkg/scraper/app.py b/pkg/scraper/app.py deleted file mode 100644 index 302e8ca..0000000 --- a/pkg/scraper/app.py +++ /dev/null @@ -1,151 +0,0 @@ -from flask import Flask, request, jsonify -from Scraper import download_documents -import os -from minio import Minio -from minio.error import S3Error -import tempfile -import shutil - -app = Flask(__name__) - -# MinIO configuration from environment variables -MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000') -MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin') -MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin') -MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true' -MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents') - -def get_minio_client(): - """Create and return MinIO client""" - return Minio( - MINIO_ENDPOINT, - access_key=MINIO_ACCESS_KEY, - secret_key=MINIO_SECRET_KEY, - secure=MINIO_USE_SSL - ) - -def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET): - """Upload a file to MinIO""" - try: - client = get_minio_client() - - # Ensure bucket exists - if not client.bucket_exists(bucket): - client.make_bucket(bucket) - - # Upload file - client.fput_object(bucket, object_name, file_path) - return True, None - except S3Error as e: - return False, str(e) - except Exception as e: - return False, str(e) - -@app.route('/health', methods=['GET']) -def health(): - """Health check endpoint""" - return jsonify({"status": "healthy"}), 200 - -@app.route('/download', methods=['POST']) -def download(): - """Download documents and upload to MinIO""" - try: - data = request.get_json() - - # Validate required fields - if not data: - return jsonify({"error": "Request body is required"}), 400 - - notice_id = data.get('notice_id') - username = data.get('username') - password = data.get('password') - login_url = data.get('login_url') - notice_url = data.get('notice_url') - category = data.get('category', 'tenders') - subcategory = data.get('subcategory', '') - - if not all([notice_id, username, password, login_url, notice_url]): - return jsonify({ - "error": "Missing required fields: notice_id, username, password, login_url, notice_url" - }), 400 - - # Create temporary directory for downloads - temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_") - - try: - # Download documents using the scraper - downloaded_files = download_documents( - username=username, - password=password, - login_url=login_url, - notice_url=notice_url, - download_dir=temp_dir - ) - - if not downloaded_files: - return jsonify({ - "error": "No files were downloaded", - "notice_id": notice_id - }), 404 - - # Upload each file to MinIO - uploaded_files = [] - errors = [] - - for file_path in downloaded_files: - try: - filename = os.path.basename(file_path) - # Create object name with hierarchical path - if subcategory: - object_name = f"{category}/{subcategory}/{notice_id}/{filename}" - else: - object_name = f"{category}/{notice_id}/{filename}" - - success, error = upload_to_minio(file_path, object_name) - - if success: - uploaded_files.append({ - "filename": filename, - "object_name": object_name, - "size": os.path.getsize(file_path) - }) - else: - errors.append({ - "filename": filename, - "error": error - }) - except Exception as e: - errors.append({ - "filename": os.path.basename(file_path), - "error": str(e) - }) - - # Clean up temporary directory - shutil.rmtree(temp_dir, ignore_errors=True) - - return jsonify({ - "success": True, - "notice_id": notice_id, - "uploaded_count": len(uploaded_files), - "uploaded_files": uploaded_files, - "errors": errors if errors else None - }), 200 - - except Exception as e: - # Clean up on error - shutil.rmtree(temp_dir, ignore_errors=True) - return jsonify({ - "error": "Failed to download documents", - "message": str(e), - "notice_id": notice_id - }), 500 - - except Exception as e: - return jsonify({ - "error": "Internal server error", - "message": str(e) - }), 500 - -if __name__ == '__main__': - port = int(os.getenv('PORT', 8000)) - app.run(host='0.0.0.0', port=port, debug=False) \ No newline at end of file diff --git a/pkg/scraper/docker-compose.scraper.yml b/pkg/scraper/docker-compose.scraper.yml deleted file mode 100644 index 48815e5..0000000 --- a/pkg/scraper/docker-compose.scraper.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: '3.8' - -services: - scraper: - build: - context: . - dockerfile: pkg/scraper/Dockerfile - container_name: tm-scraper - ports: - - "8000:8000" - environment: - - PORT=8000 - - MINIO_ENDPOINT=minio:9000 - - MINIO_ACCESS_KEY=minioadmin - - MINIO_SECRET_KEY=minioadmin - - MINIO_USE_SSL=false - - MINIO_BUCKET=documents - networks: - - tm-network - depends_on: - - minio - restart: unless-stopped - - minio: - image: minio/minio:latest - container_name: tm-minio - ports: - - "9000:9000" - - "9001:9001" - environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin - command: server /data --console-address ":9001" - volumes: - - minio-data:/data - networks: - - tm-network - restart: unless-stopped - -networks: - tm-network: - driver: bridge - -volumes: - minio-data: \ No newline at end of file diff --git a/pkg/scraper/requirements.txt b/pkg/scraper/requirements.txt deleted file mode 100644 index 8e50a69..0000000 --- a/pkg/scraper/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -playwright==1.40.0 -flask==3.0.0 -minio==7.2.0 -requests==2.31.0 -python-dotenv==1.0.0 \ No newline at end of file diff --git a/pkg/scraper/sdk.go b/pkg/scraper/sdk.go new file mode 100644 index 0000000..ef92e86 --- /dev/null +++ b/pkg/scraper/sdk.go @@ -0,0 +1,256 @@ +package scraper + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "tm/pkg/logger" +) + +// SDK represents the scraper SDK interface +type SDK interface { + // ScrapeDocuments scrapes documents for a given notice + ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) + + // ListBuckets lists all buckets with notice IDs + ListBuckets(ctx context.Context) (*ListBucketsResponse, error) + + // HealthCheck checks if the scraper service is healthy + HealthCheck(ctx context.Context) error +} + +// ScrapeRequest represents a request to scrape documents (only dynamic fields) +type ScrapeRequest struct { + NoticeID string `json:"notice_id"` + NoticeURL string `json:"notice_url"` +} + +// ScrapeResponse represents the response from scraping operation +type ScrapeResponse struct { + Success bool `json:"success"` + NoticeID string `json:"notice_id"` + UploadedCount int `json:"uploaded_count"` + UploadedFiles []UploadedFile `json:"uploaded_files"` + Errors []FileError `json:"errors,omitempty"` + Buckets []BucketInfo `json:"buckets,omitempty"` +} + +// UploadedFile represents information about an uploaded file +type UploadedFile struct { + Filename string `json:"filename"` + ObjectName string `json:"object_name"` + Size int64 `json:"size"` +} + +// FileError represents an error for a specific file +type FileError struct { + Filename string `json:"filename"` + Error string `json:"error"` +} + +// BucketInfo represents information about a bucket +type BucketInfo struct { + Name string `json:"name"` + NoticeID string `json:"notice_id"` + Count int `json:"count"` +} + +// ListBucketsResponse represents the response from listing buckets +type ListBucketsResponse struct { + Buckets []BucketInfo `json:"buckets"` + Total int `json:"total"` +} + +// Client implements the SDK interface +type Client struct { + httpClient *http.Client + baseURL string + logger logger.Logger + // Stored credentials + username string + password string + loginURL string +} + +// NewClient creates a new scraper SDK client with stored credentials +func NewClient(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) SDK { + return &Client{ + httpClient: &http.Client{ + Timeout: timeout, + }, + baseURL: baseURL, + logger: logger, + username: username, + password: password, + loginURL: loginURL, + } +} + +// ScrapeDocuments scrapes documents and uploads them to MinIO +func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) { + c.logger.Info("Starting document scraping via SDK", map[string]interface{}{ + "notice_id": req.NoticeID, + "base_url": c.baseURL, + }) + + // Prepare request body with stored credentials + dynamic fields + requestBody := map[string]interface{}{ + "notice_id": req.NoticeID, + "username": c.username, + "password": c.password, + "login_url": c.loginURL, + "notice_url": req.NoticeURL, + } + + // Use unified handler API + var response ScrapeResponse + err := c.makeRequest(ctx, "POST", "/download", requestBody, &response) + if err != nil { + return nil, err + } + + // Get buckets list after successful scraping + bucketsResp, err := c.ListBuckets(ctx) + if err != nil { + c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{ + "error": err.Error(), + }) + } else { + response.Buckets = bucketsResp.Buckets + } + + c.logger.Info("Document scraping completed via SDK", map[string]interface{}{ + "notice_id": response.NoticeID, + "uploaded_count": response.UploadedCount, + "success": response.Success, + }) + + return &response, nil +} + +// ListBuckets lists all buckets with notice IDs +func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) { + var response ListBucketsResponse + err := c.makeRequest(ctx, "GET", "/buckets", nil, &response) + return &response, err +} + +// HealthCheck checks if the scraper service is healthy +func (c *Client) HealthCheck(ctx context.Context) error { + var response map[string]interface{} + err := c.makeRequest(ctx, "GET", "/health", nil, &response) + if err != nil { + return err + } + + // Health check doesn't need to return data, just check if request succeeded + return nil +} + +// Unified HTTP Handler API +// makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling +func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error { + // Build full URL + url := fmt.Sprintf("%s%s", c.baseURL, endpoint) + + // Marshal request body to JSON if provided + var bodyReader io.Reader + if requestBody != nil { + jsonData, err := json.Marshal(requestBody) + if err != nil { + c.logger.Error("Failed to marshal request body", map[string]interface{}{ + "error": err.Error(), + "endpoint": endpoint, + }) + return fmt.Errorf("failed to marshal request: %w", err) + } + bodyReader = bytes.NewReader(jsonData) + } + + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, method, url, bodyReader) + if err != nil { + c.logger.Error("Failed to create HTTP request", map[string]interface{}{ + "error": err.Error(), + "method": method, + "url": url, + }) + return fmt.Errorf("failed to create request: %w", err) + } + + // Set headers + if requestBody != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "application/json") + + // Send request + resp, err := c.httpClient.Do(req) + if err != nil { + c.logger.Error("Failed to send HTTP request", map[string]interface{}{ + "error": err.Error(), + "method": method, + "url": url, + }) + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + // Read response body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + c.logger.Error("Failed to read response body", map[string]interface{}{ + "error": err.Error(), + "url": url, + }) + return fmt.Errorf("failed to read response: %w", err) + } + + // Handle error responses + if resp.StatusCode >= 400 { + // Try to parse error response + var errorResp map[string]interface{} + if err := json.Unmarshal(respBody, &errorResp); err == nil { + if errorMsg, ok := errorResp["error"].(string); ok { + c.logger.Error("Request failed with error response", map[string]interface{}{ + "status_code": resp.StatusCode, + "error": errorMsg, + "url": url, + }) + return fmt.Errorf("request failed: %s", errorMsg) + } + } + + c.logger.Error("Request failed with status code", map[string]interface{}{ + "status_code": resp.StatusCode, + "url": url, + "body": string(respBody), + }) + return fmt.Errorf("request failed with status %d", resp.StatusCode) + } + + // Handle successful responses + if responseBody != nil { + if err := json.Unmarshal(respBody, responseBody); err != nil { + c.logger.Error("Failed to unmarshal response", map[string]interface{}{ + "error": err.Error(), + "url": url, + "body": string(respBody), + }) + return fmt.Errorf("failed to unmarshal response: %w", err) + } + } + + c.logger.Debug("Request completed successfully", map[string]interface{}{ + "method": method, + "url": url, + "status_code": resp.StatusCode, + }) + + return nil +}