mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

scraper sdk

Browse Source
This commit is contained in:
Mazyar
2025-12-01 11:06:09 +03:30
parent d94fa34941
commit 5e777ceec8
10 changed files with 324 additions and 462 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/web/bootstrap/config.go
+3
View File
@@ -23,6 +23,9 @@ type Config struct {
type ScraperConfig struct {
BaseURL string `env:"SCRAPER_BASE_URL"`
Timeout time.Duration `env:"SCRAPER_TIMEOUT"`
Username string `env:"SCRAPER_USERNAME"`
Password string `env:"SCRAPER_PASSWORD"`
LoginURL string `env:"SCRAPER_LOGIN_URL"`
}
type AuthConfig struct {
cmd/web/main.go
+1 -1
View File
@@ -180,7 +180,7 @@ func main() {
notificationService := notification.NewService(notificationSDK, userService, customerService, logger)
contactService := contact.NewService(contactRepository, logger)
cmsService := cms.NewService(cmsRepository, logger)
scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger)
scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, conf.Scraper.Username, conf.Scraper.Password, conf.Scraper.LoginURL)
logger.Info("Services initialized successfully", map[string]interface{}{
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
})
internal/scraper/form.go
+8 -5
View File
@@ -3,12 +3,7 @@ package scraper
// ScrapeRequestForm represents the request form for scraping documents
type ScrapeRequestForm struct {
NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
Username string `json:"username" valid:"required,email" example:"feed@opplenz.com"`
Password string `json:"password" valid:"required,length(8|200)" example:"password123"`
LoginURL string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"`
NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
Category string `json:"category" valid:"optional,length(1|100)" example:"tenders"`
SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"`
}
// ScrapeResponse represents the response from scraping operation
@@ -18,6 +13,7 @@ type ScrapeResponse struct {
UploadedCount int `json:"uploaded_count"`
UploadedFiles []UploadedFileInfo `json:"uploaded_files"`
Errors []FileError `json:"errors,omitempty"`
Buckets []BucketInfo `json:"buckets,omitempty"`
}
// UploadedFileInfo represents information about an uploaded file
@@ -32,3 +28,10 @@ type FileError struct {
Filename string `json:"filename"`
Error string `json:"error"`
}
// BucketInfo represents information about a bucket
type BucketInfo struct {
Name string `json:"name"`
NoticeID string `json:"notice_id"`
Count int `json:"count"`
}
internal/scraper/service.go
+50 -85
View File
@@ -1,14 +1,11 @@
package scraper
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"tm/pkg/logger"
"tm/pkg/scraper"
)
// Service defines the interface for scraper business operations
@@ -19,18 +16,15 @@ type Service interface {
// scraperService implements Service interface
type scraperService struct {
httpClient *http.Client
baseURL string
sdk scraper.SDK
logger logger.Logger
}
// NewService creates a new scraper service
func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
// NewService creates a new scraper service with stored credentials
func NewService(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) Service {
sdk := scraper.NewClient(baseURL, timeout, logger, username, password, loginURL)
return &scraperService{
httpClient: &http.Client{
Timeout: timeout,
},
baseURL: baseURL,
sdk: sdk,
logger: logger,
}
}
@@ -39,94 +33,65 @@ func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Ser
func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
s.logger.Info("Starting document scraping", map[string]interface{}{
"notice_id": form.NoticeID,
"category": form.Category,
})
// Prepare request body
requestBody := map[string]interface{}{
// Convert form to SDK request (only dynamic fields)
req := &scraper.ScrapeRequest{
NoticeID: form.NoticeID,
NoticeURL: form.NoticeURL,
}
// Call SDK (credentials are already configured)
response, err := s.sdk.ScrapeDocuments(ctx, req)
if err != nil {
s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{
"error": err.Error(),
"notice_id": form.NoticeID,
"username": form.Username,
"password": form.Password,
"login_url": form.LoginURL,
"notice_url": form.NoticeURL,
"category": form.Category,
"subcategory": form.SubCategory,
})
return nil, err
}
bodyBytes, err := json.Marshal(requestBody)
if err != nil {
s.logger.Error("Failed to marshal request body", map[string]interface{}{
"error": err.Error(),
})
return nil, fmt.Errorf("failed to marshal request: %w", err)
// Convert SDK response to service response
serviceResp := &ScrapeResponse{
Success: response.Success,
NoticeID: response.NoticeID,
UploadedCount: response.UploadedCount,
UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)),
Errors: make([]FileError, len(response.Errors)),
Buckets: make([]BucketInfo, len(response.Buckets)),
}
// Create HTTP request
url := fmt.Sprintf("%s/download", s.baseURL)
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
if err != nil {
s.logger.Error("Failed to create HTTP request", map[string]interface{}{
"error": err.Error(),
})
return nil, fmt.Errorf("failed to create request: %w", err)
// Convert uploaded files
for i, file := range response.UploadedFiles {
serviceResp.UploadedFiles[i] = UploadedFileInfo{
Filename: file.Filename,
ObjectName: file.ObjectName,
Size: file.Size,
}
}
req.Header.Set("Content-Type", "application/json")
// Send request
resp, err := s.httpClient.Do(req)
if err != nil {
s.logger.Error("Failed to send request to scraper service", map[string]interface{}{
"error": err.Error(),
"url": url,
})
return nil, fmt.Errorf("failed to send request: %w", err)
// Convert errors
for i, err := range response.Errors {
serviceResp.Errors[i] = FileError{
Filename: err.Filename,
Error: err.Error,
}
defer resp.Body.Close()
// Read response body
respBody, err := io.ReadAll(resp.Body)
if err != nil {
s.logger.Error("Failed to read response body", map[string]interface{}{
"error": err.Error(),
})
return nil, fmt.Errorf("failed to read response: %w", err)
}
// Handle error responses
if resp.StatusCode >= 400 {
var errorResp map[string]interface{}
if err := json.Unmarshal(respBody, &errorResp); err == nil {
if errorMsg, ok := errorResp["error"].(string); ok {
s.logger.Error("Scraper service returned error", map[string]interface{}{
"status_code": resp.StatusCode,
"error": errorMsg,
})
return nil, fmt.Errorf("scraper service error: %s", errorMsg)
// Convert buckets
for i, bucket := range response.Buckets {
serviceResp.Buckets[i] = BucketInfo{
Name: bucket.Name,
NoticeID: bucket.NoticeID,
Count: bucket.Count,
}
}
s.logger.Error("Scraper service returned error", map[string]interface{}{
"status_code": resp.StatusCode,
"body": string(respBody),
})
return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode)
}
// Parse success response
var scrapeResp ScrapeResponse
if err := json.Unmarshal(respBody, &scrapeResp); err != nil {
s.logger.Error("Failed to unmarshal response", map[string]interface{}{
"error": err.Error(),
"body": string(respBody),
})
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
}
s.logger.Info("Document scraping completed", map[string]interface{}{
"notice_id": scrapeResp.NoticeID,
"uploaded_count": scrapeResp.UploadedCount,
"success": scrapeResp.Success,
"notice_id": serviceResp.NoticeID,
"uploaded_count": serviceResp.UploadedCount,
"success": serviceResp.Success,
})
return &scrapeResp, nil
return serviceResp, nil
}
pkg/scraper/Dockerfile
-43
View File
@@ -1,43 +0,0 @@
FROM python:3.11-slim
RUN apt-get update && apt-get install -y \
wget \
gnupg \
ca-certificates \
fonts-liberation \
libasound2 \
libatk-bridge2.0-0 \
libatk1.0-0 \
libatspi2.0-0 \
libcups2 \
libdbus-1-3 \
libdrm2 \
libgbm1 \
libgtk-3-0 \
libnspr4 \
libnss3 \
libx11-6 \
libx11-xcb1 \
libxcb1 \
libxcomposite1 \
libxdamage1 \
libxext6 \
libxfixes3 \
libxkbcommon0 \
libxrandr2 \
xdg-utils \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
RUN playwright install chromium
COPY Scraper.py .
COPY app.py .
EXPOSE 8000
CMD ["python", "app.py"]
pkg/scraper/Scraper.py
-121
View File
@@ -1,121 +0,0 @@
from playwright.sync_api import sync_playwright
import re
import os
import time
def clean_filename(filename):
"""Removes illegal characters from filenames."""
return re.sub(r'[<>:"/\\|?*]', '_', filename).strip()
def download_documents(username, password, login_url, notice_url, download_dir=None):
"""
Download documents from TendSign website.
Args:
username: Login username
password: Login password
login_url: URL to login page
notice_url: URL to notice entry page
download_dir: Directory to save downloads (optional, creates temp dir if not provided)
Returns:
List of downloaded file paths
"""
if download_dir is None:
download_dir = f"TendSign_Downloads_{int(time.time())}"
os.makedirs(download_dir, exist_ok=True)
print(f"[*] Downloads will be saved to: {download_dir}")
downloaded_files = []
with sync_playwright() as p:
print("[*] Starting browser...")
browser = p.chromium.launch(headless=True)
context = browser.new_context(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
viewport={"width": 1920, "height": 1080},
locale="sv-SE",
)
context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});")
page = context.new_page()
try:
# === 1. LOGIN ===
print("[*] Logging in...")
page.goto(login_url, wait_until="networkidle")
page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000)
page.fill('input[name="UcomLogin$txt_UserName"]', username)
page.fill('input[name="UcomLogin$txt_Password"]', password)
page.click('input[name="UcomLogin$btn_Submit"]')
page.wait_for_url("**/supplier/**", timeout=30000)
print("[SUCCESS] Logged in")
# === 2. NAVIGATE TO DOCUMENTS TAB ===
print(f"[*] Navigating to notice page...")
page.goto(notice_url, wait_until="networkidle", timeout=60000)
print("[*] Switching to 'Dokument' tab...")
with page.expect_navigation(wait_until="networkidle", timeout=15000):
page.click('a[href*="s_view_RFQfiles.aspx"]')
# === 3. LOCATE ALL FILE LINKS ===
print("[*] Looking for file links starting with 'Bilaga'...")
FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]"
try:
page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000)
except Exception:
raise Exception("Failed to locate any file links starting with 'Bilaga'.")
file_elements = page.locator(FILE_LINK_SELECTOR).all()
print(f"[*] Found {len(file_elements)} file links to download.")
if not file_elements:
print("[!] No file links found. Exiting.")
return downloaded_files
# === 4. DOWNLOAD LOOP ===
for i, element in enumerate(file_elements):
file_text = element.inner_text().strip()
print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...")
try:
with page.expect_download(timeout=60000) as download_info:
element.click()
download = download_info.value
suggested_name = download.suggested_filename
clean_name = clean_filename(suggested_name)
save_path = os.path.join(download_dir, clean_name)
download.save_as(save_path)
downloaded_files.append(save_path)
print(f" -> Successfully saved: {clean_name}")
except Exception as e:
print(f" -> [ERROR] Download failed for {file_text}: {e}")
time.sleep(1)
print("\n[DONE] All downloads processed.")
return downloaded_files
except Exception as e:
print(f"[CRITICAL ERROR] {e}")
page.screenshot(path=os.path.join(download_dir, "error_debug.png"))
raise
finally:
browser.close()
if __name__ == "__main__":
# Default values for backward compatibility
USERNAME = "feed@opplenz.com"
PASSWORD = "f38Dgma!rLn2VR3DZaBn"
LOGIN_URL = "https://tendsign.com/login.aspx"
NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"
download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL)
pkg/scraper/app.py
-151
View File
@@ -1,151 +0,0 @@
from flask import Flask, request, jsonify
from Scraper import download_documents
import os
from minio import Minio
from minio.error import S3Error
import tempfile
import shutil
app = Flask(__name__)
# MinIO configuration from environment variables
MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000')
MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents')
def get_minio_client():
"""Create and return MinIO client"""
return Minio(
MINIO_ENDPOINT,
access_key=MINIO_ACCESS_KEY,
secret_key=MINIO_SECRET_KEY,
secure=MINIO_USE_SSL
)
def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET):
"""Upload a file to MinIO"""
try:
client = get_minio_client()
# Ensure bucket exists
if not client.bucket_exists(bucket):
client.make_bucket(bucket)
# Upload file
client.fput_object(bucket, object_name, file_path)
return True, None
except S3Error as e:
return False, str(e)
except Exception as e:
return False, str(e)
@app.route('/health', methods=['GET'])
def health():
"""Health check endpoint"""
return jsonify({"status": "healthy"}), 200
@app.route('/download', methods=['POST'])
def download():
"""Download documents and upload to MinIO"""
try:
data = request.get_json()
# Validate required fields
if not data:
return jsonify({"error": "Request body is required"}), 400
notice_id = data.get('notice_id')
username = data.get('username')
password = data.get('password')
login_url = data.get('login_url')
notice_url = data.get('notice_url')
category = data.get('category', 'tenders')
subcategory = data.get('subcategory', '')
if not all([notice_id, username, password, login_url, notice_url]):
return jsonify({
"error": "Missing required fields: notice_id, username, password, login_url, notice_url"
}), 400
# Create temporary directory for downloads
temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_")
try:
# Download documents using the scraper
downloaded_files = download_documents(
username=username,
password=password,
login_url=login_url,
notice_url=notice_url,
download_dir=temp_dir
)
if not downloaded_files:
return jsonify({
"error": "No files were downloaded",
"notice_id": notice_id
}), 404
# Upload each file to MinIO
uploaded_files = []
errors = []
for file_path in downloaded_files:
try:
filename = os.path.basename(file_path)
# Create object name with hierarchical path
if subcategory:
object_name = f"{category}/{subcategory}/{notice_id}/{filename}"
else:
object_name = f"{category}/{notice_id}/{filename}"
success, error = upload_to_minio(file_path, object_name)
if success:
uploaded_files.append({
"filename": filename,
"object_name": object_name,
"size": os.path.getsize(file_path)
})
else:
errors.append({
"filename": filename,
"error": error
})
except Exception as e:
errors.append({
"filename": os.path.basename(file_path),
"error": str(e)
})
# Clean up temporary directory
shutil.rmtree(temp_dir, ignore_errors=True)
return jsonify({
"success": True,
"notice_id": notice_id,
"uploaded_count": len(uploaded_files),
"uploaded_files": uploaded_files,
"errors": errors if errors else None
}), 200
except Exception as e:
# Clean up on error
shutil.rmtree(temp_dir, ignore_errors=True)
return jsonify({
"error": "Failed to download documents",
"message": str(e),
"notice_id": notice_id
}), 500
except Exception as e:
return jsonify({
"error": "Internal server error",
"message": str(e)
}), 500
if __name__ == '__main__':
port = int(os.getenv('PORT', 8000))
app.run(host='0.0.0.0', port=port, debug=False)
pkg/scraper/docker-compose.scraper.yml
-45
View File
@@ -1,45 +0,0 @@
version: '3.8'
services:
scraper:
build:
context: .
dockerfile: pkg/scraper/Dockerfile
container_name: tm-scraper
ports:
- "8000:8000"
environment:
- PORT=8000
- MINIO_ENDPOINT=minio:9000
- MINIO_ACCESS_KEY=minioadmin
- MINIO_SECRET_KEY=minioadmin
- MINIO_USE_SSL=false
- MINIO_BUCKET=documents
networks:
- tm-network
depends_on:
- minio
restart: unless-stopped
minio:
image: minio/minio:latest
container_name: tm-minio
ports:
- "9000:9000"
- "9001:9001"
environment:
- MINIO_ROOT_USER=minioadmin
- MINIO_ROOT_PASSWORD=minioadmin
command: server /data --console-address ":9001"
volumes:
- minio-data:/data
networks:
- tm-network
restart: unless-stopped
networks:
tm-network:
driver: bridge
volumes:
minio-data:
pkg/scraper/requirements.txt
-5
View File
@@ -1,5 +0,0 @@
playwright==1.40.0
flask==3.0.0
minio==7.2.0
requests==2.31.0
python-dotenv==1.0.0
pkg/scraper/sdk.go
+256
View File
@@ -0,0 +1,256 @@
package scraper
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"tm/pkg/logger"
)
// SDK represents the scraper SDK interface
type SDK interface {
// ScrapeDocuments scrapes documents for a given notice
ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error)
// ListBuckets lists all buckets with notice IDs
ListBuckets(ctx context.Context) (*ListBucketsResponse, error)
// HealthCheck checks if the scraper service is healthy
HealthCheck(ctx context.Context) error
}
// ScrapeRequest represents a request to scrape documents (only dynamic fields)
type ScrapeRequest struct {
NoticeID string `json:"notice_id"`
NoticeURL string `json:"notice_url"`
}
// ScrapeResponse represents the response from scraping operation
type ScrapeResponse struct {
Success bool `json:"success"`
NoticeID string `json:"notice_id"`
UploadedCount int `json:"uploaded_count"`
UploadedFiles []UploadedFile `json:"uploaded_files"`
Errors []FileError `json:"errors,omitempty"`
Buckets []BucketInfo `json:"buckets,omitempty"`
}
// UploadedFile represents information about an uploaded file
type UploadedFile struct {
Filename string `json:"filename"`
ObjectName string `json:"object_name"`
Size int64 `json:"size"`
}
// FileError represents an error for a specific file
type FileError struct {
Filename string `json:"filename"`
Error string `json:"error"`
}
// BucketInfo represents information about a bucket
type BucketInfo struct {
Name string `json:"name"`
NoticeID string `json:"notice_id"`
Count int `json:"count"`
}
// ListBucketsResponse represents the response from listing buckets
type ListBucketsResponse struct {
Buckets []BucketInfo `json:"buckets"`
Total int `json:"total"`
}
// Client implements the SDK interface
type Client struct {
httpClient *http.Client
baseURL string
logger logger.Logger
// Stored credentials
username string
password string
loginURL string
}
// NewClient creates a new scraper SDK client with stored credentials
func NewClient(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) SDK {
return &Client{
httpClient: &http.Client{
Timeout: timeout,
},
baseURL: baseURL,
logger: logger,
username: username,
password: password,
loginURL: loginURL,
}
}
// ScrapeDocuments scrapes documents and uploads them to MinIO
func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) {
c.logger.Info("Starting document scraping via SDK", map[string]interface{}{
"notice_id": req.NoticeID,
"base_url": c.baseURL,
})
// Prepare request body with stored credentials + dynamic fields
requestBody := map[string]interface{}{
"notice_id": req.NoticeID,
"username": c.username,
"password": c.password,
"login_url": c.loginURL,
"notice_url": req.NoticeURL,
}
// Use unified handler API
var response ScrapeResponse
err := c.makeRequest(ctx, "POST", "/download", requestBody, &response)
if err != nil {
return nil, err
}
// Get buckets list after successful scraping
bucketsResp, err := c.ListBuckets(ctx)
if err != nil {
c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{
"error": err.Error(),
})
} else {
response.Buckets = bucketsResp.Buckets
}
c.logger.Info("Document scraping completed via SDK", map[string]interface{}{
"notice_id": response.NoticeID,
"uploaded_count": response.UploadedCount,
"success": response.Success,
})
return &response, nil
}
// ListBuckets lists all buckets with notice IDs
func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) {
var response ListBucketsResponse
err := c.makeRequest(ctx, "GET", "/buckets", nil, &response)
return &response, err
}
// HealthCheck checks if the scraper service is healthy
func (c *Client) HealthCheck(ctx context.Context) error {
var response map[string]interface{}
err := c.makeRequest(ctx, "GET", "/health", nil, &response)
if err != nil {
return err
}
// Health check doesn't need to return data, just check if request succeeded
return nil
}
// Unified HTTP Handler API
// makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling
func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error {
// Build full URL
url := fmt.Sprintf("%s%s", c.baseURL, endpoint)
// Marshal request body to JSON if provided
var bodyReader io.Reader
if requestBody != nil {
jsonData, err := json.Marshal(requestBody)
if err != nil {
c.logger.Error("Failed to marshal request body", map[string]interface{}{
"error": err.Error(),
"endpoint": endpoint,
})
return fmt.Errorf("failed to marshal request: %w", err)
}
bodyReader = bytes.NewReader(jsonData)
}
// Create HTTP request
req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
if err != nil {
c.logger.Error("Failed to create HTTP request", map[string]interface{}{
"error": err.Error(),
"method": method,
"url": url,
})
return fmt.Errorf("failed to create request: %w", err)
}
// Set headers
if requestBody != nil {
req.Header.Set("Content-Type", "application/json")
}
req.Header.Set("Accept", "application/json")
// Send request
resp, err := c.httpClient.Do(req)
if err != nil {
c.logger.Error("Failed to send HTTP request", map[string]interface{}{
"error": err.Error(),
"method": method,
"url": url,
})
return fmt.Errorf("failed to send request: %w", err)
}
defer resp.Body.Close()
// Read response body
respBody, err := io.ReadAll(resp.Body)
if err != nil {
c.logger.Error("Failed to read response body", map[string]interface{}{
"error": err.Error(),
"url": url,
})
return fmt.Errorf("failed to read response: %w", err)
}
// Handle error responses
if resp.StatusCode >= 400 {
// Try to parse error response
var errorResp map[string]interface{}
if err := json.Unmarshal(respBody, &errorResp); err == nil {
if errorMsg, ok := errorResp["error"].(string); ok {
c.logger.Error("Request failed with error response", map[string]interface{}{
"status_code": resp.StatusCode,
"error": errorMsg,
"url": url,
})
return fmt.Errorf("request failed: %s", errorMsg)
}
}
c.logger.Error("Request failed with status code", map[string]interface{}{
"status_code": resp.StatusCode,
"url": url,
"body": string(respBody),
})
return fmt.Errorf("request failed with status %d", resp.StatusCode)
}
// Handle successful responses
if responseBody != nil {
if err := json.Unmarshal(respBody, responseBody); err != nil {
c.logger.Error("Failed to unmarshal response", map[string]interface{}{
"error": err.Error(),
"url": url,
"body": string(respBody),
})
return fmt.Errorf("failed to unmarshal response: %w", err)
}
}
c.logger.Debug("Request completed successfully", map[string]interface{}{
"method": method,
"url": url,
"status_code": resp.StatusCode,
})
return nil
}