From b6546eb2bb8ab9de8a8229e66abdea6f9937edbb Mon Sep 17 00:00:00 2001 From: Mazyar Date: Sun, 30 Nov 2025 02:58:08 +0330 Subject: [PATCH 1/2] tendsign scraper --- cmd/web/bootstrap/config.go | 7 ++ cmd/web/main.go | 9 +- cmd/web/router/routes.go | 10 +- internal/scraper/form.go | 34 ++++++ internal/scraper/handler.go | 44 +++++++ internal/scraper/service.go | 132 +++++++++++++++++++++ pkg/scraper/Dockerfile | 43 +++++++ pkg/scraper/Scraper.py | 121 ++++++++++++++++++++ pkg/scraper/app.py | 151 +++++++++++++++++++++++++ pkg/scraper/docker-compose.scraper.yml | 45 ++++++++ pkg/scraper/requirements.txt | 5 + 11 files changed, 597 insertions(+), 4 deletions(-) create mode 100644 internal/scraper/form.go create mode 100644 internal/scraper/handler.go create mode 100644 internal/scraper/service.go create mode 100644 pkg/scraper/Dockerfile create mode 100644 pkg/scraper/Scraper.py create mode 100644 pkg/scraper/app.py create mode 100644 pkg/scraper/docker-compose.scraper.yml create mode 100644 pkg/scraper/requirements.txt diff --git a/cmd/web/bootstrap/config.go b/cmd/web/bootstrap/config.go index 51dbad5..f473da6 100644 --- a/cmd/web/bootstrap/config.go +++ b/cmd/web/bootstrap/config.go @@ -1,6 +1,7 @@ package bootstrap import ( + "time" "tm/pkg/config" ) @@ -14,10 +15,16 @@ type Config struct { Assets config.AssetsConfig Notification config.NotificationConfig Ollama config.OllamaConfig + Scraper ScraperConfig UserAuth AuthConfig CustomerAuth AuthConfig } +type ScraperConfig struct { + BaseURL string `env:"SCRAPER_BASE_URL"` + Timeout time.Duration `env:"SCRAPER_TIMEOUT"` +} + type AuthConfig struct { JWT JWTConfig } diff --git a/cmd/web/main.go b/cmd/web/main.go index 8711a0a..902cc93 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -103,6 +103,7 @@ import ( "tm/internal/feedback" "tm/internal/inquiry" "tm/internal/notification" + "tm/internal/scraper" "tm/internal/tender" "tm/internal/tender_approval" "tm/internal/user" @@ -179,8 +180,9 @@ func main() { notificationService := notification.NewService(notificationSDK, userService, customerService, logger) contactService := contact.NewService(contactRepository, logger) cmsService := cms.NewService(cmsRepository, logger) + scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger) logger.Info("Services initialized successfully", map[string]interface{}{ - "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"}, + "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"}, }) // Initialize handlers with services @@ -196,15 +198,16 @@ func main() { notificationHandler := notification.NewHandler(notificationService) contactHandler := contact.NewHandler(contactService) cmsHandler := cms.NewHandler(cmsService) + scraperHandler := scraper.NewHandler(scraperService) logger.Info("Handlers initialized successfully", map[string]interface{}{ - "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"}, + "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"}, }) // Initialize HTTP server e := bootstrap.InitHTTPServer(conf, logger) // Register routes - router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler) + router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler) router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler) // Start server diff --git a/cmd/web/router/routes.go b/cmd/web/router/routes.go index e7fb714..c584990 100644 --- a/cmd/web/router/routes.go +++ b/cmd/web/router/routes.go @@ -10,6 +10,7 @@ import ( "tm/internal/feedback" "tm/internal/inquiry" "tm/internal/notification" + "tm/internal/scraper" "tm/internal/tender" "tm/internal/tender_approval" "tm/internal/user" @@ -17,7 +18,7 @@ import ( "github.com/labstack/echo/v4" ) -func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) { +func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler) { adminV1 := e.Group("/admin/v1") // Admin Users Routes @@ -165,6 +166,13 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler cmsGP.PUT("/:id", cmsHandler.Update) cmsGP.DELETE("/:id", cmsHandler.Delete) } + + // Scraper Routes + scraperGP := adminV1.Group("/scraper") + { + scraperGP.Use(userHandler.AuthMiddleware()) + scraperGP.POST("/download", scraperHandler.ScrapeDocuments) + } } func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) { diff --git a/internal/scraper/form.go b/internal/scraper/form.go new file mode 100644 index 0000000..25c0452 --- /dev/null +++ b/internal/scraper/form.go @@ -0,0 +1,34 @@ +package scraper + +// ScrapeRequestForm represents the request form for scraping documents +type ScrapeRequestForm struct { + NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"` + Username string `json:"username" valid:"required,email" example:"feed@opplenz.com"` + Password string `json:"password" valid:"required,length(8|200)" example:"password123"` + LoginURL string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"` + NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"` + Category string `json:"category" valid:"optional,length(1|100)" example:"tenders"` + SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"` +} + +// ScrapeResponse represents the response from scraping operation +type ScrapeResponse struct { + Success bool `json:"success"` + NoticeID string `json:"notice_id"` + UploadedCount int `json:"uploaded_count"` + UploadedFiles []UploadedFileInfo `json:"uploaded_files"` + Errors []FileError `json:"errors,omitempty"` +} + +// UploadedFileInfo represents information about an uploaded file +type UploadedFileInfo struct { + Filename string `json:"filename"` + ObjectName string `json:"object_name"` + Size int64 `json:"size"` +} + +// FileError represents an error for a specific file +type FileError struct { + Filename string `json:"filename"` + Error string `json:"error"` +} diff --git a/internal/scraper/handler.go b/internal/scraper/handler.go new file mode 100644 index 0000000..6c7281e --- /dev/null +++ b/internal/scraper/handler.go @@ -0,0 +1,44 @@ +package scraper + +import ( + "tm/pkg/response" + + "github.com/labstack/echo/v4" +) + +// Handler handles HTTP requests for scraper operations +type Handler struct { + service Service +} + +// NewHandler creates a new scraper handler +func NewHandler(service Service) *Handler { + return &Handler{ + service: service, + } +} + +// ScrapeDocuments handles document scraping requests +// @Summary Scrape documents from external source +// @Description Scrapes documents from TendSign and uploads them to MinIO storage +// @Tags Admin-Scraper +// @Accept json +// @Produce json +// @Param request body ScrapeRequestForm true "Scraping request information" +// @Success 200 {object} response.Response{data=ScrapeResponse} +// @Failure 400 {object} response.Response +// @Failure 500 {object} response.Response +// @Router /admin/v1/scraper/download [post] +func (h *Handler) ScrapeDocuments(c echo.Context) error { + form, err := response.Parse[ScrapeRequestForm](c) + if err != nil { + return response.ValidationError(c, "Invalid request data", err.Error()) + } + + result, err := h.service.ScrapeDocuments(c.Request().Context(), form) + if err != nil { + return response.InternalServerError(c, "Failed to scrape documents") + } + + return response.Success(c, result, "Documents scraped and uploaded successfully") +} diff --git a/internal/scraper/service.go b/internal/scraper/service.go new file mode 100644 index 0000000..9896f11 --- /dev/null +++ b/internal/scraper/service.go @@ -0,0 +1,132 @@ +package scraper + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + "tm/pkg/logger" +) + +// Service defines the interface for scraper business operations +type Service interface { + // ScrapeDocuments scrapes documents and uploads them to MinIO + ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) +} + +// scraperService implements Service interface +type scraperService struct { + httpClient *http.Client + baseURL string + logger logger.Logger +} + +// NewService creates a new scraper service +func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service { + return &scraperService{ + httpClient: &http.Client{ + Timeout: timeout, + }, + baseURL: baseURL, + logger: logger, + } +} + +// ScrapeDocuments scrapes documents and uploads them to MinIO +func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) { + s.logger.Info("Starting document scraping", map[string]interface{}{ + "notice_id": form.NoticeID, + "category": form.Category, + }) + + // Prepare request body + requestBody := map[string]interface{}{ + "notice_id": form.NoticeID, + "username": form.Username, + "password": form.Password, + "login_url": form.LoginURL, + "notice_url": form.NoticeURL, + "category": form.Category, + "subcategory": form.SubCategory, + } + + bodyBytes, err := json.Marshal(requestBody) + if err != nil { + s.logger.Error("Failed to marshal request body", map[string]interface{}{ + "error": err.Error(), + }) + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + // Create HTTP request + url := fmt.Sprintf("%s/download", s.baseURL) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) + if err != nil { + s.logger.Error("Failed to create HTTP request", map[string]interface{}{ + "error": err.Error(), + }) + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + + // Send request + resp, err := s.httpClient.Do(req) + if err != nil { + s.logger.Error("Failed to send request to scraper service", map[string]interface{}{ + "error": err.Error(), + "url": url, + }) + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + // Read response body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + s.logger.Error("Failed to read response body", map[string]interface{}{ + "error": err.Error(), + }) + return nil, fmt.Errorf("failed to read response: %w", err) + } + + // Handle error responses + if resp.StatusCode >= 400 { + var errorResp map[string]interface{} + if err := json.Unmarshal(respBody, &errorResp); err == nil { + if errorMsg, ok := errorResp["error"].(string); ok { + s.logger.Error("Scraper service returned error", map[string]interface{}{ + "status_code": resp.StatusCode, + "error": errorMsg, + }) + return nil, fmt.Errorf("scraper service error: %s", errorMsg) + } + } + s.logger.Error("Scraper service returned error", map[string]interface{}{ + "status_code": resp.StatusCode, + "body": string(respBody), + }) + return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode) + } + + // Parse success response + var scrapeResp ScrapeResponse + if err := json.Unmarshal(respBody, &scrapeResp); err != nil { + s.logger.Error("Failed to unmarshal response", map[string]interface{}{ + "error": err.Error(), + "body": string(respBody), + }) + return nil, fmt.Errorf("failed to unmarshal response: %w", err) + } + + s.logger.Info("Document scraping completed", map[string]interface{}{ + "notice_id": scrapeResp.NoticeID, + "uploaded_count": scrapeResp.UploadedCount, + "success": scrapeResp.Success, + }) + + return &scrapeResp, nil +} diff --git a/pkg/scraper/Dockerfile b/pkg/scraper/Dockerfile new file mode 100644 index 0000000..42e6b3a --- /dev/null +++ b/pkg/scraper/Dockerfile @@ -0,0 +1,43 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y \ + wget \ + gnupg \ + ca-certificates \ + fonts-liberation \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libatspi2.0-0 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxkbcommon0 \ + libxrandr2 \ + xdg-utils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +RUN playwright install chromium + +COPY Scraper.py . +COPY app.py . + +EXPOSE 8000 + +CMD ["python", "app.py"] \ No newline at end of file diff --git a/pkg/scraper/Scraper.py b/pkg/scraper/Scraper.py new file mode 100644 index 0000000..78075d6 --- /dev/null +++ b/pkg/scraper/Scraper.py @@ -0,0 +1,121 @@ +from playwright.sync_api import sync_playwright +import re +import os +import time + +def clean_filename(filename): + """Removes illegal characters from filenames.""" + return re.sub(r'[<>:"/\\|?*]', '_', filename).strip() + +def download_documents(username, password, login_url, notice_url, download_dir=None): + """ + Download documents from TendSign website. + + Args: + username: Login username + password: Login password + login_url: URL to login page + notice_url: URL to notice entry page + download_dir: Directory to save downloads (optional, creates temp dir if not provided) + + Returns: + List of downloaded file paths + """ + if download_dir is None: + download_dir = f"TendSign_Downloads_{int(time.time())}" + + os.makedirs(download_dir, exist_ok=True) + print(f"[*] Downloads will be saved to: {download_dir}") + + downloaded_files = [] + + with sync_playwright() as p: + print("[*] Starting browser...") + browser = p.chromium.launch(headless=True) + context = browser.new_context( + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + viewport={"width": 1920, "height": 1080}, + locale="sv-SE", + ) + context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});") + page = context.new_page() + + try: + # === 1. LOGIN === + print("[*] Logging in...") + page.goto(login_url, wait_until="networkidle") + page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000) + page.fill('input[name="UcomLogin$txt_UserName"]', username) + page.fill('input[name="UcomLogin$txt_Password"]', password) + page.click('input[name="UcomLogin$btn_Submit"]') + page.wait_for_url("**/supplier/**", timeout=30000) + print("[SUCCESS] Logged in") + + # === 2. NAVIGATE TO DOCUMENTS TAB === + print(f"[*] Navigating to notice page...") + page.goto(notice_url, wait_until="networkidle", timeout=60000) + + print("[*] Switching to 'Dokument' tab...") + with page.expect_navigation(wait_until="networkidle", timeout=15000): + page.click('a[href*="s_view_RFQfiles.aspx"]') + + # === 3. LOCATE ALL FILE LINKS === + print("[*] Looking for file links starting with 'Bilaga'...") + + FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]" + + try: + page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000) + except Exception: + raise Exception("Failed to locate any file links starting with 'Bilaga'.") + + file_elements = page.locator(FILE_LINK_SELECTOR).all() + + print(f"[*] Found {len(file_elements)} file links to download.") + + if not file_elements: + print("[!] No file links found. Exiting.") + return downloaded_files + + # === 4. DOWNLOAD LOOP === + for i, element in enumerate(file_elements): + file_text = element.inner_text().strip() + print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...") + + try: + with page.expect_download(timeout=60000) as download_info: + element.click() + + download = download_info.value + + suggested_name = download.suggested_filename + clean_name = clean_filename(suggested_name) + save_path = os.path.join(download_dir, clean_name) + + download.save_as(save_path) + downloaded_files.append(save_path) + print(f" -> Successfully saved: {clean_name}") + + except Exception as e: + print(f" -> [ERROR] Download failed for {file_text}: {e}") + + time.sleep(1) + + print("\n[DONE] All downloads processed.") + return downloaded_files + + except Exception as e: + print(f"[CRITICAL ERROR] {e}") + page.screenshot(path=os.path.join(download_dir, "error_debug.png")) + raise + finally: + browser.close() + +if __name__ == "__main__": + # Default values for backward compatibility + USERNAME = "feed@opplenz.com" + PASSWORD = "f38Dgma!rLn2VR3DZaBn" + LOGIN_URL = "https://tendsign.com/login.aspx" + NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896" + + download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL) \ No newline at end of file diff --git a/pkg/scraper/app.py b/pkg/scraper/app.py new file mode 100644 index 0000000..302e8ca --- /dev/null +++ b/pkg/scraper/app.py @@ -0,0 +1,151 @@ +from flask import Flask, request, jsonify +from Scraper import download_documents +import os +from minio import Minio +from minio.error import S3Error +import tempfile +import shutil + +app = Flask(__name__) + +# MinIO configuration from environment variables +MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000') +MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin') +MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin') +MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true' +MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents') + +def get_minio_client(): + """Create and return MinIO client""" + return Minio( + MINIO_ENDPOINT, + access_key=MINIO_ACCESS_KEY, + secret_key=MINIO_SECRET_KEY, + secure=MINIO_USE_SSL + ) + +def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET): + """Upload a file to MinIO""" + try: + client = get_minio_client() + + # Ensure bucket exists + if not client.bucket_exists(bucket): + client.make_bucket(bucket) + + # Upload file + client.fput_object(bucket, object_name, file_path) + return True, None + except S3Error as e: + return False, str(e) + except Exception as e: + return False, str(e) + +@app.route('/health', methods=['GET']) +def health(): + """Health check endpoint""" + return jsonify({"status": "healthy"}), 200 + +@app.route('/download', methods=['POST']) +def download(): + """Download documents and upload to MinIO""" + try: + data = request.get_json() + + # Validate required fields + if not data: + return jsonify({"error": "Request body is required"}), 400 + + notice_id = data.get('notice_id') + username = data.get('username') + password = data.get('password') + login_url = data.get('login_url') + notice_url = data.get('notice_url') + category = data.get('category', 'tenders') + subcategory = data.get('subcategory', '') + + if not all([notice_id, username, password, login_url, notice_url]): + return jsonify({ + "error": "Missing required fields: notice_id, username, password, login_url, notice_url" + }), 400 + + # Create temporary directory for downloads + temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_") + + try: + # Download documents using the scraper + downloaded_files = download_documents( + username=username, + password=password, + login_url=login_url, + notice_url=notice_url, + download_dir=temp_dir + ) + + if not downloaded_files: + return jsonify({ + "error": "No files were downloaded", + "notice_id": notice_id + }), 404 + + # Upload each file to MinIO + uploaded_files = [] + errors = [] + + for file_path in downloaded_files: + try: + filename = os.path.basename(file_path) + # Create object name with hierarchical path + if subcategory: + object_name = f"{category}/{subcategory}/{notice_id}/{filename}" + else: + object_name = f"{category}/{notice_id}/{filename}" + + success, error = upload_to_minio(file_path, object_name) + + if success: + uploaded_files.append({ + "filename": filename, + "object_name": object_name, + "size": os.path.getsize(file_path) + }) + else: + errors.append({ + "filename": filename, + "error": error + }) + except Exception as e: + errors.append({ + "filename": os.path.basename(file_path), + "error": str(e) + }) + + # Clean up temporary directory + shutil.rmtree(temp_dir, ignore_errors=True) + + return jsonify({ + "success": True, + "notice_id": notice_id, + "uploaded_count": len(uploaded_files), + "uploaded_files": uploaded_files, + "errors": errors if errors else None + }), 200 + + except Exception as e: + # Clean up on error + shutil.rmtree(temp_dir, ignore_errors=True) + return jsonify({ + "error": "Failed to download documents", + "message": str(e), + "notice_id": notice_id + }), 500 + + except Exception as e: + return jsonify({ + "error": "Internal server error", + "message": str(e) + }), 500 + +if __name__ == '__main__': + port = int(os.getenv('PORT', 8000)) + app.run(host='0.0.0.0', port=port, debug=False) \ No newline at end of file diff --git a/pkg/scraper/docker-compose.scraper.yml b/pkg/scraper/docker-compose.scraper.yml new file mode 100644 index 0000000..48815e5 --- /dev/null +++ b/pkg/scraper/docker-compose.scraper.yml @@ -0,0 +1,45 @@ +version: '3.8' + +services: + scraper: + build: + context: . + dockerfile: pkg/scraper/Dockerfile + container_name: tm-scraper + ports: + - "8000:8000" + environment: + - PORT=8000 + - MINIO_ENDPOINT=minio:9000 + - MINIO_ACCESS_KEY=minioadmin + - MINIO_SECRET_KEY=minioadmin + - MINIO_USE_SSL=false + - MINIO_BUCKET=documents + networks: + - tm-network + depends_on: + - minio + restart: unless-stopped + + minio: + image: minio/minio:latest + container_name: tm-minio + ports: + - "9000:9000" + - "9001:9001" + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + command: server /data --console-address ":9001" + volumes: + - minio-data:/data + networks: + - tm-network + restart: unless-stopped + +networks: + tm-network: + driver: bridge + +volumes: + minio-data: \ No newline at end of file diff --git a/pkg/scraper/requirements.txt b/pkg/scraper/requirements.txt new file mode 100644 index 0000000..8e50a69 --- /dev/null +++ b/pkg/scraper/requirements.txt @@ -0,0 +1,5 @@ +playwright==1.40.0 +flask==3.0.0 +minio==7.2.0 +requests==2.31.0 +python-dotenv==1.0.0 \ No newline at end of file From 5e777ceec8fcbeb620087b0528cf46c27986bd8b Mon Sep 17 00:00:00 2001 From: Mazyar Date: Mon, 1 Dec 2025 11:06:09 +0330 Subject: [PATCH 2/2] scraper sdk --- cmd/web/bootstrap/config.go | 7 +- cmd/web/main.go | 2 +- internal/scraper/form.go | 17 +- internal/scraper/service.go | 139 +++++--------- pkg/scraper/Dockerfile | 43 ----- pkg/scraper/Scraper.py | 121 ------------ pkg/scraper/app.py | 151 --------------- pkg/scraper/docker-compose.scraper.yml | 45 ----- pkg/scraper/requirements.txt | 5 - pkg/scraper/sdk.go | 256 +++++++++++++++++++++++++ 10 files changed, 324 insertions(+), 462 deletions(-) delete mode 100644 pkg/scraper/Dockerfile delete mode 100644 pkg/scraper/Scraper.py delete mode 100644 pkg/scraper/app.py delete mode 100644 pkg/scraper/docker-compose.scraper.yml delete mode 100644 pkg/scraper/requirements.txt create mode 100644 pkg/scraper/sdk.go diff --git a/cmd/web/bootstrap/config.go b/cmd/web/bootstrap/config.go index f473da6..3bdef7f 100644 --- a/cmd/web/bootstrap/config.go +++ b/cmd/web/bootstrap/config.go @@ -21,8 +21,11 @@ type Config struct { } type ScraperConfig struct { - BaseURL string `env:"SCRAPER_BASE_URL"` - Timeout time.Duration `env:"SCRAPER_TIMEOUT"` + BaseURL string `env:"SCRAPER_BASE_URL"` + Timeout time.Duration `env:"SCRAPER_TIMEOUT"` + Username string `env:"SCRAPER_USERNAME"` + Password string `env:"SCRAPER_PASSWORD"` + LoginURL string `env:"SCRAPER_LOGIN_URL"` } type AuthConfig struct { diff --git a/cmd/web/main.go b/cmd/web/main.go index 902cc93..f5b7db2 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -180,7 +180,7 @@ func main() { notificationService := notification.NewService(notificationSDK, userService, customerService, logger) contactService := contact.NewService(contactRepository, logger) cmsService := cms.NewService(cmsRepository, logger) - scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger) + scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, conf.Scraper.Username, conf.Scraper.Password, conf.Scraper.LoginURL) logger.Info("Services initialized successfully", map[string]interface{}{ "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"}, }) diff --git a/internal/scraper/form.go b/internal/scraper/form.go index 25c0452..3c0a5b7 100644 --- a/internal/scraper/form.go +++ b/internal/scraper/form.go @@ -2,13 +2,8 @@ package scraper // ScrapeRequestForm represents the request form for scraping documents type ScrapeRequestForm struct { - NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"` - Username string `json:"username" valid:"required,email" example:"feed@opplenz.com"` - Password string `json:"password" valid:"required,length(8|200)" example:"password123"` - LoginURL string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"` - NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"` - Category string `json:"category" valid:"optional,length(1|100)" example:"tenders"` - SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"` + NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"` + NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"` } // ScrapeResponse represents the response from scraping operation @@ -18,6 +13,7 @@ type ScrapeResponse struct { UploadedCount int `json:"uploaded_count"` UploadedFiles []UploadedFileInfo `json:"uploaded_files"` Errors []FileError `json:"errors,omitempty"` + Buckets []BucketInfo `json:"buckets,omitempty"` } // UploadedFileInfo represents information about an uploaded file @@ -32,3 +28,10 @@ type FileError struct { Filename string `json:"filename"` Error string `json:"error"` } + +// BucketInfo represents information about a bucket +type BucketInfo struct { + Name string `json:"name"` + NoticeID string `json:"notice_id"` + Count int `json:"count"` +} diff --git a/internal/scraper/service.go b/internal/scraper/service.go index 9896f11..26f925e 100644 --- a/internal/scraper/service.go +++ b/internal/scraper/service.go @@ -1,14 +1,11 @@ package scraper import ( - "bytes" "context" - "encoding/json" - "fmt" - "io" - "net/http" "time" + "tm/pkg/logger" + "tm/pkg/scraper" ) // Service defines the interface for scraper business operations @@ -19,19 +16,16 @@ type Service interface { // scraperService implements Service interface type scraperService struct { - httpClient *http.Client - baseURL string - logger logger.Logger + sdk scraper.SDK + logger logger.Logger } -// NewService creates a new scraper service -func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service { +// NewService creates a new scraper service with stored credentials +func NewService(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) Service { + sdk := scraper.NewClient(baseURL, timeout, logger, username, password, loginURL) return &scraperService{ - httpClient: &http.Client{ - Timeout: timeout, - }, - baseURL: baseURL, - logger: logger, + sdk: sdk, + logger: logger, } } @@ -39,94 +33,65 @@ func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Ser func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) { s.logger.Info("Starting document scraping", map[string]interface{}{ "notice_id": form.NoticeID, - "category": form.Category, }) - // Prepare request body - requestBody := map[string]interface{}{ - "notice_id": form.NoticeID, - "username": form.Username, - "password": form.Password, - "login_url": form.LoginURL, - "notice_url": form.NoticeURL, - "category": form.Category, - "subcategory": form.SubCategory, + // Convert form to SDK request (only dynamic fields) + req := &scraper.ScrapeRequest{ + NoticeID: form.NoticeID, + NoticeURL: form.NoticeURL, } - bodyBytes, err := json.Marshal(requestBody) + // Call SDK (credentials are already configured) + response, err := s.sdk.ScrapeDocuments(ctx, req) if err != nil { - s.logger.Error("Failed to marshal request body", map[string]interface{}{ - "error": err.Error(), + s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{ + "error": err.Error(), + "notice_id": form.NoticeID, }) - return nil, fmt.Errorf("failed to marshal request: %w", err) + return nil, err } - // Create HTTP request - url := fmt.Sprintf("%s/download", s.baseURL) - req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) - if err != nil { - s.logger.Error("Failed to create HTTP request", map[string]interface{}{ - "error": err.Error(), - }) - return nil, fmt.Errorf("failed to create request: %w", err) + // Convert SDK response to service response + serviceResp := &ScrapeResponse{ + Success: response.Success, + NoticeID: response.NoticeID, + UploadedCount: response.UploadedCount, + UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)), + Errors: make([]FileError, len(response.Errors)), + Buckets: make([]BucketInfo, len(response.Buckets)), } - req.Header.Set("Content-Type", "application/json") - - // Send request - resp, err := s.httpClient.Do(req) - if err != nil { - s.logger.Error("Failed to send request to scraper service", map[string]interface{}{ - "error": err.Error(), - "url": url, - }) - return nil, fmt.Errorf("failed to send request: %w", err) - } - defer resp.Body.Close() - - // Read response body - respBody, err := io.ReadAll(resp.Body) - if err != nil { - s.logger.Error("Failed to read response body", map[string]interface{}{ - "error": err.Error(), - }) - return nil, fmt.Errorf("failed to read response: %w", err) - } - - // Handle error responses - if resp.StatusCode >= 400 { - var errorResp map[string]interface{} - if err := json.Unmarshal(respBody, &errorResp); err == nil { - if errorMsg, ok := errorResp["error"].(string); ok { - s.logger.Error("Scraper service returned error", map[string]interface{}{ - "status_code": resp.StatusCode, - "error": errorMsg, - }) - return nil, fmt.Errorf("scraper service error: %s", errorMsg) - } + // Convert uploaded files + for i, file := range response.UploadedFiles { + serviceResp.UploadedFiles[i] = UploadedFileInfo{ + Filename: file.Filename, + ObjectName: file.ObjectName, + Size: file.Size, } - s.logger.Error("Scraper service returned error", map[string]interface{}{ - "status_code": resp.StatusCode, - "body": string(respBody), - }) - return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode) } - // Parse success response - var scrapeResp ScrapeResponse - if err := json.Unmarshal(respBody, &scrapeResp); err != nil { - s.logger.Error("Failed to unmarshal response", map[string]interface{}{ - "error": err.Error(), - "body": string(respBody), - }) - return nil, fmt.Errorf("failed to unmarshal response: %w", err) + // Convert errors + for i, err := range response.Errors { + serviceResp.Errors[i] = FileError{ + Filename: err.Filename, + Error: err.Error, + } + } + + // Convert buckets + for i, bucket := range response.Buckets { + serviceResp.Buckets[i] = BucketInfo{ + Name: bucket.Name, + NoticeID: bucket.NoticeID, + Count: bucket.Count, + } } s.logger.Info("Document scraping completed", map[string]interface{}{ - "notice_id": scrapeResp.NoticeID, - "uploaded_count": scrapeResp.UploadedCount, - "success": scrapeResp.Success, + "notice_id": serviceResp.NoticeID, + "uploaded_count": serviceResp.UploadedCount, + "success": serviceResp.Success, }) - return &scrapeResp, nil + return serviceResp, nil } diff --git a/pkg/scraper/Dockerfile b/pkg/scraper/Dockerfile deleted file mode 100644 index 42e6b3a..0000000 --- a/pkg/scraper/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -FROM python:3.11-slim - -RUN apt-get update && apt-get install -y \ - wget \ - gnupg \ - ca-certificates \ - fonts-liberation \ - libasound2 \ - libatk-bridge2.0-0 \ - libatk1.0-0 \ - libatspi2.0-0 \ - libcups2 \ - libdbus-1-3 \ - libdrm2 \ - libgbm1 \ - libgtk-3-0 \ - libnspr4 \ - libnss3 \ - libx11-6 \ - libx11-xcb1 \ - libxcb1 \ - libxcomposite1 \ - libxdamage1 \ - libxext6 \ - libxfixes3 \ - libxkbcommon0 \ - libxrandr2 \ - xdg-utils \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -RUN playwright install chromium - -COPY Scraper.py . -COPY app.py . - -EXPOSE 8000 - -CMD ["python", "app.py"] \ No newline at end of file diff --git a/pkg/scraper/Scraper.py b/pkg/scraper/Scraper.py deleted file mode 100644 index 78075d6..0000000 --- a/pkg/scraper/Scraper.py +++ /dev/null @@ -1,121 +0,0 @@ -from playwright.sync_api import sync_playwright -import re -import os -import time - -def clean_filename(filename): - """Removes illegal characters from filenames.""" - return re.sub(r'[<>:"/\\|?*]', '_', filename).strip() - -def download_documents(username, password, login_url, notice_url, download_dir=None): - """ - Download documents from TendSign website. - - Args: - username: Login username - password: Login password - login_url: URL to login page - notice_url: URL to notice entry page - download_dir: Directory to save downloads (optional, creates temp dir if not provided) - - Returns: - List of downloaded file paths - """ - if download_dir is None: - download_dir = f"TendSign_Downloads_{int(time.time())}" - - os.makedirs(download_dir, exist_ok=True) - print(f"[*] Downloads will be saved to: {download_dir}") - - downloaded_files = [] - - with sync_playwright() as p: - print("[*] Starting browser...") - browser = p.chromium.launch(headless=True) - context = browser.new_context( - user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", - viewport={"width": 1920, "height": 1080}, - locale="sv-SE", - ) - context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});") - page = context.new_page() - - try: - # === 1. LOGIN === - print("[*] Logging in...") - page.goto(login_url, wait_until="networkidle") - page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000) - page.fill('input[name="UcomLogin$txt_UserName"]', username) - page.fill('input[name="UcomLogin$txt_Password"]', password) - page.click('input[name="UcomLogin$btn_Submit"]') - page.wait_for_url("**/supplier/**", timeout=30000) - print("[SUCCESS] Logged in") - - # === 2. NAVIGATE TO DOCUMENTS TAB === - print(f"[*] Navigating to notice page...") - page.goto(notice_url, wait_until="networkidle", timeout=60000) - - print("[*] Switching to 'Dokument' tab...") - with page.expect_navigation(wait_until="networkidle", timeout=15000): - page.click('a[href*="s_view_RFQfiles.aspx"]') - - # === 3. LOCATE ALL FILE LINKS === - print("[*] Looking for file links starting with 'Bilaga'...") - - FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]" - - try: - page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000) - except Exception: - raise Exception("Failed to locate any file links starting with 'Bilaga'.") - - file_elements = page.locator(FILE_LINK_SELECTOR).all() - - print(f"[*] Found {len(file_elements)} file links to download.") - - if not file_elements: - print("[!] No file links found. Exiting.") - return downloaded_files - - # === 4. DOWNLOAD LOOP === - for i, element in enumerate(file_elements): - file_text = element.inner_text().strip() - print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...") - - try: - with page.expect_download(timeout=60000) as download_info: - element.click() - - download = download_info.value - - suggested_name = download.suggested_filename - clean_name = clean_filename(suggested_name) - save_path = os.path.join(download_dir, clean_name) - - download.save_as(save_path) - downloaded_files.append(save_path) - print(f" -> Successfully saved: {clean_name}") - - except Exception as e: - print(f" -> [ERROR] Download failed for {file_text}: {e}") - - time.sleep(1) - - print("\n[DONE] All downloads processed.") - return downloaded_files - - except Exception as e: - print(f"[CRITICAL ERROR] {e}") - page.screenshot(path=os.path.join(download_dir, "error_debug.png")) - raise - finally: - browser.close() - -if __name__ == "__main__": - # Default values for backward compatibility - USERNAME = "feed@opplenz.com" - PASSWORD = "f38Dgma!rLn2VR3DZaBn" - LOGIN_URL = "https://tendsign.com/login.aspx" - NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896" - - download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL) \ No newline at end of file diff --git a/pkg/scraper/app.py b/pkg/scraper/app.py deleted file mode 100644 index 302e8ca..0000000 --- a/pkg/scraper/app.py +++ /dev/null @@ -1,151 +0,0 @@ -from flask import Flask, request, jsonify -from Scraper import download_documents -import os -from minio import Minio -from minio.error import S3Error -import tempfile -import shutil - -app = Flask(__name__) - -# MinIO configuration from environment variables -MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000') -MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin') -MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin') -MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true' -MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents') - -def get_minio_client(): - """Create and return MinIO client""" - return Minio( - MINIO_ENDPOINT, - access_key=MINIO_ACCESS_KEY, - secret_key=MINIO_SECRET_KEY, - secure=MINIO_USE_SSL - ) - -def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET): - """Upload a file to MinIO""" - try: - client = get_minio_client() - - # Ensure bucket exists - if not client.bucket_exists(bucket): - client.make_bucket(bucket) - - # Upload file - client.fput_object(bucket, object_name, file_path) - return True, None - except S3Error as e: - return False, str(e) - except Exception as e: - return False, str(e) - -@app.route('/health', methods=['GET']) -def health(): - """Health check endpoint""" - return jsonify({"status": "healthy"}), 200 - -@app.route('/download', methods=['POST']) -def download(): - """Download documents and upload to MinIO""" - try: - data = request.get_json() - - # Validate required fields - if not data: - return jsonify({"error": "Request body is required"}), 400 - - notice_id = data.get('notice_id') - username = data.get('username') - password = data.get('password') - login_url = data.get('login_url') - notice_url = data.get('notice_url') - category = data.get('category', 'tenders') - subcategory = data.get('subcategory', '') - - if not all([notice_id, username, password, login_url, notice_url]): - return jsonify({ - "error": "Missing required fields: notice_id, username, password, login_url, notice_url" - }), 400 - - # Create temporary directory for downloads - temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_") - - try: - # Download documents using the scraper - downloaded_files = download_documents( - username=username, - password=password, - login_url=login_url, - notice_url=notice_url, - download_dir=temp_dir - ) - - if not downloaded_files: - return jsonify({ - "error": "No files were downloaded", - "notice_id": notice_id - }), 404 - - # Upload each file to MinIO - uploaded_files = [] - errors = [] - - for file_path in downloaded_files: - try: - filename = os.path.basename(file_path) - # Create object name with hierarchical path - if subcategory: - object_name = f"{category}/{subcategory}/{notice_id}/{filename}" - else: - object_name = f"{category}/{notice_id}/{filename}" - - success, error = upload_to_minio(file_path, object_name) - - if success: - uploaded_files.append({ - "filename": filename, - "object_name": object_name, - "size": os.path.getsize(file_path) - }) - else: - errors.append({ - "filename": filename, - "error": error - }) - except Exception as e: - errors.append({ - "filename": os.path.basename(file_path), - "error": str(e) - }) - - # Clean up temporary directory - shutil.rmtree(temp_dir, ignore_errors=True) - - return jsonify({ - "success": True, - "notice_id": notice_id, - "uploaded_count": len(uploaded_files), - "uploaded_files": uploaded_files, - "errors": errors if errors else None - }), 200 - - except Exception as e: - # Clean up on error - shutil.rmtree(temp_dir, ignore_errors=True) - return jsonify({ - "error": "Failed to download documents", - "message": str(e), - "notice_id": notice_id - }), 500 - - except Exception as e: - return jsonify({ - "error": "Internal server error", - "message": str(e) - }), 500 - -if __name__ == '__main__': - port = int(os.getenv('PORT', 8000)) - app.run(host='0.0.0.0', port=port, debug=False) \ No newline at end of file diff --git a/pkg/scraper/docker-compose.scraper.yml b/pkg/scraper/docker-compose.scraper.yml deleted file mode 100644 index 48815e5..0000000 --- a/pkg/scraper/docker-compose.scraper.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: '3.8' - -services: - scraper: - build: - context: . - dockerfile: pkg/scraper/Dockerfile - container_name: tm-scraper - ports: - - "8000:8000" - environment: - - PORT=8000 - - MINIO_ENDPOINT=minio:9000 - - MINIO_ACCESS_KEY=minioadmin - - MINIO_SECRET_KEY=minioadmin - - MINIO_USE_SSL=false - - MINIO_BUCKET=documents - networks: - - tm-network - depends_on: - - minio - restart: unless-stopped - - minio: - image: minio/minio:latest - container_name: tm-minio - ports: - - "9000:9000" - - "9001:9001" - environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin - command: server /data --console-address ":9001" - volumes: - - minio-data:/data - networks: - - tm-network - restart: unless-stopped - -networks: - tm-network: - driver: bridge - -volumes: - minio-data: \ No newline at end of file diff --git a/pkg/scraper/requirements.txt b/pkg/scraper/requirements.txt deleted file mode 100644 index 8e50a69..0000000 --- a/pkg/scraper/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -playwright==1.40.0 -flask==3.0.0 -minio==7.2.0 -requests==2.31.0 -python-dotenv==1.0.0 \ No newline at end of file diff --git a/pkg/scraper/sdk.go b/pkg/scraper/sdk.go new file mode 100644 index 0000000..ef92e86 --- /dev/null +++ b/pkg/scraper/sdk.go @@ -0,0 +1,256 @@ +package scraper + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "tm/pkg/logger" +) + +// SDK represents the scraper SDK interface +type SDK interface { + // ScrapeDocuments scrapes documents for a given notice + ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) + + // ListBuckets lists all buckets with notice IDs + ListBuckets(ctx context.Context) (*ListBucketsResponse, error) + + // HealthCheck checks if the scraper service is healthy + HealthCheck(ctx context.Context) error +} + +// ScrapeRequest represents a request to scrape documents (only dynamic fields) +type ScrapeRequest struct { + NoticeID string `json:"notice_id"` + NoticeURL string `json:"notice_url"` +} + +// ScrapeResponse represents the response from scraping operation +type ScrapeResponse struct { + Success bool `json:"success"` + NoticeID string `json:"notice_id"` + UploadedCount int `json:"uploaded_count"` + UploadedFiles []UploadedFile `json:"uploaded_files"` + Errors []FileError `json:"errors,omitempty"` + Buckets []BucketInfo `json:"buckets,omitempty"` +} + +// UploadedFile represents information about an uploaded file +type UploadedFile struct { + Filename string `json:"filename"` + ObjectName string `json:"object_name"` + Size int64 `json:"size"` +} + +// FileError represents an error for a specific file +type FileError struct { + Filename string `json:"filename"` + Error string `json:"error"` +} + +// BucketInfo represents information about a bucket +type BucketInfo struct { + Name string `json:"name"` + NoticeID string `json:"notice_id"` + Count int `json:"count"` +} + +// ListBucketsResponse represents the response from listing buckets +type ListBucketsResponse struct { + Buckets []BucketInfo `json:"buckets"` + Total int `json:"total"` +} + +// Client implements the SDK interface +type Client struct { + httpClient *http.Client + baseURL string + logger logger.Logger + // Stored credentials + username string + password string + loginURL string +} + +// NewClient creates a new scraper SDK client with stored credentials +func NewClient(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) SDK { + return &Client{ + httpClient: &http.Client{ + Timeout: timeout, + }, + baseURL: baseURL, + logger: logger, + username: username, + password: password, + loginURL: loginURL, + } +} + +// ScrapeDocuments scrapes documents and uploads them to MinIO +func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) { + c.logger.Info("Starting document scraping via SDK", map[string]interface{}{ + "notice_id": req.NoticeID, + "base_url": c.baseURL, + }) + + // Prepare request body with stored credentials + dynamic fields + requestBody := map[string]interface{}{ + "notice_id": req.NoticeID, + "username": c.username, + "password": c.password, + "login_url": c.loginURL, + "notice_url": req.NoticeURL, + } + + // Use unified handler API + var response ScrapeResponse + err := c.makeRequest(ctx, "POST", "/download", requestBody, &response) + if err != nil { + return nil, err + } + + // Get buckets list after successful scraping + bucketsResp, err := c.ListBuckets(ctx) + if err != nil { + c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{ + "error": err.Error(), + }) + } else { + response.Buckets = bucketsResp.Buckets + } + + c.logger.Info("Document scraping completed via SDK", map[string]interface{}{ + "notice_id": response.NoticeID, + "uploaded_count": response.UploadedCount, + "success": response.Success, + }) + + return &response, nil +} + +// ListBuckets lists all buckets with notice IDs +func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) { + var response ListBucketsResponse + err := c.makeRequest(ctx, "GET", "/buckets", nil, &response) + return &response, err +} + +// HealthCheck checks if the scraper service is healthy +func (c *Client) HealthCheck(ctx context.Context) error { + var response map[string]interface{} + err := c.makeRequest(ctx, "GET", "/health", nil, &response) + if err != nil { + return err + } + + // Health check doesn't need to return data, just check if request succeeded + return nil +} + +// Unified HTTP Handler API +// makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling +func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error { + // Build full URL + url := fmt.Sprintf("%s%s", c.baseURL, endpoint) + + // Marshal request body to JSON if provided + var bodyReader io.Reader + if requestBody != nil { + jsonData, err := json.Marshal(requestBody) + if err != nil { + c.logger.Error("Failed to marshal request body", map[string]interface{}{ + "error": err.Error(), + "endpoint": endpoint, + }) + return fmt.Errorf("failed to marshal request: %w", err) + } + bodyReader = bytes.NewReader(jsonData) + } + + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, method, url, bodyReader) + if err != nil { + c.logger.Error("Failed to create HTTP request", map[string]interface{}{ + "error": err.Error(), + "method": method, + "url": url, + }) + return fmt.Errorf("failed to create request: %w", err) + } + + // Set headers + if requestBody != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "application/json") + + // Send request + resp, err := c.httpClient.Do(req) + if err != nil { + c.logger.Error("Failed to send HTTP request", map[string]interface{}{ + "error": err.Error(), + "method": method, + "url": url, + }) + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + // Read response body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + c.logger.Error("Failed to read response body", map[string]interface{}{ + "error": err.Error(), + "url": url, + }) + return fmt.Errorf("failed to read response: %w", err) + } + + // Handle error responses + if resp.StatusCode >= 400 { + // Try to parse error response + var errorResp map[string]interface{} + if err := json.Unmarshal(respBody, &errorResp); err == nil { + if errorMsg, ok := errorResp["error"].(string); ok { + c.logger.Error("Request failed with error response", map[string]interface{}{ + "status_code": resp.StatusCode, + "error": errorMsg, + "url": url, + }) + return fmt.Errorf("request failed: %s", errorMsg) + } + } + + c.logger.Error("Request failed with status code", map[string]interface{}{ + "status_code": resp.StatusCode, + "url": url, + "body": string(respBody), + }) + return fmt.Errorf("request failed with status %d", resp.StatusCode) + } + + // Handle successful responses + if responseBody != nil { + if err := json.Unmarshal(respBody, responseBody); err != nil { + c.logger.Error("Failed to unmarshal response", map[string]interface{}{ + "error": err.Error(), + "url": url, + "body": string(respBody), + }) + return fmt.Errorf("failed to unmarshal response: %w", err) + } + } + + c.logger.Debug("Request completed successfully", map[string]interface{}{ + "method": method, + "url": url, + "status_code": resp.StatusCode, + }) + + return nil +}