tendsign scraper

pkg/scraper/Dockerfile

@@ -0,0 +1,43 @@
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y \
+    wget \
+    gnupg \
+    ca-certificates \
+    fonts-liberation \
+    libasound2 \
+    libatk-bridge2.0-0 \
+    libatk1.0-0 \
+    libatspi2.0-0 \
+    libcups2 \
+    libdbus-1-3 \
+    libdrm2 \
+    libgbm1 \
+    libgtk-3-0 \
+    libnspr4 \
+    libnss3 \
+    libx11-6 \
+    libx11-xcb1 \
+    libxcb1 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxkbcommon0 \
+    libxrandr2 \
+    xdg-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN playwright install chromium
+
+COPY Scraper.py .
+COPY app.py .
+
+EXPOSE 8000
+
+CMD ["python", "app.py"]

pkg/scraper/Scraper.py

@@ -0,0 +1,121 @@
+from playwright.sync_api import sync_playwright
+import re
+import os
+import time
+
+def clean_filename(filename):
+    """Removes illegal characters from filenames."""
+    return re.sub(r'[<>:"/\\|?*]', '_', filename).strip()
+
+def download_documents(username, password, login_url, notice_url, download_dir=None):
+    """
+    Download documents from TendSign website.
+    
+    Args:
+        username: Login username
+        password: Login password
+        login_url: URL to login page
+        notice_url: URL to notice entry page
+        download_dir: Directory to save downloads (optional, creates temp dir if not provided)
+    
+    Returns:
+        List of downloaded file paths
+    """
+    if download_dir is None:
+        download_dir = f"TendSign_Downloads_{int(time.time())}"
+    
+    os.makedirs(download_dir, exist_ok=True)
+    print(f"[*] Downloads will be saved to: {download_dir}")
+    
+    downloaded_files = []
+    
+    with sync_playwright() as p:
+        print("[*] Starting browser...")
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+            viewport={"width": 1920, "height": 1080},
+            locale="sv-SE",
+        )
+        context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});")
+        page = context.new_page()
+
+        try:
+            # === 1. LOGIN ===
+            print("[*] Logging in...")
+            page.goto(login_url, wait_until="networkidle")
+            page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000)
+            page.fill('input[name="UcomLogin$txt_UserName"]', username)
+            page.fill('input[name="UcomLogin$txt_Password"]', password)
+            page.click('input[name="UcomLogin$btn_Submit"]')
+            page.wait_for_url("**/supplier/**", timeout=30000)
+            print("[SUCCESS] Logged in")
+
+            # === 2. NAVIGATE TO DOCUMENTS TAB ===
+            print(f"[*] Navigating to notice page...")
+            page.goto(notice_url, wait_until="networkidle", timeout=60000)
+
+            print("[*] Switching to 'Dokument' tab...")
+            with page.expect_navigation(wait_until="networkidle", timeout=15000):
+                page.click('a[href*="s_view_RFQfiles.aspx"]')
+
+            # === 3. LOCATE ALL FILE LINKS ===
+            print("[*] Looking for file links starting with 'Bilaga'...")
+
+            FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]"
+
+            try:
+                page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000)
+            except Exception:
+                raise Exception("Failed to locate any file links starting with 'Bilaga'.")
+
+            file_elements = page.locator(FILE_LINK_SELECTOR).all()
+            
+            print(f"[*] Found {len(file_elements)} file links to download.")
+
+            if not file_elements:
+                print("[!] No file links found. Exiting.")
+                return downloaded_files
+
+            # === 4. DOWNLOAD LOOP ===
+            for i, element in enumerate(file_elements):
+                file_text = element.inner_text().strip()
+                print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...")
+
+                try:
+                    with page.expect_download(timeout=60000) as download_info:
+                        element.click()
+
+                    download = download_info.value
+                    
+                    suggested_name = download.suggested_filename
+                    clean_name = clean_filename(suggested_name)
+                    save_path = os.path.join(download_dir, clean_name)
+
+                    download.save_as(save_path)
+                    downloaded_files.append(save_path)
+                    print(f"   -> Successfully saved: {clean_name}")
+
+                except Exception as e:
+                    print(f"   -> [ERROR] Download failed for {file_text}: {e}")
+                
+                time.sleep(1)
+
+            print("\n[DONE] All downloads processed.")
+            return downloaded_files
+
+        except Exception as e:
+            print(f"[CRITICAL ERROR] {e}")
+            page.screenshot(path=os.path.join(download_dir, "error_debug.png"))
+            raise
+        finally:
+            browser.close()
+
+if __name__ == "__main__":
+    # Default values for backward compatibility
+    USERNAME = "feed@opplenz.com"
+    PASSWORD = "f38Dgma!rLn2VR3DZaBn"
+    LOGIN_URL = "https://tendsign.com/login.aspx"
+    NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"
+    
+    download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL)

pkg/scraper/app.py

@@ -0,0 +1,151 @@
+from flask import Flask, request, jsonify
+from Scraper import download_documents
+import os
+from minio import Minio
+from minio.error import S3Error
+import tempfile
+import shutil
+
+app = Flask(__name__)
+
+# MinIO configuration from environment variables
+MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000')
+MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
+MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
+MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
+MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents')
+
+def get_minio_client():
+    """Create and return MinIO client"""
+    return Minio(
+        MINIO_ENDPOINT,
+        access_key=MINIO_ACCESS_KEY,
+        secret_key=MINIO_SECRET_KEY,
+        secure=MINIO_USE_SSL
+    )
+
+def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET):
+    """Upload a file to MinIO"""
+    try:
+        client = get_minio_client()
+        
+        # Ensure bucket exists
+        if not client.bucket_exists(bucket):
+            client.make_bucket(bucket)
+        
+        # Upload file
+        client.fput_object(bucket, object_name, file_path)
+        return True, None
+    except S3Error as e:
+        return False, str(e)
+    except Exception as e:
+        return False, str(e)
+
+@app.route('/health', methods=['GET'])
+def health():
+    """Health check endpoint"""
+    return jsonify({"status": "healthy"}), 200
+
+@app.route('/download', methods=['POST'])
+def download():
+    """Download documents and upload to MinIO"""
+    try:
+        data = request.get_json()
+        
+        # Validate required fields
+        if not data:
+            return jsonify({"error": "Request body is required"}), 400
+        
+        notice_id = data.get('notice_id')
+        username = data.get('username')
+        password = data.get('password')
+        login_url = data.get('login_url')
+        notice_url = data.get('notice_url')
+        category = data.get('category', 'tenders')
+        subcategory = data.get('subcategory', '')
+        
+        if not all([notice_id, username, password, login_url, notice_url]):
+            return jsonify({
+                "error": "Missing required fields: notice_id, username, password, login_url, notice_url"
+            }), 400
+        
+        # Create temporary directory for downloads
+        temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_")
+        
+        try:
+            # Download documents using the scraper
+            downloaded_files = download_documents(
+                username=username,
+                password=password,
+                login_url=login_url,
+                notice_url=notice_url,
+                download_dir=temp_dir
+            )
+            
+            if not downloaded_files:
+                return jsonify({
+                    "error": "No files were downloaded",
+                    "notice_id": notice_id
+                }), 404
+            
+            # Upload each file to MinIO
+            uploaded_files = []
+            errors = []
+            
+            for file_path in downloaded_files:
+                try:
+                    filename = os.path.basename(file_path)
+                    # Create object name with hierarchical path
+                    if subcategory:
+                        object_name = f"{category}/{subcategory}/{notice_id}/{filename}"
+                    else:
+                        object_name = f"{category}/{notice_id}/{filename}"
+                    
+                    success, error = upload_to_minio(file_path, object_name)
+                    
+                    if success:
+                        uploaded_files.append({
+                            "filename": filename,
+                            "object_name": object_name,
+                            "size": os.path.getsize(file_path)
+                        })
+                    else:
+                        errors.append({
+                            "filename": filename,
+                            "error": error
+                        })
+                except Exception as e:
+                    errors.append({
+                        "filename": os.path.basename(file_path),
+                        "error": str(e)
+                    })
+            
+            # Clean up temporary directory
+            shutil.rmtree(temp_dir, ignore_errors=True)
+            
+            return jsonify({
+                "success": True,
+                "notice_id": notice_id,
+                "uploaded_count": len(uploaded_files),
+                "uploaded_files": uploaded_files,
+                "errors": errors if errors else None
+            }), 200
+            
+        except Exception as e:
+            # Clean up on error
+            shutil.rmtree(temp_dir, ignore_errors=True)
+            return jsonify({
+                "error": "Failed to download documents",
+                "message": str(e),
+                "notice_id": notice_id
+            }), 500
+            
+    except Exception as e:
+        return jsonify({
+            "error": "Internal server error",
+            "message": str(e)
+        }), 500
+
+if __name__ == '__main__':
+    port = int(os.getenv('PORT', 8000))
+    app.run(host='0.0.0.0', port=port, debug=False)

pkg/scraper/docker-compose.scraper.yml

@@ -0,0 +1,45 @@
+version: '3.8'
+
+services:
+  scraper:
+    build:
+      context: .
+      dockerfile: pkg/scraper/Dockerfile
+    container_name: tm-scraper
+    ports:
+      - "8000:8000"
+    environment:
+      - PORT=8000
+      - MINIO_ENDPOINT=minio:9000
+      - MINIO_ACCESS_KEY=minioadmin
+      - MINIO_SECRET_KEY=minioadmin
+      - MINIO_USE_SSL=false
+      - MINIO_BUCKET=documents
+    networks:
+      - tm-network
+    depends_on:
+      - minio
+    restart: unless-stopped
+
+  minio:
+    image: minio/minio:latest
+    container_name: tm-minio
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+    command: server /data --console-address ":9001"
+    volumes:
+      - minio-data:/data
+    networks:
+      - tm-network
+    restart: unless-stopped
+
+networks:
+  tm-network:
+    driver: bridge
+
+volumes:
+  minio-data:

pkg/scraper/requirements.txt

@@ -0,0 +1,5 @@
+playwright==1.40.0
+flask==3.0.0
+minio==7.2.0
+requests==2.31.0
+python-dotenv==1.0.0