tendsign scraper

2025-11-30 02:58:08 +03:30
parent d57bf95c70
commit b6546eb2bb
11 changed files with 597 additions and 4 deletions
@@ -0,0 +1,121 @@
+from playwright.sync_api import sync_playwright
+import re
+import os
+import time
+
+def clean_filename(filename):
+    """Removes illegal characters from filenames."""
+    return re.sub(r'[<>:"/\\|?*]', '_', filename).strip()
+
+def download_documents(username, password, login_url, notice_url, download_dir=None):
+    """
+    Download documents from TendSign website.
+    
+    Args:
+        username: Login username
+        password: Login password
+        login_url: URL to login page
+        notice_url: URL to notice entry page
+        download_dir: Directory to save downloads (optional, creates temp dir if not provided)
+    
+    Returns:
+        List of downloaded file paths
+    """
+    if download_dir is None:
+        download_dir = f"TendSign_Downloads_{int(time.time())}"
+    
+    os.makedirs(download_dir, exist_ok=True)
+    print(f"[*] Downloads will be saved to: {download_dir}")
+    
+    downloaded_files = []
+    
+    with sync_playwright() as p:
+        print("[*] Starting browser...")
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+            viewport={"width": 1920, "height": 1080},
+            locale="sv-SE",
+        )
+        context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});")
+        page = context.new_page()
+
+        try:
+            # === 1. LOGIN ===
+            print("[*] Logging in...")
+            page.goto(login_url, wait_until="networkidle")
+            page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000)
+            page.fill('input[name="UcomLogin$txt_UserName"]', username)
+            page.fill('input[name="UcomLogin$txt_Password"]', password)
+            page.click('input[name="UcomLogin$btn_Submit"]')
+            page.wait_for_url("**/supplier/**", timeout=30000)
+            print("[SUCCESS] Logged in")
+
+            # === 2. NAVIGATE TO DOCUMENTS TAB ===
+            print(f"[*] Navigating to notice page...")
+            page.goto(notice_url, wait_until="networkidle", timeout=60000)
+
+            print("[*] Switching to 'Dokument' tab...")
+            with page.expect_navigation(wait_until="networkidle", timeout=15000):
+                page.click('a[href*="s_view_RFQfiles.aspx"]')
+
+            # === 3. LOCATE ALL FILE LINKS ===
+            print("[*] Looking for file links starting with 'Bilaga'...")
+
+            FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]"
+
+            try:
+                page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000)
+            except Exception:
+                raise Exception("Failed to locate any file links starting with 'Bilaga'.")
+
+            file_elements = page.locator(FILE_LINK_SELECTOR).all()
+            
+            print(f"[*] Found {len(file_elements)} file links to download.")
+
+            if not file_elements:
+                print("[!] No file links found. Exiting.")
+                return downloaded_files
+
+            # === 4. DOWNLOAD LOOP ===
+            for i, element in enumerate(file_elements):
+                file_text = element.inner_text().strip()
+                print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...")
+
+                try:
+                    with page.expect_download(timeout=60000) as download_info:
+                        element.click()
+
+                    download = download_info.value
+                    
+                    suggested_name = download.suggested_filename
+                    clean_name = clean_filename(suggested_name)
+                    save_path = os.path.join(download_dir, clean_name)
+
+                    download.save_as(save_path)
+                    downloaded_files.append(save_path)
+                    print(f"   -> Successfully saved: {clean_name}")
+
+                except Exception as e:
+                    print(f"   -> [ERROR] Download failed for {file_text}: {e}")
+                
+                time.sleep(1)
+
+            print("\n[DONE] All downloads processed.")
+            return downloaded_files
+
+        except Exception as e:
+            print(f"[CRITICAL ERROR] {e}")
+            page.screenshot(path=os.path.join(download_dir, "error_debug.png"))
+            raise
+        finally:
+            browser.close()
+
+if __name__ == "__main__":
+    # Default values for backward compatibility
+    USERNAME = "feed@opplenz.com"
+    PASSWORD = "f38Dgma!rLn2VR3DZaBn"
+    LOGIN_URL = "https://tendsign.com/login.aspx"
+    NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"
+    
+    download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL)