tendsign scraper

pkg/scraper/app.py

@@ -0,0 +1,151 @@
+from flask import Flask, request, jsonify
+from Scraper import download_documents
+import os
+from minio import Minio
+from minio.error import S3Error
+import tempfile
+import shutil
+
+app = Flask(__name__)
+
+# MinIO configuration from environment variables
+MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000')
+MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
+MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
+MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
+MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents')
+
+def get_minio_client():
+    """Create and return MinIO client"""
+    return Minio(
+        MINIO_ENDPOINT,
+        access_key=MINIO_ACCESS_KEY,
+        secret_key=MINIO_SECRET_KEY,
+        secure=MINIO_USE_SSL
+    )
+
+def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET):
+    """Upload a file to MinIO"""
+    try:
+        client = get_minio_client()
+        
+        # Ensure bucket exists
+        if not client.bucket_exists(bucket):
+            client.make_bucket(bucket)
+        
+        # Upload file
+        client.fput_object(bucket, object_name, file_path)
+        return True, None
+    except S3Error as e:
+        return False, str(e)
+    except Exception as e:
+        return False, str(e)
+
+@app.route('/health', methods=['GET'])
+def health():
+    """Health check endpoint"""
+    return jsonify({"status": "healthy"}), 200
+
+@app.route('/download', methods=['POST'])
+def download():
+    """Download documents and upload to MinIO"""
+    try:
+        data = request.get_json()
+        
+        # Validate required fields
+        if not data:
+            return jsonify({"error": "Request body is required"}), 400
+        
+        notice_id = data.get('notice_id')
+        username = data.get('username')
+        password = data.get('password')
+        login_url = data.get('login_url')
+        notice_url = data.get('notice_url')
+        category = data.get('category', 'tenders')
+        subcategory = data.get('subcategory', '')
+        
+        if not all([notice_id, username, password, login_url, notice_url]):
+            return jsonify({
+                "error": "Missing required fields: notice_id, username, password, login_url, notice_url"
+            }), 400
+        
+        # Create temporary directory for downloads
+        temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_")
+        
+        try:
+            # Download documents using the scraper
+            downloaded_files = download_documents(
+                username=username,
+                password=password,
+                login_url=login_url,
+                notice_url=notice_url,
+                download_dir=temp_dir
+            )
+            
+            if not downloaded_files:
+                return jsonify({
+                    "error": "No files were downloaded",
+                    "notice_id": notice_id
+                }), 404
+            
+            # Upload each file to MinIO
+            uploaded_files = []
+            errors = []
+            
+            for file_path in downloaded_files:
+                try:
+                    filename = os.path.basename(file_path)
+                    # Create object name with hierarchical path
+                    if subcategory:
+                        object_name = f"{category}/{subcategory}/{notice_id}/{filename}"
+                    else:
+                        object_name = f"{category}/{notice_id}/{filename}"
+                    
+                    success, error = upload_to_minio(file_path, object_name)
+                    
+                    if success:
+                        uploaded_files.append({
+                            "filename": filename,
+                            "object_name": object_name,
+                            "size": os.path.getsize(file_path)
+                        })
+                    else:
+                        errors.append({
+                            "filename": filename,
+                            "error": error
+                        })
+                except Exception as e:
+                    errors.append({
+                        "filename": os.path.basename(file_path),
+                        "error": str(e)
+                    })
+            
+            # Clean up temporary directory
+            shutil.rmtree(temp_dir, ignore_errors=True)
+            
+            return jsonify({
+                "success": True,
+                "notice_id": notice_id,
+                "uploaded_count": len(uploaded_files),
+                "uploaded_files": uploaded_files,
+                "errors": errors if errors else None
+            }), 200
+            
+        except Exception as e:
+            # Clean up on error
+            shutil.rmtree(temp_dir, ignore_errors=True)
+            return jsonify({
+                "error": "Failed to download documents",
+                "message": str(e),
+                "notice_id": notice_id
+            }), 500
+            
+    except Exception as e:
+        return jsonify({
+            "error": "Internal server error",
+            "message": str(e)
+        }), 500
+
+if __name__ == '__main__':
+    port = int(os.getenv('PORT', 8000))
+    app.run(host='0.0.0.0', port=port, debug=False)