Mazyar
151 lines
5.1 KiB
Python
151 lines
5.1 KiB
Python
from flask import Flask, request, jsonify
|
|
from Scraper import download_documents
|
|
import os
|
|
from minio import Minio
|
|
from minio.error import S3Error
|
|
import tempfile
|
|
import shutil
|
|
|
|
app = Flask(__name__)
|
|
|
|
# MinIO configuration from environment variables
|
|
MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000')
|
|
MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
|
|
MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
|
|
MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
|
|
MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents')
|
|
|
|
def get_minio_client():
|
|
"""Create and return MinIO client"""
|
|
return Minio(
|
|
MINIO_ENDPOINT,
|
|
access_key=MINIO_ACCESS_KEY,
|
|
secret_key=MINIO_SECRET_KEY,
|
|
secure=MINIO_USE_SSL
|
|
)
|
|
|
|
def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET):
|
|
"""Upload a file to MinIO"""
|
|
try:
|
|
client = get_minio_client()
|
|
|
|
# Ensure bucket exists
|
|
if not client.bucket_exists(bucket):
|
|
client.make_bucket(bucket)
|
|
|
|
# Upload file
|
|
client.fput_object(bucket, object_name, file_path)
|
|
return True, None
|
|
except S3Error as e:
|
|
return False, str(e)
|
|
except Exception as e:
|
|
return False, str(e)
|
|
|
|
@app.route('/health', methods=['GET'])
|
|
def health():
|
|
"""Health check endpoint"""
|
|
return jsonify({"status": "healthy"}), 200
|
|
|
|
@app.route('/download', methods=['POST'])
|
|
def download():
|
|
"""Download documents and upload to MinIO"""
|
|
try:
|
|
data = request.get_json()
|
|
|
|
# Validate required fields
|
|
if not data:
|
|
return jsonify({"error": "Request body is required"}), 400
|
|
|
|
notice_id = data.get('notice_id')
|
|
username = data.get('username')
|
|
password = data.get('password')
|
|
login_url = data.get('login_url')
|
|
notice_url = data.get('notice_url')
|
|
category = data.get('category', 'tenders')
|
|
subcategory = data.get('subcategory', '')
|
|
|
|
if not all([notice_id, username, password, login_url, notice_url]):
|
|
return jsonify({
|
|
"error": "Missing required fields: notice_id, username, password, login_url, notice_url"
|
|
}), 400
|
|
|
|
# Create temporary directory for downloads
|
|
temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_")
|
|
|
|
try:
|
|
# Download documents using the scraper
|
|
downloaded_files = download_documents(
|
|
username=username,
|
|
password=password,
|
|
login_url=login_url,
|
|
notice_url=notice_url,
|
|
download_dir=temp_dir
|
|
)
|
|
|
|
if not downloaded_files:
|
|
return jsonify({
|
|
"error": "No files were downloaded",
|
|
"notice_id": notice_id
|
|
}), 404
|
|
|
|
# Upload each file to MinIO
|
|
uploaded_files = []
|
|
errors = []
|
|
|
|
for file_path in downloaded_files:
|
|
try:
|
|
filename = os.path.basename(file_path)
|
|
# Create object name with hierarchical path
|
|
if subcategory:
|
|
object_name = f"{category}/{subcategory}/{notice_id}/{filename}"
|
|
else:
|
|
object_name = f"{category}/{notice_id}/{filename}"
|
|
|
|
success, error = upload_to_minio(file_path, object_name)
|
|
|
|
if success:
|
|
uploaded_files.append({
|
|
"filename": filename,
|
|
"object_name": object_name,
|
|
"size": os.path.getsize(file_path)
|
|
})
|
|
else:
|
|
errors.append({
|
|
"filename": filename,
|
|
"error": error
|
|
})
|
|
except Exception as e:
|
|
errors.append({
|
|
"filename": os.path.basename(file_path),
|
|
"error": str(e)
|
|
})
|
|
|
|
# Clean up temporary directory
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
|
|
return jsonify({
|
|
"success": True,
|
|
"notice_id": notice_id,
|
|
"uploaded_count": len(uploaded_files),
|
|
"uploaded_files": uploaded_files,
|
|
"errors": errors if errors else None
|
|
}), 200
|
|
|
|
except Exception as e:
|
|
# Clean up on error
|
|
shutil.rmtree(temp_dir, ignore_errors=True)
|
|
return jsonify({
|
|
"error": "Failed to download documents",
|
|
"message": str(e),
|
|
"notice_id": notice_id
|
|
}), 500
|
|
|
|
except Exception as e:
|
|
return jsonify({
|
|
"error": "Internal server error",
|
|
"message": str(e)
|
|
}), 500
|
|
|
|
if __name__ == '__main__':
|
|
port = int(os.getenv('PORT', 8000))
|
|
app.run(host='0.0.0.0', port=port, debug=False) |