tendsign scraper
This commit is contained in:
Mazyar
@@ -1,6 +1,7 @@
|
||||
package bootstrap
|
||||
|
||||
import (
|
||||
"time"
|
||||
"tm/pkg/config"
|
||||
)
|
||||
|
||||
@@ -14,10 +15,16 @@ type Config struct {
|
||||
Assets config.AssetsConfig
|
||||
Notification config.NotificationConfig
|
||||
Ollama config.OllamaConfig
|
||||
Scraper ScraperConfig
|
||||
UserAuth AuthConfig
|
||||
CustomerAuth AuthConfig
|
||||
}
|
||||
|
||||
type ScraperConfig struct {
|
||||
BaseURL string `env:"SCRAPER_BASE_URL"`
|
||||
Timeout time.Duration `env:"SCRAPER_TIMEOUT"`
|
||||
}
|
||||
|
||||
type AuthConfig struct {
|
||||
JWT JWTConfig
|
||||
}
|
||||
|
||||
+6
-3
@@ -103,6 +103,7 @@ import (
|
||||
"tm/internal/feedback"
|
||||
"tm/internal/inquiry"
|
||||
"tm/internal/notification"
|
||||
"tm/internal/scraper"
|
||||
"tm/internal/tender"
|
||||
"tm/internal/tender_approval"
|
||||
"tm/internal/user"
|
||||
@@ -179,8 +180,9 @@ func main() {
|
||||
notificationService := notification.NewService(notificationSDK, userService, customerService, logger)
|
||||
contactService := contact.NewService(contactRepository, logger)
|
||||
cmsService := cms.NewService(cmsRepository, logger)
|
||||
scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger)
|
||||
logger.Info("Services initialized successfully", map[string]interface{}{
|
||||
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"},
|
||||
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
|
||||
})
|
||||
|
||||
// Initialize handlers with services
|
||||
@@ -196,15 +198,16 @@ func main() {
|
||||
notificationHandler := notification.NewHandler(notificationService)
|
||||
contactHandler := contact.NewHandler(contactService)
|
||||
cmsHandler := cms.NewHandler(cmsService)
|
||||
scraperHandler := scraper.NewHandler(scraperService)
|
||||
logger.Info("Handlers initialized successfully", map[string]interface{}{
|
||||
"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"},
|
||||
"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
|
||||
})
|
||||
|
||||
// Initialize HTTP server
|
||||
e := bootstrap.InitHTTPServer(conf, logger)
|
||||
|
||||
// Register routes
|
||||
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler)
|
||||
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler)
|
||||
router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler)
|
||||
|
||||
// Start server
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"tm/internal/feedback"
|
||||
"tm/internal/inquiry"
|
||||
"tm/internal/notification"
|
||||
"tm/internal/scraper"
|
||||
"tm/internal/tender"
|
||||
"tm/internal/tender_approval"
|
||||
"tm/internal/user"
|
||||
@@ -17,7 +18,7 @@ import (
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) {
|
||||
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler) {
|
||||
adminV1 := e.Group("/admin/v1")
|
||||
|
||||
// Admin Users Routes
|
||||
@@ -165,6 +166,13 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
|
||||
cmsGP.PUT("/:id", cmsHandler.Update)
|
||||
cmsGP.DELETE("/:id", cmsHandler.Delete)
|
||||
}
|
||||
|
||||
// Scraper Routes
|
||||
scraperGP := adminV1.Group("/scraper")
|
||||
{
|
||||
scraperGP.Use(userHandler.AuthMiddleware())
|
||||
scraperGP.POST("/download", scraperHandler.ScrapeDocuments)
|
||||
}
|
||||
}
|
||||
|
||||
func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) {
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package scraper
|
||||
|
||||
// ScrapeRequestForm represents the request form for scraping documents
|
||||
type ScrapeRequestForm struct {
|
||||
NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
|
||||
Username string `json:"username" valid:"required,email" example:"feed@opplenz.com"`
|
||||
Password string `json:"password" valid:"required,length(8|200)" example:"password123"`
|
||||
LoginURL string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"`
|
||||
NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
|
||||
Category string `json:"category" valid:"optional,length(1|100)" example:"tenders"`
|
||||
SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"`
|
||||
}
|
||||
|
||||
// ScrapeResponse represents the response from scraping operation
|
||||
type ScrapeResponse struct {
|
||||
Success bool `json:"success"`
|
||||
NoticeID string `json:"notice_id"`
|
||||
UploadedCount int `json:"uploaded_count"`
|
||||
UploadedFiles []UploadedFileInfo `json:"uploaded_files"`
|
||||
Errors []FileError `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// UploadedFileInfo represents information about an uploaded file
|
||||
type UploadedFileInfo struct {
|
||||
Filename string `json:"filename"`
|
||||
ObjectName string `json:"object_name"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// FileError represents an error for a specific file
|
||||
type FileError struct {
|
||||
Filename string `json:"filename"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"tm/pkg/response"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
// Handler handles HTTP requests for scraper operations
|
||||
type Handler struct {
|
||||
service Service
|
||||
}
|
||||
|
||||
// NewHandler creates a new scraper handler
|
||||
func NewHandler(service Service) *Handler {
|
||||
return &Handler{
|
||||
service: service,
|
||||
}
|
||||
}
|
||||
|
||||
// ScrapeDocuments handles document scraping requests
|
||||
// @Summary Scrape documents from external source
|
||||
// @Description Scrapes documents from TendSign and uploads them to MinIO storage
|
||||
// @Tags Admin-Scraper
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body ScrapeRequestForm true "Scraping request information"
|
||||
// @Success 200 {object} response.Response{data=ScrapeResponse}
|
||||
// @Failure 400 {object} response.Response
|
||||
// @Failure 500 {object} response.Response
|
||||
// @Router /admin/v1/scraper/download [post]
|
||||
func (h *Handler) ScrapeDocuments(c echo.Context) error {
|
||||
form, err := response.Parse[ScrapeRequestForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.ScrapeDocuments(c.Request().Context(), form)
|
||||
if err != nil {
|
||||
return response.InternalServerError(c, "Failed to scrape documents")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Documents scraped and uploaded successfully")
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
"tm/pkg/logger"
|
||||
)
|
||||
|
||||
// Service defines the interface for scraper business operations
|
||||
type Service interface {
|
||||
// ScrapeDocuments scrapes documents and uploads them to MinIO
|
||||
ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
|
||||
}
|
||||
|
||||
// scraperService implements Service interface
|
||||
type scraperService struct {
|
||||
httpClient *http.Client
|
||||
baseURL string
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
// NewService creates a new scraper service
|
||||
func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
|
||||
return &scraperService{
|
||||
httpClient: &http.Client{
|
||||
Timeout: timeout,
|
||||
},
|
||||
baseURL: baseURL,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// ScrapeDocuments scrapes documents and uploads them to MinIO
|
||||
func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
|
||||
s.logger.Info("Starting document scraping", map[string]interface{}{
|
||||
"notice_id": form.NoticeID,
|
||||
"category": form.Category,
|
||||
})
|
||||
|
||||
// Prepare request body
|
||||
requestBody := map[string]interface{}{
|
||||
"notice_id": form.NoticeID,
|
||||
"username": form.Username,
|
||||
"password": form.Password,
|
||||
"login_url": form.LoginURL,
|
||||
"notice_url": form.NoticeURL,
|
||||
"category": form.Category,
|
||||
"subcategory": form.SubCategory,
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to marshal request body", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
// Create HTTP request
|
||||
url := fmt.Sprintf("%s/download", s.baseURL)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to create HTTP request", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
// Send request
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to send request to scraper service", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"url": url,
|
||||
})
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read response body
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to read response body", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
// Handle error responses
|
||||
if resp.StatusCode >= 400 {
|
||||
var errorResp map[string]interface{}
|
||||
if err := json.Unmarshal(respBody, &errorResp); err == nil {
|
||||
if errorMsg, ok := errorResp["error"].(string); ok {
|
||||
s.logger.Error("Scraper service returned error", map[string]interface{}{
|
||||
"status_code": resp.StatusCode,
|
||||
"error": errorMsg,
|
||||
})
|
||||
return nil, fmt.Errorf("scraper service error: %s", errorMsg)
|
||||
}
|
||||
}
|
||||
s.logger.Error("Scraper service returned error", map[string]interface{}{
|
||||
"status_code": resp.StatusCode,
|
||||
"body": string(respBody),
|
||||
})
|
||||
return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Parse success response
|
||||
var scrapeResp ScrapeResponse
|
||||
if err := json.Unmarshal(respBody, &scrapeResp); err != nil {
|
||||
s.logger.Error("Failed to unmarshal response", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"body": string(respBody),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
s.logger.Info("Document scraping completed", map[string]interface{}{
|
||||
"notice_id": scrapeResp.NoticeID,
|
||||
"uploaded_count": scrapeResp.UploadedCount,
|
||||
"success": scrapeResp.Success,
|
||||
})
|
||||
|
||||
return &scrapeResp, nil
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y \
|
||||
wget \
|
||||
gnupg \
|
||||
ca-certificates \
|
||||
fonts-liberation \
|
||||
libasound2 \
|
||||
libatk-bridge2.0-0 \
|
||||
libatk1.0-0 \
|
||||
libatspi2.0-0 \
|
||||
libcups2 \
|
||||
libdbus-1-3 \
|
||||
libdrm2 \
|
||||
libgbm1 \
|
||||
libgtk-3-0 \
|
||||
libnspr4 \
|
||||
libnss3 \
|
||||
libx11-6 \
|
||||
libx11-xcb1 \
|
||||
libxcb1 \
|
||||
libxcomposite1 \
|
||||
libxdamage1 \
|
||||
libxext6 \
|
||||
libxfixes3 \
|
||||
libxkbcommon0 \
|
||||
libxrandr2 \
|
||||
xdg-utils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
RUN playwright install chromium
|
||||
|
||||
COPY Scraper.py .
|
||||
COPY app.py .
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
CMD ["python", "app.py"]
|
||||
@@ -0,0 +1,121 @@
|
||||
from playwright.sync_api import sync_playwright
|
||||
import re
|
||||
import os
|
||||
import time
|
||||
|
||||
def clean_filename(filename):
|
||||
"""Removes illegal characters from filenames."""
|
||||
return re.sub(r'[<>:"/\\|?*]', '_', filename).strip()
|
||||
|
||||
def download_documents(username, password, login_url, notice_url, download_dir=None):
|
||||
"""
|
||||
Download documents from TendSign website.
|
||||
|
||||
Args:
|
||||
username: Login username
|
||||
password: Login password
|
||||
login_url: URL to login page
|
||||
notice_url: URL to notice entry page
|
||||
download_dir: Directory to save downloads (optional, creates temp dir if not provided)
|
||||
|
||||
Returns:
|
||||
List of downloaded file paths
|
||||
"""
|
||||
if download_dir is None:
|
||||
download_dir = f"TendSign_Downloads_{int(time.time())}"
|
||||
|
||||
os.makedirs(download_dir, exist_ok=True)
|
||||
print(f"[*] Downloads will be saved to: {download_dir}")
|
||||
|
||||
downloaded_files = []
|
||||
|
||||
with sync_playwright() as p:
|
||||
print("[*] Starting browser...")
|
||||
browser = p.chromium.launch(headless=True)
|
||||
context = browser.new_context(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
locale="sv-SE",
|
||||
)
|
||||
context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});")
|
||||
page = context.new_page()
|
||||
|
||||
try:
|
||||
# === 1. LOGIN ===
|
||||
print("[*] Logging in...")
|
||||
page.goto(login_url, wait_until="networkidle")
|
||||
page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000)
|
||||
page.fill('input[name="UcomLogin$txt_UserName"]', username)
|
||||
page.fill('input[name="UcomLogin$txt_Password"]', password)
|
||||
page.click('input[name="UcomLogin$btn_Submit"]')
|
||||
page.wait_for_url("**/supplier/**", timeout=30000)
|
||||
print("[SUCCESS] Logged in")
|
||||
|
||||
# === 2. NAVIGATE TO DOCUMENTS TAB ===
|
||||
print(f"[*] Navigating to notice page...")
|
||||
page.goto(notice_url, wait_until="networkidle", timeout=60000)
|
||||
|
||||
print("[*] Switching to 'Dokument' tab...")
|
||||
with page.expect_navigation(wait_until="networkidle", timeout=15000):
|
||||
page.click('a[href*="s_view_RFQfiles.aspx"]')
|
||||
|
||||
# === 3. LOCATE ALL FILE LINKS ===
|
||||
print("[*] Looking for file links starting with 'Bilaga'...")
|
||||
|
||||
FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]"
|
||||
|
||||
try:
|
||||
page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000)
|
||||
except Exception:
|
||||
raise Exception("Failed to locate any file links starting with 'Bilaga'.")
|
||||
|
||||
file_elements = page.locator(FILE_LINK_SELECTOR).all()
|
||||
|
||||
print(f"[*] Found {len(file_elements)} file links to download.")
|
||||
|
||||
if not file_elements:
|
||||
print("[!] No file links found. Exiting.")
|
||||
return downloaded_files
|
||||
|
||||
# === 4. DOWNLOAD LOOP ===
|
||||
for i, element in enumerate(file_elements):
|
||||
file_text = element.inner_text().strip()
|
||||
print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...")
|
||||
|
||||
try:
|
||||
with page.expect_download(timeout=60000) as download_info:
|
||||
element.click()
|
||||
|
||||
download = download_info.value
|
||||
|
||||
suggested_name = download.suggested_filename
|
||||
clean_name = clean_filename(suggested_name)
|
||||
save_path = os.path.join(download_dir, clean_name)
|
||||
|
||||
download.save_as(save_path)
|
||||
downloaded_files.append(save_path)
|
||||
print(f" -> Successfully saved: {clean_name}")
|
||||
|
||||
except Exception as e:
|
||||
print(f" -> [ERROR] Download failed for {file_text}: {e}")
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
print("\n[DONE] All downloads processed.")
|
||||
return downloaded_files
|
||||
|
||||
except Exception as e:
|
||||
print(f"[CRITICAL ERROR] {e}")
|
||||
page.screenshot(path=os.path.join(download_dir, "error_debug.png"))
|
||||
raise
|
||||
finally:
|
||||
browser.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Default values for backward compatibility
|
||||
USERNAME = "feed@opplenz.com"
|
||||
PASSWORD = "f38Dgma!rLn2VR3DZaBn"
|
||||
LOGIN_URL = "https://tendsign.com/login.aspx"
|
||||
NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"
|
||||
|
||||
download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL)
|
||||
@@ -0,0 +1,151 @@
|
||||
from flask import Flask, request, jsonify
|
||||
from Scraper import download_documents
|
||||
import os
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# MinIO configuration from environment variables
|
||||
MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000')
|
||||
MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
|
||||
MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
|
||||
MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
|
||||
MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents')
|
||||
|
||||
def get_minio_client():
|
||||
"""Create and return MinIO client"""
|
||||
return Minio(
|
||||
MINIO_ENDPOINT,
|
||||
access_key=MINIO_ACCESS_KEY,
|
||||
secret_key=MINIO_SECRET_KEY,
|
||||
secure=MINIO_USE_SSL
|
||||
)
|
||||
|
||||
def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET):
|
||||
"""Upload a file to MinIO"""
|
||||
try:
|
||||
client = get_minio_client()
|
||||
|
||||
# Ensure bucket exists
|
||||
if not client.bucket_exists(bucket):
|
||||
client.make_bucket(bucket)
|
||||
|
||||
# Upload file
|
||||
client.fput_object(bucket, object_name, file_path)
|
||||
return True, None
|
||||
except S3Error as e:
|
||||
return False, str(e)
|
||||
except Exception as e:
|
||||
return False, str(e)
|
||||
|
||||
@app.route('/health', methods=['GET'])
|
||||
def health():
|
||||
"""Health check endpoint"""
|
||||
return jsonify({"status": "healthy"}), 200
|
||||
|
||||
@app.route('/download', methods=['POST'])
|
||||
def download():
|
||||
"""Download documents and upload to MinIO"""
|
||||
try:
|
||||
data = request.get_json()
|
||||
|
||||
# Validate required fields
|
||||
if not data:
|
||||
return jsonify({"error": "Request body is required"}), 400
|
||||
|
||||
notice_id = data.get('notice_id')
|
||||
username = data.get('username')
|
||||
password = data.get('password')
|
||||
login_url = data.get('login_url')
|
||||
notice_url = data.get('notice_url')
|
||||
category = data.get('category', 'tenders')
|
||||
subcategory = data.get('subcategory', '')
|
||||
|
||||
if not all([notice_id, username, password, login_url, notice_url]):
|
||||
return jsonify({
|
||||
"error": "Missing required fields: notice_id, username, password, login_url, notice_url"
|
||||
}), 400
|
||||
|
||||
# Create temporary directory for downloads
|
||||
temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_")
|
||||
|
||||
try:
|
||||
# Download documents using the scraper
|
||||
downloaded_files = download_documents(
|
||||
username=username,
|
||||
password=password,
|
||||
login_url=login_url,
|
||||
notice_url=notice_url,
|
||||
download_dir=temp_dir
|
||||
)
|
||||
|
||||
if not downloaded_files:
|
||||
return jsonify({
|
||||
"error": "No files were downloaded",
|
||||
"notice_id": notice_id
|
||||
}), 404
|
||||
|
||||
# Upload each file to MinIO
|
||||
uploaded_files = []
|
||||
errors = []
|
||||
|
||||
for file_path in downloaded_files:
|
||||
try:
|
||||
filename = os.path.basename(file_path)
|
||||
# Create object name with hierarchical path
|
||||
if subcategory:
|
||||
object_name = f"{category}/{subcategory}/{notice_id}/{filename}"
|
||||
else:
|
||||
object_name = f"{category}/{notice_id}/{filename}"
|
||||
|
||||
success, error = upload_to_minio(file_path, object_name)
|
||||
|
||||
if success:
|
||||
uploaded_files.append({
|
||||
"filename": filename,
|
||||
"object_name": object_name,
|
||||
"size": os.path.getsize(file_path)
|
||||
})
|
||||
else:
|
||||
errors.append({
|
||||
"filename": filename,
|
||||
"error": error
|
||||
})
|
||||
except Exception as e:
|
||||
errors.append({
|
||||
"filename": os.path.basename(file_path),
|
||||
"error": str(e)
|
||||
})
|
||||
|
||||
# Clean up temporary directory
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
return jsonify({
|
||||
"success": True,
|
||||
"notice_id": notice_id,
|
||||
"uploaded_count": len(uploaded_files),
|
||||
"uploaded_files": uploaded_files,
|
||||
"errors": errors if errors else None
|
||||
}), 200
|
||||
|
||||
except Exception as e:
|
||||
# Clean up on error
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
return jsonify({
|
||||
"error": "Failed to download documents",
|
||||
"message": str(e),
|
||||
"notice_id": notice_id
|
||||
}), 500
|
||||
|
||||
except Exception as e:
|
||||
return jsonify({
|
||||
"error": "Internal server error",
|
||||
"message": str(e)
|
||||
}), 500
|
||||
|
||||
if __name__ == '__main__':
|
||||
port = int(os.getenv('PORT', 8000))
|
||||
app.run(host='0.0.0.0', port=port, debug=False)
|
||||
@@ -0,0 +1,45 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
scraper:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: pkg/scraper/Dockerfile
|
||||
container_name: tm-scraper
|
||||
ports:
|
||||
- "8000:8000"
|
||||
environment:
|
||||
- PORT=8000
|
||||
- MINIO_ENDPOINT=minio:9000
|
||||
- MINIO_ACCESS_KEY=minioadmin
|
||||
- MINIO_SECRET_KEY=minioadmin
|
||||
- MINIO_USE_SSL=false
|
||||
- MINIO_BUCKET=documents
|
||||
networks:
|
||||
- tm-network
|
||||
depends_on:
|
||||
- minio
|
||||
restart: unless-stopped
|
||||
|
||||
minio:
|
||||
image: minio/minio:latest
|
||||
container_name: tm-minio
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "9001:9001"
|
||||
environment:
|
||||
- MINIO_ROOT_USER=minioadmin
|
||||
- MINIO_ROOT_PASSWORD=minioadmin
|
||||
command: server /data --console-address ":9001"
|
||||
volumes:
|
||||
- minio-data:/data
|
||||
networks:
|
||||
- tm-network
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
tm-network:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
minio-data:
|
||||
@@ -0,0 +1,5 @@
|
||||
playwright==1.40.0
|
||||
flask==3.0.0
|
||||
minio==7.2.0
|
||||
requests==2.31.0
|
||||
python-dotenv==1.0.0
|
||||
Reference in New Issue
Block a user