diff --git a/cmd/web/bootstrap/config.go b/cmd/web/bootstrap/config.go index 51dbad5..f473da6 100644 --- a/cmd/web/bootstrap/config.go +++ b/cmd/web/bootstrap/config.go @@ -1,6 +1,7 @@ package bootstrap import ( + "time" "tm/pkg/config" ) @@ -14,10 +15,16 @@ type Config struct { Assets config.AssetsConfig Notification config.NotificationConfig Ollama config.OllamaConfig + Scraper ScraperConfig UserAuth AuthConfig CustomerAuth AuthConfig } +type ScraperConfig struct { + BaseURL string `env:"SCRAPER_BASE_URL"` + Timeout time.Duration `env:"SCRAPER_TIMEOUT"` +} + type AuthConfig struct { JWT JWTConfig } diff --git a/cmd/web/main.go b/cmd/web/main.go index 8711a0a..902cc93 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -103,6 +103,7 @@ import ( "tm/internal/feedback" "tm/internal/inquiry" "tm/internal/notification" + "tm/internal/scraper" "tm/internal/tender" "tm/internal/tender_approval" "tm/internal/user" @@ -179,8 +180,9 @@ func main() { notificationService := notification.NewService(notificationSDK, userService, customerService, logger) contactService := contact.NewService(contactRepository, logger) cmsService := cms.NewService(cmsRepository, logger) + scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger) logger.Info("Services initialized successfully", map[string]interface{}{ - "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"}, + "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"}, }) // Initialize handlers with services @@ -196,15 +198,16 @@ func main() { notificationHandler := notification.NewHandler(notificationService) contactHandler := contact.NewHandler(contactService) cmsHandler := cms.NewHandler(cmsService) + scraperHandler := scraper.NewHandler(scraperService) logger.Info("Handlers initialized successfully", map[string]interface{}{ - "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"}, + "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"}, }) // Initialize HTTP server e := bootstrap.InitHTTPServer(conf, logger) // Register routes - router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler) + router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler) router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler) // Start server diff --git a/cmd/web/router/routes.go b/cmd/web/router/routes.go index e7fb714..c584990 100644 --- a/cmd/web/router/routes.go +++ b/cmd/web/router/routes.go @@ -10,6 +10,7 @@ import ( "tm/internal/feedback" "tm/internal/inquiry" "tm/internal/notification" + "tm/internal/scraper" "tm/internal/tender" "tm/internal/tender_approval" "tm/internal/user" @@ -17,7 +18,7 @@ import ( "github.com/labstack/echo/v4" ) -func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) { +func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler) { adminV1 := e.Group("/admin/v1") // Admin Users Routes @@ -165,6 +166,13 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler cmsGP.PUT("/:id", cmsHandler.Update) cmsGP.DELETE("/:id", cmsHandler.Delete) } + + // Scraper Routes + scraperGP := adminV1.Group("/scraper") + { + scraperGP.Use(userHandler.AuthMiddleware()) + scraperGP.POST("/download", scraperHandler.ScrapeDocuments) + } } func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) { diff --git a/internal/scraper/form.go b/internal/scraper/form.go new file mode 100644 index 0000000..25c0452 --- /dev/null +++ b/internal/scraper/form.go @@ -0,0 +1,34 @@ +package scraper + +// ScrapeRequestForm represents the request form for scraping documents +type ScrapeRequestForm struct { + NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"` + Username string `json:"username" valid:"required,email" example:"feed@opplenz.com"` + Password string `json:"password" valid:"required,length(8|200)" example:"password123"` + LoginURL string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"` + NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"` + Category string `json:"category" valid:"optional,length(1|100)" example:"tenders"` + SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"` +} + +// ScrapeResponse represents the response from scraping operation +type ScrapeResponse struct { + Success bool `json:"success"` + NoticeID string `json:"notice_id"` + UploadedCount int `json:"uploaded_count"` + UploadedFiles []UploadedFileInfo `json:"uploaded_files"` + Errors []FileError `json:"errors,omitempty"` +} + +// UploadedFileInfo represents information about an uploaded file +type UploadedFileInfo struct { + Filename string `json:"filename"` + ObjectName string `json:"object_name"` + Size int64 `json:"size"` +} + +// FileError represents an error for a specific file +type FileError struct { + Filename string `json:"filename"` + Error string `json:"error"` +} diff --git a/internal/scraper/handler.go b/internal/scraper/handler.go new file mode 100644 index 0000000..6c7281e --- /dev/null +++ b/internal/scraper/handler.go @@ -0,0 +1,44 @@ +package scraper + +import ( + "tm/pkg/response" + + "github.com/labstack/echo/v4" +) + +// Handler handles HTTP requests for scraper operations +type Handler struct { + service Service +} + +// NewHandler creates a new scraper handler +func NewHandler(service Service) *Handler { + return &Handler{ + service: service, + } +} + +// ScrapeDocuments handles document scraping requests +// @Summary Scrape documents from external source +// @Description Scrapes documents from TendSign and uploads them to MinIO storage +// @Tags Admin-Scraper +// @Accept json +// @Produce json +// @Param request body ScrapeRequestForm true "Scraping request information" +// @Success 200 {object} response.Response{data=ScrapeResponse} +// @Failure 400 {object} response.Response +// @Failure 500 {object} response.Response +// @Router /admin/v1/scraper/download [post] +func (h *Handler) ScrapeDocuments(c echo.Context) error { + form, err := response.Parse[ScrapeRequestForm](c) + if err != nil { + return response.ValidationError(c, "Invalid request data", err.Error()) + } + + result, err := h.service.ScrapeDocuments(c.Request().Context(), form) + if err != nil { + return response.InternalServerError(c, "Failed to scrape documents") + } + + return response.Success(c, result, "Documents scraped and uploaded successfully") +} diff --git a/internal/scraper/service.go b/internal/scraper/service.go new file mode 100644 index 0000000..9896f11 --- /dev/null +++ b/internal/scraper/service.go @@ -0,0 +1,132 @@ +package scraper + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + "tm/pkg/logger" +) + +// Service defines the interface for scraper business operations +type Service interface { + // ScrapeDocuments scrapes documents and uploads them to MinIO + ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) +} + +// scraperService implements Service interface +type scraperService struct { + httpClient *http.Client + baseURL string + logger logger.Logger +} + +// NewService creates a new scraper service +func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service { + return &scraperService{ + httpClient: &http.Client{ + Timeout: timeout, + }, + baseURL: baseURL, + logger: logger, + } +} + +// ScrapeDocuments scrapes documents and uploads them to MinIO +func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) { + s.logger.Info("Starting document scraping", map[string]interface{}{ + "notice_id": form.NoticeID, + "category": form.Category, + }) + + // Prepare request body + requestBody := map[string]interface{}{ + "notice_id": form.NoticeID, + "username": form.Username, + "password": form.Password, + "login_url": form.LoginURL, + "notice_url": form.NoticeURL, + "category": form.Category, + "subcategory": form.SubCategory, + } + + bodyBytes, err := json.Marshal(requestBody) + if err != nil { + s.logger.Error("Failed to marshal request body", map[string]interface{}{ + "error": err.Error(), + }) + return nil, fmt.Errorf("failed to marshal request: %w", err) + } + + // Create HTTP request + url := fmt.Sprintf("%s/download", s.baseURL) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) + if err != nil { + s.logger.Error("Failed to create HTTP request", map[string]interface{}{ + "error": err.Error(), + }) + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + + // Send request + resp, err := s.httpClient.Do(req) + if err != nil { + s.logger.Error("Failed to send request to scraper service", map[string]interface{}{ + "error": err.Error(), + "url": url, + }) + return nil, fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + // Read response body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + s.logger.Error("Failed to read response body", map[string]interface{}{ + "error": err.Error(), + }) + return nil, fmt.Errorf("failed to read response: %w", err) + } + + // Handle error responses + if resp.StatusCode >= 400 { + var errorResp map[string]interface{} + if err := json.Unmarshal(respBody, &errorResp); err == nil { + if errorMsg, ok := errorResp["error"].(string); ok { + s.logger.Error("Scraper service returned error", map[string]interface{}{ + "status_code": resp.StatusCode, + "error": errorMsg, + }) + return nil, fmt.Errorf("scraper service error: %s", errorMsg) + } + } + s.logger.Error("Scraper service returned error", map[string]interface{}{ + "status_code": resp.StatusCode, + "body": string(respBody), + }) + return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode) + } + + // Parse success response + var scrapeResp ScrapeResponse + if err := json.Unmarshal(respBody, &scrapeResp); err != nil { + s.logger.Error("Failed to unmarshal response", map[string]interface{}{ + "error": err.Error(), + "body": string(respBody), + }) + return nil, fmt.Errorf("failed to unmarshal response: %w", err) + } + + s.logger.Info("Document scraping completed", map[string]interface{}{ + "notice_id": scrapeResp.NoticeID, + "uploaded_count": scrapeResp.UploadedCount, + "success": scrapeResp.Success, + }) + + return &scrapeResp, nil +} diff --git a/pkg/scraper/Dockerfile b/pkg/scraper/Dockerfile new file mode 100644 index 0000000..42e6b3a --- /dev/null +++ b/pkg/scraper/Dockerfile @@ -0,0 +1,43 @@ +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y \ + wget \ + gnupg \ + ca-certificates \ + fonts-liberation \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libatspi2.0-0 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxkbcommon0 \ + libxrandr2 \ + xdg-utils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +RUN playwright install chromium + +COPY Scraper.py . +COPY app.py . + +EXPOSE 8000 + +CMD ["python", "app.py"] \ No newline at end of file diff --git a/pkg/scraper/Scraper.py b/pkg/scraper/Scraper.py new file mode 100644 index 0000000..78075d6 --- /dev/null +++ b/pkg/scraper/Scraper.py @@ -0,0 +1,121 @@ +from playwright.sync_api import sync_playwright +import re +import os +import time + +def clean_filename(filename): + """Removes illegal characters from filenames.""" + return re.sub(r'[<>:"/\\|?*]', '_', filename).strip() + +def download_documents(username, password, login_url, notice_url, download_dir=None): + """ + Download documents from TendSign website. + + Args: + username: Login username + password: Login password + login_url: URL to login page + notice_url: URL to notice entry page + download_dir: Directory to save downloads (optional, creates temp dir if not provided) + + Returns: + List of downloaded file paths + """ + if download_dir is None: + download_dir = f"TendSign_Downloads_{int(time.time())}" + + os.makedirs(download_dir, exist_ok=True) + print(f"[*] Downloads will be saved to: {download_dir}") + + downloaded_files = [] + + with sync_playwright() as p: + print("[*] Starting browser...") + browser = p.chromium.launch(headless=True) + context = browser.new_context( + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36", + viewport={"width": 1920, "height": 1080}, + locale="sv-SE", + ) + context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});") + page = context.new_page() + + try: + # === 1. LOGIN === + print("[*] Logging in...") + page.goto(login_url, wait_until="networkidle") + page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000) + page.fill('input[name="UcomLogin$txt_UserName"]', username) + page.fill('input[name="UcomLogin$txt_Password"]', password) + page.click('input[name="UcomLogin$btn_Submit"]') + page.wait_for_url("**/supplier/**", timeout=30000) + print("[SUCCESS] Logged in") + + # === 2. NAVIGATE TO DOCUMENTS TAB === + print(f"[*] Navigating to notice page...") + page.goto(notice_url, wait_until="networkidle", timeout=60000) + + print("[*] Switching to 'Dokument' tab...") + with page.expect_navigation(wait_until="networkidle", timeout=15000): + page.click('a[href*="s_view_RFQfiles.aspx"]') + + # === 3. LOCATE ALL FILE LINKS === + print("[*] Looking for file links starting with 'Bilaga'...") + + FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]" + + try: + page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000) + except Exception: + raise Exception("Failed to locate any file links starting with 'Bilaga'.") + + file_elements = page.locator(FILE_LINK_SELECTOR).all() + + print(f"[*] Found {len(file_elements)} file links to download.") + + if not file_elements: + print("[!] No file links found. Exiting.") + return downloaded_files + + # === 4. DOWNLOAD LOOP === + for i, element in enumerate(file_elements): + file_text = element.inner_text().strip() + print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...") + + try: + with page.expect_download(timeout=60000) as download_info: + element.click() + + download = download_info.value + + suggested_name = download.suggested_filename + clean_name = clean_filename(suggested_name) + save_path = os.path.join(download_dir, clean_name) + + download.save_as(save_path) + downloaded_files.append(save_path) + print(f" -> Successfully saved: {clean_name}") + + except Exception as e: + print(f" -> [ERROR] Download failed for {file_text}: {e}") + + time.sleep(1) + + print("\n[DONE] All downloads processed.") + return downloaded_files + + except Exception as e: + print(f"[CRITICAL ERROR] {e}") + page.screenshot(path=os.path.join(download_dir, "error_debug.png")) + raise + finally: + browser.close() + +if __name__ == "__main__": + # Default values for backward compatibility + USERNAME = "feed@opplenz.com" + PASSWORD = "f38Dgma!rLn2VR3DZaBn" + LOGIN_URL = "https://tendsign.com/login.aspx" + NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896" + + download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL) \ No newline at end of file diff --git a/pkg/scraper/app.py b/pkg/scraper/app.py new file mode 100644 index 0000000..302e8ca --- /dev/null +++ b/pkg/scraper/app.py @@ -0,0 +1,151 @@ +from flask import Flask, request, jsonify +from Scraper import download_documents +import os +from minio import Minio +from minio.error import S3Error +import tempfile +import shutil + +app = Flask(__name__) + +# MinIO configuration from environment variables +MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000') +MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin') +MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin') +MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true' +MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents') + +def get_minio_client(): + """Create and return MinIO client""" + return Minio( + MINIO_ENDPOINT, + access_key=MINIO_ACCESS_KEY, + secret_key=MINIO_SECRET_KEY, + secure=MINIO_USE_SSL + ) + +def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET): + """Upload a file to MinIO""" + try: + client = get_minio_client() + + # Ensure bucket exists + if not client.bucket_exists(bucket): + client.make_bucket(bucket) + + # Upload file + client.fput_object(bucket, object_name, file_path) + return True, None + except S3Error as e: + return False, str(e) + except Exception as e: + return False, str(e) + +@app.route('/health', methods=['GET']) +def health(): + """Health check endpoint""" + return jsonify({"status": "healthy"}), 200 + +@app.route('/download', methods=['POST']) +def download(): + """Download documents and upload to MinIO""" + try: + data = request.get_json() + + # Validate required fields + if not data: + return jsonify({"error": "Request body is required"}), 400 + + notice_id = data.get('notice_id') + username = data.get('username') + password = data.get('password') + login_url = data.get('login_url') + notice_url = data.get('notice_url') + category = data.get('category', 'tenders') + subcategory = data.get('subcategory', '') + + if not all([notice_id, username, password, login_url, notice_url]): + return jsonify({ + "error": "Missing required fields: notice_id, username, password, login_url, notice_url" + }), 400 + + # Create temporary directory for downloads + temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_") + + try: + # Download documents using the scraper + downloaded_files = download_documents( + username=username, + password=password, + login_url=login_url, + notice_url=notice_url, + download_dir=temp_dir + ) + + if not downloaded_files: + return jsonify({ + "error": "No files were downloaded", + "notice_id": notice_id + }), 404 + + # Upload each file to MinIO + uploaded_files = [] + errors = [] + + for file_path in downloaded_files: + try: + filename = os.path.basename(file_path) + # Create object name with hierarchical path + if subcategory: + object_name = f"{category}/{subcategory}/{notice_id}/{filename}" + else: + object_name = f"{category}/{notice_id}/{filename}" + + success, error = upload_to_minio(file_path, object_name) + + if success: + uploaded_files.append({ + "filename": filename, + "object_name": object_name, + "size": os.path.getsize(file_path) + }) + else: + errors.append({ + "filename": filename, + "error": error + }) + except Exception as e: + errors.append({ + "filename": os.path.basename(file_path), + "error": str(e) + }) + + # Clean up temporary directory + shutil.rmtree(temp_dir, ignore_errors=True) + + return jsonify({ + "success": True, + "notice_id": notice_id, + "uploaded_count": len(uploaded_files), + "uploaded_files": uploaded_files, + "errors": errors if errors else None + }), 200 + + except Exception as e: + # Clean up on error + shutil.rmtree(temp_dir, ignore_errors=True) + return jsonify({ + "error": "Failed to download documents", + "message": str(e), + "notice_id": notice_id + }), 500 + + except Exception as e: + return jsonify({ + "error": "Internal server error", + "message": str(e) + }), 500 + +if __name__ == '__main__': + port = int(os.getenv('PORT', 8000)) + app.run(host='0.0.0.0', port=port, debug=False) \ No newline at end of file diff --git a/pkg/scraper/docker-compose.scraper.yml b/pkg/scraper/docker-compose.scraper.yml new file mode 100644 index 0000000..48815e5 --- /dev/null +++ b/pkg/scraper/docker-compose.scraper.yml @@ -0,0 +1,45 @@ +version: '3.8' + +services: + scraper: + build: + context: . + dockerfile: pkg/scraper/Dockerfile + container_name: tm-scraper + ports: + - "8000:8000" + environment: + - PORT=8000 + - MINIO_ENDPOINT=minio:9000 + - MINIO_ACCESS_KEY=minioadmin + - MINIO_SECRET_KEY=minioadmin + - MINIO_USE_SSL=false + - MINIO_BUCKET=documents + networks: + - tm-network + depends_on: + - minio + restart: unless-stopped + + minio: + image: minio/minio:latest + container_name: tm-minio + ports: + - "9000:9000" + - "9001:9001" + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin + command: server /data --console-address ":9001" + volumes: + - minio-data:/data + networks: + - tm-network + restart: unless-stopped + +networks: + tm-network: + driver: bridge + +volumes: + minio-data: \ No newline at end of file diff --git a/pkg/scraper/requirements.txt b/pkg/scraper/requirements.txt new file mode 100644 index 0000000..8e50a69 --- /dev/null +++ b/pkg/scraper/requirements.txt @@ -0,0 +1,5 @@ +playwright==1.40.0 +flask==3.0.0 +minio==7.2.0 +requests==2.31.0 +python-dotenv==1.0.0 \ No newline at end of file