tendsign scraper

cmd/web/bootstrap/config.go

@@ -1,6 +1,7 @@
 package bootstrap
 
 import (
+	"time"
 	"tm/pkg/config"
 )
 
@@ -14,10 +15,16 @@ type Config struct {
 	Assets       config.AssetsConfig
 	Notification config.NotificationConfig
 	Ollama       config.OllamaConfig
+	Scraper      ScraperConfig
 	UserAuth     AuthConfig
 	CustomerAuth AuthConfig
 }
 
+type ScraperConfig struct {
+	BaseURL string        `env:"SCRAPER_BASE_URL"`
+	Timeout time.Duration `env:"SCRAPER_TIMEOUT"`
+}
+
 type AuthConfig struct {
 	JWT JWTConfig
 }

cmd/web/main.go

@@ -103,6 +103,7 @@ import (
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/notification"
+	"tm/internal/scraper"
 	"tm/internal/tender"
 	"tm/internal/tender_approval"
 	"tm/internal/user"
@@ -179,8 +180,9 @@ func main() {
 	notificationService := notification.NewService(notificationSDK, userService, customerService, logger)
 	contactService := contact.NewService(contactRepository, logger)
 	cmsService := cms.NewService(cmsRepository, logger)
+	scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger)
 	logger.Info("Services initialized successfully", map[string]interface{}{
-		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"},
+		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
 	})
 
 	// Initialize handlers with services
@@ -196,15 +198,16 @@ func main() {
 	notificationHandler := notification.NewHandler(notificationService)
 	contactHandler := contact.NewHandler(contactService)
 	cmsHandler := cms.NewHandler(cmsService)
+	scraperHandler := scraper.NewHandler(scraperService)
 	logger.Info("Handlers initialized successfully", map[string]interface{}{
-		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"},
+		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
 	})
 
 	// Initialize HTTP server
 	e := bootstrap.InitHTTPServer(conf, logger)
 
 	// Register routes
-	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler)
+	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler)
 	router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler)
 
 	// Start server

cmd/web/router/routes.go

@@ -10,6 +10,7 @@ import (
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/notification"
+	"tm/internal/scraper"
 	"tm/internal/tender"
 	"tm/internal/tender_approval"
 	"tm/internal/user"
@@ -17,7 +18,7 @@ import (
 	"github.com/labstack/echo/v4"
 )
 
-func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) {
+func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler) {
 	adminV1 := e.Group("/admin/v1")
 
 	// Admin Users Routes
@@ -165,6 +166,13 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
 		cmsGP.PUT("/:id", cmsHandler.Update)
 		cmsGP.DELETE("/:id", cmsHandler.Delete)
 	}
+
+	// Scraper Routes
+	scraperGP := adminV1.Group("/scraper")
+	{
+		scraperGP.Use(userHandler.AuthMiddleware())
+		scraperGP.POST("/download", scraperHandler.ScrapeDocuments)
+	}
 }
 
 func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) {

internal/scraper/form.go

@@ -0,0 +1,34 @@
+package scraper
+
+// ScrapeRequestForm represents the request form for scraping documents
+type ScrapeRequestForm struct {
+	NoticeID    string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
+	Username    string `json:"username" valid:"required,email" example:"feed@opplenz.com"`
+	Password    string `json:"password" valid:"required,length(8|200)" example:"password123"`
+	LoginURL    string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"`
+	NoticeURL   string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
+	Category    string `json:"category" valid:"optional,length(1|100)" example:"tenders"`
+	SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"`
+}
+
+// ScrapeResponse represents the response from scraping operation
+type ScrapeResponse struct {
+	Success       bool               `json:"success"`
+	NoticeID      string             `json:"notice_id"`
+	UploadedCount int                `json:"uploaded_count"`
+	UploadedFiles []UploadedFileInfo `json:"uploaded_files"`
+	Errors        []FileError        `json:"errors,omitempty"`
+}
+
+// UploadedFileInfo represents information about an uploaded file
+type UploadedFileInfo struct {
+	Filename   string `json:"filename"`
+	ObjectName string `json:"object_name"`
+	Size       int64  `json:"size"`
+}
+
+// FileError represents an error for a specific file
+type FileError struct {
+	Filename string `json:"filename"`
+	Error    string `json:"error"`
+}

internal/scraper/handler.go

@@ -0,0 +1,44 @@
+package scraper
+
+import (
+	"tm/pkg/response"
+
+	"github.com/labstack/echo/v4"
+)
+
+// Handler handles HTTP requests for scraper operations
+type Handler struct {
+	service Service
+}
+
+// NewHandler creates a new scraper handler
+func NewHandler(service Service) *Handler {
+	return &Handler{
+		service: service,
+	}
+}
+
+// ScrapeDocuments handles document scraping requests
+// @Summary Scrape documents from external source
+// @Description Scrapes documents from TendSign and uploads them to MinIO storage
+// @Tags Admin-Scraper
+// @Accept json
+// @Produce json
+// @Param request body ScrapeRequestForm true "Scraping request information"
+// @Success 200 {object} response.Response{data=ScrapeResponse}
+// @Failure 400 {object} response.Response
+// @Failure 500 {object} response.Response
+// @Router /admin/v1/scraper/download [post]
+func (h *Handler) ScrapeDocuments(c echo.Context) error {
+	form, err := response.Parse[ScrapeRequestForm](c)
+	if err != nil {
+		return response.ValidationError(c, "Invalid request data", err.Error())
+	}
+
+	result, err := h.service.ScrapeDocuments(c.Request().Context(), form)
+	if err != nil {
+		return response.InternalServerError(c, "Failed to scrape documents")
+	}
+
+	return response.Success(c, result, "Documents scraped and uploaded successfully")
+}

internal/scraper/service.go

@@ -0,0 +1,132 @@
+package scraper
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+	"tm/pkg/logger"
+)
+
+// Service defines the interface for scraper business operations
+type Service interface {
+	// ScrapeDocuments scrapes documents and uploads them to MinIO
+	ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
+}
+
+// scraperService implements Service interface
+type scraperService struct {
+	httpClient *http.Client
+	baseURL    string
+	logger     logger.Logger
+}
+
+// NewService creates a new scraper service
+func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
+	return &scraperService{
+		httpClient: &http.Client{
+			Timeout: timeout,
+		},
+		baseURL: baseURL,
+		logger:  logger,
+	}
+}
+
+// ScrapeDocuments scrapes documents and uploads them to MinIO
+func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
+	s.logger.Info("Starting document scraping", map[string]interface{}{
+		"notice_id": form.NoticeID,
+		"category":  form.Category,
+	})
+
+	// Prepare request body
+	requestBody := map[string]interface{}{
+		"notice_id":   form.NoticeID,
+		"username":    form.Username,
+		"password":    form.Password,
+		"login_url":   form.LoginURL,
+		"notice_url":  form.NoticeURL,
+		"category":    form.Category,
+		"subcategory": form.SubCategory,
+	}
+
+	bodyBytes, err := json.Marshal(requestBody)
+	if err != nil {
+		s.logger.Error("Failed to marshal request body", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to marshal request: %w", err)
+	}
+
+	// Create HTTP request
+	url := fmt.Sprintf("%s/download", s.baseURL)
+	req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
+	if err != nil {
+		s.logger.Error("Failed to create HTTP request", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	// Send request
+	resp, err := s.httpClient.Do(req)
+	if err != nil {
+		s.logger.Error("Failed to send request to scraper service", map[string]interface{}{
+			"error": err.Error(),
+			"url":   url,
+		})
+		return nil, fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Read response body
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		s.logger.Error("Failed to read response body", map[string]interface{}{
+			"error": err.Error(),
+		})
+		return nil, fmt.Errorf("failed to read response: %w", err)
+	}
+
+	// Handle error responses
+	if resp.StatusCode >= 400 {
+		var errorResp map[string]interface{}
+		if err := json.Unmarshal(respBody, &errorResp); err == nil {
+			if errorMsg, ok := errorResp["error"].(string); ok {
+				s.logger.Error("Scraper service returned error", map[string]interface{}{
+					"status_code": resp.StatusCode,
+					"error":       errorMsg,
+				})
+				return nil, fmt.Errorf("scraper service error: %s", errorMsg)
+			}
+		}
+		s.logger.Error("Scraper service returned error", map[string]interface{}{
+			"status_code": resp.StatusCode,
+			"body":        string(respBody),
+		})
+		return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode)
+	}
+
+	// Parse success response
+	var scrapeResp ScrapeResponse
+	if err := json.Unmarshal(respBody, &scrapeResp); err != nil {
+		s.logger.Error("Failed to unmarshal response", map[string]interface{}{
+			"error": err.Error(),
+			"body":  string(respBody),
+		})
+		return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+	}
+
+	s.logger.Info("Document scraping completed", map[string]interface{}{
+		"notice_id":      scrapeResp.NoticeID,
+		"uploaded_count": scrapeResp.UploadedCount,
+		"success":        scrapeResp.Success,
+	})
+
+	return &scrapeResp, nil
+}

pkg/scraper/Dockerfile

@@ -0,0 +1,43 @@
+FROM python:3.11-slim
+
+RUN apt-get update && apt-get install -y \
+    wget \
+    gnupg \
+    ca-certificates \
+    fonts-liberation \
+    libasound2 \
+    libatk-bridge2.0-0 \
+    libatk1.0-0 \
+    libatspi2.0-0 \
+    libcups2 \
+    libdbus-1-3 \
+    libdrm2 \
+    libgbm1 \
+    libgtk-3-0 \
+    libnspr4 \
+    libnss3 \
+    libx11-6 \
+    libx11-xcb1 \
+    libxcb1 \
+    libxcomposite1 \
+    libxdamage1 \
+    libxext6 \
+    libxfixes3 \
+    libxkbcommon0 \
+    libxrandr2 \
+    xdg-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+RUN playwright install chromium
+
+COPY Scraper.py .
+COPY app.py .
+
+EXPOSE 8000
+
+CMD ["python", "app.py"]

pkg/scraper/Scraper.py

@@ -0,0 +1,121 @@
+from playwright.sync_api import sync_playwright
+import re
+import os
+import time
+
+def clean_filename(filename):
+    """Removes illegal characters from filenames."""
+    return re.sub(r'[<>:"/\\|?*]', '_', filename).strip()
+
+def download_documents(username, password, login_url, notice_url, download_dir=None):
+    """
+    Download documents from TendSign website.
+    
+    Args:
+        username: Login username
+        password: Login password
+        login_url: URL to login page
+        notice_url: URL to notice entry page
+        download_dir: Directory to save downloads (optional, creates temp dir if not provided)
+    
+    Returns:
+        List of downloaded file paths
+    """
+    if download_dir is None:
+        download_dir = f"TendSign_Downloads_{int(time.time())}"
+    
+    os.makedirs(download_dir, exist_ok=True)
+    print(f"[*] Downloads will be saved to: {download_dir}")
+    
+    downloaded_files = []
+    
+    with sync_playwright() as p:
+        print("[*] Starting browser...")
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context(
+            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+            viewport={"width": 1920, "height": 1080},
+            locale="sv-SE",
+        )
+        context.add_init_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined});")
+        page = context.new_page()
+
+        try:
+            # === 1. LOGIN ===
+            print("[*] Logging in...")
+            page.goto(login_url, wait_until="networkidle")
+            page.wait_for_selector('input[name="UcomLogin$txt_UserName"]', timeout=30000)
+            page.fill('input[name="UcomLogin$txt_UserName"]', username)
+            page.fill('input[name="UcomLogin$txt_Password"]', password)
+            page.click('input[name="UcomLogin$btn_Submit"]')
+            page.wait_for_url("**/supplier/**", timeout=30000)
+            print("[SUCCESS] Logged in")
+
+            # === 2. NAVIGATE TO DOCUMENTS TAB ===
+            print(f"[*] Navigating to notice page...")
+            page.goto(notice_url, wait_until="networkidle", timeout=60000)
+
+            print("[*] Switching to 'Dokument' tab...")
+            with page.expect_navigation(wait_until="networkidle", timeout=15000):
+                page.click('a[href*="s_view_RFQfiles.aspx"]')
+
+            # === 3. LOCATE ALL FILE LINKS ===
+            print("[*] Looking for file links starting with 'Bilaga'...")
+
+            FILE_LINK_SELECTOR = "//a[starts-with(text(), 'Bilaga ')]"
+
+            try:
+                page.wait_for_selector(FILE_LINK_SELECTOR, timeout=10000)
+            except Exception:
+                raise Exception("Failed to locate any file links starting with 'Bilaga'.")
+
+            file_elements = page.locator(FILE_LINK_SELECTOR).all()
+            
+            print(f"[*] Found {len(file_elements)} file links to download.")
+
+            if not file_elements:
+                print("[!] No file links found. Exiting.")
+                return downloaded_files
+
+            # === 4. DOWNLOAD LOOP ===
+            for i, element in enumerate(file_elements):
+                file_text = element.inner_text().strip()
+                print(f"[{i+1}/{len(file_elements)}] Attempting download: {file_text}...")
+
+                try:
+                    with page.expect_download(timeout=60000) as download_info:
+                        element.click()
+
+                    download = download_info.value
+                    
+                    suggested_name = download.suggested_filename
+                    clean_name = clean_filename(suggested_name)
+                    save_path = os.path.join(download_dir, clean_name)
+
+                    download.save_as(save_path)
+                    downloaded_files.append(save_path)
+                    print(f"   -> Successfully saved: {clean_name}")
+
+                except Exception as e:
+                    print(f"   -> [ERROR] Download failed for {file_text}: {e}")
+                
+                time.sleep(1)
+
+            print("\n[DONE] All downloads processed.")
+            return downloaded_files
+
+        except Exception as e:
+            print(f"[CRITICAL ERROR] {e}")
+            page.screenshot(path=os.path.join(download_dir, "error_debug.png"))
+            raise
+        finally:
+            browser.close()
+
+if __name__ == "__main__":
+    # Default values for backward compatibility
+    USERNAME = "feed@opplenz.com"
+    PASSWORD = "f38Dgma!rLn2VR3DZaBn"
+    LOGIN_URL = "https://tendsign.com/login.aspx"
+    NOTICE_ENTRY_URL = "https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"
+    
+    download_documents(USERNAME, PASSWORD, LOGIN_URL, NOTICE_ENTRY_URL)

pkg/scraper/app.py

@@ -0,0 +1,151 @@
+from flask import Flask, request, jsonify
+from Scraper import download_documents
+import os
+from minio import Minio
+from minio.error import S3Error
+import tempfile
+import shutil
+
+app = Flask(__name__)
+
+# MinIO configuration from environment variables
+MINIO_ENDPOINT = os.getenv('MINIO_ENDPOINT', 'localhost:9000')
+MINIO_ACCESS_KEY = os.getenv('MINIO_ACCESS_KEY', 'minioadmin')
+MINIO_SECRET_KEY = os.getenv('MINIO_SECRET_KEY', 'minioadmin')
+MINIO_USE_SSL = os.getenv('MINIO_USE_SSL', 'false').lower() == 'true'
+MINIO_BUCKET = os.getenv('MINIO_BUCKET', 'documents')
+
+def get_minio_client():
+    """Create and return MinIO client"""
+    return Minio(
+        MINIO_ENDPOINT,
+        access_key=MINIO_ACCESS_KEY,
+        secret_key=MINIO_SECRET_KEY,
+        secure=MINIO_USE_SSL
+    )
+
+def upload_to_minio(file_path, object_name, bucket=MINIO_BUCKET):
+    """Upload a file to MinIO"""
+    try:
+        client = get_minio_client()
+        
+        # Ensure bucket exists
+        if not client.bucket_exists(bucket):
+            client.make_bucket(bucket)
+        
+        # Upload file
+        client.fput_object(bucket, object_name, file_path)
+        return True, None
+    except S3Error as e:
+        return False, str(e)
+    except Exception as e:
+        return False, str(e)
+
+@app.route('/health', methods=['GET'])
+def health():
+    """Health check endpoint"""
+    return jsonify({"status": "healthy"}), 200
+
+@app.route('/download', methods=['POST'])
+def download():
+    """Download documents and upload to MinIO"""
+    try:
+        data = request.get_json()
+        
+        # Validate required fields
+        if not data:
+            return jsonify({"error": "Request body is required"}), 400
+        
+        notice_id = data.get('notice_id')
+        username = data.get('username')
+        password = data.get('password')
+        login_url = data.get('login_url')
+        notice_url = data.get('notice_url')
+        category = data.get('category', 'tenders')
+        subcategory = data.get('subcategory', '')
+        
+        if not all([notice_id, username, password, login_url, notice_url]):
+            return jsonify({
+                "error": "Missing required fields: notice_id, username, password, login_url, notice_url"
+            }), 400
+        
+        # Create temporary directory for downloads
+        temp_dir = tempfile.mkdtemp(prefix=f"scraper_{notice_id}_")
+        
+        try:
+            # Download documents using the scraper
+            downloaded_files = download_documents(
+                username=username,
+                password=password,
+                login_url=login_url,
+                notice_url=notice_url,
+                download_dir=temp_dir
+            )
+            
+            if not downloaded_files:
+                return jsonify({
+                    "error": "No files were downloaded",
+                    "notice_id": notice_id
+                }), 404
+            
+            # Upload each file to MinIO
+            uploaded_files = []
+            errors = []
+            
+            for file_path in downloaded_files:
+                try:
+                    filename = os.path.basename(file_path)
+                    # Create object name with hierarchical path
+                    if subcategory:
+                        object_name = f"{category}/{subcategory}/{notice_id}/{filename}"
+                    else:
+                        object_name = f"{category}/{notice_id}/{filename}"
+                    
+                    success, error = upload_to_minio(file_path, object_name)
+                    
+                    if success:
+                        uploaded_files.append({
+                            "filename": filename,
+                            "object_name": object_name,
+                            "size": os.path.getsize(file_path)
+                        })
+                    else:
+                        errors.append({
+                            "filename": filename,
+                            "error": error
+                        })
+                except Exception as e:
+                    errors.append({
+                        "filename": os.path.basename(file_path),
+                        "error": str(e)
+                    })
+            
+            # Clean up temporary directory
+            shutil.rmtree(temp_dir, ignore_errors=True)
+            
+            return jsonify({
+                "success": True,
+                "notice_id": notice_id,
+                "uploaded_count": len(uploaded_files),
+                "uploaded_files": uploaded_files,
+                "errors": errors if errors else None
+            }), 200
+            
+        except Exception as e:
+            # Clean up on error
+            shutil.rmtree(temp_dir, ignore_errors=True)
+            return jsonify({
+                "error": "Failed to download documents",
+                "message": str(e),
+                "notice_id": notice_id
+            }), 500
+            
+    except Exception as e:
+        return jsonify({
+            "error": "Internal server error",
+            "message": str(e)
+        }), 500
+
+if __name__ == '__main__':
+    port = int(os.getenv('PORT', 8000))
+    app.run(host='0.0.0.0', port=port, debug=False)

pkg/scraper/docker-compose.scraper.yml

@@ -0,0 +1,45 @@
+version: '3.8'
+
+services:
+  scraper:
+    build:
+      context: .
+      dockerfile: pkg/scraper/Dockerfile
+    container_name: tm-scraper
+    ports:
+      - "8000:8000"
+    environment:
+      - PORT=8000
+      - MINIO_ENDPOINT=minio:9000
+      - MINIO_ACCESS_KEY=minioadmin
+      - MINIO_SECRET_KEY=minioadmin
+      - MINIO_USE_SSL=false
+      - MINIO_BUCKET=documents
+    networks:
+      - tm-network
+    depends_on:
+      - minio
+    restart: unless-stopped
+
+  minio:
+    image: minio/minio:latest
+    container_name: tm-minio
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin
+    command: server /data --console-address ":9001"
+    volumes:
+      - minio-data:/data
+    networks:
+      - tm-network
+    restart: unless-stopped
+
+networks:
+  tm-network:
+    driver: bridge
+
+volumes:
+  minio-data:

pkg/scraper/requirements.txt

@@ -0,0 +1,5 @@
+playwright==1.40.0
+flask==3.0.0
+minio==7.2.0
+requests==2.31.0
+python-dotenv==1.0.0