Merge pull request 'tendsign scraper' (#8) from scraper into develop

Reviewed-on: https://repo.ravanertebat.com/TM/tm_back/pulls/8
Reviewed-by: Nima Nakhsotin <n.nakhostin@ravanertebat.com>

cmd/web/bootstrap/config.go

@@ -1,6 +1,7 @@
 package bootstrap
 
 import (
+	"time"
 	"tm/pkg/config"
 )
 
@@ -14,10 +15,19 @@ type Config struct {
 	Assets       config.AssetsConfig
 	Notification config.NotificationConfig
 	Ollama       config.OllamaConfig
+	Scraper      ScraperConfig
 	UserAuth     AuthConfig
 	CustomerAuth AuthConfig
 }
 
+type ScraperConfig struct {
+	BaseURL  string        `env:"SCRAPER_BASE_URL"`
+	Timeout  time.Duration `env:"SCRAPER_TIMEOUT"`
+	Username string        `env:"SCRAPER_USERNAME"`
+	Password string        `env:"SCRAPER_PASSWORD"`
+	LoginURL string        `env:"SCRAPER_LOGIN_URL"`
+}
+
 type AuthConfig struct {
 	JWT JWTConfig
 }

cmd/web/main.go

@@ -103,6 +103,7 @@ import (
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/notification"
+	"tm/internal/scraper"
 	"tm/internal/tender"
 	"tm/internal/tender_approval"
 	"tm/internal/user"
@@ -179,8 +180,9 @@ func main() {
 	notificationService := notification.NewService(notificationSDK, userService, customerService, logger)
 	contactService := contact.NewService(contactRepository, logger)
 	cmsService := cms.NewService(cmsRepository, logger)
+	scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, conf.Scraper.Username, conf.Scraper.Password, conf.Scraper.LoginURL)
 	logger.Info("Services initialized successfully", map[string]interface{}{
-		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"},
+		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
 	})
 
 	// Initialize handlers with services
@@ -196,15 +198,16 @@ func main() {
 	notificationHandler := notification.NewHandler(notificationService)
 	contactHandler := contact.NewHandler(contactService)
 	cmsHandler := cms.NewHandler(cmsService)
+	scraperHandler := scraper.NewHandler(scraperService)
 	logger.Info("Handlers initialized successfully", map[string]interface{}{
-		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"},
+		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
 	})
 
 	// Initialize HTTP server
 	e := bootstrap.InitHTTPServer(conf, logger)
 
 	// Register routes
-	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler)
+	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler)
 	router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler)
 
 	// Start server

cmd/web/router/routes.go

@@ -10,6 +10,7 @@ import (
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/notification"
+	"tm/internal/scraper"
 	"tm/internal/tender"
 	"tm/internal/tender_approval"
 	"tm/internal/user"
@@ -17,7 +18,7 @@ import (
 	"github.com/labstack/echo/v4"
 )
 
-func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) {
+func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler) {
 	adminV1 := e.Group("/admin/v1")
 
 	// Admin Users Routes
@@ -165,6 +166,13 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
 		cmsGP.PUT("/:id", cmsHandler.Update)
 		cmsGP.DELETE("/:id", cmsHandler.Delete)
 	}
+
+	// Scraper Routes
+	scraperGP := adminV1.Group("/scraper")
+	{
+		scraperGP.Use(userHandler.AuthMiddleware())
+		scraperGP.POST("/download", scraperHandler.ScrapeDocuments)
+	}
 }
 
 func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) {

internal/scraper/form.go

@@ -0,0 +1,37 @@
+package scraper
+
+// ScrapeRequestForm represents the request form for scraping documents
+type ScrapeRequestForm struct {
+	NoticeID  string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
+	NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
+}
+
+// ScrapeResponse represents the response from scraping operation
+type ScrapeResponse struct {
+	Success       bool               `json:"success"`
+	NoticeID      string             `json:"notice_id"`
+	UploadedCount int                `json:"uploaded_count"`
+	UploadedFiles []UploadedFileInfo `json:"uploaded_files"`
+	Errors        []FileError        `json:"errors,omitempty"`
+	Buckets       []BucketInfo       `json:"buckets,omitempty"`
+}
+
+// UploadedFileInfo represents information about an uploaded file
+type UploadedFileInfo struct {
+	Filename   string `json:"filename"`
+	ObjectName string `json:"object_name"`
+	Size       int64  `json:"size"`
+}
+
+// FileError represents an error for a specific file
+type FileError struct {
+	Filename string `json:"filename"`
+	Error    string `json:"error"`
+}
+
+// BucketInfo represents information about a bucket
+type BucketInfo struct {
+	Name     string `json:"name"`
+	NoticeID string `json:"notice_id"`
+	Count    int    `json:"count"`
+}

internal/scraper/handler.go

@@ -0,0 +1,44 @@
+package scraper
+
+import (
+	"tm/pkg/response"
+
+	"github.com/labstack/echo/v4"
+)
+
+// Handler handles HTTP requests for scraper operations
+type Handler struct {
+	service Service
+}
+
+// NewHandler creates a new scraper handler
+func NewHandler(service Service) *Handler {
+	return &Handler{
+		service: service,
+	}
+}
+
+// ScrapeDocuments handles document scraping requests
+// @Summary Scrape documents from external source
+// @Description Scrapes documents from TendSign and uploads them to MinIO storage
+// @Tags Admin-Scraper
+// @Accept json
+// @Produce json
+// @Param request body ScrapeRequestForm true "Scraping request information"
+// @Success 200 {object} response.Response{data=ScrapeResponse}
+// @Failure 400 {object} response.Response
+// @Failure 500 {object} response.Response
+// @Router /admin/v1/scraper/download [post]
+func (h *Handler) ScrapeDocuments(c echo.Context) error {
+	form, err := response.Parse[ScrapeRequestForm](c)
+	if err != nil {
+		return response.ValidationError(c, "Invalid request data", err.Error())
+	}
+
+	result, err := h.service.ScrapeDocuments(c.Request().Context(), form)
+	if err != nil {
+		return response.InternalServerError(c, "Failed to scrape documents")
+	}
+
+	return response.Success(c, result, "Documents scraped and uploaded successfully")
+}

internal/scraper/service.go

@@ -0,0 +1,97 @@
+package scraper
+
+import (
+	"context"
+	"time"
+
+	"tm/pkg/logger"
+	"tm/pkg/scraper"
+)
+
+// Service defines the interface for scraper business operations
+type Service interface {
+	// ScrapeDocuments scrapes documents and uploads them to MinIO
+	ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
+}
+
+// scraperService implements Service interface
+type scraperService struct {
+	sdk    scraper.SDK
+	logger logger.Logger
+}
+
+// NewService creates a new scraper service with stored credentials
+func NewService(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) Service {
+	sdk := scraper.NewClient(baseURL, timeout, logger, username, password, loginURL)
+	return &scraperService{
+		sdk:    sdk,
+		logger: logger,
+	}
+}
+
+// ScrapeDocuments scrapes documents and uploads them to MinIO
+func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
+	s.logger.Info("Starting document scraping", map[string]interface{}{
+		"notice_id": form.NoticeID,
+	})
+
+	// Convert form to SDK request (only dynamic fields)
+	req := &scraper.ScrapeRequest{
+		NoticeID:  form.NoticeID,
+		NoticeURL: form.NoticeURL,
+	}
+
+	// Call SDK (credentials are already configured)
+	response, err := s.sdk.ScrapeDocuments(ctx, req)
+	if err != nil {
+		s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{
+			"error":     err.Error(),
+			"notice_id": form.NoticeID,
+		})
+		return nil, err
+	}
+
+	// Convert SDK response to service response
+	serviceResp := &ScrapeResponse{
+		Success:       response.Success,
+		NoticeID:      response.NoticeID,
+		UploadedCount: response.UploadedCount,
+		UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)),
+		Errors:        make([]FileError, len(response.Errors)),
+		Buckets:       make([]BucketInfo, len(response.Buckets)),
+	}
+
+	// Convert uploaded files
+	for i, file := range response.UploadedFiles {
+		serviceResp.UploadedFiles[i] = UploadedFileInfo{
+			Filename:   file.Filename,
+			ObjectName: file.ObjectName,
+			Size:       file.Size,
+		}
+	}
+
+	// Convert errors
+	for i, err := range response.Errors {
+		serviceResp.Errors[i] = FileError{
+			Filename: err.Filename,
+			Error:    err.Error,
+		}
+	}
+
+	// Convert buckets
+	for i, bucket := range response.Buckets {
+		serviceResp.Buckets[i] = BucketInfo{
+			Name:     bucket.Name,
+			NoticeID: bucket.NoticeID,
+			Count:    bucket.Count,
+		}
+	}
+
+	s.logger.Info("Document scraping completed", map[string]interface{}{
+		"notice_id":      serviceResp.NoticeID,
+		"uploaded_count": serviceResp.UploadedCount,
+		"success":        serviceResp.Success,
+	})
+
+	return serviceResp, nil
+}

pkg/scraper/sdk.go

@@ -0,0 +1,256 @@
+package scraper
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"tm/pkg/logger"
+)
+
+// SDK represents the scraper SDK interface
+type SDK interface {
+	// ScrapeDocuments scrapes documents for a given notice
+	ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error)
+
+	// ListBuckets lists all buckets with notice IDs
+	ListBuckets(ctx context.Context) (*ListBucketsResponse, error)
+
+	// HealthCheck checks if the scraper service is healthy
+	HealthCheck(ctx context.Context) error
+}
+
+// ScrapeRequest represents a request to scrape documents (only dynamic fields)
+type ScrapeRequest struct {
+	NoticeID  string `json:"notice_id"`
+	NoticeURL string `json:"notice_url"`
+}
+
+// ScrapeResponse represents the response from scraping operation
+type ScrapeResponse struct {
+	Success       bool           `json:"success"`
+	NoticeID      string         `json:"notice_id"`
+	UploadedCount int            `json:"uploaded_count"`
+	UploadedFiles []UploadedFile `json:"uploaded_files"`
+	Errors        []FileError    `json:"errors,omitempty"`
+	Buckets       []BucketInfo   `json:"buckets,omitempty"`
+}
+
+// UploadedFile represents information about an uploaded file
+type UploadedFile struct {
+	Filename   string `json:"filename"`
+	ObjectName string `json:"object_name"`
+	Size       int64  `json:"size"`
+}
+
+// FileError represents an error for a specific file
+type FileError struct {
+	Filename string `json:"filename"`
+	Error    string `json:"error"`
+}
+
+// BucketInfo represents information about a bucket
+type BucketInfo struct {
+	Name     string `json:"name"`
+	NoticeID string `json:"notice_id"`
+	Count    int    `json:"count"`
+}
+
+// ListBucketsResponse represents the response from listing buckets
+type ListBucketsResponse struct {
+	Buckets []BucketInfo `json:"buckets"`
+	Total   int          `json:"total"`
+}
+
+// Client implements the SDK interface
+type Client struct {
+	httpClient *http.Client
+	baseURL    string
+	logger     logger.Logger
+	// Stored credentials
+	username string
+	password string
+	loginURL string
+}
+
+// NewClient creates a new scraper SDK client with stored credentials
+func NewClient(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) SDK {
+	return &Client{
+		httpClient: &http.Client{
+			Timeout: timeout,
+		},
+		baseURL:  baseURL,
+		logger:   logger,
+		username: username,
+		password: password,
+		loginURL: loginURL,
+	}
+}
+
+// ScrapeDocuments scrapes documents and uploads them to MinIO
+func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) {
+	c.logger.Info("Starting document scraping via SDK", map[string]interface{}{
+		"notice_id": req.NoticeID,
+		"base_url":  c.baseURL,
+	})
+
+	// Prepare request body with stored credentials + dynamic fields
+	requestBody := map[string]interface{}{
+		"notice_id":  req.NoticeID,
+		"username":   c.username,
+		"password":   c.password,
+		"login_url":  c.loginURL,
+		"notice_url": req.NoticeURL,
+	}
+
+	// Use unified handler API
+	var response ScrapeResponse
+	err := c.makeRequest(ctx, "POST", "/download", requestBody, &response)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get buckets list after successful scraping
+	bucketsResp, err := c.ListBuckets(ctx)
+	if err != nil {
+		c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{
+			"error": err.Error(),
+		})
+	} else {
+		response.Buckets = bucketsResp.Buckets
+	}
+
+	c.logger.Info("Document scraping completed via SDK", map[string]interface{}{
+		"notice_id":      response.NoticeID,
+		"uploaded_count": response.UploadedCount,
+		"success":        response.Success,
+	})
+
+	return &response, nil
+}
+
+// ListBuckets lists all buckets with notice IDs
+func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) {
+	var response ListBucketsResponse
+	err := c.makeRequest(ctx, "GET", "/buckets", nil, &response)
+	return &response, err
+}
+
+// HealthCheck checks if the scraper service is healthy
+func (c *Client) HealthCheck(ctx context.Context) error {
+	var response map[string]interface{}
+	err := c.makeRequest(ctx, "GET", "/health", nil, &response)
+	if err != nil {
+		return err
+	}
+
+	// Health check doesn't need to return data, just check if request succeeded
+	return nil
+}
+
+// Unified HTTP Handler API
+// makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling
+func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error {
+	// Build full URL
+	url := fmt.Sprintf("%s%s", c.baseURL, endpoint)
+
+	// Marshal request body to JSON if provided
+	var bodyReader io.Reader
+	if requestBody != nil {
+		jsonData, err := json.Marshal(requestBody)
+		if err != nil {
+			c.logger.Error("Failed to marshal request body", map[string]interface{}{
+				"error":    err.Error(),
+				"endpoint": endpoint,
+			})
+			return fmt.Errorf("failed to marshal request: %w", err)
+		}
+		bodyReader = bytes.NewReader(jsonData)
+	}
+
+	// Create HTTP request
+	req, err := http.NewRequestWithContext(ctx, method, url, bodyReader)
+	if err != nil {
+		c.logger.Error("Failed to create HTTP request", map[string]interface{}{
+			"error":  err.Error(),
+			"method": method,
+			"url":    url,
+		})
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+
+	// Set headers
+	if requestBody != nil {
+		req.Header.Set("Content-Type", "application/json")
+	}
+	req.Header.Set("Accept", "application/json")
+
+	// Send request
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		c.logger.Error("Failed to send HTTP request", map[string]interface{}{
+			"error":  err.Error(),
+			"method": method,
+			"url":    url,
+		})
+		return fmt.Errorf("failed to send request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	// Read response body
+	respBody, err := io.ReadAll(resp.Body)
+	if err != nil {
+		c.logger.Error("Failed to read response body", map[string]interface{}{
+			"error": err.Error(),
+			"url":   url,
+		})
+		return fmt.Errorf("failed to read response: %w", err)
+	}
+
+	// Handle error responses
+	if resp.StatusCode >= 400 {
+		// Try to parse error response
+		var errorResp map[string]interface{}
+		if err := json.Unmarshal(respBody, &errorResp); err == nil {
+			if errorMsg, ok := errorResp["error"].(string); ok {
+				c.logger.Error("Request failed with error response", map[string]interface{}{
+					"status_code": resp.StatusCode,
+					"error":       errorMsg,
+					"url":         url,
+				})
+				return fmt.Errorf("request failed: %s", errorMsg)
+			}
+		}
+
+		c.logger.Error("Request failed with status code", map[string]interface{}{
+			"status_code": resp.StatusCode,
+			"url":         url,
+			"body":        string(respBody),
+		})
+		return fmt.Errorf("request failed with status %d", resp.StatusCode)
+	}
+
+	// Handle successful responses
+	if responseBody != nil {
+		if err := json.Unmarshal(respBody, responseBody); err != nil {
+			c.logger.Error("Failed to unmarshal response", map[string]interface{}{
+				"error": err.Error(),
+				"url":   url,
+				"body":  string(respBody),
+			})
+			return fmt.Errorf("failed to unmarshal response: %w", err)
+		}
+	}
+
+	c.logger.Debug("Request completed successfully", map[string]interface{}{
+		"method":      method,
+		"url":         url,
+		"status_code": resp.StatusCode,
+	})
+
+	return nil
+}