tendsign scraper
This commit is contained in:
Mazyar
@@ -0,0 +1,34 @@
|
||||
package scraper
|
||||
|
||||
// ScrapeRequestForm represents the request form for scraping documents
|
||||
type ScrapeRequestForm struct {
|
||||
NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
|
||||
Username string `json:"username" valid:"required,email" example:"feed@opplenz.com"`
|
||||
Password string `json:"password" valid:"required,length(8|200)" example:"password123"`
|
||||
LoginURL string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"`
|
||||
NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
|
||||
Category string `json:"category" valid:"optional,length(1|100)" example:"tenders"`
|
||||
SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"`
|
||||
}
|
||||
|
||||
// ScrapeResponse represents the response from scraping operation
|
||||
type ScrapeResponse struct {
|
||||
Success bool `json:"success"`
|
||||
NoticeID string `json:"notice_id"`
|
||||
UploadedCount int `json:"uploaded_count"`
|
||||
UploadedFiles []UploadedFileInfo `json:"uploaded_files"`
|
||||
Errors []FileError `json:"errors,omitempty"`
|
||||
}
|
||||
|
||||
// UploadedFileInfo represents information about an uploaded file
|
||||
type UploadedFileInfo struct {
|
||||
Filename string `json:"filename"`
|
||||
ObjectName string `json:"object_name"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// FileError represents an error for a specific file
|
||||
type FileError struct {
|
||||
Filename string `json:"filename"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"tm/pkg/response"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
// Handler handles HTTP requests for scraper operations
|
||||
type Handler struct {
|
||||
service Service
|
||||
}
|
||||
|
||||
// NewHandler creates a new scraper handler
|
||||
func NewHandler(service Service) *Handler {
|
||||
return &Handler{
|
||||
service: service,
|
||||
}
|
||||
}
|
||||
|
||||
// ScrapeDocuments handles document scraping requests
|
||||
// @Summary Scrape documents from external source
|
||||
// @Description Scrapes documents from TendSign and uploads them to MinIO storage
|
||||
// @Tags Admin-Scraper
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body ScrapeRequestForm true "Scraping request information"
|
||||
// @Success 200 {object} response.Response{data=ScrapeResponse}
|
||||
// @Failure 400 {object} response.Response
|
||||
// @Failure 500 {object} response.Response
|
||||
// @Router /admin/v1/scraper/download [post]
|
||||
func (h *Handler) ScrapeDocuments(c echo.Context) error {
|
||||
form, err := response.Parse[ScrapeRequestForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.ScrapeDocuments(c.Request().Context(), form)
|
||||
if err != nil {
|
||||
return response.InternalServerError(c, "Failed to scrape documents")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Documents scraped and uploaded successfully")
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
"tm/pkg/logger"
|
||||
)
|
||||
|
||||
// Service defines the interface for scraper business operations
|
||||
type Service interface {
|
||||
// ScrapeDocuments scrapes documents and uploads them to MinIO
|
||||
ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
|
||||
}
|
||||
|
||||
// scraperService implements Service interface
|
||||
type scraperService struct {
|
||||
httpClient *http.Client
|
||||
baseURL string
|
||||
logger logger.Logger
|
||||
}
|
||||
|
||||
// NewService creates a new scraper service
|
||||
func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
|
||||
return &scraperService{
|
||||
httpClient: &http.Client{
|
||||
Timeout: timeout,
|
||||
},
|
||||
baseURL: baseURL,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// ScrapeDocuments scrapes documents and uploads them to MinIO
|
||||
func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
|
||||
s.logger.Info("Starting document scraping", map[string]interface{}{
|
||||
"notice_id": form.NoticeID,
|
||||
"category": form.Category,
|
||||
})
|
||||
|
||||
// Prepare request body
|
||||
requestBody := map[string]interface{}{
|
||||
"notice_id": form.NoticeID,
|
||||
"username": form.Username,
|
||||
"password": form.Password,
|
||||
"login_url": form.LoginURL,
|
||||
"notice_url": form.NoticeURL,
|
||||
"category": form.Category,
|
||||
"subcategory": form.SubCategory,
|
||||
}
|
||||
|
||||
bodyBytes, err := json.Marshal(requestBody)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to marshal request body", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to marshal request: %w", err)
|
||||
}
|
||||
|
||||
// Create HTTP request
|
||||
url := fmt.Sprintf("%s/download", s.baseURL)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to create HTTP request", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
// Send request
|
||||
resp, err := s.httpClient.Do(req)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to send request to scraper service", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"url": url,
|
||||
})
|
||||
return nil, fmt.Errorf("failed to send request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// Read response body
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to read response body", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
// Handle error responses
|
||||
if resp.StatusCode >= 400 {
|
||||
var errorResp map[string]interface{}
|
||||
if err := json.Unmarshal(respBody, &errorResp); err == nil {
|
||||
if errorMsg, ok := errorResp["error"].(string); ok {
|
||||
s.logger.Error("Scraper service returned error", map[string]interface{}{
|
||||
"status_code": resp.StatusCode,
|
||||
"error": errorMsg,
|
||||
})
|
||||
return nil, fmt.Errorf("scraper service error: %s", errorMsg)
|
||||
}
|
||||
}
|
||||
s.logger.Error("Scraper service returned error", map[string]interface{}{
|
||||
"status_code": resp.StatusCode,
|
||||
"body": string(respBody),
|
||||
})
|
||||
return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
// Parse success response
|
||||
var scrapeResp ScrapeResponse
|
||||
if err := json.Unmarshal(respBody, &scrapeResp); err != nil {
|
||||
s.logger.Error("Failed to unmarshal response", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
"body": string(respBody),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to unmarshal response: %w", err)
|
||||
}
|
||||
|
||||
s.logger.Info("Document scraping completed", map[string]interface{}{
|
||||
"notice_id": scrapeResp.NoticeID,
|
||||
"uploaded_count": scrapeResp.UploadedCount,
|
||||
"success": scrapeResp.Success,
|
||||
})
|
||||
|
||||
return &scrapeResp, nil
|
||||
}
|
||||
Reference in New Issue
Block a user