scraper sdk

2025-12-01 11:06:09 +03:30
parent d94fa34941
commit 5e777ceec8
10 changed files with 324 additions and 462 deletions
@@ -2,13 +2,8 @@ package scraper

 // ScrapeRequestForm represents the request form for scraping documents
 type ScrapeRequestForm struct {
-	NoticeID    string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
-	Username    string `json:"username" valid:"required,email" example:"feed@opplenz.com"`
-	Password    string `json:"password" valid:"required,length(8|200)" example:"password123"`
-	LoginURL    string `json:"login_url" valid:"required,url" example:"https://tendsign.com/login.aspx"`
-	NoticeURL   string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
-	Category    string `json:"category" valid:"optional,length(1|100)" example:"tenders"`
-	SubCategory string `json:"subcategory" valid:"optional,length(0|100)" example:"2024"`
+	NoticeID  string `json:"notice_id" valid:"required,length(1|100)" example:"75896"`
+	NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"`
 }

 // ScrapeResponse represents the response from scraping operation
@@ -18,6 +13,7 @@ type ScrapeResponse struct {
 	UploadedCount int                `json:"uploaded_count"`
 	UploadedFiles []UploadedFileInfo `json:"uploaded_files"`
 	Errors        []FileError        `json:"errors,omitempty"`
+	Buckets       []BucketInfo       `json:"buckets,omitempty"`
 }

 // UploadedFileInfo represents information about an uploaded file
@@ -32,3 +28,10 @@ type FileError struct {
 	Filename string `json:"filename"`
 	Error    string `json:"error"`
 }
+
+// BucketInfo represents information about a bucket
+type BucketInfo struct {
+	Name     string `json:"name"`
+	NoticeID string `json:"notice_id"`
+	Count    int    `json:"count"`
+}
@@ -1,14 +1,11 @@
 package scraper

 import (
-	"bytes"
 	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
 	"time"
+
 	"tm/pkg/logger"
+	"tm/pkg/scraper"
 )

 // Service defines the interface for scraper business operations
@@ -19,19 +16,16 @@ type Service interface {

 // scraperService implements Service interface
 type scraperService struct {
-	httpClient *http.Client
-	baseURL    string
-	logger     logger.Logger
+	sdk    scraper.SDK
+	logger logger.Logger
 }

-// NewService creates a new scraper service
-func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
+// NewService creates a new scraper service with stored credentials
+func NewService(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) Service {
+	sdk := scraper.NewClient(baseURL, timeout, logger, username, password, loginURL)
 	return &scraperService{
-		httpClient: &http.Client{
-			Timeout: timeout,
-		},
-		baseURL: baseURL,
-		logger:  logger,
+		sdk:    sdk,
+		logger: logger,
 	}
 }

@@ -39,94 +33,65 @@ func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Ser
 func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
 	s.logger.Info("Starting document scraping", map[string]interface{}{
 		"notice_id": form.NoticeID,
-		"category":  form.Category,
 	})

-	// Prepare request body
-	requestBody := map[string]interface{}{
-		"notice_id":   form.NoticeID,
-		"username":    form.Username,
-		"password":    form.Password,
-		"login_url":   form.LoginURL,
-		"notice_url":  form.NoticeURL,
-		"category":    form.Category,
-		"subcategory": form.SubCategory,
+	// Convert form to SDK request (only dynamic fields)
+	req := &scraper.ScrapeRequest{
+		NoticeID:  form.NoticeID,
+		NoticeURL: form.NoticeURL,
 	}

-	bodyBytes, err := json.Marshal(requestBody)
+	// Call SDK (credentials are already configured)
+	response, err := s.sdk.ScrapeDocuments(ctx, req)
 	if err != nil {
-		s.logger.Error("Failed to marshal request body", map[string]interface{}{
-			"error": err.Error(),
+		s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{
+			"error":     err.Error(),
+			"notice_id": form.NoticeID,
 		})
-		return nil, fmt.Errorf("failed to marshal request: %w", err)
+		return nil, err
 	}

-	// Create HTTP request
-	url := fmt.Sprintf("%s/download", s.baseURL)
-	req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
-	if err != nil {
-		s.logger.Error("Failed to create HTTP request", map[string]interface{}{
-			"error": err.Error(),
-		})
-		return nil, fmt.Errorf("failed to create request: %w", err)
+	// Convert SDK response to service response
+	serviceResp := &ScrapeResponse{
+		Success:       response.Success,
+		NoticeID:      response.NoticeID,
+		UploadedCount: response.UploadedCount,
+		UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)),
+		Errors:        make([]FileError, len(response.Errors)),
+		Buckets:       make([]BucketInfo, len(response.Buckets)),
 	}

-	req.Header.Set("Content-Type", "application/json")
-
-	// Send request
-	resp, err := s.httpClient.Do(req)
-	if err != nil {
-		s.logger.Error("Failed to send request to scraper service", map[string]interface{}{
-			"error": err.Error(),
-			"url":   url,
-		})
-		return nil, fmt.Errorf("failed to send request: %w", err)
-	}
-	defer resp.Body.Close()
-
-	// Read response body
-	respBody, err := io.ReadAll(resp.Body)
-	if err != nil {
-		s.logger.Error("Failed to read response body", map[string]interface{}{
-			"error": err.Error(),
-		})
-		return nil, fmt.Errorf("failed to read response: %w", err)
-	}
-
-	// Handle error responses
-	if resp.StatusCode >= 400 {
-		var errorResp map[string]interface{}
-		if err := json.Unmarshal(respBody, &errorResp); err == nil {
-			if errorMsg, ok := errorResp["error"].(string); ok {
-				s.logger.Error("Scraper service returned error", map[string]interface{}{
-					"status_code": resp.StatusCode,
-					"error":       errorMsg,
-				})
-				return nil, fmt.Errorf("scraper service error: %s", errorMsg)
-			}
+	// Convert uploaded files
+	for i, file := range response.UploadedFiles {
+		serviceResp.UploadedFiles[i] = UploadedFileInfo{
+			Filename:   file.Filename,
+			ObjectName: file.ObjectName,
+			Size:       file.Size,
 		}
-		s.logger.Error("Scraper service returned error", map[string]interface{}{
-			"status_code": resp.StatusCode,
-			"body":        string(respBody),
-		})
-		return nil, fmt.Errorf("scraper service returned status %d", resp.StatusCode)
 	}

-	// Parse success response
-	var scrapeResp ScrapeResponse
-	if err := json.Unmarshal(respBody, &scrapeResp); err != nil {
-		s.logger.Error("Failed to unmarshal response", map[string]interface{}{
-			"error": err.Error(),
-			"body":  string(respBody),
-		})
-		return nil, fmt.Errorf("failed to unmarshal response: %w", err)
+	// Convert errors
+	for i, err := range response.Errors {
+		serviceResp.Errors[i] = FileError{
+			Filename: err.Filename,
+			Error:    err.Error,
+		}
+	}
+
+	// Convert buckets
+	for i, bucket := range response.Buckets {
+		serviceResp.Buckets[i] = BucketInfo{
+			Name:     bucket.Name,
+			NoticeID: bucket.NoticeID,
+			Count:    bucket.Count,
+		}
 	}

 	s.logger.Info("Document scraping completed", map[string]interface{}{
-		"notice_id":      scrapeResp.NoticeID,
-		"uploaded_count": scrapeResp.UploadedCount,
-		"success":        scrapeResp.Success,
+		"notice_id":      serviceResp.NoticeID,
+		"uploaded_count": serviceResp.UploadedCount,
+		"success":        serviceResp.Success,
 	})

-	return &scrapeResp, nil
+	return serviceResp, nil
 }