diff --git a/cmd/web/bootstrap/config.go b/cmd/web/bootstrap/config.go index 51dbad5..3bdef7f 100644 --- a/cmd/web/bootstrap/config.go +++ b/cmd/web/bootstrap/config.go @@ -1,6 +1,7 @@ package bootstrap import ( + "time" "tm/pkg/config" ) @@ -14,10 +15,19 @@ type Config struct { Assets config.AssetsConfig Notification config.NotificationConfig Ollama config.OllamaConfig + Scraper ScraperConfig UserAuth AuthConfig CustomerAuth AuthConfig } +type ScraperConfig struct { + BaseURL string `env:"SCRAPER_BASE_URL"` + Timeout time.Duration `env:"SCRAPER_TIMEOUT"` + Username string `env:"SCRAPER_USERNAME"` + Password string `env:"SCRAPER_PASSWORD"` + LoginURL string `env:"SCRAPER_LOGIN_URL"` +} + type AuthConfig struct { JWT JWTConfig } diff --git a/cmd/web/main.go b/cmd/web/main.go index 8711a0a..f5b7db2 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -103,6 +103,7 @@ import ( "tm/internal/feedback" "tm/internal/inquiry" "tm/internal/notification" + "tm/internal/scraper" "tm/internal/tender" "tm/internal/tender_approval" "tm/internal/user" @@ -179,8 +180,9 @@ func main() { notificationService := notification.NewService(notificationSDK, userService, customerService, logger) contactService := contact.NewService(contactRepository, logger) cmsService := cms.NewService(cmsRepository, logger) + scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, conf.Scraper.Username, conf.Scraper.Password, conf.Scraper.LoginURL) logger.Info("Services initialized successfully", map[string]interface{}{ - "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"}, + "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"}, }) // Initialize handlers with services @@ -196,15 +198,16 @@ func main() { notificationHandler := notification.NewHandler(notificationService) contactHandler := contact.NewHandler(contactService) cmsHandler := cms.NewHandler(cmsService) + scraperHandler := scraper.NewHandler(scraperService) logger.Info("Handlers initialized successfully", map[string]interface{}{ - "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"}, + "handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"}, }) // Initialize HTTP server e := bootstrap.InitHTTPServer(conf, logger) // Register routes - router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler) + router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler) router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler) // Start server diff --git a/cmd/web/router/routes.go b/cmd/web/router/routes.go index e7fb714..c584990 100644 --- a/cmd/web/router/routes.go +++ b/cmd/web/router/routes.go @@ -10,6 +10,7 @@ import ( "tm/internal/feedback" "tm/internal/inquiry" "tm/internal/notification" + "tm/internal/scraper" "tm/internal/tender" "tm/internal/tender_approval" "tm/internal/user" @@ -17,7 +18,7 @@ import ( "github.com/labstack/echo/v4" ) -func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) { +func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler) { adminV1 := e.Group("/admin/v1") // Admin Users Routes @@ -165,6 +166,13 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler cmsGP.PUT("/:id", cmsHandler.Update) cmsGP.DELETE("/:id", cmsHandler.Delete) } + + // Scraper Routes + scraperGP := adminV1.Group("/scraper") + { + scraperGP.Use(userHandler.AuthMiddleware()) + scraperGP.POST("/download", scraperHandler.ScrapeDocuments) + } } func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) { diff --git a/internal/scraper/form.go b/internal/scraper/form.go new file mode 100644 index 0000000..3c0a5b7 --- /dev/null +++ b/internal/scraper/form.go @@ -0,0 +1,37 @@ +package scraper + +// ScrapeRequestForm represents the request form for scraping documents +type ScrapeRequestForm struct { + NoticeID string `json:"notice_id" valid:"required,length(1|100)" example:"75896"` + NoticeURL string `json:"notice_url" valid:"required,url" example:"https://tendsign.com/supplier/s_meformsnotice.aspx?MeFormsNoticeId=75896"` +} + +// ScrapeResponse represents the response from scraping operation +type ScrapeResponse struct { + Success bool `json:"success"` + NoticeID string `json:"notice_id"` + UploadedCount int `json:"uploaded_count"` + UploadedFiles []UploadedFileInfo `json:"uploaded_files"` + Errors []FileError `json:"errors,omitempty"` + Buckets []BucketInfo `json:"buckets,omitempty"` +} + +// UploadedFileInfo represents information about an uploaded file +type UploadedFileInfo struct { + Filename string `json:"filename"` + ObjectName string `json:"object_name"` + Size int64 `json:"size"` +} + +// FileError represents an error for a specific file +type FileError struct { + Filename string `json:"filename"` + Error string `json:"error"` +} + +// BucketInfo represents information about a bucket +type BucketInfo struct { + Name string `json:"name"` + NoticeID string `json:"notice_id"` + Count int `json:"count"` +} diff --git a/internal/scraper/handler.go b/internal/scraper/handler.go new file mode 100644 index 0000000..6c7281e --- /dev/null +++ b/internal/scraper/handler.go @@ -0,0 +1,44 @@ +package scraper + +import ( + "tm/pkg/response" + + "github.com/labstack/echo/v4" +) + +// Handler handles HTTP requests for scraper operations +type Handler struct { + service Service +} + +// NewHandler creates a new scraper handler +func NewHandler(service Service) *Handler { + return &Handler{ + service: service, + } +} + +// ScrapeDocuments handles document scraping requests +// @Summary Scrape documents from external source +// @Description Scrapes documents from TendSign and uploads them to MinIO storage +// @Tags Admin-Scraper +// @Accept json +// @Produce json +// @Param request body ScrapeRequestForm true "Scraping request information" +// @Success 200 {object} response.Response{data=ScrapeResponse} +// @Failure 400 {object} response.Response +// @Failure 500 {object} response.Response +// @Router /admin/v1/scraper/download [post] +func (h *Handler) ScrapeDocuments(c echo.Context) error { + form, err := response.Parse[ScrapeRequestForm](c) + if err != nil { + return response.ValidationError(c, "Invalid request data", err.Error()) + } + + result, err := h.service.ScrapeDocuments(c.Request().Context(), form) + if err != nil { + return response.InternalServerError(c, "Failed to scrape documents") + } + + return response.Success(c, result, "Documents scraped and uploaded successfully") +} diff --git a/internal/scraper/service.go b/internal/scraper/service.go new file mode 100644 index 0000000..26f925e --- /dev/null +++ b/internal/scraper/service.go @@ -0,0 +1,97 @@ +package scraper + +import ( + "context" + "time" + + "tm/pkg/logger" + "tm/pkg/scraper" +) + +// Service defines the interface for scraper business operations +type Service interface { + // ScrapeDocuments scrapes documents and uploads them to MinIO + ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) +} + +// scraperService implements Service interface +type scraperService struct { + sdk scraper.SDK + logger logger.Logger +} + +// NewService creates a new scraper service with stored credentials +func NewService(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) Service { + sdk := scraper.NewClient(baseURL, timeout, logger, username, password, loginURL) + return &scraperService{ + sdk: sdk, + logger: logger, + } +} + +// ScrapeDocuments scrapes documents and uploads them to MinIO +func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) { + s.logger.Info("Starting document scraping", map[string]interface{}{ + "notice_id": form.NoticeID, + }) + + // Convert form to SDK request (only dynamic fields) + req := &scraper.ScrapeRequest{ + NoticeID: form.NoticeID, + NoticeURL: form.NoticeURL, + } + + // Call SDK (credentials are already configured) + response, err := s.sdk.ScrapeDocuments(ctx, req) + if err != nil { + s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{ + "error": err.Error(), + "notice_id": form.NoticeID, + }) + return nil, err + } + + // Convert SDK response to service response + serviceResp := &ScrapeResponse{ + Success: response.Success, + NoticeID: response.NoticeID, + UploadedCount: response.UploadedCount, + UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)), + Errors: make([]FileError, len(response.Errors)), + Buckets: make([]BucketInfo, len(response.Buckets)), + } + + // Convert uploaded files + for i, file := range response.UploadedFiles { + serviceResp.UploadedFiles[i] = UploadedFileInfo{ + Filename: file.Filename, + ObjectName: file.ObjectName, + Size: file.Size, + } + } + + // Convert errors + for i, err := range response.Errors { + serviceResp.Errors[i] = FileError{ + Filename: err.Filename, + Error: err.Error, + } + } + + // Convert buckets + for i, bucket := range response.Buckets { + serviceResp.Buckets[i] = BucketInfo{ + Name: bucket.Name, + NoticeID: bucket.NoticeID, + Count: bucket.Count, + } + } + + s.logger.Info("Document scraping completed", map[string]interface{}{ + "notice_id": serviceResp.NoticeID, + "uploaded_count": serviceResp.UploadedCount, + "success": serviceResp.Success, + }) + + return serviceResp, nil +} diff --git a/pkg/scraper/sdk.go b/pkg/scraper/sdk.go new file mode 100644 index 0000000..ef92e86 --- /dev/null +++ b/pkg/scraper/sdk.go @@ -0,0 +1,256 @@ +package scraper + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + + "tm/pkg/logger" +) + +// SDK represents the scraper SDK interface +type SDK interface { + // ScrapeDocuments scrapes documents for a given notice + ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) + + // ListBuckets lists all buckets with notice IDs + ListBuckets(ctx context.Context) (*ListBucketsResponse, error) + + // HealthCheck checks if the scraper service is healthy + HealthCheck(ctx context.Context) error +} + +// ScrapeRequest represents a request to scrape documents (only dynamic fields) +type ScrapeRequest struct { + NoticeID string `json:"notice_id"` + NoticeURL string `json:"notice_url"` +} + +// ScrapeResponse represents the response from scraping operation +type ScrapeResponse struct { + Success bool `json:"success"` + NoticeID string `json:"notice_id"` + UploadedCount int `json:"uploaded_count"` + UploadedFiles []UploadedFile `json:"uploaded_files"` + Errors []FileError `json:"errors,omitempty"` + Buckets []BucketInfo `json:"buckets,omitempty"` +} + +// UploadedFile represents information about an uploaded file +type UploadedFile struct { + Filename string `json:"filename"` + ObjectName string `json:"object_name"` + Size int64 `json:"size"` +} + +// FileError represents an error for a specific file +type FileError struct { + Filename string `json:"filename"` + Error string `json:"error"` +} + +// BucketInfo represents information about a bucket +type BucketInfo struct { + Name string `json:"name"` + NoticeID string `json:"notice_id"` + Count int `json:"count"` +} + +// ListBucketsResponse represents the response from listing buckets +type ListBucketsResponse struct { + Buckets []BucketInfo `json:"buckets"` + Total int `json:"total"` +} + +// Client implements the SDK interface +type Client struct { + httpClient *http.Client + baseURL string + logger logger.Logger + // Stored credentials + username string + password string + loginURL string +} + +// NewClient creates a new scraper SDK client with stored credentials +func NewClient(baseURL string, timeout time.Duration, logger logger.Logger, username, password, loginURL string) SDK { + return &Client{ + httpClient: &http.Client{ + Timeout: timeout, + }, + baseURL: baseURL, + logger: logger, + username: username, + password: password, + loginURL: loginURL, + } +} + +// ScrapeDocuments scrapes documents and uploads them to MinIO +func (c *Client) ScrapeDocuments(ctx context.Context, req *ScrapeRequest) (*ScrapeResponse, error) { + c.logger.Info("Starting document scraping via SDK", map[string]interface{}{ + "notice_id": req.NoticeID, + "base_url": c.baseURL, + }) + + // Prepare request body with stored credentials + dynamic fields + requestBody := map[string]interface{}{ + "notice_id": req.NoticeID, + "username": c.username, + "password": c.password, + "login_url": c.loginURL, + "notice_url": req.NoticeURL, + } + + // Use unified handler API + var response ScrapeResponse + err := c.makeRequest(ctx, "POST", "/download", requestBody, &response) + if err != nil { + return nil, err + } + + // Get buckets list after successful scraping + bucketsResp, err := c.ListBuckets(ctx) + if err != nil { + c.logger.Warn("Failed to list buckets after scraping", map[string]interface{}{ + "error": err.Error(), + }) + } else { + response.Buckets = bucketsResp.Buckets + } + + c.logger.Info("Document scraping completed via SDK", map[string]interface{}{ + "notice_id": response.NoticeID, + "uploaded_count": response.UploadedCount, + "success": response.Success, + }) + + return &response, nil +} + +// ListBuckets lists all buckets with notice IDs +func (c *Client) ListBuckets(ctx context.Context) (*ListBucketsResponse, error) { + var response ListBucketsResponse + err := c.makeRequest(ctx, "GET", "/buckets", nil, &response) + return &response, err +} + +// HealthCheck checks if the scraper service is healthy +func (c *Client) HealthCheck(ctx context.Context) error { + var response map[string]interface{} + err := c.makeRequest(ctx, "GET", "/health", nil, &response) + if err != nil { + return err + } + + // Health check doesn't need to return data, just check if request succeeded + return nil +} + +// Unified HTTP Handler API +// makeRequest handles all HTTP requests with automatic JSON marshaling/unmarshaling +func (c *Client) makeRequest(ctx context.Context, method, endpoint string, requestBody interface{}, responseBody interface{}) error { + // Build full URL + url := fmt.Sprintf("%s%s", c.baseURL, endpoint) + + // Marshal request body to JSON if provided + var bodyReader io.Reader + if requestBody != nil { + jsonData, err := json.Marshal(requestBody) + if err != nil { + c.logger.Error("Failed to marshal request body", map[string]interface{}{ + "error": err.Error(), + "endpoint": endpoint, + }) + return fmt.Errorf("failed to marshal request: %w", err) + } + bodyReader = bytes.NewReader(jsonData) + } + + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, method, url, bodyReader) + if err != nil { + c.logger.Error("Failed to create HTTP request", map[string]interface{}{ + "error": err.Error(), + "method": method, + "url": url, + }) + return fmt.Errorf("failed to create request: %w", err) + } + + // Set headers + if requestBody != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "application/json") + + // Send request + resp, err := c.httpClient.Do(req) + if err != nil { + c.logger.Error("Failed to send HTTP request", map[string]interface{}{ + "error": err.Error(), + "method": method, + "url": url, + }) + return fmt.Errorf("failed to send request: %w", err) + } + defer resp.Body.Close() + + // Read response body + respBody, err := io.ReadAll(resp.Body) + if err != nil { + c.logger.Error("Failed to read response body", map[string]interface{}{ + "error": err.Error(), + "url": url, + }) + return fmt.Errorf("failed to read response: %w", err) + } + + // Handle error responses + if resp.StatusCode >= 400 { + // Try to parse error response + var errorResp map[string]interface{} + if err := json.Unmarshal(respBody, &errorResp); err == nil { + if errorMsg, ok := errorResp["error"].(string); ok { + c.logger.Error("Request failed with error response", map[string]interface{}{ + "status_code": resp.StatusCode, + "error": errorMsg, + "url": url, + }) + return fmt.Errorf("request failed: %s", errorMsg) + } + } + + c.logger.Error("Request failed with status code", map[string]interface{}{ + "status_code": resp.StatusCode, + "url": url, + "body": string(respBody), + }) + return fmt.Errorf("request failed with status %d", resp.StatusCode) + } + + // Handle successful responses + if responseBody != nil { + if err := json.Unmarshal(respBody, responseBody); err != nil { + c.logger.Error("Failed to unmarshal response", map[string]interface{}{ + "error": err.Error(), + "url": url, + "body": string(respBody), + }) + return fmt.Errorf("failed to unmarshal response: %w", err) + } + } + + c.logger.Debug("Request completed successfully", map[string]interface{}{ + "method": method, + "url": url, + "status_code": resp.StatusCode, + }) + + return nil +}