package scraper import ( "context" "time" "tm/pkg/logger" "tm/pkg/scraper" ) // Service defines the interface for scraper business operations type Service interface { // ScrapeDocuments scrapes documents and uploads them to MinIO ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) } // scraperService implements Service interface type scraperService struct { sdk scraper.SDK logger logger.Logger } // NewService creates a new scraper service with stored credentials func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service { sdk := scraper.NewClient(baseURL, timeout, logger) return &scraperService{ sdk: sdk, logger: logger, } } // ScrapeDocuments scrapes documents and uploads them to MinIO func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) { s.logger.Info("Starting document scraping", map[string]interface{}{ "notice_id": form.NoticeID, }) // Convert form to SDK request (only dynamic fields) req := &scraper.ScrapeRequest{ NoticeID: form.NoticeID, NoticeURL: form.NoticeURL, } // Call SDK (credentials are already configured) response, err := s.sdk.ScrapeDocuments(ctx, req) if err != nil { s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{ "error": err.Error(), "notice_id": form.NoticeID, }) return nil, err } // Convert SDK response to service response serviceResp := &ScrapeResponse{ Success: response.Success, NoticeID: response.NoticeID, UploadedCount: response.UploadedCount, UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)), Errors: make([]FileError, len(response.Errors)), Buckets: make([]BucketInfo, len(response.Buckets)), } // Convert uploaded files for i, file := range response.UploadedFiles { serviceResp.UploadedFiles[i] = UploadedFileInfo{ Filename: file.Filename, ObjectName: file.ObjectName, Size: file.Size, } } // Convert errors for i, err := range response.Errors { serviceResp.Errors[i] = FileError{ Filename: err.Filename, Error: err.Error, } } // Convert buckets for i, bucket := range response.Buckets { serviceResp.Buckets[i] = BucketInfo{ Name: bucket.Name, NoticeID: bucket.NoticeID, Count: bucket.Count, } } s.logger.Info("Document scraping completed", map[string]interface{}{ "notice_id": serviceResp.NoticeID, "uploaded_count": serviceResp.UploadedCount, "success": serviceResp.Success, }) return serviceResp, nil }