package scraper import ( "context" "fmt" "time" "tm/internal/notice" "tm/pkg/logger" "tm/pkg/mongo" "tm/pkg/notification" "tm/pkg/scraper" "tm/ted" ) // TEDConfig holds configuration for TED one-time scraping (used when creating service) type TEDConfig struct { BaseURL string Timeout time.Duration MaxRetries int RetryDelay time.Duration UserAgent string MaxConcurrency int DownloadDir string CleanupAfter time.Duration ScrapingInterval string MaxNoticesPerDay int AlertMail string } // Service defines the interface for scraper business operations type Service interface { // ScrapeDocuments scrapes documents and uploads them to MinIO ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) // TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range (runs in background) TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error } // scraperService implements Service interface type scraperService struct { sdk scraper.SDK logger logger.Logger mongoManager *mongo.ConnectionManager noticeRepo notice.Repository notificationSDK notification.SDK tedConfig *TEDConfig } // NewService creates a new scraper service with stored credentials func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service { sdk := scraper.NewClient(baseURL, timeout, logger) return &scraperService{ sdk: sdk, logger: logger, } } // NewServiceWithTED creates a new scraper service with both TendSign SDK and TED scraping support func NewServiceWithTED(baseURL string, timeout time.Duration, logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository, notificationSDK notification.SDK, tedConfig *TEDConfig) Service { sdk := scraper.NewClient(baseURL, timeout, logger) return &scraperService{ sdk: sdk, logger: logger, mongoManager: mongoManager, noticeRepo: noticeRepo, notificationSDK: notificationSDK, tedConfig: tedConfig, } } // ScrapeDocuments scrapes documents and uploads them to MinIO func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) { s.logger.Info("Starting document scraping", map[string]interface{}{ "notice_id": form.NoticeID, }) // Convert form to SDK request (only dynamic fields) req := &scraper.ScrapeRequest{ NoticeID: form.NoticeID, NoticeURL: form.NoticeURL, } // Call SDK (credentials are already configured) response, err := s.sdk.ScrapeDocuments(ctx, req) if err != nil { s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{ "error": err.Error(), "notice_id": form.NoticeID, }) return nil, err } // Convert SDK response to service response serviceResp := &ScrapeResponse{ Success: response.Success, NoticeID: response.NoticeID, UploadedCount: response.UploadedCount, UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)), Errors: make([]FileError, len(response.Errors)), Buckets: make([]BucketInfo, len(response.Buckets)), } // Convert uploaded files for i, file := range response.UploadedFiles { serviceResp.UploadedFiles[i] = UploadedFileInfo{ Filename: file.Filename, ObjectName: file.ObjectName, Size: file.Size, } } // Convert errors for i, err := range response.Errors { serviceResp.Errors[i] = FileError{ Filename: err.Filename, Error: err.Error, } } // Convert buckets for i, bucket := range response.Buckets { serviceResp.Buckets[i] = BucketInfo{ Name: bucket.Name, NoticeID: bucket.NoticeID, Count: bucket.Count, } } s.logger.Info("Document scraping completed", map[string]interface{}{ "notice_id": serviceResp.NoticeID, "uploaded_count": serviceResp.UploadedCount, "success": serviceResp.Success, }) return serviceResp, nil } // TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range. // The scraping runs in a background goroutine; the handler returns 202 Accepted immediately. func (s *scraperService) TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error { if s.tedConfig == nil || s.mongoManager == nil || s.noticeRepo == nil { return fmt.Errorf("TED scraping is not configured") } from, err := time.Parse("02/01/2006", form.FromDate) if err != nil { return fmt.Errorf("invalid from_date format: expected DD/MM/YYYY (e.g. 01/01/2026)") } to, err := time.Parse("02/01/2006", form.ToDate) if err != nil { return fmt.Errorf("invalid to_date format: expected DD/MM/YYYY (e.g. 19/02/2026)") } if from.After(to) { return fmt.Errorf("from_date cannot be after to_date") } tedConfig := s.tedConfig mongoManager := s.mongoManager noticeRepo := s.noticeRepo notificationSDK := s.notificationSDK log := s.logger go func() { tedScraperConfig := &ted.Config{ BaseURL: tedConfig.BaseURL, UserAgent: tedConfig.UserAgent, DownloadDir: tedConfig.DownloadDir, MaxRetries: tedConfig.MaxRetries, MaxConcurrency: tedConfig.MaxConcurrency, Timeout: tedConfig.Timeout, RetryDelay: tedConfig.RetryDelay, CleanupAfter: tedConfig.CleanupAfter, ScrapingInterval: tedConfig.ScrapingInterval, AlertMail: tedConfig.AlertMail, MaxNoticesPerDay: tedConfig.MaxNoticesPerDay, } tedScraper := ted.NewTEDScraper( tedScraperConfig, log, mongoManager, noticeRepo, notificationSDK, ) runCtx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) defer cancel() log.Info("Starting TED one-time scraping (triggered via API)", map[string]interface{}{ "from_date": from.Format("02/01/2006"), "to_date": to.Format("02/01/2006"), }) err := tedScraper.Run(runCtx, &from, &to) if err != nil { log.Error("TED one-time scraping failed", map[string]interface{}{ "error": err.Error(), "from_date": from.Format("02/01/2006"), "to_date": to.Format("02/01/2006"), }) return } log.Info("TED one-time scraping completed successfully", map[string]interface{}{ "from_date": from.Format("02/01/2006"), "to_date": to.Format("02/01/2006"), }) }() return nil }