tm_back/internal/scraper/service.go

package scraper

import (
	"context"
	"fmt"
	"time"

	"tm/internal/notice"
	"tm/pkg/logger"
	"tm/pkg/mongo"
	"tm/pkg/notification"
	"tm/pkg/scraper"
	"tm/ted"
)

// TEDConfig holds configuration for TED one-time scraping (used when creating service)
type TEDConfig struct {
	BaseURL          string
	Timeout          time.Duration
	MaxRetries       int
	RetryDelay       time.Duration
	UserAgent        string
	MaxConcurrency   int
	DownloadDir      string
	CleanupAfter     time.Duration
	ScrapingInterval string
	MaxNoticesPerDay int
	AlertMail        string
}

// Service defines the interface for scraper business operations
type Service interface {
	// ScrapeDocuments scrapes documents and uploads them to MinIO
	ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
	// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range (runs in background)
	TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error
}

// scraperService implements Service interface
type scraperService struct {
	sdk             scraper.SDK
	logger          logger.Logger
	mongoManager    *mongo.ConnectionManager
	noticeRepo      notice.Repository
	notificationSDK notification.SDK
	tedConfig       *TEDConfig
}

// NewService creates a new scraper service with stored credentials
func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Service {
	sdk := scraper.NewClient(baseURL, timeout, logger)
	return &scraperService{
		sdk:    sdk,
		logger: logger,
	}
}

// NewServiceWithTED creates a new scraper service with both TendSign SDK and TED scraping support
func NewServiceWithTED(baseURL string, timeout time.Duration, logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository, notificationSDK notification.SDK, tedConfig *TEDConfig) Service {
	sdk := scraper.NewClient(baseURL, timeout, logger)
	return &scraperService{
		sdk:             sdk,
		logger:          logger,
		mongoManager:    mongoManager,
		noticeRepo:      noticeRepo,
		notificationSDK: notificationSDK,
		tedConfig:       tedConfig,
	}
}

// ScrapeDocuments scrapes documents and uploads them to MinIO
func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
	s.logger.Info("Starting document scraping", map[string]interface{}{
		"notice_id": form.NoticeID,
	})

	// Convert form to SDK request (only dynamic fields)
	req := &scraper.ScrapeRequest{
		NoticeID:  form.NoticeID,
		NoticeURL: form.NoticeURL,
	}

	// Call SDK (credentials are already configured)
	response, err := s.sdk.ScrapeDocuments(ctx, req)
	if err != nil {
		s.logger.Error("Failed to scrape documents via SDK", map[string]interface{}{
			"error":     err.Error(),
			"notice_id": form.NoticeID,
		})
		return nil, err
	}

	// Convert SDK response to service response
	serviceResp := &ScrapeResponse{
		Success:       response.Success,
		NoticeID:      response.NoticeID,
		UploadedCount: response.UploadedCount,
		UploadedFiles: make([]UploadedFileInfo, len(response.UploadedFiles)),
		Errors:        make([]FileError, len(response.Errors)),
		Buckets:       make([]BucketInfo, len(response.Buckets)),
	}

	// Convert uploaded files
	for i, file := range response.UploadedFiles {
		serviceResp.UploadedFiles[i] = UploadedFileInfo{
			Filename:   file.Filename,
			ObjectName: file.ObjectName,
			Size:       file.Size,
		}
	}

	// Convert errors
	for i, err := range response.Errors {
		serviceResp.Errors[i] = FileError{
			Filename: err.Filename,
			Error:    err.Error,
		}
	}

	// Convert buckets
	for i, bucket := range response.Buckets {
		serviceResp.Buckets[i] = BucketInfo{
			Name:     bucket.Name,
			NoticeID: bucket.NoticeID,
			Count:    bucket.Count,
		}
	}

	s.logger.Info("Document scraping completed", map[string]interface{}{
		"notice_id":      serviceResp.NoticeID,
		"uploaded_count": serviceResp.UploadedCount,
		"success":        serviceResp.Success,
	})

	return serviceResp, nil
}

// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range.
// The scraping runs in a background goroutine; the handler returns 202 Accepted immediately.
func (s *scraperService) TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error {
	if s.tedConfig == nil || s.mongoManager == nil || s.noticeRepo == nil {
		return fmt.Errorf("TED scraping is not configured")
	}

	from, err := time.Parse("02/01/2006", form.FromDate)
	if err != nil {
		return fmt.Errorf("invalid from_date format: expected DD/MM/YYYY (e.g. 01/01/2026)")
	}

	to, err := time.Parse("02/01/2006", form.ToDate)
	if err != nil {
		return fmt.Errorf("invalid to_date format: expected DD/MM/YYYY (e.g. 19/02/2026)")
	}

	if from.After(to) {
		return fmt.Errorf("from_date cannot be after to_date")
	}

	tedConfig := s.tedConfig
	mongoManager := s.mongoManager
	noticeRepo := s.noticeRepo
	notificationSDK := s.notificationSDK
	log := s.logger

	go func() {
		tedScraperConfig := &ted.Config{
			BaseURL:          tedConfig.BaseURL,
			UserAgent:        tedConfig.UserAgent,
			DownloadDir:      tedConfig.DownloadDir,
			MaxRetries:       tedConfig.MaxRetries,
			MaxConcurrency:   tedConfig.MaxConcurrency,
			Timeout:          tedConfig.Timeout,
			RetryDelay:       tedConfig.RetryDelay,
			CleanupAfter:     tedConfig.CleanupAfter,
			ScrapingInterval: tedConfig.ScrapingInterval,
			AlertMail:        tedConfig.AlertMail,
			MaxNoticesPerDay: tedConfig.MaxNoticesPerDay,
		}

		tedScraper := ted.NewTEDScraper(
			tedScraperConfig,
			log,
			mongoManager,
			noticeRepo,
			notificationSDK,
		)

		runCtx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
		defer cancel()

		log.Info("Starting TED one-time scraping (triggered via API)", map[string]interface{}{
			"from_date": from.Format("02/01/2006"),
			"to_date":   to.Format("02/01/2006"),
		})

		err := tedScraper.Run(runCtx, &from, &to)
		if err != nil {
			log.Error("TED one-time scraping failed", map[string]interface{}{
				"error":     err.Error(),
				"from_date": from.Format("02/01/2006"),
				"to_date":   to.Format("02/01/2006"),
			})
			return
		}

		log.Info("TED one-time scraping completed successfully", map[string]interface{}{
			"from_date": from.Format("02/01/2006"),
			"to_date":   to.Format("02/01/2006"),
		})
	}()

	return nil
}