ted one-time api

2026-02-23 15:40:20 +03:30
parent 80befb51c7
commit e0e8b9cf04
7 changed files with 234 additions and 5 deletions
@@ -35,3 +35,9 @@ type BucketInfo struct {
 	NoticeID string `json:"notice_id"`
 	Count    int    `json:"count"`
 }
+
+// TEDOneTimeScrapeForm represents the request form for triggering TED one-time scraping
+type TEDOneTimeScrapeForm struct {
+	FromDate string `json:"from_date" valid:"required" example:"01/01/2026"`
+	ToDate   string `json:"to_date" valid:"required" example:"19/02/2026"`
+}
@@ -42,3 +42,31 @@ func (h *Handler) ScrapeDocuments(c echo.Context) error {

 	return response.Success(c, result, "Documents scraped and uploaded successfully")
 }
+
+// TriggerTEDOneTimeScraping handles TED one-time scraping requests
+// @Summary Trigger TED one-time scraping
+// @Description Starts TED one-time scraping for the specified date range. The operation runs asynchronously; check logs for progress.
+// @Tags Admin-Scraper
+// @Accept json
+// @Produce json
+// @Param request body TEDOneTimeScrapeForm true "Date range for scraping (DD/MM/YYYY)"
+// @Success 202 {object} response.APIResponse "TED scraping started"
+// @Failure 400 {object} response.APIResponse "Invalid date format or range"
+// @Failure 500 {object} response.APIResponse "TED scraping not configured"
+// @Security BearerAuth
+// @Router /admin/v1/scraper/ted/one-time [post]
+func (h *Handler) TriggerTEDOneTimeScraping(c echo.Context) error {
+	form, err := response.Parse[TEDOneTimeScrapeForm](c)
+	if err != nil {
+		return response.ValidationError(c, "Invalid request data", err.Error())
+	}
+
+	err = h.service.TriggerTEDOneTimeScraping(c.Request().Context(), form)
+	if err != nil {
+		return response.BadRequest(c, err.Error(), "")
+	}
+
+	return response.Accepted(c, map[string]string{
+		"message": "TED one-time scraping started. Check logs for progress.",
+	}, "TED one-time scraping started")
+}
@@ -2,22 +2,48 @@ package scraper

 import (
 	"context"
+	"fmt"
 	"time"

+	"tm/internal/notice"
 	"tm/pkg/logger"
+	"tm/pkg/mongo"
+	"tm/pkg/notification"
 	"tm/pkg/scraper"
+	"tm/ted"
 )

+// TEDConfig holds configuration for TED one-time scraping (used when creating service)
+type TEDConfig struct {
+	BaseURL          string
+	Timeout          time.Duration
+	MaxRetries       int
+	RetryDelay       time.Duration
+	UserAgent        string
+	MaxConcurrency   int
+	DownloadDir      string
+	CleanupAfter     time.Duration
+	ScrapingInterval string
+	MaxNoticesPerDay int
+	AlertMail        string
+}
+
 // Service defines the interface for scraper business operations
 type Service interface {
 	// ScrapeDocuments scrapes documents and uploads them to MinIO
 	ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error)
+	// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range (runs in background)
+	TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error
 }

 // scraperService implements Service interface
 type scraperService struct {
-	sdk    scraper.SDK
-	logger logger.Logger
+	sdk             scraper.SDK
+	logger          logger.Logger
+	mongoManager    *mongo.ConnectionManager
+	noticeRepo      notice.Repository
+	notificationSDK notification.SDK
+	tedConfig       *TEDConfig
 }

 // NewService creates a new scraper service with stored credentials
@@ -29,6 +55,19 @@ func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Ser
 	}
 }

+// NewServiceWithTED creates a new scraper service with both TendSign SDK and TED scraping support
+func NewServiceWithTED(baseURL string, timeout time.Duration, logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository, notificationSDK notification.SDK, tedConfig *TEDConfig) Service {
+	sdk := scraper.NewClient(baseURL, timeout, logger)
+	return &scraperService{
+		sdk:             sdk,
+		logger:          logger,
+		mongoManager:    mongoManager,
+		noticeRepo:      noticeRepo,
+		notificationSDK: notificationSDK,
+		tedConfig:       tedConfig,
+	}
+}
+
 // ScrapeDocuments scrapes documents and uploads them to MinIO
 func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) {
 	s.logger.Info("Starting document scraping", map[string]interface{}{
@@ -95,3 +134,80 @@ func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeReques

 	return serviceResp, nil
 }
+
+// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range.
+// The scraping runs in a background goroutine; the handler returns 202 Accepted immediately.
+func (s *scraperService) TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error {
+	if s.tedConfig == nil || s.mongoManager == nil || s.noticeRepo == nil {
+		return fmt.Errorf("TED scraping is not configured")
+	}
+
+	from, err := time.Parse("02/01/2006", form.FromDate)
+	if err != nil {
+		return fmt.Errorf("invalid from_date format: expected DD/MM/YYYY (e.g. 01/01/2026)")
+	}
+
+	to, err := time.Parse("02/01/2006", form.ToDate)
+	if err != nil {
+		return fmt.Errorf("invalid to_date format: expected DD/MM/YYYY (e.g. 19/02/2026)")
+	}
+
+	if from.After(to) {
+		return fmt.Errorf("from_date cannot be after to_date")
+	}
+
+	tedConfig := s.tedConfig
+	mongoManager := s.mongoManager
+	noticeRepo := s.noticeRepo
+	notificationSDK := s.notificationSDK
+	log := s.logger
+
+	go func() {
+		tedScraperConfig := &ted.Config{
+			BaseURL:          tedConfig.BaseURL,
+			UserAgent:        tedConfig.UserAgent,
+			DownloadDir:      tedConfig.DownloadDir,
+			MaxRetries:       tedConfig.MaxRetries,
+			MaxConcurrency:   tedConfig.MaxConcurrency,
+			Timeout:          tedConfig.Timeout,
+			RetryDelay:       tedConfig.RetryDelay,
+			CleanupAfter:     tedConfig.CleanupAfter,
+			ScrapingInterval: tedConfig.ScrapingInterval,
+			AlertMail:        tedConfig.AlertMail,
+			MaxNoticesPerDay: tedConfig.MaxNoticesPerDay,
+		}
+
+		tedScraper := ted.NewTEDScraper(
+			tedScraperConfig,
+			log,
+			mongoManager,
+			noticeRepo,
+			notificationSDK,
+		)
+
+		runCtx, cancel := context.WithTimeout(context.Background(), 24*time.Hour)
+		defer cancel()
+
+		log.Info("Starting TED one-time scraping (triggered via API)", map[string]interface{}{
+			"from_date": from.Format("02/01/2006"),
+			"to_date":   to.Format("02/01/2006"),
+		})
+
+		err := tedScraper.Run(runCtx, &from, &to)
+		if err != nil {
+			log.Error("TED one-time scraping failed", map[string]interface{}{
+				"error":     err.Error(),
+				"from_date": from.Format("02/01/2006"),
+				"to_date":   to.Format("02/01/2006"),
+			})
+			return
+		}
+
+		log.Info("TED one-time scraping completed successfully", map[string]interface{}{
+			"from_date": from.Format("02/01/2006"),
+			"to_date":   to.Format("02/01/2006"),
+		})
+	}()
+
+	return nil
+}