From e0e8b9cf0427bf044c1aa869d83de6bedc032508 Mon Sep 17 00:00:00 2001 From: Mazyar Date: Mon, 23 Feb 2026 15:40:20 +0330 Subject: [PATCH] ted one-time api --- cmd/web/bootstrap/config.go | 16 +++++ cmd/web/main.go | 57 ++++++++++++++++- cmd/web/router/routes.go | 3 +- internal/scraper/form.go | 6 ++ internal/scraper/handler.go | 28 +++++++++ internal/scraper/service.go | 120 +++++++++++++++++++++++++++++++++++- pkg/response/response.go | 9 +++ 7 files changed, 234 insertions(+), 5 deletions(-) diff --git a/cmd/web/bootstrap/config.go b/cmd/web/bootstrap/config.go index db3eb10..e6ab8f9 100644 --- a/cmd/web/bootstrap/config.go +++ b/cmd/web/bootstrap/config.go @@ -18,6 +18,7 @@ type Config struct { HCaptcha config.HCaptchaConfig GLM GLMConfig Scraper ScraperConfig + TED TEDConfig UserAuth AuthConfig CustomerAuth AuthConfig } @@ -56,6 +57,21 @@ type ScraperConfig struct { Timeout time.Duration `env:"SCRAPER_TIMEOUT"` } +// TEDConfig holds configuration for TED scraper (one-time scraping from admin panel) +type TEDConfig struct { + BaseURL string `env:"TED_BASE_URL" envDefault:"https://ted.europa.eu"` + Timeout time.Duration `env:"TED_TIMEOUT" envDefault:"30s"` + MaxRetries int `env:"TED_MAX_RETRIES" envDefault:"3"` + RetryDelay time.Duration `env:"TED_RETRY_DELAY" envDefault:"5s"` + UserAgent string `env:"TED_USER_AGENT" envDefault:"TM-TED-Scraper/1.0"` + MaxConcurrency int `env:"TED_MAX_CONCURRENCY" envDefault:"5"` + DownloadDir string `env:"TED_DOWNLOAD_DIR" envDefault:"./downloads"` + CleanupAfter time.Duration `env:"TED_CLEANUP_AFTER" envDefault:"24h"` + ScrapingInterval string `env:"TED_SCRAPING_INTERVAL" envDefault:"* * 10 * * *"` + MaxNoticesPerDay int `env:"TED_MAX_NOTICES_PER_DAY" envDefault:"0"` + AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"` +} + type AuthConfig struct { JWT JWTConfig } diff --git a/cmd/web/main.go b/cmd/web/main.go index 483b652..ffd2936 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -103,6 +103,7 @@ import ( "tm/internal/feedback" "tm/internal/inquiry" "tm/internal/kanban" + "tm/internal/notice" "tm/internal/notification" "tm/internal/scraper" "tm/internal/tender" @@ -115,6 +116,56 @@ import ( "tm/cmd/web/router" ) +func buildTEDConfig(c *bootstrap.TEDConfig) *scraper.TEDConfig { + if c == nil { + return &scraper.TEDConfig{ + BaseURL: "https://ted.europa.eu", + Timeout: 30 * time.Second, + MaxRetries: 3, + RetryDelay: 5 * time.Second, + UserAgent: "TM-TED-Scraper/1.0", + MaxConcurrency: 5, + DownloadDir: "./downloads", + CleanupAfter: 24 * time.Hour, + ScrapingInterval: "* * 10 * * *", + AlertMail: "alerts@opplens.com", + } + } + baseURL := c.BaseURL + if baseURL == "" { + baseURL = "https://ted.europa.eu" + } + userAgent := c.UserAgent + if userAgent == "" { + userAgent = "TM-TED-Scraper/1.0" + } + downloadDir := c.DownloadDir + if downloadDir == "" { + downloadDir = "./downloads" + } + scrapingInterval := c.ScrapingInterval + if scrapingInterval == "" { + scrapingInterval = "* * 10 * * *" + } + alertMail := c.AlertMail + if alertMail == "" { + alertMail = "alerts@opplens.com" + } + return &scraper.TEDConfig{ + BaseURL: baseURL, + Timeout: c.Timeout, + MaxRetries: c.MaxRetries, + RetryDelay: c.RetryDelay, + UserAgent: userAgent, + MaxConcurrency: c.MaxConcurrency, + DownloadDir: downloadDir, + CleanupAfter: c.CleanupAfter, + ScrapingInterval: scrapingInterval, + MaxNoticesPerDay: c.MaxNoticesPerDay, + AlertMail: alertMail, + } +} + func main() { conf := bootstrap.InitConfig() @@ -168,11 +219,12 @@ func main() { inquiryRepository := inquiry.NewRepository(mongoManager, logger) contactRepository := contact.NewContactRepository(mongoManager, logger) cmsRepository := cms.NewRepository(mongoManager, logger) + noticeRepository := notice.NewRepository(mongoManager, logger) boardRepository := kanban.NewBoardRepository(mongoManager, logger) columnRepository := kanban.NewColumnRepository(mongoManager, logger) cardRepository := kanban.NewCardRepository(mongoManager, logger) logger.Info("Repositories initialized successfully", map[string]interface{}{ - "repositories": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "contact", "cms", "kanban_board", "kanban_column", "kanban_card"}, + "repositories": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "contact", "cms", "notice", "kanban_board", "kanban_column", "kanban_card"}, }) // Initialize validation services @@ -193,7 +245,8 @@ func main() { notificationService := notification.NewService(notificationSDK, userService, customerService, logger) contactService := contact.NewService(contactRepository, logger, notificationSDK) cmsService := cms.NewService(cmsRepository, logger) - scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger) + tedConfig := buildTEDConfig(&conf.TED) + scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig) kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger) logger.Info("Services initialized successfully", map[string]interface{}{ "services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban"}, diff --git a/cmd/web/router/routes.go b/cmd/web/router/routes.go index 53cfbbe..0f82311 100644 --- a/cmd/web/router/routes.go +++ b/cmd/web/router/routes.go @@ -181,8 +181,9 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler // Scraper Routes scraperGP := adminV1.Group("/scraper") { - scraperGP.Use(userHandler.AuthMiddleware()) + scraperGP.Use(userHandler.AuthMiddleware(), userHandler.AdminMiddleware()) scraperGP.POST("/download", scraperHandler.ScrapeDocuments) + scraperGP.POST("/ted/one-time", scraperHandler.TriggerTEDOneTimeScraping) } // Kanban Routes diff --git a/internal/scraper/form.go b/internal/scraper/form.go index 3c0a5b7..7556db8 100644 --- a/internal/scraper/form.go +++ b/internal/scraper/form.go @@ -35,3 +35,9 @@ type BucketInfo struct { NoticeID string `json:"notice_id"` Count int `json:"count"` } + +// TEDOneTimeScrapeForm represents the request form for triggering TED one-time scraping +type TEDOneTimeScrapeForm struct { + FromDate string `json:"from_date" valid:"required" example:"01/01/2026"` + ToDate string `json:"to_date" valid:"required" example:"19/02/2026"` +} diff --git a/internal/scraper/handler.go b/internal/scraper/handler.go index 6c7281e..610ae84 100644 --- a/internal/scraper/handler.go +++ b/internal/scraper/handler.go @@ -42,3 +42,31 @@ func (h *Handler) ScrapeDocuments(c echo.Context) error { return response.Success(c, result, "Documents scraped and uploaded successfully") } + +// TriggerTEDOneTimeScraping handles TED one-time scraping requests +// @Summary Trigger TED one-time scraping +// @Description Starts TED one-time scraping for the specified date range. The operation runs asynchronously; check logs for progress. +// @Tags Admin-Scraper +// @Accept json +// @Produce json +// @Param request body TEDOneTimeScrapeForm true "Date range for scraping (DD/MM/YYYY)" +// @Success 202 {object} response.APIResponse "TED scraping started" +// @Failure 400 {object} response.APIResponse "Invalid date format or range" +// @Failure 500 {object} response.APIResponse "TED scraping not configured" +// @Security BearerAuth +// @Router /admin/v1/scraper/ted/one-time [post] +func (h *Handler) TriggerTEDOneTimeScraping(c echo.Context) error { + form, err := response.Parse[TEDOneTimeScrapeForm](c) + if err != nil { + return response.ValidationError(c, "Invalid request data", err.Error()) + } + + err = h.service.TriggerTEDOneTimeScraping(c.Request().Context(), form) + if err != nil { + return response.BadRequest(c, err.Error(), "") + } + + return response.Accepted(c, map[string]string{ + "message": "TED one-time scraping started. Check logs for progress.", + }, "TED one-time scraping started") +} diff --git a/internal/scraper/service.go b/internal/scraper/service.go index ec52003..5e47e03 100644 --- a/internal/scraper/service.go +++ b/internal/scraper/service.go @@ -2,22 +2,48 @@ package scraper import ( "context" + "fmt" "time" + "tm/internal/notice" "tm/pkg/logger" + "tm/pkg/mongo" + "tm/pkg/notification" "tm/pkg/scraper" + "tm/ted" ) +// TEDConfig holds configuration for TED one-time scraping (used when creating service) +type TEDConfig struct { + BaseURL string + Timeout time.Duration + MaxRetries int + RetryDelay time.Duration + UserAgent string + MaxConcurrency int + DownloadDir string + CleanupAfter time.Duration + ScrapingInterval string + MaxNoticesPerDay int + AlertMail string +} + // Service defines the interface for scraper business operations type Service interface { // ScrapeDocuments scrapes documents and uploads them to MinIO ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) + // TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range (runs in background) + TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error } // scraperService implements Service interface type scraperService struct { - sdk scraper.SDK - logger logger.Logger + sdk scraper.SDK + logger logger.Logger + mongoManager *mongo.ConnectionManager + noticeRepo notice.Repository + notificationSDK notification.SDK + tedConfig *TEDConfig } // NewService creates a new scraper service with stored credentials @@ -29,6 +55,19 @@ func NewService(baseURL string, timeout time.Duration, logger logger.Logger) Ser } } +// NewServiceWithTED creates a new scraper service with both TendSign SDK and TED scraping support +func NewServiceWithTED(baseURL string, timeout time.Duration, logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository, notificationSDK notification.SDK, tedConfig *TEDConfig) Service { + sdk := scraper.NewClient(baseURL, timeout, logger) + return &scraperService{ + sdk: sdk, + logger: logger, + mongoManager: mongoManager, + noticeRepo: noticeRepo, + notificationSDK: notificationSDK, + tedConfig: tedConfig, + } +} + // ScrapeDocuments scrapes documents and uploads them to MinIO func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeRequestForm) (*ScrapeResponse, error) { s.logger.Info("Starting document scraping", map[string]interface{}{ @@ -95,3 +134,80 @@ func (s *scraperService) ScrapeDocuments(ctx context.Context, form *ScrapeReques return serviceResp, nil } + +// TriggerTEDOneTimeScraping starts TED one-time scraping for the given date range. +// The scraping runs in a background goroutine; the handler returns 202 Accepted immediately. +func (s *scraperService) TriggerTEDOneTimeScraping(ctx context.Context, form *TEDOneTimeScrapeForm) error { + if s.tedConfig == nil || s.mongoManager == nil || s.noticeRepo == nil { + return fmt.Errorf("TED scraping is not configured") + } + + from, err := time.Parse("02/01/2006", form.FromDate) + if err != nil { + return fmt.Errorf("invalid from_date format: expected DD/MM/YYYY (e.g. 01/01/2026)") + } + + to, err := time.Parse("02/01/2006", form.ToDate) + if err != nil { + return fmt.Errorf("invalid to_date format: expected DD/MM/YYYY (e.g. 19/02/2026)") + } + + if from.After(to) { + return fmt.Errorf("from_date cannot be after to_date") + } + + tedConfig := s.tedConfig + mongoManager := s.mongoManager + noticeRepo := s.noticeRepo + notificationSDK := s.notificationSDK + log := s.logger + + go func() { + tedScraperConfig := &ted.Config{ + BaseURL: tedConfig.BaseURL, + UserAgent: tedConfig.UserAgent, + DownloadDir: tedConfig.DownloadDir, + MaxRetries: tedConfig.MaxRetries, + MaxConcurrency: tedConfig.MaxConcurrency, + Timeout: tedConfig.Timeout, + RetryDelay: tedConfig.RetryDelay, + CleanupAfter: tedConfig.CleanupAfter, + ScrapingInterval: tedConfig.ScrapingInterval, + AlertMail: tedConfig.AlertMail, + MaxNoticesPerDay: tedConfig.MaxNoticesPerDay, + } + + tedScraper := ted.NewTEDScraper( + tedScraperConfig, + log, + mongoManager, + noticeRepo, + notificationSDK, + ) + + runCtx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) + defer cancel() + + log.Info("Starting TED one-time scraping (triggered via API)", map[string]interface{}{ + "from_date": from.Format("02/01/2006"), + "to_date": to.Format("02/01/2006"), + }) + + err := tedScraper.Run(runCtx, &from, &to) + if err != nil { + log.Error("TED one-time scraping failed", map[string]interface{}{ + "error": err.Error(), + "from_date": from.Format("02/01/2006"), + "to_date": to.Format("02/01/2006"), + }) + return + } + + log.Info("TED one-time scraping completed successfully", map[string]interface{}{ + "from_date": from.Format("02/01/2006"), + "to_date": to.Format("02/01/2006"), + }) + }() + + return nil +} diff --git a/pkg/response/response.go b/pkg/response/response.go index 240c8a9..707cc37 100644 --- a/pkg/response/response.go +++ b/pkg/response/response.go @@ -68,6 +68,15 @@ func Created(c echo.Context, data interface{}, message string) error { }) } +// Accepted returns a 202 Accepted response (for async operations) +func Accepted(c echo.Context, data interface{}, message string) error { + return c.JSON(http.StatusAccepted, APIResponse{ + Success: true, + Message: message, + Data: data, + }) +} + // BadRequest returns a 400 Bad Request response func BadRequest(c echo.Context, message, details string) error { return c.JSON(http.StatusBadRequest, APIResponse{