ted one-time api

cmd/web/bootstrap/config.go

@@ -18,6 +18,7 @@ type Config struct {
 	HCaptcha     config.HCaptchaConfig
 	GLM          GLMConfig
 	Scraper      ScraperConfig
+	TED          TEDConfig
 	UserAuth     AuthConfig
 	CustomerAuth AuthConfig
 }
@@ -56,6 +57,21 @@ type ScraperConfig struct {
 	Timeout time.Duration `env:"SCRAPER_TIMEOUT"`
 }
 
+// TEDConfig holds configuration for TED scraper (one-time scraping from admin panel)
+type TEDConfig struct {
+	BaseURL          string        `env:"TED_BASE_URL" envDefault:"https://ted.europa.eu"`
+	Timeout          time.Duration `env:"TED_TIMEOUT" envDefault:"30s"`
+	MaxRetries       int           `env:"TED_MAX_RETRIES" envDefault:"3"`
+	RetryDelay       time.Duration `env:"TED_RETRY_DELAY" envDefault:"5s"`
+	UserAgent        string        `env:"TED_USER_AGENT" envDefault:"TM-TED-Scraper/1.0"`
+	MaxConcurrency   int           `env:"TED_MAX_CONCURRENCY" envDefault:"5"`
+	DownloadDir      string        `env:"TED_DOWNLOAD_DIR" envDefault:"./downloads"`
+	CleanupAfter     time.Duration `env:"TED_CLEANUP_AFTER" envDefault:"24h"`
+	ScrapingInterval string        `env:"TED_SCRAPING_INTERVAL" envDefault:"* * 10 * * *"`
+	MaxNoticesPerDay int           `env:"TED_MAX_NOTICES_PER_DAY" envDefault:"0"`
+	AlertMail        string        `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
+}
+
 type AuthConfig struct {
 	JWT JWTConfig
 }

cmd/web/main.go

@@ -103,6 +103,7 @@ import (
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/kanban"
+	"tm/internal/notice"
 	"tm/internal/notification"
 	"tm/internal/scraper"
 	"tm/internal/tender"
@@ -115,6 +116,56 @@ import (
 	"tm/cmd/web/router"
 )
 
+func buildTEDConfig(c *bootstrap.TEDConfig) *scraper.TEDConfig {
+	if c == nil {
+		return &scraper.TEDConfig{
+			BaseURL:          "https://ted.europa.eu",
+			Timeout:          30 * time.Second,
+			MaxRetries:       3,
+			RetryDelay:       5 * time.Second,
+			UserAgent:        "TM-TED-Scraper/1.0",
+			MaxConcurrency:   5,
+			DownloadDir:      "./downloads",
+			CleanupAfter:     24 * time.Hour,
+			ScrapingInterval: "* * 10 * * *",
+			AlertMail:        "alerts@opplens.com",
+		}
+	}
+	baseURL := c.BaseURL
+	if baseURL == "" {
+		baseURL = "https://ted.europa.eu"
+	}
+	userAgent := c.UserAgent
+	if userAgent == "" {
+		userAgent = "TM-TED-Scraper/1.0"
+	}
+	downloadDir := c.DownloadDir
+	if downloadDir == "" {
+		downloadDir = "./downloads"
+	}
+	scrapingInterval := c.ScrapingInterval
+	if scrapingInterval == "" {
+		scrapingInterval = "* * 10 * * *"
+	}
+	alertMail := c.AlertMail
+	if alertMail == "" {
+		alertMail = "alerts@opplens.com"
+	}
+	return &scraper.TEDConfig{
+		BaseURL:          baseURL,
+		Timeout:          c.Timeout,
+		MaxRetries:       c.MaxRetries,
+		RetryDelay:       c.RetryDelay,
+		UserAgent:        userAgent,
+		MaxConcurrency:   c.MaxConcurrency,
+		DownloadDir:      downloadDir,
+		CleanupAfter:     c.CleanupAfter,
+		ScrapingInterval: scrapingInterval,
+		MaxNoticesPerDay: c.MaxNoticesPerDay,
+		AlertMail:        alertMail,
+	}
+}
+
 func main() {
 	conf := bootstrap.InitConfig()
 
@@ -168,11 +219,12 @@ func main() {
 	inquiryRepository := inquiry.NewRepository(mongoManager, logger)
 	contactRepository := contact.NewContactRepository(mongoManager, logger)
 	cmsRepository := cms.NewRepository(mongoManager, logger)
+	noticeRepository := notice.NewRepository(mongoManager, logger)
 	boardRepository := kanban.NewBoardRepository(mongoManager, logger)
 	columnRepository := kanban.NewColumnRepository(mongoManager, logger)
 	cardRepository := kanban.NewCardRepository(mongoManager, logger)
 	logger.Info("Repositories initialized successfully", map[string]interface{}{
-		"repositories": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "contact", "cms", "kanban_board", "kanban_column", "kanban_card"},
+		"repositories": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "contact", "cms", "notice", "kanban_board", "kanban_column", "kanban_card"},
 	})
 
 	// Initialize validation services
@@ -193,7 +245,8 @@ func main() {
 	notificationService := notification.NewService(notificationSDK, userService, customerService, logger)
 	contactService := contact.NewService(contactRepository, logger, notificationSDK)
 	cmsService := cms.NewService(cmsRepository, logger)
-	scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger)
+	tedConfig := buildTEDConfig(&conf.TED)
+	scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig)
 	kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger)
 	logger.Info("Services initialized successfully", map[string]interface{}{
 		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban"},

cmd/web/router/routes.go

@@ -181,8 +181,9 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
 	// Scraper Routes
 	scraperGP := adminV1.Group("/scraper")
 	{
-		scraperGP.Use(userHandler.AuthMiddleware())
+		scraperGP.Use(userHandler.AuthMiddleware(), userHandler.AdminMiddleware())
 		scraperGP.POST("/download", scraperHandler.ScrapeDocuments)
+		scraperGP.POST("/ted/one-time", scraperHandler.TriggerTEDOneTimeScraping)
 	}
 
 	// Kanban Routes