Removed old document scraper service

cmd/web/main.go

@@ -104,9 +104,7 @@ import (
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/kanban"
-	"tm/internal/notice"
 	"tm/internal/notification"
-	"tm/internal/scraper"
 	"tm/internal/tender"
 	"tm/internal/tender_approval"
 	"tm/internal/user"
@@ -118,56 +116,6 @@ import (
 	"tm/cmd/web/router"
 )
 
-func buildTEDConfig(c *bootstrap.TEDConfig) *scraper.TEDConfig {
-	if c == nil {
-		return &scraper.TEDConfig{
-			BaseURL:          "https://ted.europa.eu",
-			Timeout:          30 * time.Second,
-			MaxRetries:       3,
-			RetryDelay:       5 * time.Second,
-			UserAgent:        "TM-TED-Scraper/1.0",
-			MaxConcurrency:   5,
-			DownloadDir:      "./downloads",
-			CleanupAfter:     24 * time.Hour,
-			ScrapingInterval: "* * 10 * * *",
-			AlertMail:        "alerts@opplens.com",
-		}
-	}
-	baseURL := c.BaseURL
-	if baseURL == "" {
-		baseURL = "https://ted.europa.eu"
-	}
-	userAgent := c.UserAgent
-	if userAgent == "" {
-		userAgent = "TM-TED-Scraper/1.0"
-	}
-	downloadDir := c.DownloadDir
-	if downloadDir == "" {
-		downloadDir = "./downloads"
-	}
-	scrapingInterval := c.ScrapingInterval
-	if scrapingInterval == "" {
-		scrapingInterval = "* * 10 * * *"
-	}
-	alertMail := c.AlertMail
-	if alertMail == "" {
-		alertMail = "alerts@opplens.com"
-	}
-	return &scraper.TEDConfig{
-		BaseURL:          baseURL,
-		Timeout:          c.Timeout,
-		MaxRetries:       c.MaxRetries,
-		RetryDelay:       c.RetryDelay,
-		UserAgent:        userAgent,
-		MaxConcurrency:   c.MaxConcurrency,
-		DownloadDir:      downloadDir,
-		CleanupAfter:     c.CleanupAfter,
-		ScrapingInterval: scrapingInterval,
-		MaxNoticesPerDay: c.MaxNoticesPerDay,
-		AlertMail:        alertMail,
-	}
-}
-
 func main() {
 	conf := bootstrap.InitConfig()
 
@@ -229,7 +177,7 @@ func main() {
 	inquiryRepository := inquiry.NewRepository(mongoManager, logger)
 	contactRepository := contact.NewContactRepository(mongoManager, logger)
 	cmsRepository := cms.NewRepository(mongoManager, logger)
-	noticeRepository := notice.NewRepository(mongoManager, logger)
+	// noticeRepository := notice.NewRepository(mongoManager, logger)
 	boardRepository := kanban.NewBoardRepository(mongoManager, logger)
 	columnRepository := kanban.NewColumnRepository(mongoManager, logger)
 	cardRepository := kanban.NewCardRepository(mongoManager, logger)
@@ -256,8 +204,6 @@ func main() {
 	notificationService := notification.NewService(notificationSDK, notificationRepository, userService, customerService, logger)
 	contactService := contact.NewService(contactRepository, logger, notificationSDK)
 	cmsService := cms.NewService(cmsRepository, logger)
-	tedConfig := buildTEDConfig(&conf.TED)
-	scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig)
 	kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger)
 	documentScraperService := document_scraper.NewService(tenderRepository, logger)
 	logger.Info("Services initialized successfully", map[string]interface{}{
@@ -277,7 +223,6 @@ func main() {
 	notificationHandler := notification.NewHandler(notificationService)
 	contactHandler := contact.NewHandler(contactService, hcaptchaVerifier)
 	cmsHandler := cms.NewHandler(cmsService)
-	scraperHandler := scraper.NewHandler(scraperService)
 	kanbanHandler := kanban.NewHandler(kanbanService)
 	fileStoreHandler := filestore.NewHandler(fileStoreService, logger)
 	documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger)
@@ -289,7 +234,7 @@ func main() {
 	e := bootstrap.InitHTTPServer(conf, logger)
 
 	// Register routes
-	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler, kanbanHandler, fileStoreHandler, xssPolicy)
+	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, xssPolicy)
 	router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, fileStoreHandler, xssPolicy)
 	router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey)
 

cmd/web/router/routes.go

@@ -12,7 +12,6 @@ import (
 	"tm/internal/inquiry"
 	"tm/internal/kanban"
 	"tm/internal/notification"
-	"tm/internal/scraper"
 	"tm/internal/tender"
 	"tm/internal/tender_approval"
 	"tm/internal/user"
@@ -22,7 +21,7 @@ import (
 	"github.com/labstack/echo/v4"
 )
 
-func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) {
+func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) {
 	adminV1 := e.Group("/admin/v1")
 
 	// Add CSP middleware to admin routes (stricter policy)
@@ -180,14 +179,6 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
 		cmsGP.DELETE("/:id", cmsHandler.Delete)
 	}
 
-	// Scraper Routes
-	scraperGP := adminV1.Group("/scraper")
-	{
-		scraperGP.Use(userHandler.AuthMiddleware(), userHandler.AdminMiddleware())
-		scraperGP.POST("/download", scraperHandler.ScrapeDocuments)
-		scraperGP.POST("/ted/one-time", scraperHandler.TriggerTEDOneTimeScraping)
-	}
-
 	// Kanban Routes
 	kanbanGP := adminV1.Group("/kanban")
 	{