tendsign scraper

cmd/web/bootstrap/config.go

@@ -1,6 +1,7 @@
 package bootstrap
 
 import (
+	"time"
 	"tm/pkg/config"
 )
 
@@ -14,10 +15,16 @@ type Config struct {
 	Assets       config.AssetsConfig
 	Notification config.NotificationConfig
 	Ollama       config.OllamaConfig
+	Scraper      ScraperConfig
 	UserAuth     AuthConfig
 	CustomerAuth AuthConfig
 }
 
+type ScraperConfig struct {
+	BaseURL string        `env:"SCRAPER_BASE_URL"`
+	Timeout time.Duration `env:"SCRAPER_TIMEOUT"`
+}
+
 type AuthConfig struct {
 	JWT JWTConfig
 }

cmd/web/main.go

@@ -103,6 +103,7 @@ import (
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/notification"
+	"tm/internal/scraper"
 	"tm/internal/tender"
 	"tm/internal/tender_approval"
 	"tm/internal/user"
@@ -179,8 +180,9 @@ func main() {
 	notificationService := notification.NewService(notificationSDK, userService, customerService, logger)
 	contactService := contact.NewService(contactRepository, logger)
 	cmsService := cms.NewService(cmsRepository, logger)
+	scraperService := scraper.NewService(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger)
 	logger.Info("Services initialized successfully", map[string]interface{}{
-		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"},
+		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
 	})
 
 	// Initialize handlers with services
@@ -196,15 +198,16 @@ func main() {
 	notificationHandler := notification.NewHandler(notificationService)
 	contactHandler := contact.NewHandler(contactService)
 	cmsHandler := cms.NewHandler(cmsService)
+	scraperHandler := scraper.NewHandler(scraperService)
 	logger.Info("Handlers initialized successfully", map[string]interface{}{
-		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms"},
+		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper"},
 	})
 
 	// Initialize HTTP server
 	e := bootstrap.InitHTTPServer(conf, logger)
 
 	// Register routes
-	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler)
+	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler)
 	router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler)
 
 	// Start server

cmd/web/router/routes.go

@@ -10,6 +10,7 @@ import (
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/notification"
+	"tm/internal/scraper"
 	"tm/internal/tender"
 	"tm/internal/tender_approval"
 	"tm/internal/user"
@@ -17,7 +18,7 @@ import (
 	"github.com/labstack/echo/v4"
 )
 
-func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) {
+func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, scraperHandler *scraper.Handler) {
 	adminV1 := e.Group("/admin/v1")
 
 	// Admin Users Routes
@@ -165,6 +166,13 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
 		cmsGP.PUT("/:id", cmsHandler.Update)
 		cmsGP.DELETE("/:id", cmsHandler.Delete)
 	}
+
+	// Scraper Routes
+	scraperGP := adminV1.Group("/scraper")
+	{
+		scraperGP.Use(userHandler.AuthMiddleware())
+		scraperGP.POST("/download", scraperHandler.ScrapeDocuments)
+	}
 }
 
 func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler) {