API for AI document scraper integration

cmd/web/bootstrap/config.go

@@ -7,20 +7,21 @@ import (
 
 // Config holds configuration for the web command
 type Config struct {
-	Server       config.ServerConfig
-	Database     config.DatabaseConfig
-	Cache        config.CacheConfig
-	Logging      config.LoggingConfig
-	RateLimit    config.RateLimitConfig
-	Assets       config.AssetsConfig
-	Notification config.NotificationConfig
-	Ollama       config.OllamaConfig
-	HCaptcha     config.HCaptchaConfig
-	GLM          GLMConfig
-	Scraper      ScraperConfig
-	TED          TEDConfig
-	UserAuth     AuthConfig
-	CustomerAuth AuthConfig
+	Server          config.ServerConfig
+	Database        config.DatabaseConfig
+	Cache           config.CacheConfig
+	Logging         config.LoggingConfig
+	RateLimit       config.RateLimitConfig
+	Assets          config.AssetsConfig
+	Notification    config.NotificationConfig
+	Ollama          config.OllamaConfig
+	HCaptcha        config.HCaptchaConfig
+	GLM             GLMConfig
+	Scraper         ScraperConfig
+	TED             TEDConfig
+	UserAuth        AuthConfig
+	CustomerAuth    AuthConfig
+	DocumentScraper DocumentScraperConfig
 }
 
 type GLMConfig struct {
@@ -82,3 +83,8 @@ type JWTConfig struct {
 	AccessExpiresIn  int    `env:"USER_AUTH_ACCESS_EXPIRES_IN"`
 	RefreshExpiresIn int    `env:"USER_AUTH_REFRESH_EXPIRES_IN"`
 }
+
+// DocumentScraperConfig holds configuration for the external document scraper service API
+type DocumentScraperConfig struct {
+	APIKey string `env:"DOCUMENT_SCRAPER_API_KEY" envDefault:""`
+}

cmd/web/main.go

@@ -100,6 +100,7 @@ import (
 	"tm/internal/company_category"
 	"tm/internal/contact"
 	"tm/internal/customer"
+	"tm/internal/document_scraper"
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/kanban"
@@ -258,8 +259,9 @@ func main() {
 	tedConfig := buildTEDConfig(&conf.TED)
 	scraperService := scraper.NewServiceWithTED(conf.Scraper.BaseURL, conf.Scraper.Timeout, logger, mongoManager, noticeRepository, notificationSDK, tedConfig)
 	kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, logger)
+	documentScraperService := document_scraper.NewService(tenderRepository, noticeRepository, logger)
 	logger.Info("Services initialized successfully", map[string]interface{}{
-		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban"},
+		"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper"},
 	})
 
 	// Initialize handlers with services
@@ -278,8 +280,9 @@ func main() {
 	scraperHandler := scraper.NewHandler(scraperService)
 	kanbanHandler := kanban.NewHandler(kanbanService)
 	fileStoreHandler := filestore.NewHandler(fileStoreService, logger)
+	documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger)
 	logger.Info("Handlers initialized successfully", map[string]interface{}{
-		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore"},
+		"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper"},
 	})
 
 	// Initialize HTTP server
@@ -288,6 +291,7 @@ func main() {
 	// Register routes
 	router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, scraperHandler, kanbanHandler, fileStoreHandler, xssPolicy)
 	router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, fileStoreHandler, xssPolicy)
+	router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey)
 
 	// Start server
 	serverAddr := fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port)

cmd/web/router/routes.go

@@ -7,6 +7,7 @@ import (
 	"tm/internal/company_category"
 	"tm/internal/contact"
 	"tm/internal/customer"
+	"tm/internal/document_scraper"
 	"tm/internal/feedback"
 	"tm/internal/inquiry"
 	"tm/internal/kanban"
@@ -317,3 +318,16 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende
 		publicFilesGP.DELETE("/:file_id", fileStoreHandler.Delete)
 	}
 }
+
+func RegisterDocumentScraperRoutes(e *echo.Echo, docScraperHandler *document_scraper.Handler, apiKey string) {
+	v1 := e.Group("/api/v1/scraper")
+
+	// Apply API key middleware to all document scraper routes
+	v1.Use(document_scraper.APIKeyMiddleware(apiKey))
+
+	// Document Scraper Routes
+	{
+		v1.GET("/tenders", docScraperHandler.ListPendingTenders)
+		v1.GET("/tenders/:notice_id", docScraperHandler.GetTenderByNoticeID)
+	}
+}