mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement TED Scraper and Configuration Management

Browse Source 
- Introduced a new TED scraper in `cmd/scraper` to handle downloading and parsing TED XML files.
- Added configuration management for the scraper, including a new `Config` struct and YAML configuration file.
- Created necessary handler, service, and repository layers for tender management, adhering to Clean Architecture principles.
- Implemented comprehensive logging and error handling throughout the scraper functionality.
- Updated API routes to include tender management operations, enhancing the overall system capabilities.
This commit is contained in:
n.nakhostin
2025-08-13 16:52:24 +03:30
parent b7fa012d13
commit 96c21b8b78
16 changed files with 6970 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/scraper/.gitkeep
View File
cmd/scraper/bootstrap.go
+140
View File
@@ -0,0 +1,140 @@
package main
import (
"context"
"fmt"
"os"
"os/signal"
"syscall"
"time"
"tm/internal/tender"
"tm/pkg/config"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/ted"
)
// Init Application Configuration
func initConfig() *Config {
conf, err := config.LoadConfig(".", &Config{})
if err != nil {
panic(fmt.Sprintf("Failed to load config: %v", err))
}
return conf
}
// Init Logger Service
func initLogger(conf config.LoggingConfig) logger.Logger {
return logger.NewLogger(&logger.Config{
Level: conf.Level,
Format: conf.Format,
Output: conf.Output,
File: logger.FileConfig{
Path: conf.File.Path,
MaxSize: conf.File.MaxSize,
MaxBackups: conf.File.MaxBackups,
MaxAge: conf.File.MaxAge,
Compress: conf.File.Compress,
},
})
}
// Init MongoDB Connection Manager
func initMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
// Convert infra.MongoConfig to mongo.ConnectionConfig
connectionConfig := mongo.ConnectionConfig{
URI: conf.URI,
Database: conf.Name,
MaxPoolSize: uint64(conf.MaxPoolSize),
MinPoolSize: 5, // Default minimum pool size
MaxConnIdleTime: 30 * time.Minute, // Default idle time
ConnectTimeout: conf.Timeout,
ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
}
// Create MongoDB connection manager
connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
if err != nil {
log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
"error": err.Error(),
"uri": conf.URI,
"db": conf.Name,
})
panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
}
log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
"database": conf.Name,
"uri": conf.URI,
"pool_size": conf.MaxPoolSize,
})
return connectionManager
}
// init TED scraper
func initTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger) {
// Initialize tender repository
tenderRepo := tender.NewTenderRepository(mongoManager, appLogger)
// Initialize TED scraper
tedScraper := ted.NewTEDScraper(
&ted.Config{
BaseURL: config.TED.BaseURL,
Timeout: config.TED.Timeout,
MaxRetries: config.TED.MaxRetries,
RetryDelay: config.TED.RetryDelay,
UserAgent: config.TED.UserAgent,
MaxConcurrency: config.TED.MaxConcurrency,
DownloadDir: config.TED.DownloadDir,
CleanupAfter: config.TED.CleanupAfter,
ScrapingInterval: config.TED.ScrapingInterval,
},
appLogger,
mongoManager,
tenderRepo,
)
// Create context with cancellation
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Set up signal handling for graceful shutdown
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
// Start the scraper in a goroutine
scraperDone := make(chan error, 1)
go func() {
appLogger.Info("Starting TED scraper", map[string]interface{}{
"interval": config.TED.ScrapingInterval.String(),
})
scraperDone <- tedScraper.RunPeriodicScraping(ctx)
}()
// Wait for shutdown signal or scraper completion
select {
case <-signalChan:
appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{})
cancel()
// Wait for scraper to finish gracefully with timeout
shutdownTimeout := time.NewTimer(30 * time.Second)
select {
case <-scraperDone:
appLogger.Info("Scraper stopped gracefully", map[string]interface{}{})
case <-shutdownTimeout.C:
appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{})
}
case err := <-scraperDone:
if err != nil {
appLogger.Error("Scraper finished with error", map[string]interface{}{
"error": err.Error(),
})
os.Exit(1)
}
appLogger.Info("Scraper finished successfully", map[string]interface{}{})
}
}
cmd/scraper/config.go
+50
View File
@@ -0,0 +1,50 @@
package main
import (
"time"
"tm/pkg/config"
)
// Config defines the scraper application configuration
type Config struct {
Database config.DatabaseConfig `mapstructure:"database"`
Logging config.LoggingConfig `mapstructure:"logging"`
TED ScraperConfig `mapstructure:"ted"`
}
type ScraperConfig struct {
BaseURL string `mapstructure:"base_url"`
Timeout time.Duration `mapstructure:"timeout"`
MaxRetries int `mapstructure:"max_retries"`
RetryDelay time.Duration `mapstructure:"retry_delay"`
UserAgent string `mapstructure:"user_agent"`
MaxConcurrency int `mapstructure:"max_concurrency"`
DownloadDir string `mapstructure:"download_dir"`
CleanupAfter time.Duration `mapstructure:"cleanup_after"`
ScrapingInterval time.Duration `mapstructure:"scraping_interval"`
}
// LoadConfig loads the configuration from file
func LoadConfig(path string) (*Config, error) {
cfg := &Config{
// Set default values
TED: ScraperConfig{
BaseURL: "https://ted.europa.eu/packages/daily",
Timeout: 30 * time.Second,
MaxRetries: 3,
RetryDelay: 5 * time.Second,
UserAgent: "TenderManagement-TED-Scraper/1.0",
MaxConcurrency: 3,
DownloadDir: "/tmp/ted_downloads",
CleanupAfter: 24 * time.Hour,
ScrapingInterval: 12 * time.Hour,
},
}
result, err := config.LoadConfig(path, cfg)
if err != nil {
return nil, err
}
return result, nil
}
cmd/scraper/config.yaml
+28
View File
@@ -0,0 +1,28 @@
database:
mongodb:
uri: "mongodb://localhost:27017"
name: "tender_management"
timeout: 10s
max_pool_size: 50
logging:
level: "info"
format: "json"
output: "file" # stdout, stderr, file
file:
path: "./logs/scraper.log"
max_size: 100 # Max size in MB before rotation
max_backups: 5 # Number of backup files to keep
max_age: 30 # Max age in days to keep log files
compress: true # Compress rotated files
ted:
base_url: "https://ted.europa.eu/packages/daily"
timeout: 300s # Increased to 5 minutes for large files
max_retries: 5
retry_delay: 10s
user_agent: "TenderManagement-TED-Scraper/1.0"
max_concurrency: 2 # Reduced to avoid overwhelming the server
download_dir: "./tmp/ted_downloads"
cleanup_after: 24h
scraping_interval: 12h # Run scraping every 12 hours
cmd/scraper/main.go
+37
View File
@@ -0,0 +1,37 @@
package main
import (
"context"
"log"
"time"
)
func main() {
// Load configuration
config, err := LoadConfig(".")
if err != nil {
log.Fatalf("Failed to load configuration: %v", err)
}
// Initialize logger
appLogger := initLogger(config.Logging)
appLogger.Info("Starting TED scraper application", map[string]interface{}{
"version": "1.0.0",
})
// Initialize MongoDB connection
mongoManager := initMongoDB(config.Database.MongoDB, appLogger)
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := mongoManager.Close(ctx); err != nil {
appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
"error": err.Error(),
})
}
}()
initTEDScraper(*config, mongoManager, appLogger)
appLogger.Info("TED scraper application stopped", map[string]interface{}{})
}
cmd/web/main.go
+18 -5
View File
@@ -35,9 +35,18 @@ package main
// @tag.name Admin-Companies
// @tag.description Administrative company management operations for web panel including CRUD operations, tag management, verification, status updates, comprehensive search and filtering, and statistical reporting
// @tag.name Admin-Tenders
// @tag.description Administrative tender management operations for web panel including CRUD operations, search, statistics, and comprehensive filtering with pagination
// @tag.name Admin-Scraping
// @tag.description Administrative scraping job management operations for TED XML data collection including job control, monitoring, and status tracking
// @tag.name Authorization
// @tag.description Customer authentication and authorization operations for mobile application including login, logout, token refresh, and profile access
// @tag.name Tenders
// @tag.description Public tender information access for mobile application including active tender listings and detailed tender information
import (
"context"
"fmt"
@@ -45,6 +54,7 @@ import (
"time"
"tm/internal/company"
"tm/internal/customer"
"tm/internal/tender"
"tm/internal/user"
_ "tm/cmd/web/docs" // This is generated by swag
@@ -87,9 +97,10 @@ func main() {
userRepository := user.NewUserRepository(mongoManager, logger)
customerRepository := customer.NewCustomerRepository(mongoManager, logger)
companyRepository := company.NewCompanyRepository(mongoManager, logger)
tenderRepository := tender.NewTenderRepository(mongoManager, logger)
logger.Info("Repositories initialized successfully", map[string]interface{}{
"repositories": []string{"customer", "user", "company"},
"repositories": []string{"customer", "user", "company", "tender"},
})
// Initialize validation service
@@ -99,26 +110,28 @@ func main() {
userService := user.NewUserService(userRepository, logger, userAuthService, userValidator)
companyService := company.NewCompanyService(companyRepository, logger)
customerService := customer.NewCustomerService(customerRepository, logger, customerAuthService, companyService)
tenderService := tender.NewTenderService(tenderRepository, logger)
logger.Info("Services initialized successfully", map[string]interface{}{
"services": []string{"customer", "user", "company"},
"services": []string{"customer", "user", "company", "tender"},
})
// Initialize handlers with services
userHandler := user.NewUserHandler(userService, logger, userValidator, userAuthService)
customerHandler := customer.NewHandler(customerService, userHandler, customerAuthService, logger)
companyHandler := company.NewHandler(companyService, userHandler, logger)
tenderHandler := tender.NewTenderHandler(tenderService, logger)
logger.Info("Handlers initialized successfully", map[string]interface{}{
"handlers": []string{"customer", "user", "company"},
"handlers": []string{"customer", "user", "company", "tender"},
})
// Initialize HTTP server
e := initHTTPServer(conf, logger)
// Register routes
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler)
router.RegisterPublicRoutes(e, customerHandler)
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler)
router.RegisterPublicRoutes(e, customerHandler, tenderHandler)
// Start server
serverAddr := fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port)
cmd/web/router/routes.go
+33 -2
View File
@@ -3,12 +3,13 @@ package router
import (
"tm/internal/company"
"tm/internal/customer"
"tm/internal/tender"
"tm/internal/user"
"github.com/labstack/echo/v4"
)
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler) {
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler) {
adminV1 := e.Group("/admin/v1")
// Admin Users Routes
@@ -87,9 +88,33 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
customersGP.POST("/:id/companies/assign", customerHandler.AssignCompaniesToCustomer)
customersGP.POST("/:id/companies/remove", customerHandler.RemoveCompaniesFromCustomer)
}
// Admin Tenders Routes
tendersGP := adminV1.Group("/tenders")
{
tendersGP.Use(userHandler.AuthMiddleware())
tendersGP.POST("", tenderHandler.CreateTender)
tendersGP.GET("", tenderHandler.ListTenders)
tendersGP.GET("/:id", tenderHandler.GetTender)
tendersGP.PUT("/:id", tenderHandler.UpdateTender)
tendersGP.DELETE("/:id", tenderHandler.DeleteTender)
tendersGP.POST("/search", tenderHandler.SearchTenders)
tendersGP.GET("/statistics", tenderHandler.GetTenderStatistics)
tendersGP.GET("/dashboard", tenderHandler.GetDashboard)
}
// Admin Scraping Routes
scrapingGP := adminV1.Group("/scraping")
{
scrapingGP.Use(userHandler.AuthMiddleware())
scrapingGP.POST("/jobs", tenderHandler.StartScrapingJob)
scrapingGP.GET("/jobs", tenderHandler.ListScrapingJobs)
scrapingGP.GET("/jobs/:id", tenderHandler.GetScrapingJob)
scrapingGP.POST("/jobs/:id/cancel", tenderHandler.CancelScrapingJob)
}
}
func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler) {
func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler) {
v1 := e.Group("/api/v1")
customerGP := v1.Group("/profile")
@@ -103,4 +128,10 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler) {
profileGP.DELETE("/logout", customerHandler.Logout)
}
// Public Tenders Routes
tendersGP := v1.Group("/tenders")
{
tendersGP.GET("", tenderHandler.GetPublicTenders)
tendersGP.GET("/:id", tenderHandler.GetPublicTenderDetails)
}
}