n.nakhostin
0c50935c42
- Deleted the TED XML parser and scraper implementation files, including parser.go, scraper.go, and ted.go, to streamline the codebase and eliminate unused components. - Commented out the initialization code in main.go to prevent execution while maintaining the structure for future reference. - Removed the README.md for the TED package, which contained documentation for the now-deleted parser and scraper functionalities.
141 lines
4.3 KiB
Go
141 lines
4.3 KiB
Go
package bootstrap
|
|
|
|
// import (
|
|
// "context"
|
|
// "fmt"
|
|
// "os"
|
|
// "os/signal"
|
|
// "syscall"
|
|
// "time"
|
|
// "tm/internal/tender"
|
|
// "tm/pkg/config"
|
|
// "tm/pkg/logger"
|
|
// "tm/pkg/mongo"
|
|
// "tm/ted"
|
|
// )
|
|
|
|
// // Init Application Configuration
|
|
// func InitConfig() (*Config, error) {
|
|
// conf, err := config.LoadConfig(".", &Config{})
|
|
// if err != nil {
|
|
// return nil, fmt.Errorf("failed to load config: %v", err)
|
|
// }
|
|
|
|
// return conf, nil
|
|
// }
|
|
|
|
// // Init Logger Service
|
|
// func InitLogger(conf config.LoggingConfig) logger.Logger {
|
|
// return logger.NewLogger(&logger.Config{
|
|
// Level: conf.Level,
|
|
// Format: conf.Format,
|
|
// Output: conf.Output,
|
|
// File: logger.FileConfig{
|
|
// Path: conf.File.Path,
|
|
// MaxSize: conf.File.MaxSize,
|
|
// MaxBackups: conf.File.MaxBackups,
|
|
// MaxAge: conf.File.MaxAge,
|
|
// Compress: conf.File.Compress,
|
|
// },
|
|
// })
|
|
// }
|
|
|
|
// // Init MongoDB Connection Manager
|
|
// func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
|
|
// // Convert infra.MongoConfig to mongo.ConnectionConfig
|
|
// connectionConfig := mongo.ConnectionConfig{
|
|
// URI: conf.URI,
|
|
// Database: conf.Name,
|
|
// MaxPoolSize: uint64(conf.MaxPoolSize),
|
|
// MinPoolSize: 5, // Default minimum pool size
|
|
// MaxConnIdleTime: 30 * time.Minute, // Default idle time
|
|
// ConnectTimeout: conf.Timeout,
|
|
// ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
|
|
// }
|
|
|
|
// // Create MongoDB connection manager
|
|
// connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
|
|
// if err != nil {
|
|
// log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
|
|
// "error": err.Error(),
|
|
// "uri": conf.URI,
|
|
// "db": conf.Name,
|
|
// })
|
|
// panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
|
|
// }
|
|
|
|
// log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
|
|
// "database": conf.Name,
|
|
// "uri": conf.URI,
|
|
// "pool_size": conf.MaxPoolSize,
|
|
// })
|
|
|
|
// return connectionManager
|
|
// }
|
|
|
|
// // init TED scraper
|
|
// func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger) {
|
|
// // Initialize tender repository
|
|
// tenderRepo := tender.NewRepository(mongoManager, appLogger)
|
|
|
|
// // Initialize TED scraper
|
|
// tedScraper := ted.NewTEDScraper(
|
|
// &ted.Config{
|
|
// BaseURL: config.TED.BaseURL,
|
|
// Timeout: config.TED.Timeout,
|
|
// MaxRetries: config.TED.MaxRetries,
|
|
// RetryDelay: config.TED.RetryDelay,
|
|
// UserAgent: config.TED.UserAgent,
|
|
// MaxConcurrency: config.TED.MaxConcurrency,
|
|
// DownloadDir: config.TED.DownloadDir,
|
|
// CleanupAfter: config.TED.CleanupAfter,
|
|
// ScrapingInterval: config.TED.ScrapingInterval,
|
|
// },
|
|
// appLogger,
|
|
// mongoManager,
|
|
// tenderRepo,
|
|
// )
|
|
|
|
// // Create context with cancellation
|
|
// ctx, cancel := context.WithCancel(context.Background())
|
|
// defer cancel()
|
|
|
|
// // Set up signal handling for graceful shutdown
|
|
// signalChan := make(chan os.Signal, 1)
|
|
// signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
|
|
|
// // Start the scraper in a goroutine
|
|
// scraperDone := make(chan error, 1)
|
|
// go func() {
|
|
// appLogger.Info("Starting TED scraper", map[string]interface{}{
|
|
// "interval": config.TED.ScrapingInterval.String(),
|
|
// })
|
|
// scraperDone <- tedScraper.RunPeriodicScraping(ctx)
|
|
// }()
|
|
|
|
// // Wait for shutdown signal or scraper completion
|
|
// select {
|
|
// case <-signalChan:
|
|
// appLogger.Info("Received shutdown signal, stopping scraper...", map[string]interface{}{})
|
|
// cancel()
|
|
|
|
// // Wait for scraper to finish gracefully with timeout
|
|
// shutdownTimeout := time.NewTimer(30 * time.Second)
|
|
// select {
|
|
// case <-scraperDone:
|
|
// appLogger.Info("Scraper stopped gracefully", map[string]interface{}{})
|
|
// case <-shutdownTimeout.C:
|
|
// appLogger.Warn("Scraper shutdown timeout, forcing exit", map[string]interface{}{})
|
|
// }
|
|
|
|
// case err := <-scraperDone:
|
|
// if err != nil {
|
|
// appLogger.Error("Scraper finished with error", map[string]interface{}{
|
|
// "error": err.Error(),
|
|
// })
|
|
// os.Exit(1)
|
|
// }
|
|
// appLogger.Info("Scraper finished successfully", map[string]interface{}{})
|
|
// }
|
|
// }
|