n.nakhostin
08116981f4
- Added command line flags for specifying a date range and a one-time scraping mode in the scraper's main function. - Implemented validation for the date inputs to ensure both dates are provided and correctly formatted. - Introduced a new RunOneTimeScraping function to handle the one-time scraping logic, including context management and error handling. - Updated the TED scraper initialization to support both scheduled and one-time scraping modes, improving flexibility in data retrieval. - Enhanced logging to provide clear feedback on the scraping process and any errors encountered.
97 lines
2.7 KiB
Go
97 lines
2.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"os/signal"
|
|
"syscall"
|
|
"time"
|
|
"tm/cmd/scraper/bootstrap"
|
|
)
|
|
|
|
func main() {
|
|
// Define command line flags
|
|
var (
|
|
fromDate = flag.String("from", "", "Start date for scraping (format: 02/01/2006)")
|
|
toDate = flag.String("to", "", "End date for scraping (format: 02/01/2006)")
|
|
oneTime = flag.Bool("onetime", false, "Run once with date range instead of scheduled mode")
|
|
)
|
|
flag.Parse()
|
|
|
|
// Load configuration
|
|
config, err := bootstrap.InitConfig()
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
|
|
// Initialize logger
|
|
appLogger := bootstrap.InitLogger(config.Logging)
|
|
appLogger.Info("Starting TED scraper application", map[string]interface{}{
|
|
"version": "1.0.0",
|
|
})
|
|
|
|
// Initialize MongoDB connection
|
|
mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
|
|
defer func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
if err := mongoManager.Close(ctx); err != nil {
|
|
appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
}
|
|
}()
|
|
|
|
// Initialize notification service
|
|
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
|
|
|
|
// Check if one-time date range scraping is requested
|
|
if *oneTime {
|
|
if *fromDate == "" || *toDate == "" {
|
|
appLogger.Error("Both -from and -to dates are required for one-time scraping", map[string]interface{}{})
|
|
fmt.Println("Usage: ./scraper -onetime -from 12/01/2025 -to 13/09/2025")
|
|
os.Exit(1)
|
|
}
|
|
|
|
// Parse dates
|
|
from, err := time.Parse("02/01/2006", *fromDate)
|
|
if err != nil {
|
|
appLogger.Error("Invalid from date format", map[string]interface{}{
|
|
"from_date": *fromDate,
|
|
"error": err.Error(),
|
|
})
|
|
fmt.Println("Date format should be: DD/MM/YYYY (e.g., 12/01/2025)")
|
|
os.Exit(1)
|
|
}
|
|
|
|
to, err := time.Parse("02/01/2006", *toDate)
|
|
if err != nil {
|
|
appLogger.Error("Invalid to date format", map[string]interface{}{
|
|
"to_date": *toDate,
|
|
"error": err.Error(),
|
|
})
|
|
fmt.Println("Date format should be: DD/MM/YYYY (e.g., 13/09/2025)")
|
|
os.Exit(1)
|
|
}
|
|
|
|
// Run one-time scraping
|
|
bootstrap.RunOneTimeScraping(*config, mongoManager, appLogger, notificationService, &from, &to)
|
|
return
|
|
}
|
|
|
|
// Initialize TED scraper for scheduled mode
|
|
bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService)
|
|
|
|
// Set up signal handling for graceful shutdown
|
|
signalChan := make(chan os.Signal, 1)
|
|
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
|
|
|
|
// Wait for shutdown signal
|
|
<-signalChan
|
|
appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{})
|
|
|
|
appLogger.Info("TED scraper application stopped", map[string]interface{}{})
|
|
}
|