Enhance Scraper with One-Time Date Range Functionality
- Added command line flags for specifying a date range and a one-time scraping mode in the scraper's main function. - Implemented validation for the date inputs to ensure both dates are provided and correctly formatted. - Introduced a new RunOneTimeScraping function to handle the one-time scraping logic, including context management and error handling. - Updated the TED scraper initialization to support both scheduled and one-time scraping modes, improving flexibility in data retrieval. - Enhanced logging to provide clear feedback on the scraping process and any errors encountered.
This commit is contained in:
+45
-1
@@ -2,6 +2,8 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
@@ -10,6 +12,14 @@ import (
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Define command line flags
|
||||
var (
|
||||
fromDate = flag.String("from", "", "Start date for scraping (format: 02/01/2006)")
|
||||
toDate = flag.String("to", "", "End date for scraping (format: 02/01/2006)")
|
||||
oneTime = flag.Bool("onetime", false, "Run once with date range instead of scheduled mode")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
// Load configuration
|
||||
config, err := bootstrap.InitConfig()
|
||||
if err != nil {
|
||||
@@ -37,7 +47,41 @@ func main() {
|
||||
// Initialize notification service
|
||||
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
|
||||
|
||||
// Initialize TED scraper
|
||||
// Check if one-time date range scraping is requested
|
||||
if *oneTime {
|
||||
if *fromDate == "" || *toDate == "" {
|
||||
appLogger.Error("Both -from and -to dates are required for one-time scraping", map[string]interface{}{})
|
||||
fmt.Println("Usage: ./scraper -onetime -from 12/01/2025 -to 13/09/2025")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Parse dates
|
||||
from, err := time.Parse("02/01/2006", *fromDate)
|
||||
if err != nil {
|
||||
appLogger.Error("Invalid from date format", map[string]interface{}{
|
||||
"from_date": *fromDate,
|
||||
"error": err.Error(),
|
||||
})
|
||||
fmt.Println("Date format should be: DD/MM/YYYY (e.g., 12/01/2025)")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
to, err := time.Parse("02/01/2006", *toDate)
|
||||
if err != nil {
|
||||
appLogger.Error("Invalid to date format", map[string]interface{}{
|
||||
"to_date": *toDate,
|
||||
"error": err.Error(),
|
||||
})
|
||||
fmt.Println("Date format should be: DD/MM/YYYY (e.g., 13/09/2025)")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Run one-time scraping
|
||||
bootstrap.RunOneTimeScraping(*config, mongoManager, appLogger, notificationService, &from, &to)
|
||||
return
|
||||
}
|
||||
|
||||
// Initialize TED scraper for scheduled mode
|
||||
bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService)
|
||||
|
||||
// Set up signal handling for graceful shutdown
|
||||
|
||||
Reference in New Issue
Block a user