Enhance Scraper with One-Time Date Range Functionality

- Added command line flags for specifying a date range and a one-time scraping mode in the scraper's main function.
- Implemented validation for the date inputs to ensure both dates are provided and correctly formatted.
- Introduced a new RunOneTimeScraping function to handle the one-time scraping logic, including context management and error handling.
- Updated the TED scraper initialization to support both scheduled and one-time scraping modes, improving flexibility in data retrieval.
- Enhanced logging to provide clear feedback on the scraping process and any errors encountered.
This commit is contained in:
n.nakhostin
2025-11-04 16:53:31 +03:30
parent fafccd0d74
commit 08116981f4
3 changed files with 110 additions and 4 deletions
+45 -1
View File
@@ -2,6 +2,8 @@ package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"syscall"
@@ -10,6 +12,14 @@ import (
)
func main() {
// Define command line flags
var (
fromDate = flag.String("from", "", "Start date for scraping (format: 02/01/2006)")
toDate = flag.String("to", "", "End date for scraping (format: 02/01/2006)")
oneTime = flag.Bool("onetime", false, "Run once with date range instead of scheduled mode")
)
flag.Parse()
// Load configuration
config, err := bootstrap.InitConfig()
if err != nil {
@@ -37,7 +47,41 @@ func main() {
// Initialize notification service
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
// Initialize TED scraper
// Check if one-time date range scraping is requested
if *oneTime {
if *fromDate == "" || *toDate == "" {
appLogger.Error("Both -from and -to dates are required for one-time scraping", map[string]interface{}{})
fmt.Println("Usage: ./scraper -onetime -from 12/01/2025 -to 13/09/2025")
os.Exit(1)
}
// Parse dates
from, err := time.Parse("02/01/2006", *fromDate)
if err != nil {
appLogger.Error("Invalid from date format", map[string]interface{}{
"from_date": *fromDate,
"error": err.Error(),
})
fmt.Println("Date format should be: DD/MM/YYYY (e.g., 12/01/2025)")
os.Exit(1)
}
to, err := time.Parse("02/01/2006", *toDate)
if err != nil {
appLogger.Error("Invalid to date format", map[string]interface{}{
"to_date": *toDate,
"error": err.Error(),
})
fmt.Println("Date format should be: DD/MM/YYYY (e.g., 13/09/2025)")
os.Exit(1)
}
// Run one-time scraping
bootstrap.RunOneTimeScraping(*config, mongoManager, appLogger, notificationService, &from, &to)
return
}
// Initialize TED scraper for scheduled mode
bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService)
// Set up signal handling for graceful shutdown