From 08116981f471bd21a59dbda3eb6a8c994c853722 Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Tue, 4 Nov 2025 16:53:31 +0330 Subject: [PATCH] Enhance Scraper with One-Time Date Range Functionality - Added command line flags for specifying a date range and a one-time scraping mode in the scraper's main function. - Implemented validation for the date inputs to ensure both dates are provided and correctly formatted. - Introduced a new RunOneTimeScraping function to handle the one-time scraping logic, including context management and error handling. - Updated the TED scraper initialization to support both scheduled and one-time scraping modes, improving flexibility in data retrieval. - Enhanced logging to provide clear feedback on the scraping process and any errors encountered. --- cmd/scraper/bootstrap/bootstrap.go | 64 +++++++++++++++++++++++++++++- cmd/scraper/main.go | 46 ++++++++++++++++++++- ted/calendar.go | 4 +- 3 files changed, 110 insertions(+), 4 deletions(-) diff --git a/cmd/scraper/bootstrap/bootstrap.go b/cmd/scraper/bootstrap/bootstrap.go index 1f866c4..1af1ed0 100644 --- a/cmd/scraper/bootstrap/bootstrap.go +++ b/cmd/scraper/bootstrap/bootstrap.go @@ -104,7 +104,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ Name: "TED Scraper Job", Func: func() { - _ = tedScraper.Run(context.Background()) + _ = tedScraper.Run(context.Background(), nil, nil) }, Expr: config.TED.ScrapingInterval, }) @@ -123,7 +123,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog appLogger.Info("Starting TED scraper", map[string]interface{}{ "interval": config.TED.ScrapingInterval, }) - scraperDone <- tedScraper.Run(ctx) + scraperDone <- tedScraper.Run(ctx, nil, nil) }() // Wait for shutdown signal or scraper completion @@ -183,3 +183,63 @@ func InitNotificationService(conf config.NotificationConfig, log logger.Logger) return *notificationSDK } + +// RunOneTimeScraping runs the TED scraper once for a specific date range +func RunOneTimeScraping( + config Config, + mongoManager *mongo.ConnectionManager, + appLogger logger.Logger, + notificationService notification.SDK, + from, to *time.Time, +) { + // Initialize notice repository + noticeRepo := notice.NewRepository(mongoManager, appLogger) + + // Convert scraper config to TED config + tedConfig := &ted.Config{ + BaseURL: config.TED.BaseURL, + UserAgent: config.TED.UserAgent, + DownloadDir: config.TED.DownloadDir, + MaxRetries: config.TED.MaxRetries, + MaxConcurrency: config.TED.MaxConcurrency, + Timeout: config.TED.Timeout, + RetryDelay: config.TED.RetryDelay, + CleanupAfter: config.TED.CleanupAfter, + ScrapingInterval: config.TED.ScrapingInterval, + AlertMail: config.AlertMail, + } + + // Initialize TED scraper + tedScraper := ted.NewTEDScraper( + tedConfig, + appLogger, + mongoManager, + noticeRepo, + notificationService, + ) + + appLogger.Info("Starting one-time TED scraping", map[string]interface{}{ + "from_date": from.Format("02/01/2006"), + "to_date": to.Format("02/01/2006"), + }) + + // Create context with timeout for the entire operation + ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) // 24 hour timeout + defer cancel() + + // Run the scraper with date range + err := tedScraper.Run(ctx, from, to) + if err != nil { + appLogger.Error("One-time scraping failed", map[string]interface{}{ + "error": err.Error(), + "from_date": from.Format("02/01/2006"), + "to_date": to.Format("02/01/2006"), + }) + os.Exit(1) + } + + appLogger.Info("One-time TED scraping completed successfully", map[string]interface{}{ + "from_date": from.Format("02/01/2006"), + "to_date": to.Format("02/01/2006"), + }) +} diff --git a/cmd/scraper/main.go b/cmd/scraper/main.go index ac77140..f70560f 100644 --- a/cmd/scraper/main.go +++ b/cmd/scraper/main.go @@ -2,6 +2,8 @@ package main import ( "context" + "flag" + "fmt" "os" "os/signal" "syscall" @@ -10,6 +12,14 @@ import ( ) func main() { + // Define command line flags + var ( + fromDate = flag.String("from", "", "Start date for scraping (format: 02/01/2006)") + toDate = flag.String("to", "", "End date for scraping (format: 02/01/2006)") + oneTime = flag.Bool("onetime", false, "Run once with date range instead of scheduled mode") + ) + flag.Parse() + // Load configuration config, err := bootstrap.InitConfig() if err != nil { @@ -37,7 +47,41 @@ func main() { // Initialize notification service notificationService := bootstrap.InitNotificationService(config.Notification, appLogger) - // Initialize TED scraper + // Check if one-time date range scraping is requested + if *oneTime { + if *fromDate == "" || *toDate == "" { + appLogger.Error("Both -from and -to dates are required for one-time scraping", map[string]interface{}{}) + fmt.Println("Usage: ./scraper -onetime -from 12/01/2025 -to 13/09/2025") + os.Exit(1) + } + + // Parse dates + from, err := time.Parse("02/01/2006", *fromDate) + if err != nil { + appLogger.Error("Invalid from date format", map[string]interface{}{ + "from_date": *fromDate, + "error": err.Error(), + }) + fmt.Println("Date format should be: DD/MM/YYYY (e.g., 12/01/2025)") + os.Exit(1) + } + + to, err := time.Parse("02/01/2006", *toDate) + if err != nil { + appLogger.Error("Invalid to date format", map[string]interface{}{ + "to_date": *toDate, + "error": err.Error(), + }) + fmt.Println("Date format should be: DD/MM/YYYY (e.g., 13/09/2025)") + os.Exit(1) + } + + // Run one-time scraping + bootstrap.RunOneTimeScraping(*config, mongoManager, appLogger, notificationService, &from, &to) + return + } + + // Initialize TED scraper for scheduled mode bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService) // Set up signal handling for graceful shutdown diff --git a/ted/calendar.go b/ted/calendar.go index 0c4cc4a..4491b1f 100644 --- a/ted/calendar.go +++ b/ted/calendar.go @@ -45,7 +45,9 @@ func GetOJS(baseURL string, year int, date string) (string, error) { for _, row := range records[1:] { rowDate := strings.TrimSpace(row[1]) if rowDate == strings.TrimSpace(date) { - ojs = fmt.Sprintf("%v00%v", year, strings.TrimSpace(fmt.Sprint(row[0]))) + // Format: YEAR + zero-padded OJS number to make total length 9 digits + // Examples: 202500004, 202500014, 202500196 + ojs = fmt.Sprintf("%d%05s", year, strings.TrimSpace(fmt.Sprint(row[0]))) break } }