Enhance Scraper with One-Time Date Range Functionality

- Added command line flags for specifying a date range and a one-time scraping mode in the scraper's main function.
- Implemented validation for the date inputs to ensure both dates are provided and correctly formatted.
- Introduced a new RunOneTimeScraping function to handle the one-time scraping logic, including context management and error handling.
- Updated the TED scraper initialization to support both scheduled and one-time scraping modes, improving flexibility in data retrieval.
- Enhanced logging to provide clear feedback on the scraping process and any errors encountered.

cmd/scraper/bootstrap/bootstrap.go

@@ -104,7 +104,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
 	schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
 		Name: "TED Scraper Job",
 		Func: func() {
-			_ = tedScraper.Run(context.Background())
+			_ = tedScraper.Run(context.Background(), nil, nil)
 		},
 		Expr: config.TED.ScrapingInterval,
 	})
@@ -123,7 +123,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
 		appLogger.Info("Starting TED scraper", map[string]interface{}{
 			"interval": config.TED.ScrapingInterval,
 		})
-		scraperDone <- tedScraper.Run(ctx)
+		scraperDone <- tedScraper.Run(ctx, nil, nil)
 	}()
 
 	// Wait for shutdown signal or scraper completion
@@ -183,3 +183,63 @@ func InitNotificationService(conf config.NotificationConfig, log logger.Logger)
 
 	return *notificationSDK
 }
+
+// RunOneTimeScraping runs the TED scraper once for a specific date range
+func RunOneTimeScraping(
+	config Config,
+	mongoManager *mongo.ConnectionManager,
+	appLogger logger.Logger,
+	notificationService notification.SDK,
+	from, to *time.Time,
+) {
+	// Initialize notice repository
+	noticeRepo := notice.NewRepository(mongoManager, appLogger)
+
+	// Convert scraper config to TED config
+	tedConfig := &ted.Config{
+		BaseURL:          config.TED.BaseURL,
+		UserAgent:        config.TED.UserAgent,
+		DownloadDir:      config.TED.DownloadDir,
+		MaxRetries:       config.TED.MaxRetries,
+		MaxConcurrency:   config.TED.MaxConcurrency,
+		Timeout:          config.TED.Timeout,
+		RetryDelay:       config.TED.RetryDelay,
+		CleanupAfter:     config.TED.CleanupAfter,
+		ScrapingInterval: config.TED.ScrapingInterval,
+		AlertMail:        config.AlertMail,
+	}
+
+	// Initialize TED scraper
+	tedScraper := ted.NewTEDScraper(
+		tedConfig,
+		appLogger,
+		mongoManager,
+		noticeRepo,
+		notificationService,
+	)
+
+	appLogger.Info("Starting one-time TED scraping", map[string]interface{}{
+		"from_date": from.Format("02/01/2006"),
+		"to_date":   to.Format("02/01/2006"),
+	})
+
+	// Create context with timeout for the entire operation
+	ctx, cancel := context.WithTimeout(context.Background(), 24*time.Hour) // 24 hour timeout
+	defer cancel()
+
+	// Run the scraper with date range
+	err := tedScraper.Run(ctx, from, to)
+	if err != nil {
+		appLogger.Error("One-time scraping failed", map[string]interface{}{
+			"error":     err.Error(),
+			"from_date": from.Format("02/01/2006"),
+			"to_date":   to.Format("02/01/2006"),
+		})
+		os.Exit(1)
+	}
+
+	appLogger.Info("One-time TED scraping completed successfully", map[string]interface{}{
+		"from_date": from.Format("02/01/2006"),
+		"to_date":   to.Format("02/01/2006"),
+	})
+}

cmd/scraper/main.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"context"
+	"flag"
+	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
@@ -10,6 +12,14 @@ import (
 )
 
 func main() {
+	// Define command line flags
+	var (
+		fromDate = flag.String("from", "", "Start date for scraping (format: 02/01/2006)")
+		toDate   = flag.String("to", "", "End date for scraping (format: 02/01/2006)")
+		oneTime  = flag.Bool("onetime", false, "Run once with date range instead of scheduled mode")
+	)
+	flag.Parse()
+
 	// Load configuration
 	config, err := bootstrap.InitConfig()
 	if err != nil {
@@ -37,7 +47,41 @@ func main() {
 	// Initialize notification service
 	notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
 
-	// Initialize TED scraper
+	// Check if one-time date range scraping is requested
+	if *oneTime {
+		if *fromDate == "" || *toDate == "" {
+			appLogger.Error("Both -from and -to dates are required for one-time scraping", map[string]interface{}{})
+			fmt.Println("Usage: ./scraper -onetime -from 12/01/2025 -to 13/09/2025")
+			os.Exit(1)
+		}
+
+		// Parse dates
+		from, err := time.Parse("02/01/2006", *fromDate)
+		if err != nil {
+			appLogger.Error("Invalid from date format", map[string]interface{}{
+				"from_date": *fromDate,
+				"error":     err.Error(),
+			})
+			fmt.Println("Date format should be: DD/MM/YYYY (e.g., 12/01/2025)")
+			os.Exit(1)
+		}
+
+		to, err := time.Parse("02/01/2006", *toDate)
+		if err != nil {
+			appLogger.Error("Invalid to date format", map[string]interface{}{
+				"to_date": *toDate,
+				"error":   err.Error(),
+			})
+			fmt.Println("Date format should be: DD/MM/YYYY (e.g., 13/09/2025)")
+			os.Exit(1)
+		}
+
+		// Run one-time scraping
+		bootstrap.RunOneTimeScraping(*config, mongoManager, appLogger, notificationService, &from, &to)
+		return
+	}
+
+	// Initialize TED scraper for scheduled mode
 	bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService)
 
 	// Set up signal handling for graceful shutdown

ted/calendar.go

@@ -45,7 +45,9 @@ func GetOJS(baseURL string, year int, date string) (string, error) {
 	for _, row := range records[1:] {
 		rowDate := strings.TrimSpace(row[1])
 		if rowDate == strings.TrimSpace(date) {
-			ojs = fmt.Sprintf("%v00%v", year, strings.TrimSpace(fmt.Sprint(row[0])))
+			// Format: YEAR + zero-padded OJS number to make total length 9 digits
+			// Examples: 202500004, 202500014, 202500196
+			ojs = fmt.Sprintf("%d%05s", year, strings.TrimSpace(fmt.Sprint(row[0])))
 			break
 		}
 	}