From 10fd53af41b1f5c4c5acf98b209631c95f259ee2 Mon Sep 17 00:00:00 2001
From: Nima Nakhostin <nakhostin.nima1998@gmail.com>
Date: Sun, 23 Nov 2025 14:17:45 +0330
Subject: [PATCH] Refactor Scraper Configuration and Command Line Handling

- Removed command line flag parsing for date range scraping in the scraper's main function.
- Introduced configuration fields for one-time scraping and date range (FromDate, ToDate) in the ScraperConfig struct.
- Updated the main function to utilize the new configuration fields, improving the clarity and maintainability of the scraping logic.
- This change enhances the scraper's functionality by centralizing configuration management and reducing command line complexity.
---
 cmd/scraper/bootstrap/config.go |  3 +++
 cmd/scraper/main.go             | 23 +++++++----------------
 2 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/cmd/scraper/bootstrap/config.go b/cmd/scraper/bootstrap/config.go
index 28fc584..472f3dd 100644
--- a/cmd/scraper/bootstrap/config.go
+++ b/cmd/scraper/bootstrap/config.go
@@ -24,4 +24,7 @@ type ScraperConfig struct {
 	DownloadDir      string        `env:"TED_DOWNLOAD_DIR" envDefault:"./downloads"`
 	CleanupAfter     time.Duration `env:"TED_CLEANUP_AFTER" envDefault:"24h"`
 	ScrapingInterval string        `env:"TED_SCRAPING_INTERVAL" envDefault:"* * 10 * * *"`
+	OneTime          bool          `env:"TED_ONE_TIME" envDefault:"false"`
+	FromDate         string        `env:"TED_FROM_DATE" envDefault:"02/01/2025"`
+	ToDate           string        `env:"TED_TO_DATE" envDefault:"13/09/2025"`
 }
diff --git a/cmd/scraper/main.go b/cmd/scraper/main.go
index f70560f..3242069 100644
--- a/cmd/scraper/main.go
+++ b/cmd/scraper/main.go
@@ -2,7 +2,6 @@ package main
 
 import (
 	"context"
-	"flag"
 	"fmt"
 	"os"
 	"os/signal"
@@ -12,14 +11,6 @@ import (
 )
 
 func main() {
-	// Define command line flags
-	var (
-		fromDate = flag.String("from", "", "Start date for scraping (format: 02/01/2006)")
-		toDate   = flag.String("to", "", "End date for scraping (format: 02/01/2006)")
-		oneTime  = flag.Bool("onetime", false, "Run once with date range instead of scheduled mode")
-	)
-	flag.Parse()
-
 	// Load configuration
 	config, err := bootstrap.InitConfig()
 	if err != nil {
@@ -48,28 +39,28 @@ func main() {
 	notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
 
 	// Check if one-time date range scraping is requested
-	if *oneTime {
-		if *fromDate == "" || *toDate == "" {
-			appLogger.Error("Both -from and -to dates are required for one-time scraping", map[string]interface{}{})
+	if config.TED.OneTime {
+		if config.TED.FromDate == "" || config.TED.ToDate == "" {
+			appLogger.Error("Both TED_FROM_DATE and TED_TO_DATE are required for one-time scraping", map[string]interface{}{})
 			fmt.Println("Usage: ./scraper -onetime -from 12/01/2025 -to 13/09/2025")
 			os.Exit(1)
 		}
 
 		// Parse dates
-		from, err := time.Parse("02/01/2006", *fromDate)
+		from, err := time.Parse("02/01/2006", config.TED.FromDate)
 		if err != nil {
 			appLogger.Error("Invalid from date format", map[string]interface{}{
-				"from_date": *fromDate,
+				"from_date": config.TED.FromDate,
 				"error":     err.Error(),
 			})
 			fmt.Println("Date format should be: DD/MM/YYYY (e.g., 12/01/2025)")
 			os.Exit(1)
 		}
 
-		to, err := time.Parse("02/01/2006", *toDate)
+		to, err := time.Parse("02/01/2006", config.TED.ToDate)
 		if err != nil {
 			appLogger.Error("Invalid to date format", map[string]interface{}{
-				"to_date": *toDate,
+				"to_date": config.TED.ToDate,
 				"error":   err.Error(),
 			})
 			fmt.Println("Date format should be: DD/MM/YYYY (e.g., 13/09/2025)")