Refactor Worker Initialization to Integrate GLM SDK

- Updated the worker initialization process to include the new GLM SDK, enhancing the worker's capabilities for translation tasks.
- Modified the InitWorker function and NewNoticeWorker constructor to accept the GLM service, ensuring a cohesive integration.
- Implemented the GLM service initialization and logging for successful setup, improving maintainability and usability.
- Updated the NoticeWorker to utilize the GLM SDK for translating notice titles and descriptions, enhancing functionality and user experience.

ted/scraper.go

@@ -345,7 +345,6 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 	// 	// Create new tender
 	s.logger.Info("Creating new tender", map[string]interface{}{
 		"contract_notice_id": t.ContractNoticeID,
-		"tender_id":          t.TenderID,
 	})
 
 	t.CreatedAt = time.Now().Unix()
@@ -355,14 +354,12 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 	if err != nil {
 		s.logger.Error("Failed to create new tender", map[string]interface{}{
 			"contract_notice_id": t.ContractNoticeID,
-			"tender_id":          t.TenderID,
 			"error":              err.Error(),
 		})
 		return fmt.Errorf("failed to create tender: %w", err)
 	}
 
 	s.logger.Info("Successfully created new tender", map[string]interface{}{
-		"tender_id":          t.TenderID,
 		"contract_notice_id": t.ContractNoticeID,
 	})
 	// }
@@ -372,7 +369,6 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 		"notice_id":   id,
 		"notice_type": noticeType,
 		"file_name":   fileName,
-		"tender_id":   t.TenderID,
 		"tender_url":  t.TenderURL,
 	})
 
@@ -380,34 +376,118 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
 	return nil
 }
 
-func (s *TEDScraper) Run(ctx context.Context) error {
-	now := time.Now().Local()
-	s.logger.Info("Running periodic scraping", map[string]interface{}{
-		"time": now.Format(time.RFC3339),
+func (s *TEDScraper) Run(ctx context.Context, from, to *time.Time) error {
+	// If no date range is provided, use current date
+	if from == nil && to == nil {
+		now := time.Now().Local()
+		s.logger.Info("Running periodic scraping for current date", map[string]interface{}{
+			"date": now.Format("02/01/2006"),
+		})
+		return s.processSingleDate(ctx, now)
+	}
+
+	// Validate date range
+	if from == nil || to == nil {
+		return fmt.Errorf("both from and to dates must be provided for date range scraping")
+	}
+
+	if from.After(*to) {
+		return fmt.Errorf("from date cannot be after to date")
+	}
+
+	s.logger.Info("Running date range scraping", map[string]interface{}{
+		"from_date": from.Format("02/01/2006"),
+		"to_date":   to.Format("02/01/2006"),
+	})
+
+	// Process each date in the range
+	current := *from
+	totalDays := 0
+	successfulDays := 0
+	var errors []string
+
+	for !current.After(*to) {
+		totalDays++
+
+		s.logger.Info("Processing date", map[string]interface{}{
+			"date":     current.Format("02/01/2006"),
+			"progress": fmt.Sprintf("%d/%d", totalDays, int(to.Sub(*from).Hours()/24)+1),
+		})
+
+		err := s.processSingleDate(ctx, current)
+		if err != nil {
+			errorMsg := fmt.Sprintf("failed to process date %s: %v", current.Format("02/01/2006"), err)
+			errors = append(errors, errorMsg)
+			s.logger.Error("Failed to process date", map[string]interface{}{
+				"date":  current.Format("02/01/2006"),
+				"error": err.Error(),
+			})
+		} else {
+			successfulDays++
+		}
+
+		// Check for context cancellation
+		select {
+		case <-ctx.Done():
+			s.logger.Warn("Date range scraping cancelled", map[string]interface{}{
+				"processed_days":  totalDays,
+				"successful_days": successfulDays,
+				"cancelled_at":    current.Format("02/01/2006"),
+			})
+			return ctx.Err()
+		default:
+		}
+
+		// Move to next day
+		current = current.AddDate(0, 0, 1)
+	}
+
+	s.logger.Info("Date range scraping completed", map[string]interface{}{
+		"total_days":      totalDays,
+		"successful_days": successfulDays,
+		"failed_days":     len(errors),
+		"errors":          errors,
+	})
+
+	// Return error if no days were processed successfully
+	if successfulDays == 0 && len(errors) > 0 {
+		return fmt.Errorf("failed to process any dates in range: %v", errors)
+	}
+
+	return nil
+}
+
+// processSingleDate processes data for a single date
+func (s *TEDScraper) processSingleDate(ctx context.Context, date time.Time) error {
+	s.logger.Info("Processing single date", map[string]interface{}{
+		"date": date.Format("02/01/2006"),
 	})
 
 	// get ojs
-	ojs, err := GetOJS(s.config.BaseURL, now.Year(), now.Format("02/01/2006"))
+	ojs, err := GetOJS(s.config.BaseURL, date.Year(), date.Format("02/01/2006"))
 	if err != nil {
 		s.logger.Error("Failed to get OJS", map[string]interface{}{
+			"date":  date.Format("02/01/2006"),
 			"error": err.Error(),
 		})
-		return nil
+		return fmt.Errorf("failed to get OJS for date %s: %w", date.Format("02/01/2006"), err)
 	}
 
 	// download daily package
 	result, err := s.DownloadDailyPackage(ctx, ojs)
 	if err != nil {
 		s.logger.Error("Failed to download daily package", map[string]interface{}{
+			"date":  date.Format("02/01/2006"),
+			"ojs":   ojs,
 			"error": err.Error(),
 		})
-		return err
+		return fmt.Errorf("failed to download daily package for date %s: %w", date.Format("02/01/2006"), err)
 	}
 
 	// process daily package
 	s.logger.Info("Processed daily package successfully", map[string]interface{}{
 		"result": result,
-		"time":   now.Format(time.RFC3339),
+		"date":   date.Format("02/01/2006"),
 		"ojs":    ojs,
 	})