Refactor queue management by removing deprecated components and updating configuration

- Removed the QueueConfig structure and related queue management files as they are no longer in use. - Updated the worker initialization to reflect the removal of queue-related configurations. - Cleaned up the bootstrap process by eliminating deprecated logging related to the queue system. This update streamlines the worker's configuration and prepares the codebase for future enhancements without the legacy queue management components.
2026-06-08 23:49:51 +03:30
parent d07e1c9cf0
commit 9cd22c24b6
8 changed files with 0 additions and 735 deletions
@@ -1,165 +0,0 @@
-package workers
-
-import (
-	"context"
-	"time"
-	"tm/internal/tender"
-	"tm/pkg/logger"
-	"tm/pkg/mongo"
-	"tm/pkg/queue"
-)
-
-// QueueProducerWorker handles enqueuing unscraped tenders to the job queue
-type QueueProducerWorker struct {
-	Mongo      *mongo.ConnectionManager
-	Logger     logger.Logger
-	TenderRepo tender.TenderRepository
-	Queue      queue.Queue
-	BatchSize  int
-}
-
-// NewQueueProducerWorker creates a new queue producer worker
-func NewQueueProducerWorker(
-	mongo *mongo.ConnectionManager,
-	logger logger.Logger,
-	tenderRepo tender.TenderRepository,
-	queue queue.Queue,
-	batchSize int,
-) *QueueProducerWorker {
-	if batchSize <= 0 {
-		batchSize = 100 // Default batch size
-	}
-
-	logger.Info("Creating new queue producer worker", map[string]interface{}{
-		"batch_size": batchSize,
-	})
-
-	return &QueueProducerWorker{
-		Mongo:      mongo,
-		Logger:     logger,
-		TenderRepo: tenderRepo,
-		Queue:      queue,
-		BatchSize:  batchSize,
-	}
-}
-
-// Run continuously processes and enqueues unscraped tenders
-func (w *QueueProducerWorker) Run() {
-	w.Logger.Info("Queue producer worker started", map[string]interface{}{})
-
-	defer func() {
-		w.Logger.Info("Queue producer worker stopped", map[string]interface{}{})
-	}()
-
-	batchIndex := 0
-	hasMore := true
-
-	for hasMore {
-		w.Logger.Info("Queue producer fetching batch of tenders", map[string]interface{}{
-			"batch_index": batchIndex,
-			"batch_size":  w.BatchSize,
-		})
-
-		// Get batch of unscraped tenders
-		tenders, totalCount, err := w.TenderRepo.GetUnScrapedTenders(
-			context.Background(),
-			w.BatchSize,
-			batchIndex*w.BatchSize,
-		)
-
-		if err != nil {
-			w.Logger.Error("Failed to fetch unscraped tenders", map[string]interface{}{
-				"batch_index": batchIndex,
-				"error":       err.Error(),
-			})
-			// Wait before retrying
-			time.Sleep(30 * time.Second)
-			continue
-		}
-
-		w.Logger.Info("Fetched tender batch", map[string]interface{}{
-			"batch_size":   len(tenders),
-			"total_count":  totalCount,
-			"batch_index":  batchIndex,
-		})
-
-		if len(tenders) == 0 {
-			// No more tenders in this batch
-			hasMore = false
-			w.Logger.Info("No more unscraped tenders found", map[string]interface{}{
-				"total_processed": totalCount,
-			})
-			break
-		}
-
-		// Enqueue each tender
-		successCount := 0
-		failCount := 0
-
-		for _, t := range tenders {
-			if err := w.enqueueTender(&t); err != nil {
-				w.Logger.Error("Failed to enqueue tender", map[string]interface{}{
-					"tender_id":             t.ID.Hex(),
-					"notice_publication_id": t.NoticePublicationID,
-					"error":                 err.Error(),
-				})
-				failCount++
-			} else {
-				successCount++
-			}
-		}
-
-		w.Logger.Info("Batch enqueuing completed", map[string]interface{}{
-			"success_count": successCount,
-			"fail_count":    failCount,
-			"batch_index":   batchIndex,
-		})
-
-		// Move to next batch
-		batchIndex++
-
-		// Check if there are more tenders
-		if int64(batchIndex*w.BatchSize) >= totalCount {
-			hasMore = false
-		}
-
-		// Avoid tight loop - brief pause between batches
-		time.Sleep(1 * time.Second)
-	}
-}
-
-// enqueueTender adds a single tender to the queue
-func (w *QueueProducerWorker) enqueueTender(t *tender.Tender) error {
-	// Skip if missing required fields
-	if t.NoticePublicationID == "" || t.DocumentURI == "" {
-		w.Logger.Warn("Skipping tender - missing required fields", map[string]interface{}{
-			"tender_id":             t.ID.Hex(),
-			"notice_publication_id": t.NoticePublicationID,
-			"document_uri":          t.DocumentURI,
-		})
-		return nil
-	}
-
-	// Create job
-	job := queue.NewDocumentScraperJob(
-		t.NoticePublicationID,
-		t.DocumentURI,
-		t.ID.Hex(),
-		queue.DefaultMaxRetries,
-	)
-
-	// Enqueue job
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	if err := w.Queue.Enqueue(ctx, job); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-// RunOnce processes all unscraped tenders in a single run (for scheduled use)
-func (w *QueueProducerWorker) RunOnce() {
-	w.Run()
-}