mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor queue management by removing deprecated components and updating configuration

Browse Source 
- Removed the QueueConfig structure and related queue management files as they are no longer in use.
- Updated the worker initialization to reflect the removal of queue-related configurations.
- Cleaned up the bootstrap process by eliminating deprecated logging related to the queue system.

This update streamlines the worker's configuration and prepares the codebase for future enhancements without the legacy queue management components.
This commit is contained in:
Mazyar
2026-06-08 23:49:51 +03:30
parent d07e1c9cf0
commit 9cd22c24b6
8 changed files with 0 additions and 735 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/bootstrap/bootstrap.go
-8
View File
@@ -85,8 +85,6 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
"notice_processing_limit": config.Worker.NoticeProcessingLimit,
"notice_batch_pause": config.Worker.NoticeBatchPause.String(),
"notice_fetch_error_backoff": config.Worker.NoticeFetchErrorBackoff.String(),
"queue_enabled": config.Queue.Enabled,
"queue_mode": config.Queue.Mode,
})
// Initialize repositories
noticeRepo := notice.NewRepository(mongoManager, appLogger)
@@ -95,12 +93,6 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
// Create a single shared cron scheduler for all recurring jobs
scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
if config.Queue.Enabled {
appLogger.Warn("Redis document-scraper queue is deprecated; document scraping is owned by the AI service pipeline", map[string]interface{}{
"queue_mode": config.Queue.Mode,
})
}
// Document summarization via external AI service (requires AI_SUMMARIZER_API_BASE_URL)
if aiClient != nil {
scheduler.AddJob(schedule.Job{
cmd/worker/bootstrap/config.go
-16
View File
@@ -15,25 +15,9 @@ type Config struct {
Scraper config.ScraperConfig
MinIO config.MinIOConfig
AISummarizer AISummarizerConfig
Queue QueueConfig `env:"QUEUE_" envPrefix:"QUEUE_"`
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
}
// QueueConfig defines the queue configuration
type QueueConfig struct {
Enabled bool `env:"QUEUE_ENABLED" envDefault:"true"`
RedisURL string `env:"QUEUE_REDIS_URL" envDefault:"redis://localhost:6379"`
Mode string `env:"QUEUE_MODE" envDefault:"consumer"` // "producer", "consumer", or "both"
MaxRetries int `env:"QUEUE_MAX_RETRIES" envDefault:"3"`
RetryBackoff time.Duration `env:"QUEUE_RETRY_BACKOFF" envDefault:"10s"`
JobTimeout time.Duration `env:"QUEUE_JOB_TIMEOUT" envDefault:"5m"`
PoolSize int `env:"QUEUE_POOL_SIZE" envDefault:"10"`
IsDLQEnabled bool `env:"QUEUE_DLQ_ENABLED" envDefault:"true"`
Concurrency int `env:"QUEUE_CONCURRENCY" envDefault:"5"`
ProducerBatch int `env:"QUEUE_PRODUCER_BATCH" envDefault:"100"`
ProducerInterval string `env:"QUEUE_PRODUCER_INTERVAL" envDefault:"0 */6 * * * *"` // Every 6 hours
}
type WorkerConfig struct {
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
cmd/worker/workers/queue_producer.go
-165
View File
@@ -1,165 +0,0 @@
package workers
import (
"context"
"time"
"tm/internal/tender"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/queue"
)
// QueueProducerWorker handles enqueuing unscraped tenders to the job queue
type QueueProducerWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tender.TenderRepository
Queue queue.Queue
BatchSize int
}
// NewQueueProducerWorker creates a new queue producer worker
func NewQueueProducerWorker(
mongo *mongo.ConnectionManager,
logger logger.Logger,
tenderRepo tender.TenderRepository,
queue queue.Queue,
batchSize int,
) *QueueProducerWorker {
if batchSize <= 0 {
batchSize = 100 // Default batch size
}
logger.Info("Creating new queue producer worker", map[string]interface{}{
"batch_size": batchSize,
})
return &QueueProducerWorker{
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
Queue: queue,
BatchSize: batchSize,
}
}
// Run continuously processes and enqueues unscraped tenders
func (w *QueueProducerWorker) Run() {
w.Logger.Info("Queue producer worker started", map[string]interface{}{})
defer func() {
w.Logger.Info("Queue producer worker stopped", map[string]interface{}{})
}()
batchIndex := 0
hasMore := true
for hasMore {
w.Logger.Info("Queue producer fetching batch of tenders", map[string]interface{}{
"batch_index": batchIndex,
"batch_size": w.BatchSize,
})
// Get batch of unscraped tenders
tenders, totalCount, err := w.TenderRepo.GetUnScrapedTenders(
context.Background(),
w.BatchSize,
batchIndex*w.BatchSize,
)
if err != nil {
w.Logger.Error("Failed to fetch unscraped tenders", map[string]interface{}{
"batch_index": batchIndex,
"error": err.Error(),
})
// Wait before retrying
time.Sleep(30 * time.Second)
continue
}
w.Logger.Info("Fetched tender batch", map[string]interface{}{
"batch_size": len(tenders),
"total_count": totalCount,
"batch_index": batchIndex,
})
if len(tenders) == 0 {
// No more tenders in this batch
hasMore = false
w.Logger.Info("No more unscraped tenders found", map[string]interface{}{
"total_processed": totalCount,
})
break
}
// Enqueue each tender
successCount := 0
failCount := 0
for _, t := range tenders {
if err := w.enqueueTender(&t); err != nil {
w.Logger.Error("Failed to enqueue tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
"error": err.Error(),
})
failCount++
} else {
successCount++
}
}
w.Logger.Info("Batch enqueuing completed", map[string]interface{}{
"success_count": successCount,
"fail_count": failCount,
"batch_index": batchIndex,
})
// Move to next batch
batchIndex++
// Check if there are more tenders
if int64(batchIndex*w.BatchSize) >= totalCount {
hasMore = false
}
// Avoid tight loop - brief pause between batches
time.Sleep(1 * time.Second)
}
}
// enqueueTender adds a single tender to the queue
func (w *QueueProducerWorker) enqueueTender(t *tender.Tender) error {
// Skip if missing required fields
if t.NoticePublicationID == "" || t.DocumentURI == "" {
w.Logger.Warn("Skipping tender - missing required fields", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_publication_id": t.NoticePublicationID,
"document_uri": t.DocumentURI,
})
return nil
}
// Create job
job := queue.NewDocumentScraperJob(
t.NoticePublicationID,
t.DocumentURI,
t.ID.Hex(),
queue.DefaultMaxRetries,
)
// Enqueue job
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := w.Queue.Enqueue(ctx, job); err != nil {
return err
}
return nil
}
// RunOnce processes all unscraped tenders in a single run (for scheduled use)
func (w *QueueProducerWorker) RunOnce() {
w.Run()
}