worker concurrency config

cmd/worker/workers/notice.go

@@ -43,6 +43,10 @@ type NoticeWorker struct {
 	Concurrency int
 	// FetchBatchSize is how many unprocessed notices to load from MongoDB per batch (default 50).
 	FetchBatchSize int
+	// BatchPause is slept after each batch completes, before fetching the next batch (0 = disabled).
+	BatchPause time.Duration
+	// FetchErrorBackoff is slept after a failed notice fetch before retrying (0 = disabled).
+	FetchErrorBackoff time.Duration
 
 	// tenderIDSeedMu / tenderIDSeeded track which (sourceCode+year) counters have been seeded
 	// from the current tenders collection during this process, so we only run the seeding
@@ -51,7 +55,7 @@ type NoticeWorker struct {
 	tenderIDSeeded map[string]bool
 }
 
-func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, processingLimit, concurrency, fetchBatchSize int, deleteProcessedNotices bool) *NoticeWorker {
+func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, processingLimit, concurrency, fetchBatchSize int, deleteProcessedNotices bool, batchPause, fetchErrorBackoff time.Duration) *NoticeWorker {
 	if processingLimit < 0 {
 		processingLimit = 5 // Default limit
 	}
@@ -61,6 +65,12 @@ func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notif
 	if fetchBatchSize < 1 {
 		fetchBatchSize = 50
 	}
+	if batchPause < 0 {
+		batchPause = 0
+	}
+	if fetchErrorBackoff < 0 {
+		fetchErrorBackoff = 0
+	}
 	return &NoticeWorker{
 		Mongo:                  mongo,
 		Logger:                 logger,
@@ -71,6 +81,8 @@ func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notif
 		DeleteProcessedNotices: deleteProcessedNotices,
 		Concurrency:            concurrency,
 		FetchBatchSize:         fetchBatchSize,
+		BatchPause:             batchPause,
+		FetchErrorBackoff:      fetchErrorBackoff,
 		tenderIDSeeded:         make(map[string]bool),
 	}
 }
@@ -130,9 +142,11 @@ func (w *NoticeWorker) cleanupProcessedNotices() {
 
 func (w *NoticeWorker) Run() {
 	w.Logger.Info("Notice worker started", map[string]interface{}{
-		"concurrency":      w.Concurrency,
-		"fetch_batch":      w.FetchBatchSize,
-		"processing_limit": w.ProcessingLimit,
+		"concurrency":         w.Concurrency,
+		"fetch_batch":         w.FetchBatchSize,
+		"processing_limit":    w.ProcessingLimit,
+		"batch_pause":         w.BatchPause.String(),
+		"fetch_error_backoff": w.FetchErrorBackoff.String(),
 	})
 
 	var processedTotal atomic.Int64
@@ -155,6 +169,9 @@ func (w *NoticeWorker) Run() {
 			w.Logger.Error("Failed to get notices", map[string]interface{}{
 				"error": err.Error(),
 			})
+			if w.FetchErrorBackoff > 0 {
+				time.Sleep(w.FetchErrorBackoff)
+			}
 			continue
 		}
 
@@ -224,6 +241,10 @@ func (w *NoticeWorker) Run() {
 			})
 		}
 
+		if w.BatchPause > 0 && len(notices) > 0 {
+			time.Sleep(w.BatchPause)
+		}
+
 		if maxToProcess > 0 && int(processedTotal.Load()) >= maxToProcess {
 			w.Logger.Info("Reached maximum processing limit, stopping", map[string]interface{}{
 				"processed_count": processedTotal.Load(),
@@ -909,9 +930,9 @@ func (w *NoticeWorker) ensureTenderIDCounterSeeded(ctx context.Context, sourceCo
 
 	w.tenderIDSeeded[key] = true
 	w.Logger.Info("Tender id counter seeded", map[string]interface{}{
-		"key":         key,
-		"seed_value":  currentMax,
-		"next_id":     fmt.Sprintf("%s%d", key, currentMax+1),
+		"key":        key,
+		"seed_value": currentMax,
+		"next_id":    fmt.Sprintf("%s%d", key, currentMax+1),
 	})
 	return nil
 }