AI scraper prefix

This commit is contained in:
Mazyar
2026-05-13 17:32:37 +03:30
parent 584992f0b6
commit 5ab6b2714e
3 changed files with 117 additions and 76 deletions
+9 -2
View File
@@ -21,6 +21,10 @@ import (
"golang.org/x/sync/errgroup"
)
// noticeRunSerialMu ensures only one NoticeWorker.Run executes at a time (startup
// goroutine + cron ticks can otherwise overlap and double load on MongoDB).
var noticeRunSerialMu sync.Mutex
const (
// tenderIDCountersCollection holds per-(sourceCode+year) atomic counters used to allocate
// unique tender_id values (e.g. PTD26827). Using a Mongo counter document is the standard way
@@ -39,7 +43,7 @@ type NoticeWorker struct {
TenderRepo tender.TenderRepository
ProcessingLimit int
DeleteProcessedNotices bool
// Concurrency is the maximum number of notices processed in parallel per batch (default 8).
// Concurrency is the maximum number of notices processed in parallel per batch (default 1).
Concurrency int
// FetchBatchSize is how many unprocessed notices to load from MongoDB per batch (default 50).
FetchBatchSize int
@@ -60,7 +64,7 @@ func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notif
processingLimit = 5 // Default limit
}
if concurrency < 1 {
concurrency = 8
concurrency = 1
}
if fetchBatchSize < 1 {
fetchBatchSize = 50
@@ -141,6 +145,9 @@ func (w *NoticeWorker) cleanupProcessedNotices() {
}
func (w *NoticeWorker) Run() {
noticeRunSerialMu.Lock()
defer noticeRunSerialMu.Unlock()
w.Logger.Info("Notice worker started", map[string]interface{}{
"concurrency": w.Concurrency,
"fetch_batch": w.FetchBatchSize,