Mazyar
492f9ba3c8
continuous-integration/drone/push Build is passing
- Introduced AIPipelineAutoWorker to manage the execution of the AI pipeline auto run, including startup catch-up and scheduled tasks. - Enhanced WorkerConfig to include AIPipelineAutoEnabled and AIPipelineAutoInterval settings for better control over AI pipeline execution. - Added logging for AI pipeline auto run status, including success and error handling, to improve observability. - Updated daily job tracker to include AIPipelineAutoJobName for tracking AI pipeline job completions. This update enhances the system's capability to automate AI pipeline executions, improving efficiency and reliability in processing AI tasks.
60 lines
3.5 KiB
Go
60 lines
3.5 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"time"
|
|
"tm/pkg/config"
|
|
)
|
|
|
|
// Config defines the worker application configuration
|
|
type Config struct {
|
|
Database config.DatabaseConfig
|
|
Logging config.LoggingConfig
|
|
Notification config.NotificationConfig
|
|
Ollama config.OllamaConfig
|
|
Worker WorkerConfig
|
|
Scraper config.ScraperConfig
|
|
MinIO config.MinIOConfig
|
|
AISummarizer AISummarizerConfig
|
|
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
|
|
}
|
|
|
|
type WorkerConfig struct {
|
|
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
|
|
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
|
|
DeleteProcessedNotices bool `env:"WORKER_DELETE_PROCESSED_NOTICES" envDefault:"true"`
|
|
NoticeWorkerConcurrency int `env:"WORKER_NOTICE_CONCURRENCY" envDefault:"1"`
|
|
NoticeFetchBatchSize int `env:"WORKER_NOTICE_FETCH_BATCH" envDefault:"50"`
|
|
// NoticeBatchPause is slept after each notice batch finishes, before loading the next batch (0 = no pause).
|
|
NoticeBatchPause time.Duration `env:"WORKER_NOTICE_BATCH_PAUSE" envDefault:"0s"`
|
|
// NoticeFetchErrorBackoff is slept after a failed notice fetch before retrying the loop (0 = retry immediately).
|
|
NoticeFetchErrorBackoff time.Duration `env:"WORKER_NOTICE_FETCH_ERROR_BACKOFF" envDefault:"2s"`
|
|
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
|
|
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
|
|
// TranslationEnabled schedules the automatic batch translation cron job on the worker.
|
|
// On-demand translation (admin/public tender endpoints and AI pipeline routes on web) is unaffected.
|
|
TranslationEnabled bool `env:"WORKER_TRANSLATION_ENABLED" envDefault:"true"`
|
|
// NotificationInterval schedules promotion of due scheduled notifications from pending to sent.
|
|
NotificationInterval string `env:"WORKER_NOTIFICATION_INTERVAL" envDefault:"0 * * * * *"`
|
|
// AIPipelineAutoEnabled schedules the daily AI pipeline auto run on the worker.
|
|
// On-demand pipeline routes on web are unaffected.
|
|
AIPipelineAutoEnabled bool `env:"WORKER_AI_PIPELINE_AUTO_ENABLED" envDefault:"true"`
|
|
// AIPipelineAutoInterval runs once per day by default at 11:00 UTC (after TED scrape at 10:00).
|
|
AIPipelineAutoInterval string `env:"WORKER_AI_PIPELINE_AUTO_INTERVAL" envDefault:"0 0 11 * * *"`
|
|
}
|
|
|
|
// AISummarizerConfig holds configuration for the external AI summarizer service.
|
|
type AISummarizerConfig struct {
|
|
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
|
|
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
|
|
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
|
|
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
|
|
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
|
|
TranslationLanguages string `env:"AI_SUMMARIZER_TRANSLATION_LANGUAGES" envDefault:"en"`
|
|
MinioEndpoint string `env:"AI_SUMMARIZER_MINIO_ENDPOINT" envDefault:""`
|
|
MinioAccessKey string `env:"AI_SUMMARIZER_MINIO_ACCESS_KEY" envDefault:""`
|
|
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
|
|
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
|
|
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
|
|
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens"`
|
|
}
|