Mazyar
0b74e9ad23
continuous-integration/drone/push Build is passing
- Introduced a new `RecommendedTendersPageCacheRefresher` interface to manage the asynchronous refresh of recommendation pages in Redis, improving cache management. - Updated the `companyService` to support setting page cache languages and refreshing the recommended tenders page cache based on company IDs. - Enhanced the `tenderService` to build and invalidate recommended tenders page caches, ensuring timely updates and efficient retrieval of cached recommendations. - Added configuration options for recommendation page cache languages in the `AISummarizerConfig`, allowing for flexible language support. - Implemented unit tests for the new caching logic and page refresh functionality, ensuring robust validation of the recommendation caching process. This update significantly improves the efficiency and responsiveness of the tender recommendation service by integrating enhanced caching mechanisms and page refresh capabilities.
65 lines
4.1 KiB
Go
65 lines
4.1 KiB
Go
package bootstrap
|
|
|
|
import (
|
|
"time"
|
|
"tm/pkg/config"
|
|
)
|
|
|
|
// Config defines the worker application configuration
|
|
type Config struct {
|
|
Database config.DatabaseConfig
|
|
Redis config.RedisConfig
|
|
Logging config.LoggingConfig
|
|
Notification config.NotificationConfig
|
|
Ollama config.OllamaConfig
|
|
Worker WorkerConfig
|
|
Scraper config.ScraperConfig
|
|
MinIO config.MinIOConfig
|
|
AISummarizer AISummarizerConfig
|
|
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
|
|
}
|
|
|
|
type WorkerConfig struct {
|
|
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
|
|
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
|
|
DeleteProcessedNotices bool `env:"WORKER_DELETE_PROCESSED_NOTICES" envDefault:"true"`
|
|
NoticeWorkerConcurrency int `env:"WORKER_NOTICE_CONCURRENCY" envDefault:"1"`
|
|
NoticeFetchBatchSize int `env:"WORKER_NOTICE_FETCH_BATCH" envDefault:"50"`
|
|
// NoticeBatchPause is slept after each notice batch finishes, before loading the next batch (0 = no pause).
|
|
NoticeBatchPause time.Duration `env:"WORKER_NOTICE_BATCH_PAUSE" envDefault:"0s"`
|
|
// NoticeFetchErrorBackoff is slept after a failed notice fetch before retrying the loop (0 = retry immediately).
|
|
NoticeFetchErrorBackoff time.Duration `env:"WORKER_NOTICE_FETCH_ERROR_BACKOFF" envDefault:"2s"`
|
|
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
|
|
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
|
|
// TranslationEnabled schedules the automatic batch translation cron job on the worker.
|
|
// On-demand translation (admin/public tender endpoints and AI pipeline routes on web) is unaffected.
|
|
TranslationEnabled bool `env:"WORKER_TRANSLATION_ENABLED" envDefault:"true"`
|
|
// NotificationInterval schedules promotion of due scheduled notifications from pending to sent.
|
|
NotificationInterval string `env:"WORKER_NOTIFICATION_INTERVAL" envDefault:"0 * * * * *"`
|
|
// AIPipelineAutoEnabled schedules the daily AI pipeline daily-run on the worker.
|
|
// On-demand pipeline routes on web are unaffected.
|
|
AIPipelineAutoEnabled bool `env:"WORKER_AI_PIPELINE_AUTO_ENABLED" envDefault:"true"`
|
|
// AIPipelineAutoInterval runs daily-run once per day by default at 11:00 UTC (after TED scrape at 10:00).
|
|
AIPipelineAutoInterval string `env:"WORKER_AI_PIPELINE_AUTO_INTERVAL" envDefault:"0 0 11 * * *"`
|
|
// RecommendationRefreshAfterPipelineEnabled refreshes cached company recommendations after the daily AI pipeline run.
|
|
RecommendationRefreshAfterPipelineEnabled bool `env:"WORKER_RECOMMENDATION_REFRESH" envDefault:"true"`
|
|
}
|
|
|
|
// AISummarizerConfig holds configuration for the external AI summarizer service.
|
|
type AISummarizerConfig struct {
|
|
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
|
|
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
|
|
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
|
|
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
|
|
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
|
|
TranslationLanguages string `env:"AI_SUMMARIZER_TRANSLATION_LANGUAGES" envDefault:"en"`
|
|
MinioEndpoint string `env:"AI_SUMMARIZER_MINIO_ENDPOINT" envDefault:""`
|
|
MinioAccessKey string `env:"AI_SUMMARIZER_MINIO_ACCESS_KEY" envDefault:""`
|
|
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
|
|
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
|
|
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
|
|
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens"`
|
|
RecommendationCacheTTL time.Duration `env:"AI_RECOMMENDATION_CACHE_TTL" envDefault:"0"`
|
|
RecommendationPageCacheLanguages string `env:"AI_RECOMMENDATION_PAGE_CACHE_LANGUAGES" envDefault:"en"`
|
|
}
|