n.nakhostin
05c7eae8a2
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
30 lines
554 B
YAML
30 lines
554 B
YAML
# TED Scraper Configuration
|
|
database:
|
|
mongodb:
|
|
uri: "mongodb://localhost:27017"
|
|
name: "tender_management"
|
|
max_pool_size: 100
|
|
timeout: 30s
|
|
|
|
logging:
|
|
level: "info"
|
|
format: "json"
|
|
output: "stdout"
|
|
file:
|
|
path: "./logs/scraper.log"
|
|
max_size: 100
|
|
max_backups: 10
|
|
max_age: 30
|
|
compress: true
|
|
|
|
ted:
|
|
base_url: "https://ted.europa.eu"
|
|
timeout: 30s
|
|
max_retries: 3
|
|
retry_delay: 5s
|
|
user_agent: "TM-TED-Scraper/1.0"
|
|
max_concurrency: 5
|
|
download_dir: "./downloads"
|
|
cleanup_after: 24h
|
|
scraping_interval: 24h
|