n.nakhostin
b1438870f0
- Introduced iat.yaml configuration files for both the scraper and web applications, defining essential settings such as database connections, logging configurations, and scraping parameters. - Updated the Dockerfiles for both applications to include the new iat.yaml files, ensuring they are available in the container environment for runtime configuration. - This enhancement improves the maintainability and configurability of both applications, allowing for easier adjustments to settings without code changes.
29 lines
841 B
YAML
29 lines
841 B
YAML
database:
|
|
mongodb:
|
|
uri: "mongodb://mongo:27017"
|
|
name: "tender_management"
|
|
timeout: 10s
|
|
max_pool_size: 50
|
|
|
|
logging:
|
|
level: "info"
|
|
format: "json"
|
|
output: "file" # stdout, stderr, file
|
|
file:
|
|
path: "./logs/scraper.log"
|
|
max_size: 100 # Max size in MB before rotation
|
|
max_backups: 5 # Number of backup files to keep
|
|
max_age: 30 # Max age in days to keep log files
|
|
compress: true # Compress rotated files
|
|
|
|
ted:
|
|
base_url: "https://ted.europa.eu/packages/daily"
|
|
timeout: 300s # Increased to 5 minutes for large files
|
|
max_retries: 5
|
|
retry_delay: 10s
|
|
user_agent: "TenderManagement-TED-Scraper/1.0"
|
|
max_concurrency: 2 # Reduced to avoid overwhelming the server
|
|
download_dir: "./ted_downloads"
|
|
cleanup_after: 72h
|
|
scraping_interval: 12h # Run scraping every 12 hours
|