n.nakhostin
f67e179ffc
- Added 'ted_samples/' to .gitignore to exclude TED sample files from version control. - Updated the TED scraper configuration to extend the cleanup duration from 24 hours to 72 hours, allowing for better resource management. - Enhanced the Tender entity by adding new fields: TenderDeadline, SubmissionDeadline, ApplicationDeadline, and SubmissionURL, improving the data model for tender management. - Updated the TenderResponse structure to include the new fields, ensuring consistency in API responses. - Implemented logic in the TED scraper to calculate submission and application deadlines based on tender deadlines, enhancing the scraping functionality.
29 lines
849 B
YAML
29 lines
849 B
YAML
database:
|
|
mongodb:
|
|
uri: "mongodb://localhost:27017"
|
|
name: "tender_management"
|
|
timeout: 10s
|
|
max_pool_size: 50
|
|
|
|
logging:
|
|
level: "info"
|
|
format: "json"
|
|
output: "file" # stdout, stderr, file
|
|
file:
|
|
path: "./logs/scraper.log"
|
|
max_size: 100 # Max size in MB before rotation
|
|
max_backups: 5 # Number of backup files to keep
|
|
max_age: 30 # Max age in days to keep log files
|
|
compress: true # Compress rotated files
|
|
|
|
ted:
|
|
base_url: "https://ted.europa.eu/packages/daily"
|
|
timeout: 300s # Increased to 5 minutes for large files
|
|
max_retries: 5
|
|
retry_delay: 10s
|
|
user_agent: "TenderManagement-TED-Scraper/1.0"
|
|
max_concurrency: 2 # Reduced to avoid overwhelming the server
|
|
download_dir: "./tmp/ted_downloads"
|
|
cleanup_after: 72h
|
|
scraping_interval: 12h # Run scraping every 12 hours
|