Add Configuration Files for Scraper and Web Applications

- Introduced iat.yaml configuration files for both the scraper and web applications, defining essential settings such as database connections, logging configurations, and scraping parameters.
- Updated the Dockerfiles for both applications to include the new iat.yaml files, ensuring they are available in the container environment for runtime configuration.
- This enhancement improves the maintainability and configurability of both applications, allowing for easier adjustments to settings without code changes.

cmd/scraper/Dockerfile

@@ -6,6 +6,7 @@ FROM golang:${GO_VERSION} AS go-builder
 WORKDIR /go/src/company/tm_back
 
 COPY . .
+COPY cmd/scraper/iat.yaml /go/bin/config.yaml
 
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o scraper-build ./cmd/scraper/main.go
 

cmd/scraper/iat.yaml

@@ -0,0 +1,28 @@
+database:
+  mongodb:
+    uri: "mongodb://mongo:27017"
+    name: "tender_management"
+    timeout: 10s
+    max_pool_size: 50
+
+logging:
+  level: "info"
+  format: "json"
+  output: "file"  # stdout, stderr, file
+  file:
+    path: "./logs/scraper.log"
+    max_size: 100      # Max size in MB before rotation
+    max_backups: 5     # Number of backup files to keep
+    max_age: 30        # Max age in days to keep log files
+    compress: true     # Compress rotated files
+
+ted:
+  base_url: "https://ted.europa.eu/packages/daily"
+  timeout: 300s  # Increased to 5 minutes for large files
+  max_retries: 5
+  retry_delay: 10s
+  user_agent: "TenderManagement-TED-Scraper/1.0"
+  max_concurrency: 2  # Reduced to avoid overwhelming the server
+  download_dir: "./ted_downloads"
+  cleanup_after: 72h
+  scraping_interval: 12h  # Run scraping every 12 hours

cmd/web/Dockerfile

@@ -6,6 +6,7 @@ FROM golang:${GO_VERSION} AS go-builder
 WORKDIR /go/src/company/tm_back
 
 COPY . .
+COPY cmd/web/iat.yaml /go/bin/config.yaml
 
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o web-build ./cmd/web/main.go
 

cmd/web/iat.yaml

@@ -0,0 +1,50 @@
+server:
+  host: "0.0.0.0"
+  port: 8081
+  timeout: 30s
+  read_timeout: 10s
+  write_timeout: 10s
+
+database:
+  mongodb:
+    uri: "mongodb://mongo:27017"
+    name: "tender_management"
+    timeout: 10s
+    max_pool_size: 100
+
+cache:
+  redis:
+    host: "redis"
+    port: 6379
+    password: ""
+    db: 0
+    pool_size: 10
+
+user_authorization:
+  jwt:
+    access_secret: "44a51a556e5e987fe35559d40cefc08dde894df6f70a7512caa444e72bee7b1b"
+    refresh_secret: "c99009dd58f9a398118afd8f3eb7f1ba38eba33001c93cb45aa6cb88e0582f36"
+    access_expires_in: 3600 # 1 hour in seconds
+    refresh_expires_in: 2592000 # 30 days in seconds
+
+customer_authorization:
+  jwt:
+    access_secret: "78db1387c47a97a7faea439fc66806282d661fe4c0cc0e893147c677d2b89aa8"
+    refresh_secret: "84e6d4e375d3acfb40030e08756249217955aa40a6816369c64b06666ea74cba"
+    access_expires_in: 3600 # 1 hour in seconds
+    refresh_expires_in: 2592000 # 30 days in seconds
+
+logging:
+  level: "info"
+  format: "json"
+  output: "file"  # stdout, stderr, file
+  file:
+    path: "./logs/app.log"
+    max_size: 100      # Max size in MB before rotation
+    max_backups: 5     # Number of backup files to keep
+    max_age: 30        # Max age in days to keep log files
+    compress: true     # Compress rotated files
+
+rate_limiting:
+  requests_per_minute: 100
+  burst: 20