Add TED Scraper Configuration and Cron Scheduler Implementation

- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters.
- Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling.
- Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality.
- Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling.
- Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
This commit is contained in:
n.nakhostin
2025-09-30 16:03:53 +03:30
parent 0c50935c42
commit 05c7eae8a2
36 changed files with 3427 additions and 311 deletions
+46
View File
@@ -0,0 +1,46 @@
package ted
import (
"bytes"
"encoding/csv"
"fmt"
"io"
"net/http"
"strings"
)
const (
calendarUrl = "https://ted.europa.eu/en/release-calendar/-/download/file/CSV/%d"
)
// Get OJS Number
func GetOJS(year int, date string) (string, error) {
var (
ojs string
)
url := fmt.Sprintf(calendarUrl, year)
resp, err := http.Get(url)
if err != nil {
return ojs, err
}
defer resp.Body.Close()
// read CSV
body, _ := io.ReadAll(resp.Body)
reader := csv.NewReader(bytes.NewReader(body))
records, _ := reader.ReadAll()
for _, row := range records[1:] {
rowDate := strings.TrimSpace(row[1])
if rowDate == strings.TrimSpace(date) {
ojs = fmt.Sprintf("%v00%v", year, strings.TrimSpace(fmt.Sprint(row[0])))
break
}
}
if ojs == "" {
return ojs, fmt.Errorf("can not find ojs number")
}
return ojs, nil
}