Add TED Scraper Configuration and Cron Scheduler Implementation
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
package ted
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
calendarUrl = "https://ted.europa.eu/en/release-calendar/-/download/file/CSV/%d"
|
||||
)
|
||||
|
||||
// Get OJS Number
|
||||
func GetOJS(year int, date string) (string, error) {
|
||||
var (
|
||||
ojs string
|
||||
)
|
||||
url := fmt.Sprintf(calendarUrl, year)
|
||||
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return ojs, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// read CSV
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
reader := csv.NewReader(bytes.NewReader(body))
|
||||
records, _ := reader.ReadAll()
|
||||
|
||||
for _, row := range records[1:] {
|
||||
rowDate := strings.TrimSpace(row[1])
|
||||
if rowDate == strings.TrimSpace(date) {
|
||||
ojs = fmt.Sprintf("%v00%v", year, strings.TrimSpace(fmt.Sprint(row[0])))
|
||||
break
|
||||
}
|
||||
}
|
||||
if ojs == "" {
|
||||
return ojs, fmt.Errorf("can not find ojs number")
|
||||
}
|
||||
|
||||
return ojs, nil
|
||||
}
|
||||
Reference in New Issue
Block a user