n.nakhostin
05c7eae8a2
- Introduced a new configuration file for the TED scraper, defining database connection settings, logging preferences, and scraping parameters. - Implemented a CronScheduler to manage scheduled tasks for TED operations, utilizing the robfig/cron library for scheduling. - Added new entities and structures for handling TED XML data, including eForms and tender-related information, enhancing the scraper's functionality. - Updated the main application to initialize the TED scraper with the new configuration and set up graceful shutdown handling. - Removed the deprecated scraping implementation to streamline the codebase and focus on the new architecture.
47 lines
832 B
Go
47 lines
832 B
Go
package ted
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/csv"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
)
|
|
|
|
const (
|
|
calendarUrl = "https://ted.europa.eu/en/release-calendar/-/download/file/CSV/%d"
|
|
)
|
|
|
|
// Get OJS Number
|
|
func GetOJS(year int, date string) (string, error) {
|
|
var (
|
|
ojs string
|
|
)
|
|
url := fmt.Sprintf(calendarUrl, year)
|
|
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
return ojs, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
// read CSV
|
|
body, _ := io.ReadAll(resp.Body)
|
|
reader := csv.NewReader(bytes.NewReader(body))
|
|
records, _ := reader.ReadAll()
|
|
|
|
for _, row := range records[1:] {
|
|
rowDate := strings.TrimSpace(row[1])
|
|
if rowDate == strings.TrimSpace(date) {
|
|
ojs = fmt.Sprintf("%v00%v", year, strings.TrimSpace(fmt.Sprint(row[0])))
|
|
break
|
|
}
|
|
}
|
|
if ojs == "" {
|
|
return ojs, fmt.Errorf("can not find ojs number")
|
|
}
|
|
|
|
return ojs, nil
|
|
}
|