Refactor MongoDB index creation to use NonEmptyStringPartialFilter for improved clarity and consistency. Update GetOJS function to utilize context and user agent for HTTP requests, enhancing error handling and response management.

2026-06-06 21:36:43 +03:30
parent 5af3bfaa1a
commit c91b525e7e
5 changed files with 164 additions and 28 deletions
@@ -2,58 +2,183 @@ package ted

 import (
 	"bytes"
+	"context"
 	"encoding/csv"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
+	"time"
 )

 const (
-	calendarUrl = "%s/en/release-calendar/-/download/file/CSV/%d"
+	calendarURL  = "%s/en/release-calendar/-/download/file/CSV/%d"
+	searchAPIURL = "https://api.ted.europa.eu/v3/notices/search"
 )

-// Get OJS Number
-func GetOJS(baseURL string, year int, date string, client *http.Client) (string, error) {
-	var (
-		ojs string
-	)
-	url := fmt.Sprintf(calendarUrl, baseURL, year)
+type searchAPIRequest struct {
+	Query  string   `json:"query"`
+	Fields []string `json:"fields"`
+	Limit  int      `json:"limit"`
+}

-	resp, err := client.Get(url)
+type searchAPIResponse struct {
+	Notices []struct {
+		OJSNumber string `json:"ojs-number"`
+	} `json:"notices"`
+}
+
+// GetOJS resolves the OJS package id for a publication date.
+// It tries the TED release calendar CSV first, then falls back to the TED Search API
+// when the calendar download is blocked (e.g. AWS WAF HTTP 202).
+func GetOJS(ctx context.Context, baseURL string, year int, date, userAgent string, client *http.Client) (string, error) {
+	if ojs, err := getOJSFromCalendarCSV(ctx, baseURL, year, date, userAgent, client); err == nil {
+		return ojs, nil
+	} else if !isCalendarUnavailable(err) {
+		return "", err
+	}
+
+	return getOJSFromSearchAPIAt(ctx, date, userAgent, client, searchAPIURL)
+}
+
+func getOJSFromCalendarCSV(ctx context.Context, baseURL string, year int, date, userAgent string, client *http.Client) (string, error) {
+	url := fmt.Sprintf(calendarURL, baseURL, year)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		return ojs, err
+		return "", fmt.Errorf("create calendar request: %w", err)
+	}
+	if strings.TrimSpace(userAgent) != "" {
+		req.Header.Set("User-Agent", userAgent)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
 	}
 	defer resp.Body.Close()

-	// read CSV
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return ojs, err
+		return "", err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("calendar HTTP error: %d %s (body: %s)", resp.StatusCode, resp.Status, previewBody(body))
 	}

 	reader := csv.NewReader(bytes.NewReader(body))
 	records, err := reader.ReadAll()
 	if err != nil {
-		return ojs, err
+		return "", fmt.Errorf("parse calendar CSV: %w", err)
 	}

 	if len(records) < 2 {
-		return ojs, fmt.Errorf("ojs calendar is empty")
+		return "", fmt.Errorf("ojs calendar is empty (rows=%d, body=%s)", len(records), previewBody(body))
 	}

+	targetDate := strings.TrimSpace(date)
 	for _, row := range records[1:] {
+		if len(row) < 2 {
+			continue
+		}
 		rowDate := strings.TrimSpace(row[1])
-		if rowDate == strings.TrimSpace(date) {
-			// Format: YEAR + zero-padded OJS number to make total length 9 digits
-			// Examples: 202500004, 202500014, 202500196
-			ojs = fmt.Sprintf("%d%05s", year, strings.TrimSpace(fmt.Sprint(row[0])))
-			break
+		if rowDate == targetDate {
+			return fmt.Sprintf("%d%05s", year, strings.TrimSpace(row[0])), nil
 		}
 	}
-	if ojs == "" {
-		return ojs, fmt.Errorf("can not find ojs number")
+
+	return "", fmt.Errorf("can not find ojs number for date %s", date)
+}
+
+func getOJSFromSearchAPIAt(ctx context.Context, date, userAgent string, client *http.Client, apiURL string) (string, error) {
+	parsed, err := time.Parse("02/01/2006", strings.TrimSpace(date))
+	if err != nil {
+		return "", fmt.Errorf("parse publication date %s: %w", date, err)
 	}

-	return ojs, nil
+	payload, err := json.Marshal(searchAPIRequest{
+		Query:  fmt.Sprintf("publication-date=%s", parsed.Format("20060102")),
+		Fields: []string{"ojs-number"},
+		Limit:  1,
+	})
+	if err != nil {
+		return "", fmt.Errorf("marshal search request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(payload))
+	if err != nil {
+		return "", fmt.Errorf("create search request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+	if strings.TrimSpace(userAgent) != "" {
+		req.Header.Set("User-Agent", userAgent)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("search API HTTP error: %d %s (body: %s)", resp.StatusCode, resp.Status, previewBody(body))
+	}
+
+	var result searchAPIResponse
+	if err := json.Unmarshal(body, &result); err != nil {
+		return "", fmt.Errorf("decode search API response: %w", err)
+	}
+	if len(result.Notices) == 0 || strings.TrimSpace(result.Notices[0].OJSNumber) == "" {
+		return "", fmt.Errorf("can not find ojs number for date %s", date)
+	}
+
+	return formatOJSPackageID(parsed.Year(), result.Notices[0].OJSNumber)
+}
+
+func formatOJSPackageID(fallbackYear int, ojsNumber string) (string, error) {
+	parts := strings.Split(strings.TrimSpace(ojsNumber), "/")
+	if len(parts) != 2 {
+		return "", fmt.Errorf("unexpected ojs-number format %q", ojsNumber)
+	}
+
+	seq := strings.TrimSpace(parts[0])
+	yearStr := strings.TrimSpace(parts[1])
+	if seq == "" || yearStr == "" {
+		return "", fmt.Errorf("unexpected ojs-number format %q", ojsNumber)
+	}
+
+	year := fallbackYear
+	if parsedYear, err := time.Parse("2006", yearStr); err == nil {
+		year = parsedYear.Year()
+	}
+
+	return fmt.Sprintf("%d%05s", year, seq), nil
+}
+
+func isCalendarUnavailable(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	return strings.Contains(msg, "calendar HTTP error: 202") ||
+		strings.Contains(msg, "ojs calendar is empty")
+}
+
+func previewBody(body []byte) string {
+	const maxLen = 160
+	s := strings.TrimSpace(string(body))
+	s = strings.ReplaceAll(s, "\n", " ")
+	s = strings.ReplaceAll(s, "\r", " ")
+	if len(s) <= maxLen {
+		return s
+	}
+	return s[:maxLen] + "..."
 }