Refactor MongoDB index creation to use NonEmptyStringPartialFilter for improved clarity and consistency. Update GetOJS function to utilize context and user agent for HTTP requests, enhancing error handling and response management.

2026-06-06 21:36:43 +03:30
parent 5af3bfaa1a
commit c91b525e7e
5 changed files with 164 additions and 28 deletions
@@ -43,7 +43,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
 		*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
 		// One row per TED ContractNoticeID when present (parallel scrape races).
 		*orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"contract_notice_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_notice_id")),
 	}

 	// Create indexes
@@ -94,16 +94,16 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
 		*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
 		// One tender row per TED ContractNoticeID when present (parallel ingest / lookup gaps).
 		*orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"contract_notice_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_notice_id")),
 		*orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}),
 		// One tender row per TED publication / procedure when ids are present (prevents parallel worker races).
 		*orm.CreateUniqueIndex("notice_publication_id_unique", bson.D{{Key: "notice_publication_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"notice_publication_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("notice_publication_id")),
 		*orm.CreateUniqueIndex("contract_folder_id_unique", bson.D{{Key: "contract_folder_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"contract_folder_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_folder_id")),
 		// One tender per UBL procurement project (merges per-lot notices that share ProcurementProject/ID).
 		*orm.CreateUniqueIndex("procurement_project_id_unique", bson.D{{Key: "procurement_project_id", Value: 1}}).
-			WithPartialFilterExpression(bson.M{"procurement_project_id": bson.M{"$type": "string", "$ne": ""}}),
+			WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("procurement_project_id")),
 		*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
 		*orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}),
 		*orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}),
@@ -17,7 +17,7 @@ func NewIndex(name string, keys bson.D) *Index {
 	return &Index{
 		Name:    name,
 		Keys:    keys,
-		Options: options.Index(),
+		Options: options.Index().SetName(name),
 	}
 }

@@ -39,6 +39,17 @@ func (i *Index) WithExpireAfterSeconds(seconds int32) *Index {
 	return i
 }

+// NonEmptyStringPartialFilter returns a partial-index filter for non-empty string field values.
+// MongoDB partial indexes do not support $ne; use $gt "" instead.
+func NonEmptyStringPartialFilter(field string) bson.M {
+	return bson.M{
+		field: bson.M{
+			"$type": "string",
+			"$gt":   "",
+		},
+	}
+}
+
 // WithPartialFilterExpression sets the partial filter expression for the index
 func (i *Index) WithPartialFilterExpression(filter bson.M) *Index {
 	i.Options.SetPartialFilterExpression(filter)
@@ -2,58 +2,183 @@ package ted

 import (
 	"bytes"
+	"context"
 	"encoding/csv"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"strings"
+	"time"
 )

 const (
-	calendarUrl = "%s/en/release-calendar/-/download/file/CSV/%d"
+	calendarURL  = "%s/en/release-calendar/-/download/file/CSV/%d"
+	searchAPIURL = "https://api.ted.europa.eu/v3/notices/search"
 )

-// Get OJS Number
-func GetOJS(baseURL string, year int, date string, client *http.Client) (string, error) {
-	var (
-		ojs string
-	)
-	url := fmt.Sprintf(calendarUrl, baseURL, year)
+type searchAPIRequest struct {
+	Query  string   `json:"query"`
+	Fields []string `json:"fields"`
+	Limit  int      `json:"limit"`
+}

-	resp, err := client.Get(url)
+type searchAPIResponse struct {
+	Notices []struct {
+		OJSNumber string `json:"ojs-number"`
+	} `json:"notices"`
+}
+
+// GetOJS resolves the OJS package id for a publication date.
+// It tries the TED release calendar CSV first, then falls back to the TED Search API
+// when the calendar download is blocked (e.g. AWS WAF HTTP 202).
+func GetOJS(ctx context.Context, baseURL string, year int, date, userAgent string, client *http.Client) (string, error) {
+	if ojs, err := getOJSFromCalendarCSV(ctx, baseURL, year, date, userAgent, client); err == nil {
+		return ojs, nil
+	} else if !isCalendarUnavailable(err) {
+		return "", err
+	}
+
+	return getOJSFromSearchAPIAt(ctx, date, userAgent, client, searchAPIURL)
+}
+
+func getOJSFromCalendarCSV(ctx context.Context, baseURL string, year int, date, userAgent string, client *http.Client) (string, error) {
+	url := fmt.Sprintf(calendarURL, baseURL, year)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
-		return ojs, err
+		return "", fmt.Errorf("create calendar request: %w", err)
+	}
+	if strings.TrimSpace(userAgent) != "" {
+		req.Header.Set("User-Agent", userAgent)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
 	}
 	defer resp.Body.Close()

-	// read CSV
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return ojs, err
+		return "", err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("calendar HTTP error: %d %s (body: %s)", resp.StatusCode, resp.Status, previewBody(body))
 	}

 	reader := csv.NewReader(bytes.NewReader(body))
 	records, err := reader.ReadAll()
 	if err != nil {
-		return ojs, err
+		return "", fmt.Errorf("parse calendar CSV: %w", err)
 	}

 	if len(records) < 2 {
-		return ojs, fmt.Errorf("ojs calendar is empty")
+		return "", fmt.Errorf("ojs calendar is empty (rows=%d, body=%s)", len(records), previewBody(body))
 	}

+	targetDate := strings.TrimSpace(date)
 	for _, row := range records[1:] {
+		if len(row) < 2 {
+			continue
+		}
 		rowDate := strings.TrimSpace(row[1])
-		if rowDate == strings.TrimSpace(date) {
-			// Format: YEAR + zero-padded OJS number to make total length 9 digits
-			// Examples: 202500004, 202500014, 202500196
-			ojs = fmt.Sprintf("%d%05s", year, strings.TrimSpace(fmt.Sprint(row[0])))
-			break
+		if rowDate == targetDate {
+			return fmt.Sprintf("%d%05s", year, strings.TrimSpace(row[0])), nil
 		}
 	}
-	if ojs == "" {
-		return ojs, fmt.Errorf("can not find ojs number")
+
+	return "", fmt.Errorf("can not find ojs number for date %s", date)
+}
+
+func getOJSFromSearchAPIAt(ctx context.Context, date, userAgent string, client *http.Client, apiURL string) (string, error) {
+	parsed, err := time.Parse("02/01/2006", strings.TrimSpace(date))
+	if err != nil {
+		return "", fmt.Errorf("parse publication date %s: %w", date, err)
 	}

-	return ojs, nil
+	payload, err := json.Marshal(searchAPIRequest{
+		Query:  fmt.Sprintf("publication-date=%s", parsed.Format("20060102")),
+		Fields: []string{"ojs-number"},
+		Limit:  1,
+	})
+	if err != nil {
+		return "", fmt.Errorf("marshal search request: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(payload))
+	if err != nil {
+		return "", fmt.Errorf("create search request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+	if strings.TrimSpace(userAgent) != "" {
+		req.Header.Set("User-Agent", userAgent)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("search API HTTP error: %d %s (body: %s)", resp.StatusCode, resp.Status, previewBody(body))
+	}
+
+	var result searchAPIResponse
+	if err := json.Unmarshal(body, &result); err != nil {
+		return "", fmt.Errorf("decode search API response: %w", err)
+	}
+	if len(result.Notices) == 0 || strings.TrimSpace(result.Notices[0].OJSNumber) == "" {
+		return "", fmt.Errorf("can not find ojs number for date %s", date)
+	}
+
+	return formatOJSPackageID(parsed.Year(), result.Notices[0].OJSNumber)
+}
+
+func formatOJSPackageID(fallbackYear int, ojsNumber string) (string, error) {
+	parts := strings.Split(strings.TrimSpace(ojsNumber), "/")
+	if len(parts) != 2 {
+		return "", fmt.Errorf("unexpected ojs-number format %q", ojsNumber)
+	}
+
+	seq := strings.TrimSpace(parts[0])
+	yearStr := strings.TrimSpace(parts[1])
+	if seq == "" || yearStr == "" {
+		return "", fmt.Errorf("unexpected ojs-number format %q", ojsNumber)
+	}
+
+	year := fallbackYear
+	if parsedYear, err := time.Parse("2006", yearStr); err == nil {
+		year = parsedYear.Year()
+	}
+
+	return fmt.Sprintf("%d%05s", year, seq), nil
+}
+
+func isCalendarUnavailable(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	return strings.Contains(msg, "calendar HTTP error: 202") ||
+		strings.Contains(msg, "ojs calendar is empty")
+}
+
+func previewBody(body []byte) string {
+	const maxLen = 160
+	s := strings.TrimSpace(string(body))
+	s = strings.ReplaceAll(s, "\n", " ")
+	s = strings.ReplaceAll(s, "\r", " ")
+	if len(s) <= maxLen {
+		return s
+	}
+	return s[:maxLen] + "..."
 }
@@ -525,7 +525,7 @@ func (s *TEDScraper) processSingleDate(ctx context.Context, date time.Time) erro
 	})

 	// get ojs
-	ojs, err := GetOJS(s.config.BaseURL, date.Year(), date.Format("02/01/2006"), s.httpClient)
+	ojs, err := GetOJS(ctx, s.config.BaseURL, date.Year(), date.Format("02/01/2006"), s.config.UserAgent, s.httpClient)
 	if err != nil {
 		s.logger.Error("Failed to get OJS", map[string]interface{}{
 			"date":  date.Format("02/01/2006"),