From c91b525e7efbf0575b248c57ce08a173e78d49c6 Mon Sep 17 00:00:00 2001 From: Mazyar Date: Sat, 6 Jun 2026 21:36:43 +0330 Subject: [PATCH] Refactor MongoDB index creation to use NonEmptyStringPartialFilter for improved clarity and consistency. Update GetOJS function to utilize context and user agent for HTTP requests, enhancing error handling and response management. --- internal/notice/repository.go | 2 +- internal/tender/repository.go | 8 +- pkg/mongo/utils.go | 13 ++- ted/calendar.go | 167 +++++++++++++++++++++++++++++----- ted/scraper.go | 2 +- 5 files changed, 164 insertions(+), 28 deletions(-) diff --git a/internal/notice/repository.go b/internal/notice/repository.go index 8e9e2b2..2d10bcf 100644 --- a/internal/notice/repository.go +++ b/internal/notice/repository.go @@ -43,7 +43,7 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re *orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}), // One row per TED ContractNoticeID when present (parallel scrape races). *orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}). - WithPartialFilterExpression(bson.M{"contract_notice_id": bson.M{"$type": "string", "$ne": ""}}), + WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_notice_id")), } // Create indexes diff --git a/internal/tender/repository.go b/internal/tender/repository.go index b8a12a3..3b03226 100644 --- a/internal/tender/repository.go +++ b/internal/tender/repository.go @@ -94,16 +94,16 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te *orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}), // One tender row per TED ContractNoticeID when present (parallel ingest / lookup gaps). *orm.CreateUniqueIndex("contract_notice_id_unique", bson.D{{Key: "contract_notice_id", Value: 1}}). - WithPartialFilterExpression(bson.M{"contract_notice_id": bson.M{"$type": "string", "$ne": ""}}), + WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_notice_id")), *orm.NewIndex("contract_folder_id_idx", bson.D{{Key: "contract_folder_id", Value: 1}}), // One tender row per TED publication / procedure when ids are present (prevents parallel worker races). *orm.CreateUniqueIndex("notice_publication_id_unique", bson.D{{Key: "notice_publication_id", Value: 1}}). - WithPartialFilterExpression(bson.M{"notice_publication_id": bson.M{"$type": "string", "$ne": ""}}), + WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("notice_publication_id")), *orm.CreateUniqueIndex("contract_folder_id_unique", bson.D{{Key: "contract_folder_id", Value: 1}}). - WithPartialFilterExpression(bson.M{"contract_folder_id": bson.M{"$type": "string", "$ne": ""}}), + WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("contract_folder_id")), // One tender per UBL procurement project (merges per-lot notices that share ProcurementProject/ID). *orm.CreateUniqueIndex("procurement_project_id_unique", bson.D{{Key: "procurement_project_id", Value: 1}}). - WithPartialFilterExpression(bson.M{"procurement_project_id": bson.M{"$type": "string", "$ne": ""}}), + WithPartialFilterExpression(orm.NonEmptyStringPartialFilter("procurement_project_id")), *orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}), *orm.NewIndex("source_idx", bson.D{{Key: "source", Value: 1}}), *orm.NewIndex("country_code_idx", bson.D{{Key: "country_code", Value: 1}}), diff --git a/pkg/mongo/utils.go b/pkg/mongo/utils.go index d52a62a..f57b170 100644 --- a/pkg/mongo/utils.go +++ b/pkg/mongo/utils.go @@ -17,7 +17,7 @@ func NewIndex(name string, keys bson.D) *Index { return &Index{ Name: name, Keys: keys, - Options: options.Index(), + Options: options.Index().SetName(name), } } @@ -39,6 +39,17 @@ func (i *Index) WithExpireAfterSeconds(seconds int32) *Index { return i } +// NonEmptyStringPartialFilter returns a partial-index filter for non-empty string field values. +// MongoDB partial indexes do not support $ne; use $gt "" instead. +func NonEmptyStringPartialFilter(field string) bson.M { + return bson.M{ + field: bson.M{ + "$type": "string", + "$gt": "", + }, + } +} + // WithPartialFilterExpression sets the partial filter expression for the index func (i *Index) WithPartialFilterExpression(filter bson.M) *Index { i.Options.SetPartialFilterExpression(filter) diff --git a/ted/calendar.go b/ted/calendar.go index 0015204..ffc4d61 100644 --- a/ted/calendar.go +++ b/ted/calendar.go @@ -2,58 +2,183 @@ package ted import ( "bytes" + "context" "encoding/csv" + "encoding/json" "fmt" "io" "net/http" "strings" + "time" ) const ( - calendarUrl = "%s/en/release-calendar/-/download/file/CSV/%d" + calendarURL = "%s/en/release-calendar/-/download/file/CSV/%d" + searchAPIURL = "https://api.ted.europa.eu/v3/notices/search" ) -// Get OJS Number -func GetOJS(baseURL string, year int, date string, client *http.Client) (string, error) { - var ( - ojs string - ) - url := fmt.Sprintf(calendarUrl, baseURL, year) +type searchAPIRequest struct { + Query string `json:"query"` + Fields []string `json:"fields"` + Limit int `json:"limit"` +} - resp, err := client.Get(url) +type searchAPIResponse struct { + Notices []struct { + OJSNumber string `json:"ojs-number"` + } `json:"notices"` +} + +// GetOJS resolves the OJS package id for a publication date. +// It tries the TED release calendar CSV first, then falls back to the TED Search API +// when the calendar download is blocked (e.g. AWS WAF HTTP 202). +func GetOJS(ctx context.Context, baseURL string, year int, date, userAgent string, client *http.Client) (string, error) { + if ojs, err := getOJSFromCalendarCSV(ctx, baseURL, year, date, userAgent, client); err == nil { + return ojs, nil + } else if !isCalendarUnavailable(err) { + return "", err + } + + return getOJSFromSearchAPIAt(ctx, date, userAgent, client, searchAPIURL) +} + +func getOJSFromCalendarCSV(ctx context.Context, baseURL string, year int, date, userAgent string, client *http.Client) (string, error) { + url := fmt.Sprintf(calendarURL, baseURL, year) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { - return ojs, err + return "", fmt.Errorf("create calendar request: %w", err) + } + if strings.TrimSpace(userAgent) != "" { + req.Header.Set("User-Agent", userAgent) + } + + resp, err := client.Do(req) + if err != nil { + return "", err } defer resp.Body.Close() - // read CSV body, err := io.ReadAll(resp.Body) if err != nil { - return ojs, err + return "", err + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("calendar HTTP error: %d %s (body: %s)", resp.StatusCode, resp.Status, previewBody(body)) } reader := csv.NewReader(bytes.NewReader(body)) records, err := reader.ReadAll() if err != nil { - return ojs, err + return "", fmt.Errorf("parse calendar CSV: %w", err) } if len(records) < 2 { - return ojs, fmt.Errorf("ojs calendar is empty") + return "", fmt.Errorf("ojs calendar is empty (rows=%d, body=%s)", len(records), previewBody(body)) } + targetDate := strings.TrimSpace(date) for _, row := range records[1:] { + if len(row) < 2 { + continue + } rowDate := strings.TrimSpace(row[1]) - if rowDate == strings.TrimSpace(date) { - // Format: YEAR + zero-padded OJS number to make total length 9 digits - // Examples: 202500004, 202500014, 202500196 - ojs = fmt.Sprintf("%d%05s", year, strings.TrimSpace(fmt.Sprint(row[0]))) - break + if rowDate == targetDate { + return fmt.Sprintf("%d%05s", year, strings.TrimSpace(row[0])), nil } } - if ojs == "" { - return ojs, fmt.Errorf("can not find ojs number") + + return "", fmt.Errorf("can not find ojs number for date %s", date) +} + +func getOJSFromSearchAPIAt(ctx context.Context, date, userAgent string, client *http.Client, apiURL string) (string, error) { + parsed, err := time.Parse("02/01/2006", strings.TrimSpace(date)) + if err != nil { + return "", fmt.Errorf("parse publication date %s: %w", date, err) } - return ojs, nil + payload, err := json.Marshal(searchAPIRequest{ + Query: fmt.Sprintf("publication-date=%s", parsed.Format("20060102")), + Fields: []string{"ojs-number"}, + Limit: 1, + }) + if err != nil { + return "", fmt.Errorf("marshal search request: %w", err) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(payload)) + if err != nil { + return "", fmt.Errorf("create search request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + if strings.TrimSpace(userAgent) != "" { + req.Header.Set("User-Agent", userAgent) + } + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("search API HTTP error: %d %s (body: %s)", resp.StatusCode, resp.Status, previewBody(body)) + } + + var result searchAPIResponse + if err := json.Unmarshal(body, &result); err != nil { + return "", fmt.Errorf("decode search API response: %w", err) + } + if len(result.Notices) == 0 || strings.TrimSpace(result.Notices[0].OJSNumber) == "" { + return "", fmt.Errorf("can not find ojs number for date %s", date) + } + + return formatOJSPackageID(parsed.Year(), result.Notices[0].OJSNumber) +} + +func formatOJSPackageID(fallbackYear int, ojsNumber string) (string, error) { + parts := strings.Split(strings.TrimSpace(ojsNumber), "/") + if len(parts) != 2 { + return "", fmt.Errorf("unexpected ojs-number format %q", ojsNumber) + } + + seq := strings.TrimSpace(parts[0]) + yearStr := strings.TrimSpace(parts[1]) + if seq == "" || yearStr == "" { + return "", fmt.Errorf("unexpected ojs-number format %q", ojsNumber) + } + + year := fallbackYear + if parsedYear, err := time.Parse("2006", yearStr); err == nil { + year = parsedYear.Year() + } + + return fmt.Sprintf("%d%05s", year, seq), nil +} + +func isCalendarUnavailable(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.Contains(msg, "calendar HTTP error: 202") || + strings.Contains(msg, "ojs calendar is empty") +} + +func previewBody(body []byte) string { + const maxLen = 160 + s := strings.TrimSpace(string(body)) + s = strings.ReplaceAll(s, "\n", " ") + s = strings.ReplaceAll(s, "\r", " ") + if len(s) <= maxLen { + return s + } + return s[:maxLen] + "..." } diff --git a/ted/scraper.go b/ted/scraper.go index 4bbb054..4224b4e 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -525,7 +525,7 @@ func (s *TEDScraper) processSingleDate(ctx context.Context, date time.Time) erro }) // get ojs - ojs, err := GetOJS(s.config.BaseURL, date.Year(), date.Format("02/01/2006"), s.httpClient) + ojs, err := GetOJS(ctx, s.config.BaseURL, date.Year(), date.Format("02/01/2006"), s.config.UserAgent, s.httpClient) if err != nil { s.logger.Error("Failed to get OJS", map[string]interface{}{ "date": date.Format("02/01/2006"),