68b170126d
- Updated `InitAISummarizerClient` to accept `mongoManager` for tracking translation success. - Introduced new `Statistics` endpoint in the dashboard to fetch scraping and translation statistics. - Enhanced `TranslationWorker` to utilize the new success counter for tracking successful translations. - Added necessary data structures and query forms for statistics reporting. This refactor improves the tracking of AI translation success and provides new insights through the dashboard statistics.
291 lines
9.2 KiB
Go
291 lines
9.2 KiB
Go
package ai_summarizer
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"tm/pkg/logger"
|
|
)
|
|
|
|
// Client is the HTTP client for the AI summarization on-demand API.
|
|
type Client struct {
|
|
config *Config
|
|
httpClient *http.Client
|
|
logger logger.Logger
|
|
}
|
|
|
|
// NewClient creates a new AI summarizer HTTP client.
|
|
func NewClient(config *Config, log logger.Logger) (*Client, error) {
|
|
if config == nil {
|
|
config = DefaultConfig()
|
|
}
|
|
if err := config.ValidateAPI(); err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: invalid config: %w", err)
|
|
}
|
|
|
|
return &Client{
|
|
config: config,
|
|
httpClient: &http.Client{
|
|
Timeout: config.APITimeout,
|
|
},
|
|
logger: log,
|
|
}, nil
|
|
}
|
|
|
|
// FetchSummaryOnDemand calls the external AI service POST /ai/summarize endpoint
|
|
// to generate a summary on demand. It retries on transient failures up to the
|
|
// configured number of attempts.
|
|
func (c *Client) FetchSummaryOnDemand(ctx context.Context, reqBody SummarizeRequest) (*SummarizeResponse, error) {
|
|
jsonBody, err := json.Marshal(reqBody)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to marshal request body: %w", err)
|
|
}
|
|
|
|
url := c.config.APIBaseURL + "/ai/summarize"
|
|
|
|
var lastErr error
|
|
for attempt := 0; attempt <= c.config.APIRetryCount; attempt++ {
|
|
if attempt > 0 {
|
|
c.logger.Warn("Retrying AI summarize request", map[string]interface{}{
|
|
"contract_folder_id": reqBody.ContractFolderID,
|
|
"notice_publication_id": reqBody.NoticePublicationID,
|
|
"attempt": attempt,
|
|
})
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-time.After(c.config.APIRetryDelay):
|
|
}
|
|
}
|
|
|
|
resp, err := c.doPost(ctx, url, jsonBody)
|
|
if err != nil {
|
|
lastErr = err
|
|
c.logger.Error("AI summarize request failed", map[string]interface{}{
|
|
"contract_folder_id": reqBody.ContractFolderID,
|
|
"notice_publication_id": reqBody.NoticePublicationID,
|
|
"attempt": attempt,
|
|
"error": err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
return nil, fmt.Errorf("ai_summarizer: all %d attempts failed, last error: %w", c.config.APIRetryCount+1, lastErr)
|
|
}
|
|
|
|
// FetchTranslationOnDemand calls the external AI service POST /ai/translate endpoint
|
|
// to generate title/description translation on demand.
|
|
func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody TranslateRequest) (*TranslateResponse, error) {
|
|
jsonBody, err := json.Marshal(reqBody)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to marshal translation request body: %w", err)
|
|
}
|
|
|
|
url := c.config.APIBaseURL + "/ai/translate"
|
|
|
|
var lastErr error
|
|
for attempt := 0; attempt <= c.config.APIRetryCount; attempt++ {
|
|
if attempt > 0 {
|
|
c.logger.Warn("Retrying AI translate request", map[string]interface{}{
|
|
"notice_publication_id": reqBody.NoticePublicationID,
|
|
"language": reqBody.Language,
|
|
"attempt": attempt,
|
|
})
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil, ctx.Err()
|
|
case <-time.After(c.config.APIRetryDelay):
|
|
}
|
|
}
|
|
|
|
resp, err := c.doTranslatePost(ctx, url, jsonBody, reqBody.RequestSource)
|
|
if err != nil {
|
|
lastErr = err
|
|
c.logger.Error("AI translate request failed", map[string]interface{}{
|
|
"notice_publication_id": reqBody.NoticePublicationID,
|
|
"language": reqBody.Language,
|
|
"attempt": attempt,
|
|
"error": err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
return nil, fmt.Errorf("ai_summarizer: all %d translation attempts failed, last error: %w", c.config.APIRetryCount+1, lastErr)
|
|
}
|
|
|
|
// TriggerPipelineTranslate calls POST /pipeline/translate to enqueue batch translation
|
|
// for the given target languages.
|
|
func (c *Client) TriggerPipelineTranslate(ctx context.Context, languages []string) (*PipelineTranslateResponse, error) {
|
|
reqBody := PipelineTranslateRequest{Languages: languages}
|
|
jsonBody, err := json.Marshal(reqBody)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to marshal pipeline translate request: %w", err)
|
|
}
|
|
|
|
url := c.config.APIBaseURL + "/pipeline/translate"
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to create pipeline translate request: %w", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Accept", "application/json")
|
|
|
|
httpResp, err := c.httpClient.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: pipeline translate request error: %w", err)
|
|
}
|
|
defer httpResp.Body.Close()
|
|
|
|
bodyBytes, err := io.ReadAll(httpResp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to read pipeline translate response: %w", err)
|
|
}
|
|
if httpResp.StatusCode >= 400 {
|
|
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
|
}
|
|
|
|
var result PipelineTranslateResponse
|
|
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to decode pipeline translate response: %w", err)
|
|
}
|
|
|
|
c.logger.Info("Pipeline translate triggered", map[string]interface{}{
|
|
"status": result.Status,
|
|
"languages": result.Languages,
|
|
})
|
|
|
|
return &result, nil
|
|
}
|
|
|
|
// doPost performs a single POST request and parses the response.
|
|
func (c *Client) doPost(ctx context.Context, url string, jsonBody []byte) (*SummarizeResponse, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to create request: %w", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Accept", "application/json")
|
|
|
|
c.logger.Debug("Sending AI summarize request", map[string]interface{}{
|
|
"url": url,
|
|
})
|
|
|
|
httpResp, err := c.httpClient.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: request error: %w", err)
|
|
}
|
|
defer httpResp.Body.Close()
|
|
|
|
bodyBytes, err := io.ReadAll(httpResp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to read response body: %w", err)
|
|
}
|
|
|
|
if httpResp.StatusCode == http.StatusNotFound {
|
|
return nil, fmt.Errorf("%w: status 404, body: %s", ErrObjectNotFound, string(bodyBytes))
|
|
}
|
|
|
|
if httpResp.StatusCode >= 400 {
|
|
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
|
}
|
|
|
|
var result SummarizeResponse
|
|
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to decode response JSON: %w", err)
|
|
}
|
|
|
|
c.logger.Info("AI summarize request succeeded", map[string]interface{}{
|
|
"contract_folder_id": result.ContractFolderID,
|
|
"notice_publication_id": result.ResolvedNoticePublicationID(),
|
|
"summaries_count": len(result.Summaries),
|
|
})
|
|
|
|
return &result, nil
|
|
}
|
|
|
|
// doTranslatePost performs a single translation POST request and parses the response.
|
|
func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byte, requestSource string) (*TranslateResponse, error) {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to create translation request: %w", err)
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Accept", "application/json")
|
|
|
|
c.logger.Debug("Sending AI translate request", map[string]interface{}{
|
|
"url": url,
|
|
})
|
|
|
|
httpResp, err := c.httpClient.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: translation request error: %w", err)
|
|
}
|
|
defer httpResp.Body.Close()
|
|
|
|
bodyBytes, err := io.ReadAll(httpResp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to read translation response body: %w", err)
|
|
}
|
|
|
|
if httpResp.StatusCode >= 400 {
|
|
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
|
}
|
|
|
|
result, err := decodeTranslateResponse(bodyBytes)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
c.logger.Info("AI translate request succeeded", map[string]interface{}{
|
|
"notice_publication_id": result.NoticePublicationID,
|
|
"language": result.Language,
|
|
})
|
|
|
|
if c.config.OnSuccessfulTranslation != nil {
|
|
if err := c.config.OnSuccessfulTranslation(ctx, requestSource); err != nil {
|
|
c.logger.Warn("Failed to increment AI translation success counter", map[string]interface{}{
|
|
"notice_publication_id": result.NoticePublicationID,
|
|
"language": result.Language,
|
|
"request_source": requestSource,
|
|
"error": err.Error(),
|
|
})
|
|
}
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
type translateAPIEnvelope struct {
|
|
Success bool `json:"success"`
|
|
Message string `json:"message"`
|
|
Data TranslateResponse `json:"data"`
|
|
}
|
|
|
|
func decodeTranslateResponse(bodyBytes []byte) (*TranslateResponse, error) {
|
|
var envelope translateAPIEnvelope
|
|
if err := json.Unmarshal(bodyBytes, &envelope); err == nil && envelope.Success && envelope.Data.Language != "" {
|
|
return &envelope.Data, nil
|
|
}
|
|
|
|
var direct TranslateResponse
|
|
if err := json.Unmarshal(bodyBytes, &direct); err != nil {
|
|
return nil, fmt.Errorf("ai_summarizer: failed to decode translation response JSON: %w", err)
|
|
}
|
|
if strings.TrimSpace(direct.Language) == "" {
|
|
return nil, fmt.Errorf("ai_summarizer: translation response missing language")
|
|
}
|
|
return &direct, nil
|
|
}
|