Integrated AI summary

pkg/ai_summarizer/client.go

@@ -0,0 +1,125 @@
+package ai_summarizer
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"tm/pkg/logger"
+)
+
+// Client is the HTTP client for the AI summarization on-demand API.
+type Client struct {
+	config     *Config
+	httpClient *http.Client
+	logger     logger.Logger
+}
+
+// NewClient creates a new AI summarizer HTTP client.
+func NewClient(config *Config, log logger.Logger) (*Client, error) {
+	if config == nil {
+		config = DefaultConfig()
+	}
+	if err := config.ValidateAPI(); err != nil {
+		return nil, fmt.Errorf("ai_summarizer: invalid config: %w", err)
+	}
+
+	return &Client{
+		config: config,
+		httpClient: &http.Client{
+			Timeout: config.APITimeout,
+		},
+		logger: log,
+	}, nil
+}
+
+// FetchSummaryOnDemand calls the external AI service POST /ai/summarize endpoint
+// to generate a summary on demand. It retries on transient failures up to the
+// configured number of attempts.
+func (c *Client) FetchSummaryOnDemand(ctx context.Context, reqBody SummarizeRequest) (*SummarizeResponse, error) {
+	jsonBody, err := json.Marshal(reqBody)
+	if err != nil {
+		return nil, fmt.Errorf("ai_summarizer: failed to marshal request body: %w", err)
+	}
+
+	url := c.config.APIBaseURL + "/ai/summarize"
+
+	var lastErr error
+	for attempt := 0; attempt <= c.config.APIRetryCount; attempt++ {
+		if attempt > 0 {
+			c.logger.Warn("Retrying AI summarize request", map[string]interface{}{
+				"notice_id": reqBody.NoticeID,
+				"attempt":   attempt,
+			})
+			select {
+			case <-ctx.Done():
+				return nil, ctx.Err()
+			case <-time.After(c.config.APIRetryDelay):
+			}
+		}
+
+		resp, err := c.doPost(ctx, url, jsonBody)
+		if err != nil {
+			lastErr = err
+			c.logger.Error("AI summarize request failed", map[string]interface{}{
+				"notice_id": reqBody.NoticeID,
+				"attempt":   attempt,
+				"error":     err.Error(),
+			})
+			continue
+		}
+
+		return resp, nil
+	}
+
+	return nil, fmt.Errorf("ai_summarizer: all %d attempts failed, last error: %w", c.config.APIRetryCount+1, lastErr)
+}
+
+// doPost performs a single POST request and parses the response.
+func (c *Client) doPost(ctx context.Context, url string, jsonBody []byte) (*SummarizeResponse, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
+	if err != nil {
+		return nil, fmt.Errorf("ai_summarizer: failed to create request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "application/json")
+
+	c.logger.Debug("Sending AI summarize request", map[string]interface{}{
+		"url": url,
+	})
+
+	httpResp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("ai_summarizer: request error: %w", err)
+	}
+	defer httpResp.Body.Close()
+
+	bodyBytes, err := io.ReadAll(httpResp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("ai_summarizer: failed to read response body: %w", err)
+	}
+
+	if httpResp.StatusCode == http.StatusNotFound {
+		return nil, fmt.Errorf("%w: status 404, body: %s", ErrObjectNotFound, string(bodyBytes))
+	}
+
+	if httpResp.StatusCode >= 400 {
+		return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
+	}
+
+	var result SummarizeResponse
+	if err := json.Unmarshal(bodyBytes, &result); err != nil {
+		return nil, fmt.Errorf("ai_summarizer: failed to decode response JSON: %w", err)
+	}
+
+	c.logger.Info("AI summarize request succeeded", map[string]interface{}{
+		"notice_id":       result.NoticeID,
+		"summaries_count": len(result.Summaries),
+	})
+
+	return &result, nil
+}

pkg/ai_summarizer/config.go

@@ -0,0 +1,76 @@
+package ai_summarizer
+
+import (
+	"errors"
+	"fmt"
+	"time"
+)
+
+// Config holds the configuration for the AI Summarizer service.
+// It contains settings for both the HTTP API and the MinIO storage integration.
+type Config struct {
+	// HTTP API settings
+	APIBaseURL    string        // Base URL of the AI service, e.g. "http://ai-host:8000"
+	APITimeout    time.Duration // Timeout for HTTP requests to the AI API
+	APIRetryCount int           // Number of retry attempts for failed API requests
+	APIRetryDelay time.Duration // Delay between retry attempts
+
+	// MinIO storage settings
+	MinioEndpoint  string        // MinIO endpoint, e.g. "ai-host:9000"
+	MinioAccessKey string        // MinIO access key ID
+	MinioSecretKey string        // MinIO secret access key
+	MinioUseSSL    bool          // Whether to use SSL for MinIO connection
+	MinioRegion    string        // MinIO region
+	MinioBucket    string        // MinIO bucket name (default: "opplens-documents")
+	MinioTimeout   time.Duration // Timeout for MinIO operations
+}
+
+// DefaultConfig returns a Config with sensible defaults.
+func DefaultConfig() *Config {
+	return &Config{
+		APITimeout:    120 * time.Second,
+		APIRetryCount: 2,
+		APIRetryDelay: 3 * time.Second,
+
+		MinioBucket:  "opplens-documents",
+		MinioRegion:  "us-east-1",
+		MinioUseSSL:  false,
+		MinioTimeout: 30 * time.Second,
+	}
+}
+
+// Validate checks that all required configuration fields are present.
+// Use this when both HTTP API and MinIO storage are needed.
+func (c *Config) Validate() error {
+	if c.APIBaseURL == "" {
+		return errors.New("ai_summarizer: APIBaseURL is required")
+	}
+	return c.ValidateStorage()
+}
+
+// ValidateStorage checks only the MinIO storage-related configuration fields.
+// Use this when only the storage client is needed (no HTTP API).
+func (c *Config) ValidateStorage() error {
+	if c.MinioEndpoint == "" {
+		return errors.New("ai_summarizer: MinioEndpoint is required")
+	}
+	if c.MinioAccessKey == "" {
+		return errors.New("ai_summarizer: MinioAccessKey is required")
+	}
+	if c.MinioSecretKey == "" {
+		return errors.New("ai_summarizer: MinioSecretKey is required")
+	}
+	if c.MinioBucket == "" {
+		return fmt.Errorf("ai_summarizer: MinioBucket is required")
+	}
+	return nil
+}
+
+// ValidateAPI checks only the HTTP API-related configuration fields.
+// Use this when only the HTTP client is needed (no MinIO storage).
+func (c *Config) ValidateAPI() error {
+	if c.APIBaseURL == "" {
+		return errors.New("ai_summarizer: APIBaseURL is required")
+	}
+	return nil
+}

pkg/ai_summarizer/entities.go

@@ -0,0 +1,43 @@
+package ai_summarizer
+
+// SummarizeRequest represents the request payload sent to the AI service
+// POST /ai/summarize endpoint for on-demand summarization.
+type SummarizeRequest struct {
+	NoticeID           string   `json:"notice_id"`                     // required
+	DocumentURL        string   `json:"document_url"`                  // required
+	Title              string   `json:"title"`                         // required
+	Description        string   `json:"description"`                   // required
+	Country            string   `json:"country,omitempty"`             // optional
+	EstimatedValue     *float64 `json:"estimated_value,omitempty"`     // optional (pointer to distinguish 0 from absent)
+	Currency           string   `json:"currency,omitempty"`            // optional
+	SubmissionDeadline string   `json:"submission_deadline,omitempty"` // optional
+	AuthorityName      string   `json:"authority_name,omitempty"`      // optional
+}
+
+// SummarizeResponse represents the response from the AI summarization API.
+type SummarizeResponse struct {
+	NoticeID       string            `json:"notice_id"`
+	Summaries      []DocumentSummary `json:"summaries"`
+	OverallSummary *string           `json:"overall_summary"` // nil when not available
+}
+
+// DocumentSummary represents a summary for a single document within the AI response.
+type DocumentSummary struct {
+	DocumentName string `json:"document_name"`
+	DocumentType string `json:"document_type"`
+	Summary      string `json:"summary"`
+	SummaryModel string `json:"summary_model"`
+	Error        string `json:"error"` // empty string when no error
+}
+
+// TenderJSON represents the structure of the <notice_id>/tender.json file
+// stored in the AI service's MinIO bucket.
+type TenderJSON struct {
+	OverallSummary  *string         `json:"overall_summary"` // nil if not yet generated
+	SummarizeStatus SummarizeStatus `json:"summarize_status"`
+}
+
+// SummarizeStatus represents the summarization status inside tender.json.
+type SummarizeStatus struct {
+	Done bool `json:"done"` // true when the pipeline has finished
+}

pkg/ai_summarizer/errors.go

@@ -0,0 +1,24 @@
+package ai_summarizer
+
+import "errors"
+
+// Common sentinel errors for the AI summarizer package.
+var (
+	// ErrSummaryNotReady is returned when the tender.json exists but
+	// summarize_status.done is still false.
+	ErrSummaryNotReady = errors.New("ai_summarizer: summary not ready yet")
+
+	// ErrNoOverallSummary is returned when the pipeline is done but
+	// overall_summary is empty or null.
+	ErrNoOverallSummary = errors.New("ai_summarizer: overall_summary is empty")
+
+	// ErrObjectNotFound is returned when the tender.json object does not
+	// exist in the MinIO bucket (HTTP 404 / NoSuchKey).
+	ErrObjectNotFound = errors.New("ai_summarizer: tender.json not found in storage")
+
+	// ErrBucketNotFound is returned when the MinIO bucket does not exist.
+	ErrBucketNotFound = errors.New("ai_summarizer: bucket not found")
+
+	// ErrAPINonSuccess is returned when the AI API returns a non-2xx status code.
+	ErrAPINonSuccess = errors.New("ai_summarizer: API returned non-success status")
+)

pkg/ai_summarizer/storage.go

@@ -0,0 +1,235 @@
+package ai_summarizer
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+
+	"tm/pkg/logger"
+)
+
+// StorageClient wraps a MinIO client configured to connect to the AI service's
+// MinIO bucket and retrieve tender.json files.
+type StorageClient struct {
+	client *minio.Client
+	config *Config
+	logger logger.Logger
+}
+
+// TenderSummaryItem represents a single tender's summary retrieved from storage.
+type TenderSummaryItem struct {
+	NoticeID       string `json:"notice_id"`
+	OverallSummary string `json:"overall_summary"`
+	Done           bool   `json:"done"`
+}
+
+// NewStorageClient creates a new MinIO storage client for the AI service.
+// It establishes a connection to the external MinIO instance and verifies
+// that the configured bucket exists.
+func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error) {
+	if config == nil {
+		config = DefaultConfig()
+	}
+	if err := config.ValidateStorage(); err != nil {
+		return nil, fmt.Errorf("ai_summarizer: invalid storage config: %w", err)
+	}
+
+	minioClient, err := minio.New(config.MinioEndpoint, &minio.Options{
+		Creds:  credentials.NewStaticV4(config.MinioAccessKey, config.MinioSecretKey, ""),
+		Secure: config.MinioUseSSL,
+		Region: config.MinioRegion,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("ai_summarizer: failed to create MinIO client: %w", err)
+	}
+
+	return &StorageClient{
+		client: minioClient,
+		config: config,
+		logger: log,
+	}, nil
+}
+
+// GetSummaryFromStorage fetches the <noticeID>/tender.json file from the AI
+// service's MinIO bucket, parses it, and returns the overall_summary string.
+//
+// It returns:
+//   - (summary, nil) on success
+//   - ("", ErrObjectNotFound) if the tender.json does not exist (404)
+//   - ("", ErrSummaryNotReady) if the file exists but summarize_status.done is false
+//   - ("", ErrNoOverallSummary) if done is true but overall_summary is empty/null
+//   - ("", error) for any other failure
+func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error) {
+	objectPath := fmt.Sprintf("%s/tender.json", noticeID)
+
+	s.logger.Debug("Fetching tender.json from MinIO", map[string]interface{}{
+		"bucket": s.config.MinioBucket,
+		"path":   objectPath,
+	})
+
+	// Attempt to get the object
+	object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectPath, minio.GetObjectOptions{})
+	if err != nil {
+		// minio-go returns an error response that we can inspect
+		var minioErr minio.ErrorResponse
+		if errors.As(err, &minioErr) {
+			switch minioErr.StatusCode {
+			case http.StatusNotFound:
+				return "", ErrObjectNotFound
+			case http.StatusForbidden:
+				return "", fmt.Errorf("ai_summarizer: access denied to bucket %s: %w", s.config.MinioBucket, err)
+			default:
+				return "", fmt.Errorf("ai_summarizer: MinIO error (code=%s, status=%d): %w",
+					minioErr.Code, minioErr.StatusCode, err)
+			}
+		}
+		return "", fmt.Errorf("ai_summarizer: failed to get object: %w", err)
+	}
+	defer object.Close()
+
+	// Read the object content
+	data, err := io.ReadAll(object)
+	if err != nil {
+		return "", fmt.Errorf("ai_summarizer: failed to read tender.json: %w", err)
+	}
+
+	// Parse the JSON
+	var tenderJSON TenderJSON
+	if err := json.Unmarshal(data, &tenderJSON); err != nil {
+		return "", fmt.Errorf("ai_summarizer: failed to parse tender.json: %w", err)
+	}
+
+	// Check if summarization is complete
+	if !tenderJSON.SummarizeStatus.Done {
+		return "", ErrSummaryNotReady
+	}
+
+	// Check if overall_summary is available
+	if tenderJSON.OverallSummary == nil || *tenderJSON.OverallSummary == "" {
+		return "", ErrNoOverallSummary
+	}
+
+	s.logger.Info("Retrieved overall_summary from storage", map[string]interface{}{
+		"notice_id":      noticeID,
+		"summary_length": len(*tenderJSON.OverallSummary),
+	})
+
+	return *tenderJSON.OverallSummary, nil
+}
+
+// IsSummaryReady is a convenience method that checks whether the tender.json
+// exists and the summarize_status.done flag is true, without returning the
+// actual summary text.
+func (s *StorageClient) IsSummaryReady(ctx context.Context, noticeID string) (bool, error) {
+	objectPath := fmt.Sprintf("%s/tender.json", noticeID)
+
+	object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectPath, minio.GetObjectOptions{})
+	if err != nil {
+		var minioErr minio.ErrorResponse
+		if errors.As(err, &minioErr) && minioErr.StatusCode == http.StatusNotFound {
+			return false, nil // not found means not ready, but not an error
+		}
+		return false, fmt.Errorf("ai_summarizer: failed to check summary readiness: %w", err)
+	}
+	defer object.Close()
+
+	data, err := io.ReadAll(object)
+	if err != nil {
+		return false, fmt.Errorf("ai_summarizer: failed to read tender.json: %w", err)
+	}
+
+	var tenderJSON TenderJSON
+	if err := json.Unmarshal(data, &tenderJSON); err != nil {
+		return false, fmt.Errorf("ai_summarizer: failed to parse tender.json: %w", err)
+	}
+
+	return tenderJSON.SummarizeStatus.Done, nil
+}
+
+// GetAllSummaries lists all tender.json files in the MinIO bucket and returns
+// the overall_summary for each tender where summarize_status.done is true.
+// This is useful for batch-retrieving all summaries that the AI pipeline has
+// already processed.
+func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryItem, error) {
+	s.logger.Debug("Listing all tender.json files from MinIO", map[string]interface{}{
+		"bucket": s.config.MinioBucket,
+	})
+
+	// List all objects ending with /tender.json
+	objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
+		Prefix:    "",
+		Recursive: true,
+	})
+
+	var results []TenderSummaryItem
+
+	for object := range objectCh {
+		if object.Err != nil {
+			s.logger.Error("Error listing MinIO objects", map[string]interface{}{
+				"error": object.Err.Error(),
+			})
+			continue
+		}
+
+		// Only process tender.json files (path pattern: <notice_id>/tender.json)
+		if !strings.HasSuffix(object.Key, "/tender.json") {
+			continue
+		}
+
+		// Extract notice_id from the path
+		noticeID := strings.TrimSuffix(object.Key, "/tender.json")
+		if noticeID == "" {
+			continue
+		}
+
+		// Fetch and parse the tender.json
+		summary, err := s.GetSummaryFromStorage(ctx, noticeID)
+		if err != nil {
+			// Log but skip tenders that aren't ready or have errors
+			s.logger.Debug("Skipping tender summary", map[string]interface{}{
+				"notice_id": noticeID,
+				"error":     err.Error(),
+			})
+
+			// Still include items that aren't done so the caller knows they exist
+			if errors.Is(err, ErrSummaryNotReady) {
+				results = append(results, TenderSummaryItem{
+					NoticeID: noticeID,
+					Done:     false,
+				})
+			}
+			continue
+		}
+
+		results = append(results, TenderSummaryItem{
+			NoticeID:       noticeID,
+			OverallSummary: summary,
+			Done:           true,
+		})
+	}
+
+	s.logger.Info("Retrieved all summaries from storage", map[string]interface{}{
+		"total_count":     len(results),
+		"completed_count": countDone(results),
+	})
+
+	return results, nil
+}
+
+// countDone counts how many items have Done=true
+func countDone(items []TenderSummaryItem) int {
+	count := 0
+	for _, item := range items {
+		if item.Done {
+			count++
+		}
+	}
+	return count
+}