From 154610e2a09af4950c5df09a23cd5d64fea13d8d Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Sat, 4 Oct 2025 15:21:35 +0330 Subject: [PATCH 1/6] Add Ollama SDK with Core Functionality and Documentation - Introduced the Ollama SDK, providing a comprehensive interface for interacting with Ollama AI models, including text generation, chat conversations, embeddings, and model management. - Implemented core components such as Client, Config, and various request/response entities to facilitate API interactions. - Added structured error handling with specific error types for improved clarity and debugging. - Included a README and usage guide to assist developers in integrating the SDK into their applications, ensuring a smooth onboarding experience. - Established a fluent API for chat interactions and streaming responses, enhancing usability and flexibility for developers. --- pkg/ollama/README.md | 394 +++++++++++++++++++++++++++++++++++++++ pkg/ollama/USAGE.md | 263 ++++++++++++++++++++++++++ pkg/ollama/client.go | 290 ++++++++++++++++++++++++++++ pkg/ollama/config.go | 91 +++++++++ pkg/ollama/entities.go | 182 ++++++++++++++++++ pkg/ollama/errors.go | 146 +++++++++++++++ pkg/ollama/interfaces.go | 121 ++++++++++++ pkg/ollama/sdk.go | 232 +++++++++++++++++++++++ 8 files changed, 1719 insertions(+) create mode 100644 pkg/ollama/README.md create mode 100644 pkg/ollama/USAGE.md create mode 100644 pkg/ollama/client.go create mode 100644 pkg/ollama/config.go create mode 100644 pkg/ollama/entities.go create mode 100644 pkg/ollama/errors.go create mode 100644 pkg/ollama/interfaces.go create mode 100644 pkg/ollama/sdk.go diff --git a/pkg/ollama/README.md b/pkg/ollama/README.md new file mode 100644 index 0000000..ad86132 --- /dev/null +++ b/pkg/ollama/README.md @@ -0,0 +1,394 @@ +# Ollama SDK + +A comprehensive Go SDK for interacting with Ollama AI models. This SDK provides a clean, type-safe interface for text generation, chat conversations, embeddings, and model management. + +## Features + +- **Text Generation**: Generate text using various AI models +- **Chat Conversations**: Interactive chat with support for system prompts and conversation history +- **Embeddings**: Generate embeddings for text similarity and search +- **Model Management**: Pull, list, delete, and inspect models +- **Streaming Support**: Real-time streaming of generated text and chat responses +- **Automatic Retries**: Configurable retry mechanism with exponential backoff +- **Structured Logging**: Built-in logging with structured fields +- **Fluent API**: Builder pattern for constructing complex chat conversations +- **Type Safety**: Full type safety with comprehensive validation +- **Error Handling**: Detailed error types and proper error wrapping +- **Configuration**: Flexible configuration via environment variables or code +- **Health Checks**: Built-in health checking capabilities + +## Installation + +The SDK is already part of your `tm` module. Import it as: +```go +import "tm/pkg/ollama" +``` + +## Quick Start + +### Basic Setup + +```go +// 1. Configuration +config := ollama.DefaultConfig() +config.BaseURL = "http://localhost:11434" // Your Ollama server URL +config.DefaultModel = "llama2" // Default model to use + +// 2. Logger (using project's logger) +loggerConfig := &logger.Config{Level: "info", Format: "json", Output: "stdout"} +projectLogger := logger.NewLogger(loggerConfig) + +// 3. Create SDK +sdk, err := ollama.New(config, projectLogger) +if err != nil { + log.Fatal("Failed to create Ollama SDK:", err) +} +``` + +### Environment Variables + +Set environment variables for automatic configuration: + +```bash +export OLLAMA_BASE_URL="http://localhost:11434" +export OLLAMA_TIMEOUT="300s" +export OLLAMA_RETRY_ATTEMPTS="3" +export OLLAMA_RETRY_DELAY="2s" +export OLLAMA_ENABLE_LOGGING="true" +export OLLAMA_DEFAULT_MODEL="llama2" +export OLLAMA_DEFAULT_TEMPERATURE="0.7" +export OLLAMA_DEFAULT_TOP_P="0.9" +export OLLAMA_DEFAULT_MAX_TOKENS="1000" +``` + +## API Reference + +### Configuration + +```go +type Config struct { + BaseURL string `env:"OLLAMA_BASE_URL"` + Timeout time.Duration `env:"OLLAMA_TIMEOUT"` + RetryAttempts int `env:"OLLAMA_RETRY_ATTEMPTS"` + RetryDelay time.Duration `env:"OLLAMA_RETRY_DELAY"` + EnableLogging bool `env:"OLLAMA_ENABLE_LOGGING"` + UserAgent string `env:"OLLAMA_USER_AGENT"` + DefaultModel string `env:"OLLAMA_DEFAULT_MODEL"` + DefaultTemperature float64 `env:"OLLAMA_DEFAULT_TEMPERATURE"` + DefaultTopP float64 `env:"OLLAMA_DEFAULT_TOP_P"` + DefaultMaxTokens int `env:"OLLAMA_DEFAULT_MAX_TOKENS"` + EnableStreaming bool `env:"OLLAMA_ENABLE_STREAMING"` +} +``` + +### Creating SDK Instance + +```go +// With default configuration +sdk, err := ollama.New(nil, logger) + +// With custom configuration +config := &ollama.Config{ + BaseURL: "http://localhost:11434", + Timeout: 300 * time.Second, + RetryAttempts: 3, + RetryDelay: 2 * time.Second, + EnableLogging: true, + DefaultModel: "llama2", + DefaultTemperature: 0.7, + DefaultTopP: 0.9, + DefaultMaxTokens: 1000, +} +sdk, err := ollama.New(config, logger) + +// With custom HTTP client +httpClient := &http.Client{Timeout: 10 * time.Second} +sdk, err := ollama.NewWithClient(config, httpClient, logger) +``` + +## Usage Examples + +### Simple Text Generation + +```go +ctx := context.Background() + +// Quick generation with default model +response, err := sdk.QuickGenerate(ctx, "Write a short story about a robot") +if err != nil { + log.Printf("Generation failed: %v", err) +} +fmt.Println(response) + +// Generation with specific model +response, err := sdk.QuickGenerateWithModel(ctx, "codellama", "Write a Python function to sort a list") +``` + +### Advanced Text Generation + +```go +req := &ollama.GenerateRequest{ + Model: "llama2", + Prompt: "Explain quantum computing in simple terms", + Options: map[string]interface{}{ + "temperature": 0.7, + "top_p": 0.9, + "max_tokens": 500, + }, +} + +resp, err := sdk.Generate(ctx, req) +if err != nil { + log.Printf("Generation failed: %v", err) +} +fmt.Printf("Generated: %s\n", resp.Response) +``` + +### Chat Conversations + +```go +// Simple chat +response, err := sdk.QuickChat(ctx, "What is the capital of France?") +fmt.Println(response) + +// Chat with system prompt +response, err := sdk.QuickChatWithSystem(ctx, + "You are a helpful coding assistant.", + "How do I implement a binary search in Go?") +``` + +### Advanced Chat with Conversation History + +```go +req := &ollama.ChatRequest{ + Model: "llama2", + Messages: []ollama.Message{ + { + Role: ollama.MessageRoleSystem, + Content: "You are a helpful assistant.", + }, + { + Role: ollama.MessageRoleUser, + Content: "Hello, how are you?", + }, + { + Role: ollama.MessageRoleAssistant, + Content: "I'm doing well, thank you! How can I help you today?", + }, + { + Role: ollama.MessageRoleUser, + Content: "Can you help me with a coding problem?", + }, + }, +} + +resp, err := sdk.Chat(ctx, req) +fmt.Printf("Assistant: %s\n", resp.Message.Content) +``` + +### Fluent Chat Builder + +```go +response, err := sdk.NewChatBuilder(). + SetModel("llama2"). + SetSystem("You are a helpful coding assistant."). + AddMessage(ollama.MessageRoleUser, "How do I create a struct in Go?"). + SetTemperature(0.7). + SetMaxTokens(500). + Send(ctx) + +fmt.Printf("Assistant: %s\n", response.Message.Content) + +// Continue the conversation +response, err = sdk.NewChatBuilder(). + SetModel("llama2"). + SetSystem("You are a helpful coding assistant."). + AddMessage(ollama.MessageRoleUser, "How do I create a struct in Go?"). + SetTemperature(0.7). + SetMaxTokens(500). + Continue(ctx, "Can you show me an example with methods?") + +fmt.Printf("Assistant: %s\n", response.Message.Content) +``` + +### Streaming Responses + +```go +// Streaming text generation +err := sdk.StreamGenerate(ctx, &ollama.GenerateRequest{ + Model: "llama2", + Prompt: "Write a poem about programming", +}, func(chunk string) error { + fmt.Print(chunk) + return nil +}) + +// Streaming chat +err := sdk.StreamChat(ctx, &ollama.ChatRequest{ + Model: "llama2", + Messages: []ollama.Message{ + { + Role: ollama.MessageRoleUser, + Content: "Tell me a story", + }, + }, +}, func(chunk string) error { + fmt.Print(chunk) + return nil +}) +``` + +### Embeddings + +```go +// Generate embeddings +embeddings, err := sdk.QuickEmbed(ctx, "This is some text to embed") +if err != nil { + log.Printf("Embedding failed: %v", err) +} + +// Advanced embedding request +req := &ollama.EmbedRequest{ + Model: "llama2", + Prompt: "Text to generate embeddings for", +} + +resp, err := sdk.Embed(ctx, req) +fmt.Printf("Embedding dimension: %d\n", len(resp.Embedding)) +``` + +### Model Management + +```go +// List available models +models, err := sdk.ListModels(ctx) +for _, model := range models.Models { + fmt.Printf("Model: %s, Size: %d bytes\n", model.Name, model.Size) +} + +// Pull a new model +pullResp, err := sdk.PullModel(ctx, "codellama") +fmt.Printf("Pull status: %s\n", pullResp.Status) + +// Show model information +modelInfo, err := sdk.ShowModel(ctx, "llama2") +fmt.Printf("Model details: %+v\n", modelInfo.Details) + +// Delete a model +deleteResp, err := sdk.DeleteModel(ctx, "unwanted-model") +fmt.Printf("Delete status: %s\n", deleteResp.Status) +``` + +### Health Check + +```go +err := sdk.Health(ctx) +if err != nil { + log.Printf("Ollama service is not available: %v", err) +} else { + fmt.Println("Ollama service is healthy") +} +``` + +## Error Handling + +The SDK provides specific error types for different failure scenarios: + +```go +resp, err := sdk.Generate(ctx, req) +if err != nil { + switch e := err.(type) { + case *ollama.ConfigError: + fmt.Printf("Configuration error: %s\n", e.Message) + case *ollama.ModelError: + fmt.Printf("Model error for %s: %s\n", e.Model, e.Message) + case *ollama.GenerationError: + fmt.Printf("Generation error: %s\n", e.Message) + case *ollama.ConnectionError: + fmt.Printf("Connection error: %s\n", e.Message) + case *ollama.TimeoutError: + fmt.Printf("Timeout during %s: %s\n", e.Operation, e.Message) + default: + fmt.Printf("Unexpected error: %v\n", err) + } +} +``` + +## Advanced Usage + +### Custom HTTP Client + +```go +httpClient := &http.Client{ + Timeout: 60 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + }, +} + +sdk, err := ollama.NewWithClient(config, httpClient, logger) +``` + +### Custom Options + +```go +req := &ollama.GenerateRequest{ + Model: "llama2", + Prompt: "Write a haiku", +} + +opts := &ollama.GenerateOptions{ + Temperature: &[]float64{0.5}[0], + TopP: &[]float64{0.8}[0], + TopK: &[]int{40}[0], + RepeatPenalty: &[]float64{1.1}[0], + Seed: &[]int{42}[0], + Stop: []string{"", "\n\n"}, + Format: "json", +} + +resp, err := sdk.GenerateWithOptions(ctx, req, opts) +``` + +## Best Practices + +1. **Always use context**: Pass context for cancellation and timeouts +2. **Handle errors properly**: Use type assertions for specific error handling +3. **Configure timeouts**: Set appropriate timeouts for your use case +4. **Use streaming for long responses**: Enable streaming for better user experience +5. **Monitor model usage**: Keep track of token usage and costs +6. **Validate inputs**: Ensure prompts and parameters are within limits +7. **Use structured logging**: Leverage the built-in logging for debugging + +## Troubleshooting + +### Common Issues + +1. **Connection refused**: Ensure Ollama server is running on the correct port +2. **Model not found**: Pull the model first using `PullModel()` +3. **Timeout errors**: Increase the timeout configuration +4. **Rate limiting**: Implement backoff strategies for high-volume usage + +### Debug Mode + +Enable debug logging to troubleshoot issues: + +```go +config := ollama.DefaultConfig() +config.EnableLogging = true + +loggerConfig := &logger.Config{ + Level: "debug", + Format: "json", + Output: "stdout", +} +projectLogger := logger.NewLogger(loggerConfig) + +sdk, err := ollama.New(config, projectLogger) +``` + +## License + +This SDK is part of the Tender Management system and follows the same licensing terms. diff --git a/pkg/ollama/USAGE.md b/pkg/ollama/USAGE.md new file mode 100644 index 0000000..ef1dc96 --- /dev/null +++ b/pkg/ollama/USAGE.md @@ -0,0 +1,263 @@ +# Ollama SDK - Quick Usage Guide + +## Installation +The SDK is already part of your `tm` module. Import it as: +```go +import "tm/pkg/ollama" +``` + +## Basic Setup +```go +// 1. Configuration +config := ollama.DefaultConfig() +config.BaseURL = "http://localhost:11434" // Your Ollama server URL +config.DefaultModel = "llama2" // Default model to use + +// 2. Logger (using project's logger) +loggerConfig := &logger.Config{Level: "info", Format: "json", Output: "stdout"} +projectLogger := logger.NewLogger(loggerConfig) + +// 3. Create SDK +sdk, err := ollama.New(config, projectLogger) +if err != nil { + log.Fatal("Failed to create Ollama SDK:", err) +} +``` + +## Environment Variables +```bash +export OLLAMA_BASE_URL="http://localhost:11434" +export OLLAMA_TIMEOUT="300s" +export OLLAMA_RETRY_ATTEMPTS="3" +export OLLAMA_RETRY_DELAY="2s" +export OLLAMA_ENABLE_LOGGING="true" +export OLLAMA_DEFAULT_MODEL="llama2" +export OLLAMA_DEFAULT_TEMPERATURE="0.7" +export OLLAMA_DEFAULT_TOP_P="0.9" +export OLLAMA_DEFAULT_MAX_TOKENS="1000" +``` + +## Quick Examples + +### Simple Text Generation +```go +ctx := context.Background() + +// Generate text with default model +response, err := sdk.QuickGenerate(ctx, "Write a short story about a robot") +if err != nil { + log.Printf("Generation failed: %v", err) +} +fmt.Println(response) + +// Generate with specific model +response, err := sdk.QuickGenerateWithModel(ctx, "codellama", "Write a Python function to sort a list") +``` + +### Chat Conversations +```go +// Simple chat +response, err := sdk.QuickChat(ctx, "What is the capital of France?") +fmt.Println(response) + +// Chat with system prompt +response, err := sdk.QuickChatWithSystem(ctx, + "You are a helpful coding assistant.", + "How do I implement a binary search in Go?") +``` + +### Advanced Chat with Builder +```go +// Create a chat builder +builder := sdk.NewChatBuilder(). + SetModel("llama2"). + SetSystem("You are a helpful coding assistant."). + SetTemperature(0.7). + SetMaxTokens(500) + +// Start conversation +response, err := builder. + AddMessage(ollama.MessageRoleUser, "How do I create a struct in Go?"). + Send(ctx) + +fmt.Printf("Assistant: %s\n", response.Message.Content) + +// Continue the conversation +response, err = builder.Continue(ctx, "Can you show me an example with methods?") +fmt.Printf("Assistant: %s\n", response.Message.Content) +``` + +### Streaming Responses +```go +// Streaming text generation +err := sdk.StreamGenerate(ctx, &ollama.GenerateRequest{ + Model: "llama2", + Prompt: "Write a poem about programming", +}, func(chunk string) error { + fmt.Print(chunk) // Print each chunk as it arrives + return nil +}) + +// Streaming chat +err := sdk.StreamChat(ctx, &ollama.ChatRequest{ + Model: "llama2", + Messages: []ollama.Message{ + { + Role: ollama.MessageRoleUser, + Content: "Tell me a story", + }, + }, +}, func(chunk string) error { + fmt.Print(chunk) + return nil +}) +``` + +### Embeddings +```go +// Generate embeddings +embeddings, err := sdk.QuickEmbed(ctx, "This is some text to embed") +if err != nil { + log.Printf("Embedding failed: %v", err) +} + +fmt.Printf("Embedding dimension: %d\n", len(embeddings)) +``` + +### Model Management +```go +// List available models +models, err := sdk.ListModels(ctx) +for _, model := range models.Models { + fmt.Printf("Model: %s, Size: %d bytes\n", model.Name, model.Size) +} + +// Pull a new model +pullResp, err := sdk.PullModel(ctx, "codellama") +fmt.Printf("Pull status: %s\n", pullResp.Status) + +// Show model information +modelInfo, err := sdk.ShowModel(ctx, "llama2") +fmt.Printf("Model family: %s\n", modelInfo.Details.Family) + +// Delete a model +deleteResp, err := sdk.DeleteModel(ctx, "unwanted-model") +fmt.Printf("Delete status: %s\n", deleteResp.Status) +``` + +### Health Check +```go +err := sdk.Health(ctx) +if err != nil { + log.Printf("Ollama service is not available: %v", err) +} else { + fmt.Println("Ollama service is healthy") +} +``` + +## Error Handling +```go +resp, err := sdk.Generate(ctx, req) +if err != nil { + switch e := err.(type) { + case *ollama.ConfigError: + fmt.Printf("Configuration error: %s\n", e.Message) + case *ollama.ModelError: + fmt.Printf("Model error for %s: %s\n", e.Model, e.Message) + case *ollama.GenerationError: + fmt.Printf("Generation error: %s\n", e.Message) + case *ollama.ConnectionError: + fmt.Printf("Connection error: %s\n", e.Message) + case *ollama.TimeoutError: + fmt.Printf("Timeout during %s: %s\n", e.Operation, e.Message) + default: + fmt.Printf("Unexpected error: %v\n", err) + } +} +``` + +## Advanced Configuration +```go +config := &ollama.Config{ + BaseURL: "http://localhost:11434", + Timeout: 300 * time.Second, + RetryAttempts: 3, + RetryDelay: 2 * time.Second, + EnableLogging: true, + UserAgent: "MyApp-OllamaSDK/1.0", + DefaultModel: "llama2", + DefaultTemperature: 0.7, + DefaultTopP: 0.9, + DefaultMaxTokens: 1000, + EnableStreaming: false, +} + +sdk, err := ollama.New(config, logger) +``` + +## Custom HTTP Client +```go +httpClient := &http.Client{ + Timeout: 60 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + }, +} + +sdk, err := ollama.NewWithClient(config, httpClient, logger) +``` + +## Integration with Project Logger +```go +// Use the project's logger system +loggerConfig := &logger.Config{ + Level: "info", + Format: "json", + Output: "stdout", +} +projectLogger := logger.NewLogger(loggerConfig) + +sdk, err := ollama.New(config, projectLogger) +``` + +## Best Practices + +1. **Always use context**: Pass context for cancellation and timeouts +2. **Handle errors properly**: Use type assertions for specific error handling +3. **Configure timeouts**: Set appropriate timeouts for your use case +4. **Use streaming for long responses**: Enable streaming for better user experience +5. **Monitor model usage**: Keep track of token usage and costs +6. **Validate inputs**: Ensure prompts and parameters are within limits +7. **Use structured logging**: Leverage the built-in logging for debugging + +## Common Use Cases + +### Document Summarization +```go +response, err := sdk.QuickChatWithSystem(ctx, + "You are a helpful assistant that summarizes documents concisely.", + "Please summarize this document: [document content]") +``` + +### Code Review Assistant +```go +response, err := sdk.QuickChatWithSystem(ctx, + "You are a senior software engineer reviewing code. Provide constructive feedback.", + "Please review this Go code: [code snippet]") +``` + +### Translation Service +```go +response, err := sdk.QuickChatWithSystem(ctx, + "You are a professional translator. Translate the following text accurately.", + "Translate this to French: Hello, how are you?") +``` + +### Question Answering +```go +response, err := sdk.QuickChatWithSystem(ctx, + "You are a helpful assistant that answers questions accurately and concisely.", + "What is the difference between HTTP and HTTPS?") +``` diff --git a/pkg/ollama/client.go b/pkg/ollama/client.go new file mode 100644 index 0000000..a070ed6 --- /dev/null +++ b/pkg/ollama/client.go @@ -0,0 +1,290 @@ +package ollama + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + "tm/pkg/logger" +) + +// Client implements the HTTPClient interface for Ollama API +type Client struct { + config *Config + httpClient *http.Client + logger logger.Logger +} + +// NewClient creates a new Ollama HTTP client +func NewClient(config *Config, logger logger.Logger) (*Client, error) { + if config == nil { + config = DefaultConfig() + } + + if err := config.Validate(); err != nil { + return nil, err + } + + httpClient := &http.Client{ + Timeout: config.Timeout, + } + + return &Client{ + config: config, + httpClient: httpClient, + logger: logger, + }, nil +} + +// NewClientWithHTTPClient creates a new client with a custom HTTP client +func NewClientWithHTTPClient(config *Config, httpClient *http.Client, logger logger.Logger) (*Client, error) { + if config == nil { + config = DefaultConfig() + } + + if err := config.Validate(); err != nil { + return nil, err + } + + return &Client{ + config: config, + httpClient: httpClient, + logger: logger, + }, nil +} + +// Post performs a POST request to the Ollama API +func (c *Client) Post(ctx context.Context, url string, body interface{}, result interface{}) error { + return c.makeRequest(ctx, "POST", url, body, result) +} + +// Get performs a GET request to the Ollama API +func (c *Client) Get(ctx context.Context, url string, result interface{}) error { + return c.makeRequest(ctx, "GET", url, nil, result) +} + +// Delete performs a DELETE request to the Ollama API +func (c *Client) Delete(ctx context.Context, url string, body interface{}, result interface{}) error { + return c.makeRequest(ctx, "DELETE", url, body, result) +} + +// makeRequest performs an HTTP request with retry logic and logging +func (c *Client) makeRequest(ctx context.Context, method, endpoint string, body interface{}, result interface{}) error { + url := c.config.BaseURL + endpoint + + var bodyReader io.Reader + if body != nil { + bodyBytes, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + bodyReader = bytes.NewReader(bodyBytes) + } + + var lastErr error + for attempt := 0; attempt <= c.config.RetryAttempts; attempt++ { + if attempt > 0 { + // Wait before retry + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(c.config.RetryDelay): + } + } + + err := c.doRequest(ctx, method, url, bodyReader, result) + if err == nil { + return nil + } + + lastErr = err + + // Don't retry on certain errors + if c.shouldNotRetry(err) { + break + } + + if c.logger != nil { + c.logger.Warn("Request failed, retrying", map[string]interface{}{ + "attempt": attempt + 1, + "max_attempts": c.config.RetryAttempts + 1, + "error": err.Error(), + "url": url, + "method": method, + }) + } + } + + return fmt.Errorf("request failed after %d attempts: %w", c.config.RetryAttempts+1, lastErr) +} + +// doRequest performs a single HTTP request +func (c *Client) doRequest(ctx context.Context, method, url string, body io.Reader, result interface{}) error { + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", c.config.UserAgent) + + if c.logger != nil && c.config.EnableLogging { + c.logger.Debug("Making HTTP request", map[string]interface{}{ + "method": method, + "url": url, + "user_agent": c.config.UserAgent, + }) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to execute request: %w", err) + } + defer resp.Body.Close() + + if c.logger != nil && c.config.EnableLogging { + c.logger.Debug("Received HTTP response", map[string]interface{}{ + "status_code": resp.StatusCode, + "url": url, + }) + } + + if resp.StatusCode >= 400 { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(bodyBytes)) + } + + if result != nil { + if err := json.NewDecoder(resp.Body).Decode(result); err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + } + + return nil +} + +// shouldNotRetry determines if an error should not be retried +func (c *Client) shouldNotRetry(err error) bool { + // Don't retry on context cancellation + if err == context.Canceled || err == context.DeadlineExceeded { + return true + } + + // Don't retry on certain HTTP status codes + if err.Error() == "HTTP 400" || err.Error() == "HTTP 401" || err.Error() == "HTTP 403" || err.Error() == "HTTP 404" { + return true + } + + return false +} + +// StreamGenerate performs a streaming text generation request +func (c *Client) StreamGenerate(ctx context.Context, req *GenerateRequest, onChunk func(*StreamChunk) error) error { + url := c.config.BaseURL + "/api/generate" + + // Force streaming + stream := true + req.Stream = &stream + + bodyBytes, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("User-Agent", c.config.UserAgent) + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(bodyBytes)) + } + + decoder := json.NewDecoder(resp.Body) + for { + var chunk StreamChunk + if err := decoder.Decode(&chunk); err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("failed to decode stream chunk: %w", err) + } + + if err := onChunk(&chunk); err != nil { + return fmt.Errorf("error processing stream chunk: %w", err) + } + + if chunk.Done { + break + } + } + + return nil +} + +// StreamChat performs a streaming chat request +func (c *Client) StreamChat(ctx context.Context, req *ChatRequest, onChunk func(*StreamChunk) error) error { + url := c.config.BaseURL + "/api/chat" + + // Force streaming + stream := true + req.Stream = &stream + + bodyBytes, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("User-Agent", c.config.UserAgent) + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(bodyBytes)) + } + + decoder := json.NewDecoder(resp.Body) + for { + var chunk StreamChunk + if err := decoder.Decode(&chunk); err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("failed to decode stream chunk: %w", err) + } + + if err := onChunk(&chunk); err != nil { + return fmt.Errorf("error processing stream chunk: %w", err) + } + + if chunk.Done { + break + } + } + + return nil +} diff --git a/pkg/ollama/config.go b/pkg/ollama/config.go new file mode 100644 index 0000000..2bb1d80 --- /dev/null +++ b/pkg/ollama/config.go @@ -0,0 +1,91 @@ +package ollama + +import ( + "time" +) + +// Config represents the configuration for the Ollama SDK +type Config struct { + // BaseURL is the base URL of the Ollama API server + BaseURL string `env:"OLLAMA_BASE_URL" default:"http://localhost:11434"` + + // Timeout is the timeout for HTTP requests + Timeout time.Duration `env:"OLLAMA_TIMEOUT" default:"300s"` + + // RetryAttempts is the number of retry attempts for failed requests + RetryAttempts int `env:"OLLAMA_RETRY_ATTEMPTS" default:"3"` + + // RetryDelay is the delay between retry attempts + RetryDelay time.Duration `env:"OLLAMA_RETRY_DELAY" default:"2s"` + + // EnableLogging enables request/response logging + EnableLogging bool `env:"OLLAMA_ENABLE_LOGGING" default:"true"` + + // UserAgent is the user agent string for HTTP requests + UserAgent string `env:"OLLAMA_USER_AGENT" default:"TenderManagement-OllamaSDK/1.0"` + + // DefaultModel is the default model to use if none is specified + DefaultModel string `env:"OLLAMA_DEFAULT_MODEL" default:"llama2"` + + // DefaultTemperature is the default temperature for text generation + DefaultTemperature float64 `env:"OLLAMA_DEFAULT_TEMPERATURE" default:"0.7"` + + // DefaultTopP is the default top_p value for text generation + DefaultTopP float64 `env:"OLLAMA_DEFAULT_TOP_P" default:"0.9"` + + // DefaultMaxTokens is the default maximum number of tokens to generate + DefaultMaxTokens int `env:"OLLAMA_DEFAULT_MAX_TOKENS" default:"1000"` + + // EnableStreaming enables streaming responses by default + EnableStreaming bool `env:"OLLAMA_ENABLE_STREAMING" default:"false"` +} + +// DefaultConfig returns a default configuration for the Ollama SDK +func DefaultConfig() *Config { + return &Config{ + BaseURL: "http://localhost:11434", + Timeout: 300 * time.Second, + RetryAttempts: 3, + RetryDelay: 2 * time.Second, + EnableLogging: true, + UserAgent: "TenderManagement-OllamaSDK/1.0", + DefaultModel: "llama2", + DefaultTemperature: 0.7, + DefaultTopP: 0.9, + DefaultMaxTokens: 1000, + EnableStreaming: false, + } +} + +// Validate validates the configuration +func (c *Config) Validate() error { + if c.BaseURL == "" { + return ErrInvalidConfig("BaseURL cannot be empty") + } + + if c.Timeout <= 0 { + return ErrInvalidConfig("Timeout must be positive") + } + + if c.RetryAttempts < 0 { + return ErrInvalidConfig("RetryAttempts cannot be negative") + } + + if c.RetryDelay < 0 { + return ErrInvalidConfig("RetryDelay cannot be negative") + } + + if c.DefaultTemperature < 0 || c.DefaultTemperature > 2 { + return ErrInvalidConfig("DefaultTemperature must be between 0 and 2") + } + + if c.DefaultTopP < 0 || c.DefaultTopP > 1 { + return ErrInvalidConfig("DefaultTopP must be between 0 and 1") + } + + if c.DefaultMaxTokens < 0 { + return ErrInvalidConfig("DefaultMaxTokens cannot be negative") + } + + return nil +} diff --git a/pkg/ollama/entities.go b/pkg/ollama/entities.go new file mode 100644 index 0000000..fb8615e --- /dev/null +++ b/pkg/ollama/entities.go @@ -0,0 +1,182 @@ +package ollama + +import ( + "time" +) + +// GenerateRequest represents a request to generate text +type GenerateRequest struct { + Model string `json:"model" validate:"required"` + Prompt string `json:"prompt" validate:"required"` + Stream *bool `json:"stream,omitempty"` + Format string `json:"format,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` + System string `json:"system,omitempty"` + Template string `json:"template,omitempty"` + Context []int `json:"context,omitempty"` + Raw bool `json:"raw,omitempty"` + KeepAlive string `json:"keep_alive,omitempty"` +} + +// GenerateResponse represents the response from a text generation request +type GenerateResponse struct { + Model string `json:"model"` + CreatedAt time.Time `json:"created_at"` + Response string `json:"response"` + Done bool `json:"done"` + Context []int `json:"context,omitempty"` + TotalDuration int64 `json:"total_duration,omitempty"` + LoadDuration int64 `json:"load_duration,omitempty"` + PromptEvalCount int `json:"prompt_eval_count,omitempty"` + PromptEvalDuration int64 `json:"prompt_eval_duration,omitempty"` + EvalCount int `json:"eval_count,omitempty"` + EvalDuration int64 `json:"eval_duration,omitempty"` +} + +// ChatRequest represents a request for a chat conversation +type ChatRequest struct { + Model string `json:"model" validate:"required"` + Messages []Message `json:"messages" validate:"required,min=1"` + Stream *bool `json:"stream,omitempty"` + Format string `json:"format,omitempty"` + Options ChatOptions `json:"options,omitempty"` + KeepAlive string `json:"keep_alive,omitempty"` +} + +// Message represents a message in a chat conversation +type Message struct { + Role MessageRole `json:"role" validate:"required,oneof=system user assistant"` + Content string `json:"content" validate:"required"` + Images []string `json:"images,omitempty"` +} + +// ChatOptions represents options for chat requests +type ChatOptions struct { + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + TopK *int `json:"top_k,omitempty"` + RepeatPenalty *float64 `json:"repeat_penalty,omitempty"` + Seed *int `json:"seed,omitempty"` + Stop []string `json:"stop,omitempty"` + NumCtx *int `json:"num_ctx,omitempty"` + NumPredict *int `json:"num_predict,omitempty"` + NumKeep *int `json:"num_keep,omitempty"` + NumThread *int `json:"num_thread,omitempty"` + RepeatLastN *int `json:"repeat_last_n,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` +} + +// ChatResponse represents the response from a chat request +type ChatResponse struct { + Model string `json:"model"` + CreatedAt time.Time `json:"created_at"` + Message Message `json:"message"` + Done bool `json:"done"` + TotalDuration int64 `json:"total_duration,omitempty"` + LoadDuration int64 `json:"load_duration,omitempty"` + PromptEvalCount int `json:"prompt_eval_count,omitempty"` + PromptEvalDuration int64 `json:"prompt_eval_duration,omitempty"` + EvalCount int `json:"eval_count,omitempty"` + EvalDuration int64 `json:"eval_duration,omitempty"` +} + +// EmbedRequest represents a request to generate embeddings +type EmbedRequest struct { + Model string `json:"model" validate:"required"` + Prompt string `json:"prompt" validate:"required"` + Options map[string]interface{} `json:"options,omitempty"` +} + +// EmbedResponse represents the response from an embedding request +type EmbedResponse struct { + Embedding []float64 `json:"embedding"` +} + +// PullRequest represents a request to pull a model +type PullRequest struct { + Name string `json:"name" validate:"required"` + Insecure bool `json:"insecure,omitempty"` + Stream *bool `json:"stream,omitempty"` +} + +// PullResponse represents the response from a pull request +type PullResponse struct { + Status string `json:"status"` + Digest string `json:"digest,omitempty"` + Total int64 `json:"total,omitempty"` + Completed int64 `json:"completed,omitempty"` +} + +// ListModelsResponse represents the response from listing models +type ListModelsResponse struct { + Models []ModelInfo `json:"models"` +} + +// ModelInfo represents information about a model +type ModelInfo struct { + Name string `json:"name"` + ModifiedAt time.Time `json:"modified_at"` + Size int64 `json:"size"` + Digest string `json:"digest"` + Details ModelDetails `json:"details,omitempty"` +} + +// ModelDetails represents detailed information about a model +type ModelDetails struct { + Format string `json:"format,omitempty"` + Family string `json:"family,omitempty"` + Families []string `json:"families,omitempty"` + ParameterSize string `json:"parameter_size,omitempty"` + QuantizationLevel string `json:"quantization_level,omitempty"` + ParentModel string `json:"parent_model,omitempty"` + Templates map[string]interface{} `json:"templates,omitempty"` + System string `json:"system,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` +} + +// DeleteRequest represents a request to delete a model +type DeleteRequest struct { + Name string `json:"name" validate:"required"` +} + +// DeleteResponse represents the response from a delete request +type DeleteResponse struct { + Status string `json:"status"` +} + +// ShowModelRequest represents a request to show model information +type ShowModelRequest struct { + Name string `json:"name" validate:"required"` + Verbose bool `json:"verbose,omitempty"` +} + +// ShowModelResponse represents the response from a show model request +type ShowModelResponse struct { + License string `json:"license,omitempty"` + Modelfile string `json:"modelfile,omitempty"` + Parameters string `json:"parameters,omitempty"` + Template string `json:"template,omitempty"` + Details ModelDetails `json:"details,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` +} + +// HealthResponse represents the health check response +type HealthResponse struct { + Status string `json:"status"` +} + +// StreamChunk represents a chunk in a streaming response +type StreamChunk struct { + Model string `json:"model,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + Response string `json:"response,omitempty"` + Done bool `json:"done"` + Context []int `json:"context,omitempty"` + TotalDuration int64 `json:"total_duration,omitempty"` + LoadDuration int64 `json:"load_duration,omitempty"` + PromptEvalCount int `json:"prompt_eval_count,omitempty"` + PromptEvalDuration int64 `json:"prompt_eval_duration,omitempty"` + EvalCount int `json:"eval_count,omitempty"` + EvalDuration int64 `json:"eval_duration,omitempty"` + Message *Message `json:"message,omitempty"` +} diff --git a/pkg/ollama/errors.go b/pkg/ollama/errors.go new file mode 100644 index 0000000..e121c07 --- /dev/null +++ b/pkg/ollama/errors.go @@ -0,0 +1,146 @@ +package ollama + +import ( + "fmt" +) + +// Error types for the Ollama SDK +var ( + // ErrInvalidConfig is returned when configuration is invalid + ErrInvalidConfig = func(msg string) error { + return &ConfigError{Message: msg} + } + + // ErrInvalidRequest is returned when a request is invalid + ErrInvalidRequest = func(msg string) error { + return &RequestError{Message: msg} + } + + // ErrModelNotFound is returned when a model is not found + ErrModelNotFound = func(model string) error { + return &ModelError{Message: fmt.Sprintf("model %s not found", model), Model: model} + } + + // ErrModelPullFailed is returned when pulling a model fails + ErrModelPullFailed = func(model string, reason string) error { + return &ModelError{Message: fmt.Sprintf("failed to pull model %s: %s", model, reason), Model: model} + } + + // ErrModelDeleteFailed is returned when deleting a model fails + ErrModelDeleteFailed = func(model string, reason string) error { + return &ModelError{Message: fmt.Sprintf("failed to delete model %s: %s", model, reason), Model: model} + } + + // ErrGenerationFailed is returned when text generation fails + ErrGenerationFailed = func(reason string) error { + return &GenerationError{Message: fmt.Sprintf("generation failed: %s", reason)} + } + + // ErrChatFailed is returned when chat fails + ErrChatFailed = func(reason string) error { + return &ChatError{Message: fmt.Sprintf("chat failed: %s", reason)} + } + + // ErrEmbeddingFailed is returned when embedding generation fails + ErrEmbeddingFailed = func(reason string) error { + return &EmbeddingError{Message: fmt.Sprintf("embedding failed: %s", reason)} + } + + // ErrConnectionFailed is returned when connection to Ollama fails + ErrConnectionFailed = func(reason string) error { + return &ConnectionError{Message: fmt.Sprintf("connection failed: %s", reason)} + } + + // ErrTimeout is returned when a request times out + ErrTimeout = func(operation string) error { + return &TimeoutError{Message: fmt.Sprintf("timeout during %s", operation), Operation: operation} + } + + // ErrRateLimitExceeded is returned when rate limit is exceeded + ErrRateLimitExceeded = func() error { + return &RateLimitError{Message: "rate limit exceeded"} + } +) + +// ConfigError represents a configuration error +type ConfigError struct { + Message string +} + +func (e *ConfigError) Error() string { + return fmt.Sprintf("config error: %s", e.Message) +} + +// RequestError represents a request validation error +type RequestError struct { + Message string +} + +func (e *RequestError) Error() string { + return fmt.Sprintf("request error: %s", e.Message) +} + +// ModelError represents a model-related error +type ModelError struct { + Message string + Model string +} + +func (e *ModelError) Error() string { + return fmt.Sprintf("model error: %s", e.Message) +} + +// GenerationError represents a text generation error +type GenerationError struct { + Message string +} + +func (e *GenerationError) Error() string { + return fmt.Sprintf("generation error: %s", e.Message) +} + +// ChatError represents a chat error +type ChatError struct { + Message string +} + +func (e *ChatError) Error() string { + return fmt.Sprintf("chat error: %s", e.Message) +} + +// EmbeddingError represents an embedding generation error +type EmbeddingError struct { + Message string +} + +func (e *EmbeddingError) Error() string { + return fmt.Sprintf("embedding error: %s", e.Message) +} + +// ConnectionError represents a connection error +type ConnectionError struct { + Message string +} + +func (e *ConnectionError) Error() string { + return fmt.Sprintf("connection error: %s", e.Message) +} + +// TimeoutError represents a timeout error +type TimeoutError struct { + Message string + Operation string +} + +func (e *TimeoutError) Error() string { + return fmt.Sprintf("timeout error: %s", e.Message) +} + +// RateLimitError represents a rate limit error +type RateLimitError struct { + Message string +} + +func (e *RateLimitError) Error() string { + return fmt.Sprintf("rate limit error: %s", e.Message) +} diff --git a/pkg/ollama/interfaces.go b/pkg/ollama/interfaces.go new file mode 100644 index 0000000..b9695e1 --- /dev/null +++ b/pkg/ollama/interfaces.go @@ -0,0 +1,121 @@ +package ollama + +import ( + "context" +) + +// OllamaService defines the interface for Ollama operations +type OllamaService interface { + // Generate generates text using a specified model and prompt + Generate(ctx context.Context, req *GenerateRequest) (*GenerateResponse, error) + + // GenerateWithOptions generates text with additional options + GenerateWithOptions(ctx context.Context, req *GenerateRequest, opts *GenerateOptions) (*GenerateResponse, error) + + // Chat performs a conversational chat with the model + Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) + + // Embed generates embeddings for the given text + Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) + + // PullModel pulls a model from the registry + PullModel(ctx context.Context, model string) (*PullResponse, error) + + // ListModels lists available models + ListModels(ctx context.Context) (*ListModelsResponse, error) + + // DeleteModel deletes a model + DeleteModel(ctx context.Context, model string) (*DeleteResponse, error) + + // ShowModel shows model information + ShowModel(ctx context.Context, model string) (*ShowModelResponse, error) + + // Health checks if the Ollama service is available + Health(ctx context.Context) error + + // NewChatBuilder creates a new chat builder for conversational interactions + NewChatBuilder() ChatBuilder +} + +// HTTPClient defines the interface for HTTP operations +type HTTPClient interface { + // Post performs a POST request + Post(ctx context.Context, url string, body interface{}, result interface{}) error + + // Get performs a GET request + Get(ctx context.Context, url string, result interface{}) error + + // Delete performs a DELETE request + Delete(ctx context.Context, url string, result interface{}) error +} + +// Logger defines the interface for logging operations +type Logger interface { + Debug(msg string, fields map[string]interface{}) + Info(msg string, fields map[string]interface{}) + Warn(msg string, fields map[string]interface{}) + Error(msg string, fields map[string]interface{}) + Fatal(msg string, fields map[string]interface{}) + WithFields(fields map[string]interface{}) Logger + Sync() error +} + +// ChatBuilder provides a fluent interface for building chat requests +type ChatBuilder interface { + // SetModel sets the model to use + SetModel(model string) ChatBuilder + + // SetSystem sets the system prompt + SetSystem(system string) ChatBuilder + + // AddMessage adds a message to the conversation + AddMessage(role MessageRole, content string) ChatBuilder + + // SetTemperature sets the temperature for generation + SetTemperature(temperature float64) ChatBuilder + + // SetTopP sets the top_p parameter + SetTopP(topP float64) ChatBuilder + + // SetMaxTokens sets the maximum number of tokens to generate + SetMaxTokens(maxTokens int) ChatBuilder + + // SetStream sets whether to stream the response + SetStream(stream bool) ChatBuilder + + // SetFormat sets the response format + SetFormat(format string) ChatBuilder + + // SetOptions sets custom options + SetOptions(options map[string]interface{}) ChatBuilder + + // Build creates the chat request + Build() (*ChatRequest, error) + + // Send builds and sends the chat request + Send(ctx context.Context) (*ChatResponse, error) + + // Continue continues the conversation with the current context + Continue(ctx context.Context, userMessage string) (*ChatResponse, error) +} + +// MessageRole represents the role of a message in a chat +type MessageRole string + +const ( + MessageRoleSystem MessageRole = "system" + MessageRoleUser MessageRole = "user" + MessageRoleAssistant MessageRole = "assistant" +) + +// GenerateOptions represents additional options for text generation +type GenerateOptions struct { + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + TopK *int `json:"top_k,omitempty"` + RepeatPenalty *float64 `json:"repeat_penalty,omitempty"` + Seed *int `json:"seed,omitempty"` + Stop []string `json:"stop,omitempty"` + Format string `json:"format,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` +} diff --git a/pkg/ollama/sdk.go b/pkg/ollama/sdk.go new file mode 100644 index 0000000..eab9b0c --- /dev/null +++ b/pkg/ollama/sdk.go @@ -0,0 +1,232 @@ +package ollama + +import ( + "context" + "fmt" + "net/http" + "tm/pkg/logger" +) + +// SDK provides a high-level interface for the Ollama service +type SDK struct { + service OllamaService + client *Client + config *Config +} + +// New creates a new Ollama SDK with the provided configuration and logger +func New(config *Config, logger logger.Logger) (*SDK, error) { + if config == nil { + config = DefaultConfig() + } + + client, err := NewClient(config, logger) + if err != nil { + return nil, fmt.Errorf("failed to create ollama client: %w", err) + } + + service := NewService(client) + + return &SDK{ + service: service, + client: client, + config: config, + }, nil +} + +// NewWithClient creates a new Ollama SDK with a custom HTTP client +func NewWithClient(config *Config, httpClient *http.Client, logger logger.Logger) (*SDK, error) { + if config == nil { + config = DefaultConfig() + } + + client, err := NewClientWithHTTPClient(config, httpClient, logger) + if err != nil { + return nil, fmt.Errorf("failed to create ollama client: %w", err) + } + + service := NewService(client) + + return &SDK{ + service: service, + client: client, + config: config, + }, nil +} + +// Generate generates text using a specified model and prompt +func (s *SDK) Generate(ctx context.Context, req *GenerateRequest) (*GenerateResponse, error) { + return s.service.Generate(ctx, req) +} + +// GenerateWithOptions generates text with additional options +func (s *SDK) GenerateWithOptions(ctx context.Context, req *GenerateRequest, opts *GenerateOptions) (*GenerateResponse, error) { + return s.service.GenerateWithOptions(ctx, req, opts) +} + +// Chat performs a conversational chat with the model +func (s *SDK) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) { + return s.service.Chat(ctx, req) +} + +// Embed generates embeddings for the given text +func (s *SDK) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) { + return s.service.Embed(ctx, req) +} + +// PullModel pulls a model from the registry +func (s *SDK) PullModel(ctx context.Context, model string) (*PullResponse, error) { + return s.service.PullModel(ctx, model) +} + +// ListModels lists available models +func (s *SDK) ListModels(ctx context.Context) (*ListModelsResponse, error) { + return s.service.ListModels(ctx) +} + +// DeleteModel deletes a model +func (s *SDK) DeleteModel(ctx context.Context, model string) (*DeleteResponse, error) { + return s.service.DeleteModel(ctx, model) +} + +// ShowModel shows model information +func (s *SDK) ShowModel(ctx context.Context, model string) (*ShowModelResponse, error) { + return s.service.ShowModel(ctx, model) +} + +// Health checks if the Ollama service is available +func (s *SDK) Health(ctx context.Context) error { + return s.service.Health(ctx) +} + +// NewChatBuilder creates a new chat builder for conversational interactions +func (s *SDK) NewChatBuilder() ChatBuilder { + return s.service.NewChatBuilder() +} + +// QuickGenerate is a convenience method for simple text generation +func (s *SDK) QuickGenerate(ctx context.Context, prompt string) (string, error) { + req := &GenerateRequest{ + Model: s.config.DefaultModel, + Prompt: prompt, + } + + resp, err := s.Generate(ctx, req) + if err != nil { + return "", err + } + + return resp.Response, nil +} + +// QuickGenerateWithModel is a convenience method for text generation with a specific model +func (s *SDK) QuickGenerateWithModel(ctx context.Context, model, prompt string) (string, error) { + req := &GenerateRequest{ + Model: model, + Prompt: prompt, + } + + resp, err := s.Generate(ctx, req) + if err != nil { + return "", err + } + + return resp.Response, nil +} + +// QuickChat is a convenience method for simple chat interactions +func (s *SDK) QuickChat(ctx context.Context, userMessage string) (string, error) { + req := &ChatRequest{ + Model: s.config.DefaultModel, + Messages: []Message{ + { + Role: MessageRoleUser, + Content: userMessage, + }, + }, + } + + resp, err := s.Chat(ctx, req) + if err != nil { + return "", err + } + + return resp.Message.Content, nil +} + +// QuickChatWithSystem is a convenience method for chat with system prompt +func (s *SDK) QuickChatWithSystem(ctx context.Context, systemPrompt, userMessage string) (string, error) { + req := &ChatRequest{ + Model: s.config.DefaultModel, + Messages: []Message{ + { + Role: MessageRoleSystem, + Content: systemPrompt, + }, + { + Role: MessageRoleUser, + Content: userMessage, + }, + }, + } + + resp, err := s.Chat(ctx, req) + if err != nil { + return "", err + } + + return resp.Message.Content, nil +} + +// QuickEmbed is a convenience method for generating embeddings +func (s *SDK) QuickEmbed(ctx context.Context, text string) ([]float64, error) { + req := &EmbedRequest{ + Model: s.config.DefaultModel, + Prompt: text, + } + + resp, err := s.Embed(ctx, req) + if err != nil { + return nil, err + } + + return resp.Embedding, nil +} + +// StreamGenerate performs streaming text generation +func (s *SDK) StreamGenerate(ctx context.Context, req *GenerateRequest, onChunk func(string) error) error { + // Force streaming + stream := true + req.Stream = &stream + + return s.client.StreamGenerate(ctx, req, func(chunk *StreamChunk) error { + if chunk.Response != "" { + return onChunk(chunk.Response) + } + return nil + }) +} + +// StreamChat performs streaming chat +func (s *SDK) StreamChat(ctx context.Context, req *ChatRequest, onChunk func(string) error) error { + // Force streaming + stream := true + req.Stream = &stream + + return s.client.StreamChat(ctx, req, func(chunk *StreamChunk) error { + if chunk.Message != nil && chunk.Message.Content != "" { + return onChunk(chunk.Message.Content) + } + return nil + }) +} + +// GetService returns the underlying service for advanced usage +func (s *SDK) GetService() OllamaService { + return s.service +} + +// GetConfig returns the current configuration +func (s *SDK) GetConfig() *Config { + return s.config +} From cc3d6163ed34f1a0c09236434c7f3dabc59290ff Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Sat, 4 Oct 2025 15:22:49 +0330 Subject: [PATCH 2/6] Refactor Tender Management to Introduce Notice Entity and Repository - Replaced the tender repository with a new notice repository, encapsulating notice-related data access methods. - Introduced the Notice entity to represent tender/contract notices, including relevant fields and methods for managing notice data. - Updated the TED scraper to utilize the new notice repository for creating and managing notices, enhancing the integration with the tender management system. - Implemented Ollama SDK initialization in the web bootstrap process, allowing for improved AI interactions. - Enhanced the tender service to include Ollama SDK for additional functionality, ensuring a more robust service layer. --- cmd/scraper/bootstrap/bootstrap.go | 8 +- cmd/web/bootstrap/bootstrap.go | 36 +++ cmd/web/bootstrap/config.go | 1 + cmd/web/main.go | 5 +- internal/notice/entity.go | 276 ++++++++++++++++++ internal/notice/repository.go | 160 +++++++++++ internal/notice/service.go | 36 +++ internal/tender/service.go | 5 +- pkg/config/config.go | 15 + pkg/mongo/repository.go | 29 ++ pkg/ollama/service.go | 444 +++++++++++++++++++++++++++++ pkg/schedule/cron.go | 8 +- ted/mapper.go | 42 +-- ted/scraper.go | 104 +++---- ted/service.go | 4 +- 15 files changed, 1088 insertions(+), 85 deletions(-) create mode 100644 internal/notice/entity.go create mode 100644 internal/notice/repository.go create mode 100644 internal/notice/service.go create mode 100644 pkg/ollama/service.go diff --git a/cmd/scraper/bootstrap/bootstrap.go b/cmd/scraper/bootstrap/bootstrap.go index 2fa2a2c..9cbdf52 100644 --- a/cmd/scraper/bootstrap/bootstrap.go +++ b/cmd/scraper/bootstrap/bootstrap.go @@ -7,7 +7,7 @@ import ( "os/signal" "syscall" "time" - "tm/internal/tender" + "tm/internal/notice" "tm/pkg/config" "tm/pkg/logger" "tm/pkg/mongo" @@ -78,7 +78,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa // init TED scraper func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) { // Initialize tender repository - tenderRepo := tender.NewRepository(mongoManager, appLogger) + noticeRepo := notice.NewRepository(mongoManager, appLogger) // Initialize TED scraper tedScraper := ted.NewTEDScraper( @@ -95,12 +95,12 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog }, appLogger, mongoManager, - tenderRepo, + noticeRepo, notify, ) // start TED scraper job - schedule.NewCronScheduler(appLogger, mongoManager, tenderRepo).AddJob(schedule.Job{ + schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ Name: "TED Scraper Job", Func: func() { _ = tedScraper.Run(context.Background()) diff --git a/cmd/web/bootstrap/bootstrap.go b/cmd/web/bootstrap/bootstrap.go index c42c98c..63402b0 100644 --- a/cmd/web/bootstrap/bootstrap.go +++ b/cmd/web/bootstrap/bootstrap.go @@ -8,6 +8,7 @@ import ( "tm/pkg/logger" "tm/pkg/mongo" "tm/pkg/notification" + "tm/pkg/ollama" "tm/pkg/redis" "github.com/labstack/echo/v4" @@ -213,3 +214,38 @@ func InitNotificationService(conf config.NotificationConfig, log logger.Logger) return *notificationSDK } + +// Init Ollama Service +func InitOllamaService(conf config.OllamaConfig, log logger.Logger) ollama.SDK { + ollamaConfig := &ollama.Config{ + BaseURL: conf.BaseURL, + Timeout: conf.Timeout, + RetryAttempts: conf.RetryAttempts, + RetryDelay: conf.RetryDelay, + EnableLogging: conf.EnableLogging, + UserAgent: conf.UserAgent, + DefaultModel: conf.DefaultModel, + DefaultTemperature: conf.DefaultTemperature, + DefaultTopP: conf.DefaultTopP, + DefaultMaxTokens: conf.DefaultMaxTokens, + EnableStreaming: conf.EnableStreaming, + } + + ollamaSDK, err := ollama.New(ollamaConfig, log) + if err != nil { + log.Error("Failed to initialize Ollama SDK", map[string]interface{}{ + "error": err.Error(), + }) + } + + log.Info("Ollama SDK initialized successfully", map[string]interface{}{ + "base_url": conf.BaseURL, + "timeout": conf.Timeout, + "retry_attempts": conf.RetryAttempts, + "retry_delay": conf.RetryDelay, + "enable_logging": conf.EnableLogging, + "user_agent": conf.UserAgent, + }) + + return *ollamaSDK +} diff --git a/cmd/web/bootstrap/config.go b/cmd/web/bootstrap/config.go index 0f9f9b7..51dbad5 100644 --- a/cmd/web/bootstrap/config.go +++ b/cmd/web/bootstrap/config.go @@ -13,6 +13,7 @@ type Config struct { RateLimit config.RateLimitConfig Assets config.AssetsConfig Notification config.NotificationConfig + Ollama config.OllamaConfig UserAuth AuthConfig CustomerAuth AuthConfig } diff --git a/cmd/web/main.go b/cmd/web/main.go index c03f5ab..f300820 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -129,6 +129,9 @@ func main() { // Initialize notification service notificationSDK := bootstrap.InitNotificationService(conf.Notification, logger) + // Initialize ollama service + ollamaSDK := bootstrap.InitOllamaService(conf.Ollama, logger) + // Initialize authorization service userAuthService := bootstrap.InitAuthorizationService(conf.UserAuth.JWT, redisClient, logger) customerAuthService := bootstrap.InitAuthorizationService(conf.CustomerAuth.JWT, redisClient, logger) @@ -155,7 +158,7 @@ func main() { categoryService := company_category.NewService(categoryRepository, logger) companyService := company.NewService(companyRepository, categoryService, logger) customerService := customer.NewService(customerRepository, logger, customerAuthService, companyService, redisClient, notificationSDK, customerValidator) - tenderService := tender.NewService(tenderRepository, companyService, logger) + tenderService := tender.NewService(tenderRepository, companyService, ollamaSDK, logger) feedbackService := feedback.NewService(feedbackRepo, tenderService, companyService, logger) tenderApprovalService := tender_approval.NewService(tenderApprovalRepository, tenderService, logger) inquiryService := inquiry.NewService(inquiryRepository, notificationSDK, logger) diff --git a/internal/notice/entity.go b/internal/notice/entity.go new file mode 100644 index 0000000..5f8b0bf --- /dev/null +++ b/internal/notice/entity.go @@ -0,0 +1,276 @@ +package notice + +import ( + "fmt" + "strconv" + "strings" + "time" + "tm/pkg/mongo" +) + +// Tender represents a tender/contract notice entity +type Notice struct { + mongo.Model `bson:",inline"` + ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"` + NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"` + ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"` + NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"` + NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"` + NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"` + IssueDate int64 `bson:"issue_date" json:"issue_date"` + IssueTime int64 `bson:"issue_time" json:"issue_time"` + TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` + PublicationDate int64 `bson:"publication_date" json:"publication_date"` + SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` + ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` + GazetteID string `bson:"gazette_id" json:"gazette_id"` + Title string `bson:"title" json:"title"` + Description string `bson:"description" json:"description"` + ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"` + ProcedureCode string `bson:"procedure_code" json:"procedure_code"` + MainClassification string `bson:"main_classification" json:"main_classification"` + AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"` + EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"` + Currency string `bson:"currency" json:"currency"` + Duration string `bson:"duration" json:"duration"` + DurationUnit string `bson:"duration_unit" json:"duration_unit"` + PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"` + CountryCode string `bson:"country_code" json:"country_code"` + RegionCode string `bson:"region_code" json:"region_code"` + CityName string `bson:"city_name" json:"city_name"` + PostalCode string `bson:"postal_code" json:"postal_code"` + DocumentURI string `bson:"document_uri" json:"document_uri"` + TenderURL string `bson:"tender_url" json:"tender_url"` + SubmissionURL string `bson:"submission_url" json:"submission_url"` + BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"` + ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"` + Organizations []Organization `bson:"organizations" json:"organizations"` + SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"` + OfficialLanguages []string `bson:"official_languages" json:"official_languages"` + Status TenderStatus `bson:"status" json:"status"` + Source TenderSource `bson:"source" json:"source"` + SourceFileURL string `bson:"source_file_url" json:"source_file_url"` + SourceFileName string `bson:"source_file_name" json:"source_file_name"` + ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` + TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} + + // Status-specific fields + CancellationReason string `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"` + CancellationDate int64 `bson:"cancellation_date,omitempty" json:"cancellation_date,omitempty"` + AwardDate int64 `bson:"award_date,omitempty" json:"award_date,omitempty"` + AwardedValue float64 `bson:"awarded_value,omitempty" json:"awarded_value,omitempty"` + WinningTenderer *Organization `bson:"winning_tenderer,omitempty" json:"winning_tenderer,omitempty"` + ContractNumber string `bson:"contract_number,omitempty" json:"contract_number,omitempty"` + SuspensionReason string `bson:"suspension_reason,omitempty" json:"suspension_reason,omitempty"` + SuspensionDate int64 `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"` + Modifications []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"` + AwardedEntities []Awarded `bson:"awarded,omitempty" json:"awarded,omitempty"` +} + +// Organization represents organization information +type Organization struct { + ID string `bson:"id" json:"id"` + Name string `bson:"name" json:"name"` + CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"` + WebsiteURI string `bson:"website_uri,omitempty" json:"website_uri,omitempty"` + ContactName string `bson:"contact_name,omitempty" json:"contact_name,omitempty"` + ContactTelephone string `bson:"contact_telephone,omitempty" json:"contact_telephone,omitempty"` + ContactEmail string `bson:"contact_email,omitempty" json:"contact_email,omitempty"` + ContactFax string `bson:"contact_fax,omitempty" json:"contact_fax,omitempty"` + Address Address `bson:"address" json:"address"` + Role string `bson:"role,omitempty" json:"role,omitempty"` +} + +// Address represents address information +type Address struct { + StreetName string `bson:"street_name,omitempty" json:"street_name,omitempty"` + CityName string `bson:"city_name,omitempty" json:"city_name,omitempty"` + PostalZone string `bson:"postal_zone,omitempty" json:"postal_zone,omitempty"` + CountrySubentityCode string `bson:"country_subentity_code,omitempty" json:"country_subentity_code,omitempty"` + Department string `bson:"department,omitempty" json:"department,omitempty"` + Region string `bson:"region,omitempty" json:"region,omitempty"` + CountryCode string `bson:"country_code" json:"country_code"` +} + +// SelectionCriterion represents selection criteria +type SelectionCriterion struct { + TypeCode string `bson:"type_code" json:"type_code"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"` +} + +// ProcessingMetadata contains metadata about how the tender was processed +type ProcessingMetadata struct { + ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` + ProcessedAt int64 `bson:"processed_at" json:"processed_at"` + ProcessingVersion string `bson:"processing_version" json:"processing_version"` + ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"` + ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"` + EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"` + TranslatedData map[string]string `bson:"translated_data,omitempty" json:"translated_data,omitempty"` + TranslatedAt int64 `bson:"translated_at,omitempty" json:"translated_at,omitempty"` + Processed bool `bson:"processed" json:"processed"` +} + +// TenderModification represents a modification made to a tender +type TenderModification struct { + ModificationDate int64 `bson:"modification_date" json:"modification_date"` // Unix milliseconds + ModificationReason string `bson:"modification_reason" json:"modification_reason"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"` +} + +// AwardedEntity represents an entity that has been awarded a contract +type Awarded struct { + Name string `bson:"name" json:"name"` + Address string `bson:"address,omitempty" json:"address,omitempty"` + Country string `bson:"country,omitempty" json:"country,omitempty"` + Amount float64 `bson:"amount" json:"amount"` + Currency string `bson:"currency,omitempty" json:"currency,omitempty"` + Share float64 `bson:"share,omitempty" json:"share,omitempty"` // Percentage share for multiple winners + AwardDate int64 `bson:"award_date" json:"award_date"` // Unix milliseconds + ContractID string `bson:"contract_id,omitempty" json:"contract_id,omitempty"` // Contract reference ID + TenderID string `bson:"tender_id,omitempty" json:"tender_id,omitempty"` // Tender reference ID + LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` // Lot reference ID + CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"` // Company registration ID + OrganizationID string `bson:"organization_id,omitempty" json:"organization_id,omitempty"` // Organization reference ID +} + +// TenderStatus represents the status of a tender +type TenderStatus string + +const ( + TenderStatusActive TenderStatus = "active" + TenderStatusExpired TenderStatus = "expired" + TenderStatusCancelled TenderStatus = "cancelled" + TenderStatusAwarded TenderStatus = "awarded" + TenderStatusDraft TenderStatus = "draft" + TenderStatusClosed TenderStatus = "closed" + TenderStatusModified TenderStatus = "modified" + TenderStatusSuspended TenderStatus = "suspended" + TenderStatusPublished TenderStatus = "published" +) + +// TenderSource represents the source of tender data +type TenderSource string + +const ( + TenderSourceTEDScraper TenderSource = "ted_scraper" + TenderSourceManual TenderSource = "manual" + TenderSourceAPI TenderSource = "api" + TenderSourceBulkImport TenderSource = "bulk_import" +) + +// TenderStatistics represents tender statistics +type TenderStatistics struct { + TotalTenders int64 `json:"total_tenders"` + ActiveTenders int64 `json:"active_tenders"` + ExpiredTenders int64 `json:"expired_tenders"` + TendersByCountry map[string]int64 `json:"tenders_by_country"` + TendersByType map[string]int64 `json:"tenders_by_type"` + TendersByClassification map[string]int64 `json:"tenders_by_classification"` + AverageEstimatedValue float64 `json:"average_estimated_value"` + TotalEstimatedValue float64 `json:"total_estimated_value"` + LastUpdated int64 `json:"last_updated"` // Unix milliseconds +} + +// GetID returns the tender ID +func (t *Notice) GetID() string { + return t.ID.Hex() +} + +// IsActive returns true if the tender is active +func (t *Notice) IsActive() bool { + return t.Status == TenderStatusActive +} + +// IsExpired returns true if the tender deadline has passed +func (t *Notice) IsExpired() bool { + if t.Status == TenderStatusExpired { + return true + } + + if t.TenderDeadline == 0 { + return false + } + + // Check if deadline (in milliseconds) is in the past + now := time.Now().Unix() + return t.TenderDeadline < now +} + +// GetTenderURL returns the TED tender URL +func (t *Notice) GetTenderURL() string { + if t.TenderURL != "" { + return t.TenderURL + } + + if t.NoticePublicationID != "" { + return generateTenderURL(t.NoticePublicationID) + } + + return "" +} + +// generateTenderURL generates the TED tender URL from NoticePublicationID +func generateTenderURL(noticePublicationID string) string { + // Remove leading zeros and format for URL + // Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025 + parts := strings.Split(noticePublicationID, "-") + if len(parts) != 2 { + return "" + } + + // Convert to int and back to string to remove leading zeros + if num, err := strconv.Atoi(parts[0]); err == nil { + cleanID := fmt.Sprintf("%d-%s", num, parts[1]) + return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID) + } + + return "" +} + +// GetMainOrganization returns the buyer organization +func (t *Notice) GetMainOrganization() *Organization { + return t.BuyerOrganization +} + +// HasEstimatedValue returns true if the tender has an estimated value +func (t *Notice) HasEstimatedValue() bool { + return t.EstimatedValue > 0 +} + +// GetFormattedEstimatedValue returns formatted estimated value with currency +func (t *Notice) GetFormattedEstimatedValue() string { + if !t.HasEstimatedValue() { + return "" + } + + if t.Currency != "" { + return fmt.Sprintf("%.2f %s", t.EstimatedValue, t.Currency) + } + + return fmt.Sprintf("%.2f", t.EstimatedValue) +} + +// UpdateStatus updates the tender status and updated timestamp +func (t *Notice) UpdateStatus(status TenderStatus) { + t.Status = status + t.UpdatedAt = time.Now().Unix() +} + +// GetDaysUntilDeadline returns the number of days until the tender deadline +func (t *Notice) GetDaysUntilDeadline() int { + if t.TenderDeadline == 0 { + return 0 + } + + now := time.Now().Unix() + if t.TenderDeadline <= now { + return 0 // Already expired + } + + diffMillis := t.TenderDeadline - now + diffDays := diffMillis / (24 * 60 * 60 * 1000) + return int(diffDays) +} diff --git a/internal/notice/repository.go b/internal/notice/repository.go new file mode 100644 index 0000000..ac235ad --- /dev/null +++ b/internal/notice/repository.go @@ -0,0 +1,160 @@ +package notice + +import ( + "context" + "time" + "tm/pkg/logger" + orm "tm/pkg/mongo" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// NoticeRepository interface defines notice data access methods +type Repository interface { + Import(ctx context.Context, notice *Notice) error + BulkImport(ctx context.Context, notices []Notice) error + GetByID(ctx context.Context, id string) (*Notice, error) + GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error) + Delete(ctx context.Context, id string) error +} + +func collectionName() string { + return "notices" +} + +// noticeRepository implements NoticeRepository interface using MongoDB ORM +type noticeRepository struct { + ormRepo orm.Repository[Notice] + mongoManager *orm.ConnectionManager + logger logger.Logger +} + +// NewRepository creates a new notice repository +func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Repository { + // Create indexes for notices collection + noticeIndexes := []orm.Index{} + + // Create indexes + if err := mongoManager.CreateIndexes(collectionName(), noticeIndexes); err != nil { + logger.Warn("Failed to create notice indexes", map[string]interface{}{ + "error": err.Error(), + }) + } + + // Create ORM repositories + noticeOrmRepo := orm.NewRepository[Notice](mongoManager.GetCollection(collectionName()), logger) + + return ¬iceRepository{ + ormRepo: noticeOrmRepo, + mongoManager: mongoManager, + logger: logger, + } +} + +// Import imports a new notice +func (r *noticeRepository) Import(ctx context.Context, notice *Notice) error { + now := time.Now().Unix() + notice.SetCreatedAt(now) + + err := r.ormRepo.Create(ctx, notice) + if err != nil { + r.logger.Error("Failed to create notice", map[string]interface{}{ + "error": err.Error(), + "notice_publication_id": notice.NoticePublicationID, + }) + return err + } + + r.logger.Info("Notice created successfully", map[string]interface{}{ + "notice_id": notice.ID, + "notice_publication_id": notice.NoticePublicationID, + }) + + return nil +} + +// BulkImport imports a new notice +func (r *noticeRepository) BulkImport(ctx context.Context, notices []Notice) error { + err := r.ormRepo.CreateMany(ctx, notices) + if err != nil { + r.logger.Error("Failed to bulk import notices", map[string]interface{}{ + "error": err.Error(), + }) + return err + } + + return nil +} + +// GetByID retrieves a notice by ID +func (r *noticeRepository) GetByID(ctx context.Context, id string) (*Notice, error) { + notice, err := r.ormRepo.FindByID(ctx, id) + if err != nil { + r.logger.Error("Failed to get notice by ID", map[string]interface{}{ + "notice_id": id, + "error": err.Error(), + }) + return nil, err + } + + return notice, nil +} + +// GetByContractNoticeID retrieves a notice by contract notice ID +func (r *noticeRepository) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error) { + filter := bson.M{"contract_notice_id": contractNoticeID} + pagination := orm.Pagination{Limit: 1} + + result, err := r.ormRepo.FindAll(ctx, filter, pagination) + if err != nil { + r.logger.Error("Failed to get notice by contract notice ID", map[string]interface{}{ + "contract_notice_id": contractNoticeID, + "error": err.Error(), + }) + return nil, err + } + + if len(result.Items) == 0 { + return nil, orm.ErrDocumentNotFound + } + + return &result.Items[0], nil +} + +// Update updates an existing notice +func (r *noticeRepository) Update(ctx context.Context, notice *Notice) error { + notice.UpdatedAt = time.Now().Unix() + + err := r.ormRepo.Update(ctx, notice) + if err != nil { + r.logger.Error("Failed to update notice", map[string]interface{}{ + "notice_id": notice.ID, + "error": err.Error(), + }) + return err + } + + r.logger.Info("Notice updated successfully", map[string]interface{}{ + "notice_id": notice.ID, + }) + + return nil +} + +// Delete deletes a notice by ID +func (r *noticeRepository) Delete(ctx context.Context, id string) error { + err := r.ormRepo.Delete(ctx, id) + if err != nil { + r.logger.Error("Failed to delete notice", map[string]interface{}{ + "notice_id": id, + "error": err.Error(), + }) + return err + } + + r.logger.Info("Notice deleted successfully", map[string]interface{}{ + "notice_id": id, + }) + + return nil +} diff --git a/internal/notice/service.go b/internal/notice/service.go new file mode 100644 index 0000000..6a8d5c6 --- /dev/null +++ b/internal/notice/service.go @@ -0,0 +1,36 @@ +package notice + +import ( + "context" + "tm/pkg/logger" +) + +// Service defines business logic for notice operations +type Service interface { + Import(ctx context.Context, form *Notice) error + BulkImport(ctx context.Context, form []Notice) error +} + +// noticeService implements the Service interface +type noticeService struct { + repository Repository + logger logger.Logger +} + +// NewService creates a new notice service +func NewService(repository Repository, logger logger.Logger) Service { + return ¬iceService{ + repository: repository, + logger: logger, + } +} + +// Import imports a new notice +func (s *noticeService) Import(ctx context.Context, form *Notice) error { + return s.repository.Import(ctx, form) +} + +// BulkImport imports a new notice +func (s *noticeService) BulkImport(ctx context.Context, form []Notice) error { + return s.repository.BulkImport(ctx, form) +} diff --git a/internal/tender/service.go b/internal/tender/service.go index e21f4ea..77faef6 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -6,6 +6,7 @@ import ( "time" "tm/internal/company" "tm/pkg/logger" + "tm/pkg/ollama" "tm/pkg/response" ) @@ -27,14 +28,16 @@ type Service interface { type tenderService struct { repository TenderRepository companyService company.Service + ollamaSDK ollama.SDK logger logger.Logger } // NewService creates a new tender service -func NewService(repository TenderRepository, companyService company.Service, logger logger.Logger) Service { +func NewService(repository TenderRepository, companyService company.Service, ollamaSDK ollama.SDK, logger logger.Logger) Service { return &tenderService{ repository: repository, companyService: companyService, + ollamaSDK: ollamaSDK, logger: logger, } } diff --git a/pkg/config/config.go b/pkg/config/config.go index 85ce26f..e785796 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -74,6 +74,21 @@ type NotificationConfig struct { UserAgent string `env:"NOTIFICATION_USER_AGENT"` } +// OllamaConfig represents the configuration for the Ollama SDK +type OllamaConfig struct { + BaseURL string `env:"OLLAMA_BASE_URL" default:"http://localhost:11434"` + Timeout time.Duration `env:"OLLAMA_TIMEOUT" default:"300s"` + RetryAttempts int `env:"OLLAMA_RETRY_ATTEMPTS" default:"3"` + RetryDelay time.Duration `env:"OLLAMA_RETRY_DELAY" default:"2s"` + EnableLogging bool `env:"OLLAMA_ENABLE_LOGGING" default:"true"` + UserAgent string `env:"OLLAMA_USER_AGENT" default:"TenderManagement-OllamaSDK/1.0"` + DefaultModel string `env:"OLLAMA_DEFAULT_MODEL" default:"llama2"` + DefaultTemperature float64 `env:"OLLAMA_DEFAULT_TEMPERATURE" default:"0.7"` + DefaultTopP float64 `env:"OLLAMA_DEFAULT_TOP_P" default:"0.9"` + DefaultMaxTokens int `env:"OLLAMA_DEFAULT_MAX_TOKENS" default:"1000"` + EnableStreaming bool `env:"OLLAMA_ENABLE_STREAMING" default:"false"` +} + // LoadConfig loads configuration with priority: OS environment variables > .env file // OS environment variables take precedence over .env file values func LoadConfig[T any](path string, config T) (T, error) { diff --git a/pkg/mongo/repository.go b/pkg/mongo/repository.go index f9553b3..3ee1538 100644 --- a/pkg/mongo/repository.go +++ b/pkg/mongo/repository.go @@ -47,6 +47,9 @@ type Repository[T any] interface { // Create inserts a new document Create(ctx context.Context, model *T) error + // CreateMany inserts multiple documents + CreateMany(ctx context.Context, models []T) error + // Update updates an existing document Update(ctx context.Context, model *T) error @@ -239,6 +242,32 @@ func (r *repository[T]) Create(ctx context.Context, model *T) error { return nil } +// CreateMany inserts multiple documents +func (r *repository[T]) CreateMany(ctx context.Context, models []T) error { + // Set timestamps if the model implements Timestampable + for _, v := range models { + if timestampable, ok := any(v).(Timestampable); ok { + now := time.Now().Unix() + timestampable.SetCreatedAt(now) + timestampable.SetUpdatedAt(now) + } + } + + result, err := r.collection.InsertMany(ctx, models) + if err != nil { + r.logger.Error("Failed to create multiple documents", map[string]interface{}{ + "error": err.Error(), + }) + return fmt.Errorf("failed to create multiple documents: %w", err) + } + + r.logger.Info("Documents created successfully", map[string]interface{}{ + "ids": result.InsertedIDs, + }) + + return nil +} + // Update updates an existing document func (r *repository[T]) Update(ctx context.Context, model *T) error { // Set updated timestamp if the model implements Timestampable diff --git a/pkg/ollama/service.go b/pkg/ollama/service.go new file mode 100644 index 0000000..53e4bd4 --- /dev/null +++ b/pkg/ollama/service.go @@ -0,0 +1,444 @@ +package ollama + +import ( + "context" + "fmt" + "tm/pkg/logger" +) + +// Service implements the OllamaService interface +type Service struct { + client *Client + config *Config + logger logger.Logger +} + +// NewService creates a new Ollama service +func NewService(client *Client) OllamaService { + return &Service{ + client: client, + config: client.config, + logger: client.logger, + } +} + +// Generate generates text using a specified model and prompt +func (s *Service) Generate(ctx context.Context, req *GenerateRequest) (*GenerateResponse, error) { + if s.logger != nil { + s.logger.Info("Generating text", map[string]interface{}{ + "model": req.Model, + "prompt_length": len(req.Prompt), + }) + } + + // Set defaults + if req.Model == "" { + req.Model = s.config.DefaultModel + } + + var resp GenerateResponse + err := s.client.Post(ctx, "/api/generate", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Text generation failed", map[string]interface{}{ + "model": req.Model, + "error": err.Error(), + }) + } + return nil, ErrGenerationFailed(err.Error()) + } + + if s.logger != nil { + s.logger.Info("Text generation completed", map[string]interface{}{ + "model": req.Model, + "response_length": len(resp.Response), + "eval_count": resp.EvalCount, + }) + } + + return &resp, nil +} + +// GenerateWithOptions generates text with additional options +func (s *Service) GenerateWithOptions(ctx context.Context, req *GenerateRequest, opts *GenerateOptions) (*GenerateResponse, error) { + if s.logger != nil { + s.logger.Info("Generating text with options", map[string]interface{}{ + "model": req.Model, + "prompt_length": len(req.Prompt), + }) + } + + // Apply options to request + if opts != nil { + if req.Options == nil { + req.Options = make(map[string]interface{}) + } + + if opts.Temperature != nil { + req.Options["temperature"] = *opts.Temperature + } + if opts.TopP != nil { + req.Options["top_p"] = *opts.TopP + } + if opts.TopK != nil { + req.Options["top_k"] = *opts.TopK + } + if opts.RepeatPenalty != nil { + req.Options["repeat_penalty"] = *opts.RepeatPenalty + } + if opts.Seed != nil { + req.Options["seed"] = *opts.Seed + } + if opts.Stop != nil { + req.Options["stop"] = opts.Stop + } + if opts.Format != "" { + req.Format = opts.Format + } + + // Merge additional options + for k, v := range opts.Options { + req.Options[k] = v + } + } + + return s.Generate(ctx, req) +} + +// Chat performs a conversational chat with the model +func (s *Service) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) { + if s.logger != nil { + s.logger.Info("Starting chat", map[string]interface{}{ + "model": req.Model, + "message_count": len(req.Messages), + }) + } + + // Set defaults + if req.Model == "" { + req.Model = s.config.DefaultModel + } + + var resp ChatResponse + err := s.client.Post(ctx, "/api/chat", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Chat failed", map[string]interface{}{ + "model": req.Model, + "error": err.Error(), + }) + } + return nil, ErrChatFailed(err.Error()) + } + + if s.logger != nil { + s.logger.Info("Chat completed", map[string]interface{}{ + "model": req.Model, + "response_length": len(resp.Message.Content), + "eval_count": resp.EvalCount, + }) + } + + return &resp, nil +} + +// Embed generates embeddings for the given text +func (s *Service) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) { + if s.logger != nil { + s.logger.Info("Generating embeddings", map[string]interface{}{ + "model": req.Model, + "text_length": len(req.Prompt), + }) + } + + var resp EmbedResponse + err := s.client.Post(ctx, "/api/embeddings", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Embedding generation failed", map[string]interface{}{ + "model": req.Model, + "error": err.Error(), + }) + } + return nil, ErrEmbeddingFailed(err.Error()) + } + + if s.logger != nil { + s.logger.Info("Embeddings generated", map[string]interface{}{ + "model": req.Model, + "embedding_dimension": len(resp.Embedding), + }) + } + + return &resp, nil +} + +// PullModel pulls a model from the registry +func (s *Service) PullModel(ctx context.Context, model string) (*PullResponse, error) { + if s.logger != nil { + s.logger.Info("Pulling model", map[string]interface{}{ + "model": model, + }) + } + + req := &PullRequest{ + Name: model, + } + + var resp PullResponse + err := s.client.Post(ctx, "/api/pull", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Model pull failed", map[string]interface{}{ + "model": model, + "error": err.Error(), + }) + } + return nil, ErrModelPullFailed(model, err.Error()) + } + + if s.logger != nil { + s.logger.Info("Model pulled successfully", map[string]interface{}{ + "model": model, + "status": resp.Status, + }) + } + + return &resp, nil +} + +// ListModels lists available models +func (s *Service) ListModels(ctx context.Context) (*ListModelsResponse, error) { + if s.logger != nil { + s.logger.Debug("Listing models", map[string]interface{}{}) + } + + var resp ListModelsResponse + err := s.client.Get(ctx, "/api/tags", &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Failed to list models", map[string]interface{}{ + "error": err.Error(), + }) + } + return nil, fmt.Errorf("failed to list models: %w", err) + } + + if s.logger != nil { + s.logger.Debug("Models listed", map[string]interface{}{ + "model_count": len(resp.Models), + }) + } + + return &resp, nil +} + +// DeleteModel deletes a model +func (s *Service) DeleteModel(ctx context.Context, model string) (*DeleteResponse, error) { + if s.logger != nil { + s.logger.Info("Deleting model", map[string]interface{}{ + "model": model, + }) + } + + req := &DeleteRequest{ + Name: model, + } + + var resp DeleteResponse + err := s.client.Delete(ctx, "/api/delete", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Model deletion failed", map[string]interface{}{ + "model": model, + "error": err.Error(), + }) + } + return nil, ErrModelDeleteFailed(model, err.Error()) + } + + if s.logger != nil { + s.logger.Info("Model deleted successfully", map[string]interface{}{ + "model": model, + "status": resp.Status, + }) + } + + return &resp, nil +} + +// ShowModel shows model information +func (s *Service) ShowModel(ctx context.Context, model string) (*ShowModelResponse, error) { + if s.logger != nil { + s.logger.Debug("Showing model information", map[string]interface{}{ + "model": model, + }) + } + + req := &ShowModelRequest{ + Name: model, + } + + var resp ShowModelResponse + err := s.client.Post(ctx, "/api/show", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Failed to show model", map[string]interface{}{ + "model": model, + "error": err.Error(), + }) + } + return nil, fmt.Errorf("failed to show model %s: %w", model, err) + } + + if s.logger != nil { + s.logger.Debug("Model information retrieved", map[string]interface{}{ + "model": model, + }) + } + + return &resp, nil +} + +// Health checks if the Ollama service is available +func (s *Service) Health(ctx context.Context) error { + if s.logger != nil { + s.logger.Debug("Checking Ollama health", map[string]interface{}{}) + } + + var resp HealthResponse + err := s.client.Get(ctx, "/api/version", &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Health check failed", map[string]interface{}{ + "error": err.Error(), + }) + } + return ErrConnectionFailed(err.Error()) + } + + if s.logger != nil { + s.logger.Debug("Health check passed", map[string]interface{}{}) + } + + return nil +} + +// NewChatBuilder creates a new chat builder for conversational interactions +func (s *Service) NewChatBuilder() ChatBuilder { + return &chatBuilder{ + service: s, + request: &ChatRequest{ + Model: s.config.DefaultModel, + Messages: make([]Message, 0), + Options: ChatOptions{ + Temperature: &s.config.DefaultTemperature, + TopP: &s.config.DefaultTopP, + }, + }, + } +} + +// chatBuilder implements the ChatBuilder interface +type chatBuilder struct { + service OllamaService + request *ChatRequest +} + +func (b *chatBuilder) SetModel(model string) ChatBuilder { + b.request.Model = model + return b +} + +func (b *chatBuilder) SetSystem(system string) ChatBuilder { + // Add or update system message + for i, msg := range b.request.Messages { + if msg.Role == MessageRoleSystem { + b.request.Messages[i].Content = system + return b + } + } + + // Add new system message at the beginning + systemMsg := Message{ + Role: MessageRoleSystem, + Content: system, + } + b.request.Messages = append([]Message{systemMsg}, b.request.Messages...) + return b +} + +func (b *chatBuilder) AddMessage(role MessageRole, content string) ChatBuilder { + msg := Message{ + Role: role, + Content: content, + } + b.request.Messages = append(b.request.Messages, msg) + return b +} + +func (b *chatBuilder) SetTemperature(temperature float64) ChatBuilder { + b.request.Options.Temperature = &temperature + return b +} + +func (b *chatBuilder) SetTopP(topP float64) ChatBuilder { + b.request.Options.TopP = &topP + return b +} + +func (b *chatBuilder) SetMaxTokens(maxTokens int) ChatBuilder { + b.request.Options.NumPredict = &maxTokens + return b +} + +func (b *chatBuilder) SetStream(stream bool) ChatBuilder { + b.request.Stream = &stream + return b +} + +func (b *chatBuilder) SetFormat(format string) ChatBuilder { + b.request.Format = format + return b +} + +func (b *chatBuilder) SetOptions(options map[string]interface{}) ChatBuilder { + if b.request.Options.Options == nil { + b.request.Options.Options = make(map[string]interface{}) + } + for k, v := range options { + b.request.Options.Options[k] = v + } + return b +} + +func (b *chatBuilder) Build() (*ChatRequest, error) { + if len(b.request.Messages) == 0 { + return nil, ErrInvalidRequest("at least one message is required") + } + return b.request, nil +} + +func (b *chatBuilder) Send(ctx context.Context) (*ChatResponse, error) { + req, err := b.Build() + if err != nil { + return nil, err + } + return b.service.Chat(ctx, req) +} + +func (b *chatBuilder) Continue(ctx context.Context, userMessage string) (*ChatResponse, error) { + // Add user message + b.AddMessage(MessageRoleUser, userMessage) + + // Send chat request + resp, err := b.Send(ctx) + if err != nil { + return nil, err + } + + // Add assistant response to conversation history + b.AddMessage(MessageRoleAssistant, resp.Message.Content) + + return resp, nil +} diff --git a/pkg/schedule/cron.go b/pkg/schedule/cron.go index 2f9220d..ab9ac46 100644 --- a/pkg/schedule/cron.go +++ b/pkg/schedule/cron.go @@ -2,7 +2,7 @@ package schedule import ( "time" - "tm/internal/tender" + "tm/internal/notice" "tm/pkg/logger" "tm/pkg/mongo" @@ -13,7 +13,7 @@ import ( type CronScheduler struct { cron *cron.Cron mongoManager *mongo.ConnectionManager - tenderRepo tender.TenderRepository + noticeRepo notice.Repository logger logger.Logger } @@ -31,14 +31,14 @@ type Job struct { } // NewCronScheduler creates a new cron scheduler instance -func NewCronScheduler(logger logger.Logger, mongoManager *mongo.ConnectionManager, tenderRepo tender.TenderRepository) *CronScheduler { +func NewCronScheduler(logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository) *CronScheduler { // Create cron with timezone support and seconds precision c := cron.New(cron.WithLocation(time.UTC), cron.WithSeconds()) return &CronScheduler{ cron: c, mongoManager: mongoManager, - tenderRepo: tenderRepo, + noticeRepo: noticeRepo, logger: logger, } } diff --git a/ted/mapper.go b/ted/mapper.go index 2134d86..2d254c4 100644 --- a/ted/mapper.go +++ b/ted/mapper.go @@ -4,11 +4,11 @@ import ( "strconv" "strings" "time" - "tm/internal/tender" + "tm/internal/notice" ) // mapToTenderFromParsedDoc maps a parsed TED document to tender entity -func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *tender.Tender { +func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice { switch parsedDoc.Type { case DocumentTypeContractNotice: if parsedDoc.ContractNotice == nil { @@ -45,7 +45,7 @@ func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceF } // mapGenericNoticeToTender maps any TED document type to tender entity using the common interface -func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *tender.Tender { +func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *notice.Notice { if doc == nil { s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{}) return nil @@ -53,7 +53,7 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so now := time.Now().Unix() - t := &tender.Tender{ + t := ¬ice.Notice{ ContractNoticeID: doc.GetID(), ContractFolderID: "", // Not available in generic interface NoticeTypeCode: doc.GetNoticeType(), @@ -68,11 +68,11 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so // Basic tender information Title: "Generic Notice " + doc.GetNoticeType(), Description: "Processed from " + doc.GetNoticeType() + " document", - Status: tender.TenderStatusActive, - Source: tender.TenderSourceTEDScraper, + Status: notice.TenderStatusActive, + Source: notice.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, - ProcessingMetadata: tender.ProcessingMetadata{ + ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", @@ -86,10 +86,10 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so } // mapToTender maps a TED ContractNotice to tender entity -func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *tender.Tender { +func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *notice.Notice { now := time.Now().Unix() - t := &tender.Tender{ + t := ¬ice.Notice{ ContractNoticeID: cn.ID, ContractFolderID: cn.ContractFolderID, NoticeTypeCode: cn.GetNoticeSubType(), @@ -105,11 +105,11 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa MainClassification: cn.GetMainClassification(), PlaceOfPerformance: cn.GetPlaceOfPerformance(), DocumentURI: cn.GetDocumentURI(), - Status: tender.TenderStatusActive, - Source: tender.TenderSourceTEDScraper, + Status: notice.TenderStatusActive, + Source: notice.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, - ProcessingMetadata: tender.ProcessingMetadata{ + ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", @@ -215,7 +215,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa // Set selection criteria if criteria := cn.GetSelectionCriteria(); criteria != nil { for _, criterion := range criteria { - t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{ + t.SelectionCriteria = append(t.SelectionCriteria, notice.SelectionCriterion{ TypeCode: criterion.CriterionTypeCode, Description: criterion.Description, LanguageID: criterion.LanguageID, @@ -235,10 +235,10 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa } // mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity -func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *tender.Tender { +func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *notice.Notice { now := time.Now().Unix() - t := &tender.Tender{ + t := ¬ice.Notice{ ContractNoticeID: can.ID, ContractFolderID: can.ContractFolderID, NoticeTypeCode: can.NoticeTypeCode, @@ -252,11 +252,11 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so ProcurementTypeCode: can.GetContractNature(), ProcedureCode: can.GetProcedureCode(), MainClassification: can.GetMainClassification(), - Status: tender.TenderStatusAwarded, // Award notices are for completed tenders - Source: tender.TenderSourceTEDScraper, + Status: notice.TenderStatusAwarded, // Award notices are for completed tenders + Source: notice.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, - ProcessingMetadata: tender.ProcessingMetadata{ + ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", @@ -315,13 +315,13 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so } // mapOrganization maps a TED Organization to tender Organization -func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.Organization { +func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.Organization { if tedOrg == nil || tedOrg.Company == nil { return nil } company := tedOrg.Company - org := &tender.Organization{ + org := ¬ice.Organization{ Role: role, } @@ -351,7 +351,7 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender. // Set address if company.PostalAddress != nil { - org.Address = tender.Address{ + org.Address = notice.Address{ StreetName: company.PostalAddress.StreetName, CityName: company.PostalAddress.CityName, PostalZone: company.PostalAddress.PostalZone, diff --git a/ted/scraper.go b/ted/scraper.go index 8f5a18a..c37bd6e 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" "time" - "tm/internal/tender" + "tm/internal/notice" "tm/pkg/logger" "tm/pkg/mongo" "tm/pkg/notification" @@ -37,7 +37,7 @@ type TEDScraper struct { httpClient *http.Client xmlParser *TEDParser mongoManager *mongo.ConnectionManager - tenderRepo tender.TenderRepository + tenderRepo notice.Repository } // NewTEDScraper creates a new TED scraper instance @@ -45,7 +45,7 @@ func NewTEDScraper( config *Config, logger logger.Logger, mongoManager *mongo.ConnectionManager, - tenderRepo tender.TenderRepository, + tenderRepo notice.Repository, notify notification.SDK, ) *TEDScraper { httpClient := &http.Client{ @@ -275,7 +275,7 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o return result, nil } -func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*tender.Tender, error) { +func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) { existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID) if err != nil { if err.Error() == "document not found" { @@ -308,60 +308,60 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file // Map to tender entity t := s.mapToTenderFromParsedDoc(notice, "", fileName) - existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID) + // existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID) + // if err != nil { + // s.logger.Warn("Error checking for existing tender", map[string]interface{}{ + // "contract_notice_id": t.ContractNoticeID, + // "error": err.Error(), + // }) + // } + + // if err == nil && existingTender != nil { + // // Update existing tender + // s.logger.Info("Found existing tender, updating", map[string]interface{}{ + // "existing_tender_id": existingTender.ID.Hex(), + // "contract_notice_id": t.ContractNoticeID, + // }) + + // t.ID = existingTender.ID + // t.UpdatedAt = time.Now().Unix() + // err = s.tenderRepo.Update(ctx, t) + // if err != nil { + // s.logger.Error("Failed to update existing tender", map[string]interface{}{ + // "tender_id": t.ID.Hex(), + // "error": err.Error(), + // }) + // return fmt.Errorf("failed to update existing tender: %w", err) + // } + + // s.logger.Info("Successfully updated tender", map[string]interface{}{ + // "tender_id": t.ID.Hex(), + // }) + // } else { + // // Create new tender + s.logger.Info("Creating new tender", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "tender_id": t.TenderID, + }) + + t.CreatedAt = time.Now().Unix() + t.UpdatedAt = t.CreatedAt + + err = s.tenderRepo.Import(ctx, t) if err != nil { - s.logger.Warn("Error checking for existing tender", map[string]interface{}{ + s.logger.Error("Failed to create new tender", map[string]interface{}{ "contract_notice_id": t.ContractNoticeID, + "tender_id": t.TenderID, "error": err.Error(), }) + return fmt.Errorf("failed to create tender: %w", err) } - if err == nil && existingTender != nil { - // Update existing tender - s.logger.Info("Found existing tender, updating", map[string]interface{}{ - "existing_tender_id": existingTender.ID.Hex(), - "contract_notice_id": t.ContractNoticeID, - }) - - t.ID = existingTender.ID - t.UpdatedAt = time.Now().Unix() - err = s.tenderRepo.Update(ctx, t) - if err != nil { - s.logger.Error("Failed to update existing tender", map[string]interface{}{ - "tender_id": t.ID.Hex(), - "error": err.Error(), - }) - return fmt.Errorf("failed to update existing tender: %w", err) - } - - s.logger.Info("Successfully updated tender", map[string]interface{}{ - "tender_id": t.ID.Hex(), - }) - } else { - // Create new tender - s.logger.Info("Creating new tender", map[string]interface{}{ - "contract_notice_id": t.ContractNoticeID, - "tender_id": t.TenderID, - }) - - t.CreatedAt = time.Now().Unix() - t.UpdatedAt = t.CreatedAt - - err = s.tenderRepo.Create(ctx, t) - if err != nil { - s.logger.Error("Failed to create new tender", map[string]interface{}{ - "contract_notice_id": t.ContractNoticeID, - "tender_id": t.TenderID, - "error": err.Error(), - }) - return fmt.Errorf("failed to create tender: %w", err) - } - - s.logger.Info("Successfully created new tender", map[string]interface{}{ - "tender_id": t.TenderID, - "contract_notice_id": t.ContractNoticeID, - }) - } + s.logger.Info("Successfully created new tender", map[string]interface{}{ + "tender_id": t.TenderID, + "contract_notice_id": t.ContractNoticeID, + }) + // } // Log detailed parsed information based on notice type s.logger.Info("Successfully parsed notice", map[string]interface{}{ diff --git a/ted/service.go b/ted/service.go index 622b16e..ef691d0 100644 --- a/ted/service.go +++ b/ted/service.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" "time" - "tm/internal/tender" + "tm/internal/notice" ) // parseDate parses various date formats used in TED XML @@ -146,7 +146,7 @@ func GenerateTenderURL(noticePublicationID string) string { // generateTenderID generates a unique tender ID using the format: // {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} -func GenerateTenderID(t *tender.Tender) string { +func GenerateTenderID(t *notice.Notice) string { var parts []string // Source (TED) From 02157693af19b107ea8f7e7290756002da355084 Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Sat, 4 Oct 2025 15:25:40 +0330 Subject: [PATCH 3/6] Refactor TED Scraper Initialization in Main Application - Moved the initialization of the notification service to follow the MongoDB connection setup for better organization. - Added comments to clarify the initialization sequence of services, including the TED scraper. - Removed commented-out code related to stopping the scheduler to clean up the main function. - Enhanced the overall readability and structure of the main application entry point. --- cmd/scraper/main.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/scraper/main.go b/cmd/scraper/main.go index 4bf7395..ac77140 100644 --- a/cmd/scraper/main.go +++ b/cmd/scraper/main.go @@ -22,8 +22,6 @@ func main() { "version": "1.0.0", }) - // Initialize notification service - // Initialize MongoDB connection mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger) defer func() { @@ -36,7 +34,10 @@ func main() { } }() + // Initialize notification service notificationService := bootstrap.InitNotificationService(config.Notification, appLogger) + + // Initialize TED scraper bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService) // Set up signal handling for graceful shutdown @@ -47,8 +48,5 @@ func main() { <-signalChan appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{}) - // Stop the scheduler - // scheduler.Stop() - appLogger.Info("TED scraper application stopped", map[string]interface{}{}) } From 1c3ad648c14931bf8fce7ecf7be0e91ac1b1ca06 Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Sun, 5 Oct 2025 16:17:41 +0330 Subject: [PATCH 4/6] Implement Worker Service and Refactor Tender Initialization - Introduced a new worker service with a dedicated main entry point for handling background tasks, including MongoDB connection management and notification service initialization. - Added a bootstrap package to manage application configuration, logging, and service initialization for the worker. - Implemented a NoticeWorker to process unprocessed notices and create corresponding tender entities, enhancing the integration of notice management within the tender system. - Refactored the tender service to remove the Ollama SDK dependency, streamlining the service initialization in the main application. - Enhanced the notice repository with a method to retrieve unprocessed notices, improving data handling capabilities. --- cmd/web/main.go | 5 +- cmd/worker/bootstrap/bootstrap.go | 147 +++++++++++++++ cmd/worker/bootstrap/config.go | 18 ++ cmd/worker/main.go | 55 ++++++ cmd/worker/workers/notice.go | 300 ++++++++++++++++++++++++++++++ internal/notice/repository.go | 22 +++ internal/tender/service.go | 5 +- 7 files changed, 544 insertions(+), 8 deletions(-) create mode 100644 cmd/worker/bootstrap/bootstrap.go create mode 100644 cmd/worker/bootstrap/config.go create mode 100644 cmd/worker/main.go create mode 100644 cmd/worker/workers/notice.go diff --git a/cmd/web/main.go b/cmd/web/main.go index f300820..c03f5ab 100644 --- a/cmd/web/main.go +++ b/cmd/web/main.go @@ -129,9 +129,6 @@ func main() { // Initialize notification service notificationSDK := bootstrap.InitNotificationService(conf.Notification, logger) - // Initialize ollama service - ollamaSDK := bootstrap.InitOllamaService(conf.Ollama, logger) - // Initialize authorization service userAuthService := bootstrap.InitAuthorizationService(conf.UserAuth.JWT, redisClient, logger) customerAuthService := bootstrap.InitAuthorizationService(conf.CustomerAuth.JWT, redisClient, logger) @@ -158,7 +155,7 @@ func main() { categoryService := company_category.NewService(categoryRepository, logger) companyService := company.NewService(companyRepository, categoryService, logger) customerService := customer.NewService(customerRepository, logger, customerAuthService, companyService, redisClient, notificationSDK, customerValidator) - tenderService := tender.NewService(tenderRepository, companyService, ollamaSDK, logger) + tenderService := tender.NewService(tenderRepository, companyService, logger) feedbackService := feedback.NewService(feedbackRepo, tenderService, companyService, logger) tenderApprovalService := tender_approval.NewService(tenderApprovalRepository, tenderService, logger) inquiryService := inquiry.NewService(inquiryRepository, notificationSDK, logger) diff --git a/cmd/worker/bootstrap/bootstrap.go b/cmd/worker/bootstrap/bootstrap.go new file mode 100644 index 0000000..96fbea5 --- /dev/null +++ b/cmd/worker/bootstrap/bootstrap.go @@ -0,0 +1,147 @@ +package bootstrap + +import ( + "fmt" + "time" + "tm/cmd/worker/workers" + "tm/internal/notice" + "tm/internal/tender" + "tm/pkg/config" + "tm/pkg/logger" + "tm/pkg/mongo" + "tm/pkg/notification" + "tm/pkg/ollama" + "tm/pkg/schedule" +) + +// Init Application Configuration +func InitConfig() (*Config, error) { + conf, err := config.LoadConfig(".", &Config{}) + if err != nil { + return nil, fmt.Errorf("failed to load config: %v", err) + } + + return conf, nil +} + +// Init Logger Service +func InitLogger(conf config.LoggingConfig) logger.Logger { + return logger.NewLogger(&logger.Config{ + Level: conf.Level, + Format: conf.Format, + Output: conf.Output, + File: logger.FileConfig{ + Path: conf.File.Path, + MaxSize: conf.File.MaxSize, + MaxBackups: conf.File.MaxBackups, + MaxAge: conf.File.MaxAge, + Compress: conf.File.Compress, + }, + }) +} + +// Init MongoDB Connection Manager +func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager { + // Convert infra.MongoConfig to mongo.ConnectionConfig + connectionConfig := mongo.ConnectionConfig{ + URI: conf.URI, + Database: conf.Name, + MaxPoolSize: uint64(conf.MaxPoolSize), + MinPoolSize: 5, // Default minimum pool size + MaxConnIdleTime: 30 * time.Minute, // Default idle time + ConnectTimeout: conf.Timeout, + ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout + } + + // Create MongoDB connection manager + connectionManager, err := mongo.NewConnectionManager(connectionConfig, log) + if err != nil { + log.Error("Failed to initialize MongoDB connection", map[string]interface{}{ + "error": err.Error(), + "uri": conf.URI, + "db": conf.Name, + }) + panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err)) + } + + log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{ + "database": conf.Name, + "uri": conf.URI, + "pool_size": conf.MaxPoolSize, + }) + + return connectionManager +} + +// Init Worker +func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, ollamaSDK *ollama.SDK) { + // Initialize repositories + noticeRepo := notice.NewRepository(mongoManager, appLogger) + tenderRepo := tender.NewRepository(mongoManager, appLogger) + + // start Worker job + schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ + Name: "Worker Job", + Func: func() { + worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK) + worker.Run() + }, + Expr: config.Worker.Interval, + }) + +} + +// Init Notification Service +func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK { + notificationConfig := ¬ification.Config{ + BaseURL: conf.BaseURL, + Timeout: conf.Timeout, + RetryAttempts: conf.RetryAttempts, + RetryDelay: conf.RetryDelay, + EnableLogging: conf.EnableLogging, + UserAgent: conf.UserAgent, + } + + notificationSDK, err := notification.New(notificationConfig, log) + if err != nil { + log.Error("Failed to initialize Notification SDK", map[string]interface{}{ + "error": err.Error(), + }) + fmt.Printf("Failed to initialize Notification SDK: %v\n", err.Error()) + } + + log.Info("Notification SDK initialized successfully", map[string]interface{}{ + "base_url": conf.BaseURL, + "timeout": conf.Timeout, + "retry_attempts": conf.RetryAttempts, + "retry_delay": conf.RetryDelay, + "enable_logging": conf.EnableLogging, + "user_agent": conf.UserAgent, + }) + + return *notificationSDK +} + +// Init Ollama Service +func InitOllamaService(conf config.OllamaConfig, log logger.Logger) *ollama.SDK { + ollamaSDK, err := ollama.New(&ollama.Config{ + BaseURL: conf.BaseURL, + Timeout: conf.Timeout, + RetryAttempts: conf.RetryAttempts, + RetryDelay: conf.RetryDelay, + EnableLogging: conf.EnableLogging, + UserAgent: conf.UserAgent, + DefaultModel: conf.DefaultModel, + DefaultTemperature: conf.DefaultTemperature, + DefaultTopP: conf.DefaultTopP, + DefaultMaxTokens: conf.DefaultMaxTokens, + EnableStreaming: conf.EnableStreaming, + }, log) + if err != nil { + log.Error("Failed to initialize Ollama SDK", map[string]interface{}{ + "error": err.Error(), + }) + } + + return ollamaSDK +} diff --git a/cmd/worker/bootstrap/config.go b/cmd/worker/bootstrap/config.go new file mode 100644 index 0000000..7d219ff --- /dev/null +++ b/cmd/worker/bootstrap/config.go @@ -0,0 +1,18 @@ +package bootstrap + +import ( + "tm/pkg/config" +) + +// Config defines the scraper application configuration +type Config struct { + Database config.DatabaseConfig + Logging config.LoggingConfig + Notification config.NotificationConfig + Ollama config.OllamaConfig + Worker WorkerConfig +} + +type WorkerConfig struct { + Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"` +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 0000000..8fdb9ee --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + "time" + "tm/cmd/worker/bootstrap" +) + +func main() { + // Load configuration + config, err := bootstrap.InitConfig() + if err != nil { + panic(err) + } + + // Initialize logger + appLogger := bootstrap.InitLogger(config.Logging) + appLogger.Info("Starting worker application", map[string]interface{}{ + "version": "1.0.0", + }) + + // Initialize MongoDB connection + mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger) + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := mongoManager.Close(ctx); err != nil { + appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{ + "error": err.Error(), + }) + } + }() + + // Initialize notification service + notificationService := bootstrap.InitNotificationService(config.Notification, appLogger) + + // Initialize ollama service + ollamaSDK := bootstrap.InitOllamaService(config.Ollama, appLogger) + + // Initialize Worker + bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, ollamaSDK) + + // Set up signal handling for graceful shutdown + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + + // Wait for shutdown signal + <-signalChan + appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{}) + + appLogger.Info("Worker application stopped", map[string]interface{}{}) +} diff --git a/cmd/worker/workers/notice.go b/cmd/worker/workers/notice.go new file mode 100644 index 0000000..977480b --- /dev/null +++ b/cmd/worker/workers/notice.go @@ -0,0 +1,300 @@ +package workers + +import ( + "context" + "fmt" + "tm/internal/notice" + "tm/internal/tender" + "tm/pkg/logger" + "tm/pkg/mongo" + "tm/pkg/notification" + "tm/pkg/ollama" +) + +type NoticeWorker struct { + Mongo *mongo.ConnectionManager + Logger logger.Logger + Notify notification.SDK + Ollama *ollama.SDK + NoticeRepo notice.Repository + TenderRepo tender.TenderRepository +} + +func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, ollama *ollama.SDK) *NoticeWorker { + return &NoticeWorker{ + Mongo: mongo, + Logger: logger, + Notify: notify, + Ollama: ollama, + NoticeRepo: noticeRepo, + TenderRepo: tenderRepo, + } +} + +func (w *NoticeWorker) Run() { + w.Logger.Info("Notice worker started", map[string]interface{}{}) + + limit := 10 + skip := 0 + for { + notices, _, err := w.NoticeRepo.GetUnProcessedNotices(context.Background(), limit, skip) + if err != nil { + w.Logger.Error("Failed to get notices", map[string]interface{}{ + "error": err.Error(), + }) + continue + } + + if len(notices) == 0 { + break + } + + for _, n := range notices { + w.Logger.Info("Notice", map[string]interface{}{ + "notice": n.ID.Hex(), + }) + + // exists, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID) + // if err != nil { + // w.Logger.Error("Failed to get tender", map[string]interface{}{ + // "error": err.Error(), + // }) + // continue + // } + + t, err := w.ToTender(&n) + if err != nil { + w.Logger.Error("Failed to create tender", map[string]interface{}{ + "error": err.Error(), + }) + continue + } + + err = w.TenderRepo.Create(context.Background(), t) + if err != nil { + w.Logger.Error("Failed to create tender", map[string]interface{}{ + "error": err.Error(), + }) + } + + } + + skip += limit + } +} + +func (w *NoticeWorker) translateAI(msg string) (string, error) { + builder := w.Ollama.NewChatBuilder().SetStream(false).SetMaxTokens(200000) + chat, err := builder.AddMessage(ollama.MessageRoleUser, fmt.Sprintf("Translate the following text into fluent, natural English while keeping the original meaning and tone. Do not summarize, shorten, or add extra explanations. Just provide the translation without other additional values. %s", msg)).Build() + if err != nil { + return "", err + } + resp, err := w.Ollama.Chat(context.Background(), chat) + if err != nil { + return "", err + } + + return resp.Message.Content, nil +} + +func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { + // Buyer Organization + buyerOrg := &tender.Organization{} + if n.BuyerOrganization != nil { + buyerOrg = &tender.Organization{ + Name: n.BuyerOrganization.Name, + CompanyID: n.BuyerOrganization.CompanyID, + WebsiteURI: n.BuyerOrganization.WebsiteURI, + ContactName: n.BuyerOrganization.ContactName, + ContactTelephone: n.BuyerOrganization.ContactTelephone, + ContactEmail: n.BuyerOrganization.ContactEmail, + ContactFax: n.BuyerOrganization.ContactFax, + Role: n.BuyerOrganization.Role, + Address: tender.Address{ + StreetName: n.BuyerOrganization.Address.StreetName, + CityName: n.BuyerOrganization.Address.CityName, + PostalZone: n.BuyerOrganization.Address.PostalZone, + CountrySubentityCode: n.BuyerOrganization.Address.CountrySubentityCode, + Department: n.BuyerOrganization.Address.Department, + Region: n.BuyerOrganization.Address.Region, + CountryCode: n.BuyerOrganization.Address.CountryCode, + }, + } + } + + // Review Organization + reviewOrg := &tender.Organization{} + if n.ReviewOrganization != nil { + reviewOrg = &tender.Organization{ + Name: n.ReviewOrganization.Name, + CompanyID: n.ReviewOrganization.CompanyID, + WebsiteURI: n.ReviewOrganization.WebsiteURI, + ContactName: n.ReviewOrganization.ContactName, + ContactTelephone: n.ReviewOrganization.ContactTelephone, + ContactEmail: n.ReviewOrganization.ContactEmail, + ContactFax: n.ReviewOrganization.ContactFax, + Role: n.ReviewOrganization.Role, + Address: tender.Address{ + StreetName: n.ReviewOrganization.Address.StreetName, + CityName: n.ReviewOrganization.Address.CityName, + PostalZone: n.ReviewOrganization.Address.PostalZone, + CountrySubentityCode: n.ReviewOrganization.Address.CountrySubentityCode, + Department: n.ReviewOrganization.Address.Department, + Region: n.ReviewOrganization.Address.Region, + CountryCode: n.ReviewOrganization.Address.CountryCode, + }, + } + } + + // Organizations + organizations := make([]tender.Organization, len(n.Organizations)) + for i, org := range n.Organizations { + organizations[i] = tender.Organization{ + Name: org.Name, + CompanyID: org.CompanyID, + WebsiteURI: org.WebsiteURI, + ContactName: org.ContactName, + ContactTelephone: org.ContactTelephone, + ContactEmail: org.ContactEmail, + ContactFax: org.ContactFax, + Role: org.Role, + Address: tender.Address{ + StreetName: org.Address.StreetName, + CityName: org.Address.CityName, + PostalZone: org.Address.PostalZone, + CountrySubentityCode: org.Address.CountrySubentityCode, + Department: org.Address.Department, + Region: org.Address.Region, + CountryCode: org.Address.CountryCode, + }, + } + } + + // Selection Criteria + selectionCriteria := make([]tender.SelectionCriterion, len(n.SelectionCriteria)) + for i, criterion := range n.SelectionCriteria { + selectionCriteria[i] = tender.SelectionCriterion{ + TypeCode: criterion.TypeCode, + Description: criterion.Description, + LanguageID: criterion.LanguageID, + } + } + + // Winning Tenderer + winningTenderer := &tender.Organization{} + if n.WinningTenderer != nil { + winningTenderer = &tender.Organization{ + Name: n.WinningTenderer.Name, + CompanyID: n.WinningTenderer.CompanyID, + WebsiteURI: n.WinningTenderer.WebsiteURI, + ContactName: n.WinningTenderer.ContactName, + ContactTelephone: n.WinningTenderer.ContactTelephone, + ContactEmail: n.WinningTenderer.ContactEmail, + ContactFax: n.WinningTenderer.ContactFax, + Role: n.WinningTenderer.Role, + Address: tender.Address{ + StreetName: n.WinningTenderer.Address.StreetName, + CityName: n.WinningTenderer.Address.CityName, + PostalZone: n.WinningTenderer.Address.PostalZone, + CountrySubentityCode: n.WinningTenderer.Address.CountrySubentityCode, + Department: n.WinningTenderer.Address.Department, + Region: n.WinningTenderer.Address.Region, + CountryCode: n.WinningTenderer.Address.CountryCode, + }, + } + } + + // Modifications + modifications := make([]tender.TenderModification, len(n.Modifications)) + for i, modification := range n.Modifications { + modifications[i] = tender.TenderModification{ + ModificationDate: modification.ModificationDate, + ModificationReason: modification.ModificationReason, + Description: modification.Description, + LanguageID: modification.LanguageID, + } + } + + // Awarded Entities + awardedEntities := make([]tender.Awarded, len(n.AwardedEntities)) + for i, awarded := range n.AwardedEntities { + awardedEntities[i] = tender.Awarded{ + Name: awarded.Name, + Address: awarded.Address, + Country: awarded.Country, + Amount: awarded.Amount, + Currency: awarded.Currency, + Share: awarded.Share, + AwardDate: awarded.AwardDate, + ContractID: awarded.ContractID, + TenderID: awarded.TenderID, + LotID: awarded.LotID, + CompanyID: awarded.CompanyID, + OrganizationID: awarded.OrganizationID, + } + } + + title, err := w.translateAI(n.Title) + if err != nil { + return nil, fmt.Errorf("failed to translate title: %w", err) + } + + description, err := w.translateAI(n.Description) + if err != nil { + return nil, fmt.Errorf("failed to translate description: %w", err) + } + + return &tender.Tender{ + NoticePublicationID: n.NoticePublicationID, + Title: title, + Description: description, + ProcurementTypeCode: n.ProcurementTypeCode, + ProcedureCode: n.ProcedureCode, + EstimatedValue: n.EstimatedValue, + Currency: n.Currency, + TenderDeadline: n.TenderDeadline, + SubmissionURL: n.SubmissionURL, + BuyerOrganization: buyerOrg, + ReviewOrganization: reviewOrg, + Organizations: organizations, + SelectionCriteria: selectionCriteria, + Status: tender.TenderStatus(n.Status), + Source: tender.TenderSource(n.Source), + SourceFileURL: n.SourceFileURL, + SourceFileName: n.SourceFileName, + TenderID: n.TenderID, + ContractNoticeID: n.ContractNoticeID, + ContractFolderID: n.ContractFolderID, + NoticeTypeCode: n.NoticeTypeCode, + NoticeSubTypeCode: n.NoticeSubTypeCode, + NoticeLanguageCode: n.NoticeLanguageCode, + IssueDate: n.IssueDate, + IssueTime: n.IssueTime, + PublicationDate: n.PublicationDate, + SubmissionDeadline: n.SubmissionDeadline, + ApplicationDeadline: n.ApplicationDeadline, + GazetteID: n.GazetteID, + Duration: n.Duration, + DurationUnit: n.DurationUnit, + PlaceOfPerformance: n.PlaceOfPerformance, + CountryCode: n.CountryCode, + RegionCode: n.RegionCode, + CityName: n.CityName, + PostalCode: n.PostalCode, + DocumentURI: n.DocumentURI, + TenderURL: n.TenderURL, + MainClassification: n.MainClassification, + AdditionalClassifications: n.AdditionalClassifications, + OfficialLanguages: n.OfficialLanguages, + CancellationReason: n.CancellationReason, + CancellationDate: n.CancellationDate, + AwardDate: n.AwardDate, + AwardedValue: n.AwardedValue, + ContractNumber: n.ContractNumber, + SuspensionReason: n.SuspensionReason, + SuspensionDate: n.SuspensionDate, + WinningTenderer: winningTenderer, + Modifications: modifications, + AwardedEntities: awardedEntities, + }, nil +} diff --git a/internal/notice/repository.go b/internal/notice/repository.go index ac235ad..bcf9551 100644 --- a/internal/notice/repository.go +++ b/internal/notice/repository.go @@ -15,6 +15,7 @@ type Repository interface { BulkImport(ctx context.Context, notices []Notice) error GetByID(ctx context.Context, id string) (*Notice, error) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error) + GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) Delete(ctx context.Context, id string) error } @@ -121,6 +122,27 @@ func (r *noticeRepository) GetByContractNoticeID(ctx context.Context, contractNo return &result.Items[0], nil } +// GetUnProcessedNotices retrieves unprocessed notices + +func (r *noticeRepository) GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) { + filter := bson.M{"processing_metadata.processed": false} + + pagination := orm.Pagination{ + Limit: limit, + Skip: skip, + } + + result, err := r.ormRepo.FindAll(ctx, filter, pagination) + if err != nil { + r.logger.Error("Failed to get unprocessed notices", map[string]interface{}{ + "error": err.Error(), + }) + return nil, 0, err + } + + return result.Items, result.TotalCount, nil +} + // Update updates an existing notice func (r *noticeRepository) Update(ctx context.Context, notice *Notice) error { notice.UpdatedAt = time.Now().Unix() diff --git a/internal/tender/service.go b/internal/tender/service.go index 77faef6..e21f4ea 100644 --- a/internal/tender/service.go +++ b/internal/tender/service.go @@ -6,7 +6,6 @@ import ( "time" "tm/internal/company" "tm/pkg/logger" - "tm/pkg/ollama" "tm/pkg/response" ) @@ -28,16 +27,14 @@ type Service interface { type tenderService struct { repository TenderRepository companyService company.Service - ollamaSDK ollama.SDK logger logger.Logger } // NewService creates a new tender service -func NewService(repository TenderRepository, companyService company.Service, ollamaSDK ollama.SDK, logger logger.Logger) Service { +func NewService(repository TenderRepository, companyService company.Service, logger logger.Logger) Service { return &tenderService{ repository: repository, companyService: companyService, - ollamaSDK: ollamaSDK, logger: logger, } } From 07479088737934b8d0e2f9bf38f338a7a89ed451 Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Thu, 16 Oct 2025 16:20:23 +0330 Subject: [PATCH 5/6] Add AlertMail Configuration and Enhance TED Scraper Notification Logic - Introduced AlertMail configuration in both scraper and worker bootstrap files, allowing for customizable email notifications. - Updated the TED scraper to utilize the AlertMail configuration for sending completion notifications, improving flexibility in notification management. - Enhanced error logging in the worker's main function to capture issues when listing Ollama models, ensuring better visibility into potential failures. - Refactored notification sending logic to check for a valid AlertMail before dispatching emails, ensuring notifications are only sent when configured. - Improved overall structure and readability of the bootstrap configuration files, aligning with best practices for maintainability. --- cmd/scraper/bootstrap/bootstrap.go | 1 + cmd/scraper/bootstrap/config.go | 1 + cmd/worker/bootstrap/bootstrap.go | 3 ++ cmd/worker/bootstrap/config.go | 1 + cmd/worker/main.go | 7 +++++ ted/scraper.go | 44 ++++++++++++++++-------------- 6 files changed, 37 insertions(+), 20 deletions(-) diff --git a/cmd/scraper/bootstrap/bootstrap.go b/cmd/scraper/bootstrap/bootstrap.go index 9cbdf52..1f866c4 100644 --- a/cmd/scraper/bootstrap/bootstrap.go +++ b/cmd/scraper/bootstrap/bootstrap.go @@ -92,6 +92,7 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog DownloadDir: config.TED.DownloadDir, CleanupAfter: config.TED.CleanupAfter, ScrapingInterval: config.TED.ScrapingInterval, + AlertMail: config.AlertMail, }, appLogger, mongoManager, diff --git a/cmd/scraper/bootstrap/config.go b/cmd/scraper/bootstrap/config.go index 7766ae2..28fc584 100644 --- a/cmd/scraper/bootstrap/config.go +++ b/cmd/scraper/bootstrap/config.go @@ -11,6 +11,7 @@ type Config struct { Logging config.LoggingConfig TED ScraperConfig Notification config.NotificationConfig + AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"` } type ScraperConfig struct { diff --git a/cmd/worker/bootstrap/bootstrap.go b/cmd/worker/bootstrap/bootstrap.go index 96fbea5..844ac09 100644 --- a/cmd/worker/bootstrap/bootstrap.go +++ b/cmd/worker/bootstrap/bootstrap.go @@ -79,6 +79,9 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger noticeRepo := notice.NewRepository(mongoManager, appLogger) tenderRepo := tender.NewRepository(mongoManager, appLogger) + worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK) + worker.Run() + // start Worker job schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ Name: "Worker Job", diff --git a/cmd/worker/bootstrap/config.go b/cmd/worker/bootstrap/config.go index 7d219ff..8c3485c 100644 --- a/cmd/worker/bootstrap/config.go +++ b/cmd/worker/bootstrap/config.go @@ -11,6 +11,7 @@ type Config struct { Notification config.NotificationConfig Ollama config.OllamaConfig Worker WorkerConfig + AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"` } type WorkerConfig struct { diff --git a/cmd/worker/main.go b/cmd/worker/main.go index 8fdb9ee..e01131e 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -39,6 +39,13 @@ func main() { // Initialize ollama service ollamaSDK := bootstrap.InitOllamaService(config.Ollama, appLogger) + aiModels, err := ollamaSDK.ListModels(context.Background()) + if err != nil { + appLogger.Error("Failed to list ollama models", map[string]interface{}{ + "error": err.Error(), + }) + } + appLogger.Info("Ollama models", map[string]interface{}{"models": aiModels}) // Initialize Worker bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, ollamaSDK) diff --git a/ted/scraper.go b/ted/scraper.go index c37bd6e..3f28d15 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -27,6 +27,7 @@ type Config struct { RetryDelay time.Duration `mapstructure:"retry_delay"` CleanupAfter time.Duration `mapstructure:"cleanup_after"` ScrapingInterval string `mapstructure:"scraping_interval"` + AlertMail string `mapstructure:"ALERT_MAIL"` } // TEDScraper handles downloading and parsing TED XML files @@ -251,26 +252,29 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o } } - go s.notify.SendNotification( - context.Background(), - ¬ification.NotificationRequest{ - EventType: notification.EventTypeEmail, - Title: "TED scraper completed", - Message: GenerateFileProcessingEmailTemplate( - &FileProcessingEmailTemplateData{ - OJS: ojs, - ProcessedCount: result.ProcessedCount, - SuccessCount: result.SuccessCount, - Errors: result.Errors, - ProcessedAt: result.ProcessedAt, - }), - Priority: "important", - Methods: notification.NotificationMethods{ - Email: "nakhostin.nima1998@gmail.com", - }, - UserID: "ted-scraper", - Type: "alert", - }) + if s.config.AlertMail != "" { + + go s.notify.SendNotification( + context.Background(), + ¬ification.NotificationRequest{ + EventType: notification.EventTypeEmail, + Title: "TED scraper completed", + Message: GenerateFileProcessingEmailTemplate( + &FileProcessingEmailTemplateData{ + OJS: ojs, + ProcessedCount: result.ProcessedCount, + SuccessCount: result.SuccessCount, + Errors: result.Errors, + ProcessedAt: result.ProcessedAt, + }), + Priority: "important", + Methods: notification.NotificationMethods{ + Email: s.config.AlertMail, + }, + UserID: "ted-scraper", + Type: "alert", + }) + } return result, nil } From 8e5308b70b5bc75dc30f8d407725cf7de60f4965 Mon Sep 17 00:00:00 2001 From: "n.nakhostin" Date: Sat, 18 Oct 2025 09:02:11 +0330 Subject: [PATCH 6/6] Refactor NoticeWorker to Enhance Tender Creation Logic - Updated the ToTender method to accept a tender instance, allowing for direct population of tender fields from the notice entity. - Simplified the tender creation and update process by checking if the tender ID is zero, streamlining the logic for handling tender entities. - Removed the commented-out code related to AI translation, improving code clarity and maintainability. - Enhanced error logging for tender creation and update failures, ensuring better visibility into potential issues during processing. --- cmd/worker/workers/notice.go | 159 +++++++++++++++-------------------- 1 file changed, 66 insertions(+), 93 deletions(-) diff --git a/cmd/worker/workers/notice.go b/cmd/worker/workers/notice.go index 977480b..a9e7d82 100644 --- a/cmd/worker/workers/notice.go +++ b/cmd/worker/workers/notice.go @@ -2,7 +2,6 @@ package workers import ( "context" - "fmt" "tm/internal/notice" "tm/internal/tender" "tm/pkg/logger" @@ -54,15 +53,12 @@ func (w *NoticeWorker) Run() { "notice": n.ID.Hex(), }) - // exists, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID) - // if err != nil { - // w.Logger.Error("Failed to get tender", map[string]interface{}{ - // "error": err.Error(), - // }) - // continue - // } + t, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID) + if err == nil { + continue + } - t, err := w.ToTender(&n) + err = w.ToTender(&n, t) if err != nil { w.Logger.Error("Failed to create tender", map[string]interface{}{ "error": err.Error(), @@ -70,34 +66,28 @@ func (w *NoticeWorker) Run() { continue } - err = w.TenderRepo.Create(context.Background(), t) - if err != nil { - w.Logger.Error("Failed to create tender", map[string]interface{}{ - "error": err.Error(), - }) + if t.ID.IsZero() { + err = w.TenderRepo.Create(context.Background(), t) + if err != nil { + w.Logger.Error("Failed to create tender", map[string]interface{}{ + "error": err.Error(), + }) + } + } else { + err = w.TenderRepo.Update(context.Background(), t) + if err != nil { + w.Logger.Error("Failed to update tender", map[string]interface{}{ + "error": err.Error(), + }) + } } - } skip += limit } } -func (w *NoticeWorker) translateAI(msg string) (string, error) { - builder := w.Ollama.NewChatBuilder().SetStream(false).SetMaxTokens(200000) - chat, err := builder.AddMessage(ollama.MessageRoleUser, fmt.Sprintf("Translate the following text into fluent, natural English while keeping the original meaning and tone. Do not summarize, shorten, or add extra explanations. Just provide the translation without other additional values. %s", msg)).Build() - if err != nil { - return "", err - } - resp, err := w.Ollama.Chat(context.Background(), chat) - if err != nil { - return "", err - } - - return resp.Message.Content, nil -} - -func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { +func (w *NoticeWorker) ToTender(n *notice.Notice, t *tender.Tender) error { // Buyer Organization buyerOrg := &tender.Organization{} if n.BuyerOrganization != nil { @@ -234,67 +224,50 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) { } } - title, err := w.translateAI(n.Title) - if err != nil { - return nil, fmt.Errorf("failed to translate title: %w", err) - } - - description, err := w.translateAI(n.Description) - if err != nil { - return nil, fmt.Errorf("failed to translate description: %w", err) - } - - return &tender.Tender{ - NoticePublicationID: n.NoticePublicationID, - Title: title, - Description: description, - ProcurementTypeCode: n.ProcurementTypeCode, - ProcedureCode: n.ProcedureCode, - EstimatedValue: n.EstimatedValue, - Currency: n.Currency, - TenderDeadline: n.TenderDeadline, - SubmissionURL: n.SubmissionURL, - BuyerOrganization: buyerOrg, - ReviewOrganization: reviewOrg, - Organizations: organizations, - SelectionCriteria: selectionCriteria, - Status: tender.TenderStatus(n.Status), - Source: tender.TenderSource(n.Source), - SourceFileURL: n.SourceFileURL, - SourceFileName: n.SourceFileName, - TenderID: n.TenderID, - ContractNoticeID: n.ContractNoticeID, - ContractFolderID: n.ContractFolderID, - NoticeTypeCode: n.NoticeTypeCode, - NoticeSubTypeCode: n.NoticeSubTypeCode, - NoticeLanguageCode: n.NoticeLanguageCode, - IssueDate: n.IssueDate, - IssueTime: n.IssueTime, - PublicationDate: n.PublicationDate, - SubmissionDeadline: n.SubmissionDeadline, - ApplicationDeadline: n.ApplicationDeadline, - GazetteID: n.GazetteID, - Duration: n.Duration, - DurationUnit: n.DurationUnit, - PlaceOfPerformance: n.PlaceOfPerformance, - CountryCode: n.CountryCode, - RegionCode: n.RegionCode, - CityName: n.CityName, - PostalCode: n.PostalCode, - DocumentURI: n.DocumentURI, - TenderURL: n.TenderURL, - MainClassification: n.MainClassification, - AdditionalClassifications: n.AdditionalClassifications, - OfficialLanguages: n.OfficialLanguages, - CancellationReason: n.CancellationReason, - CancellationDate: n.CancellationDate, - AwardDate: n.AwardDate, - AwardedValue: n.AwardedValue, - ContractNumber: n.ContractNumber, - SuspensionReason: n.SuspensionReason, - SuspensionDate: n.SuspensionDate, - WinningTenderer: winningTenderer, - Modifications: modifications, - AwardedEntities: awardedEntities, - }, nil + t.NoticePublicationID = n.NoticePublicationID + t.Title = n.Title + t.Description = n.Description + t.ProcurementTypeCode = n.ProcurementTypeCode + t.ProcedureCode = n.ProcedureCode + t.EstimatedValue = n.EstimatedValue + t.Currency = n.Currency + t.TenderDeadline = n.TenderDeadline + t.SubmissionURL = n.SubmissionURL + t.BuyerOrganization = buyerOrg + t.ReviewOrganization = reviewOrg + t.Organizations = organizations + t.SelectionCriteria = selectionCriteria + t.WinningTenderer = winningTenderer + t.Modifications = modifications + t.AwardedEntities = awardedEntities + t.NoticeTypeCode = n.NoticeTypeCode + t.NoticeSubTypeCode = n.NoticeSubTypeCode + t.NoticeLanguageCode = n.NoticeLanguageCode + t.IssueDate = n.IssueDate + t.IssueTime = n.IssueTime + t.PublicationDate = n.PublicationDate + t.SubmissionDeadline = n.SubmissionDeadline + t.ApplicationDeadline = n.ApplicationDeadline + t.GazetteID = n.GazetteID + t.PlaceOfPerformance = n.PlaceOfPerformance + t.CountryCode = n.CountryCode + t.RegionCode = n.RegionCode + t.CityName = n.CityName + t.PostalCode = n.PostalCode + t.DocumentURI = n.DocumentURI + t.TenderURL = n.TenderURL + t.MainClassification = n.MainClassification + t.AdditionalClassifications = n.AdditionalClassifications + t.OfficialLanguages = n.OfficialLanguages + t.CancellationReason = n.CancellationReason + t.CancellationDate = n.CancellationDate + t.AwardDate = n.AwardDate + t.AwardedValue = n.AwardedValue + t.ContractNumber = n.ContractNumber + t.SuspensionReason = n.SuspensionReason + t.SuspensionDate = n.SuspensionDate + t.WinningTenderer = winningTenderer + t.Modifications = modifications + t.AwardedEntities = awardedEntities + return nil }