mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/ai-translator' into develop

Browse Source
This commit is contained in:
n.nakhostin
2025-10-18 09:04:32 +03:30
parent 952986331a 8e5308b70b
commit 69d201479f
27 changed files with 3354 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/scraper/bootstrap/bootstrap.go
+5 -4
View File
@@ -7,7 +7,7 @@ import (
"os/signal" "os/signal"
"syscall" "syscall"
"time" "time"
"tm/internal/tender" "tm/internal/notice"
"tm/pkg/config" "tm/pkg/config"
"tm/pkg/logger" "tm/pkg/logger"
"tm/pkg/mongo" "tm/pkg/mongo"
@@ -78,7 +78,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
// init TED scraper // init TED scraper
func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) { func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) {
// Initialize tender repository // Initialize tender repository
tenderRepo := tender.NewRepository(mongoManager, appLogger) noticeRepo := notice.NewRepository(mongoManager, appLogger)
// Initialize TED scraper // Initialize TED scraper
tedScraper := ted.NewTEDScraper( tedScraper := ted.NewTEDScraper(
@@ -92,15 +92,16 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
DownloadDir: config.TED.DownloadDir, DownloadDir: config.TED.DownloadDir,
CleanupAfter: config.TED.CleanupAfter, CleanupAfter: config.TED.CleanupAfter,
ScrapingInterval: config.TED.ScrapingInterval, ScrapingInterval: config.TED.ScrapingInterval,
AlertMail: config.AlertMail,
}, },
appLogger, appLogger,
mongoManager, mongoManager,
tenderRepo, noticeRepo,
notify, notify,
) )
// start TED scraper job // start TED scraper job
schedule.NewCronScheduler(appLogger, mongoManager, tenderRepo).AddJob(schedule.Job{ schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
Name: "TED Scraper Job", Name: "TED Scraper Job",
Func: func() { Func: func() {
_ = tedScraper.Run(context.Background()) _ = tedScraper.Run(context.Background())
cmd/scraper/bootstrap/config.go
+1
View File
@@ -11,6 +11,7 @@ type Config struct {
Logging config.LoggingConfig Logging config.LoggingConfig
TED ScraperConfig TED ScraperConfig
Notification config.NotificationConfig Notification config.NotificationConfig
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
} }
type ScraperConfig struct { type ScraperConfig struct {
cmd/scraper/main.go
+3 -5
View File
@@ -22,8 +22,6 @@ func main() {
"version": "1.0.0", "version": "1.0.0",
}) })
// Initialize notification service
// Initialize MongoDB connection // Initialize MongoDB connection
mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger) mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
defer func() { defer func() {
@@ -36,7 +34,10 @@ func main() {
} }
}() }()
// Initialize notification service
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger) notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
// Initialize TED scraper
bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService) bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService)
// Set up signal handling for graceful shutdown // Set up signal handling for graceful shutdown
@@ -47,8 +48,5 @@ func main() {
<-signalChan <-signalChan
appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{}) appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{})
// Stop the scheduler
// scheduler.Stop()
appLogger.Info("TED scraper application stopped", map[string]interface{}{}) appLogger.Info("TED scraper application stopped", map[string]interface{}{})
} }
cmd/web/bootstrap/bootstrap.go
+36
View File
@@ -8,6 +8,7 @@ import (
"tm/pkg/logger" "tm/pkg/logger"
"tm/pkg/mongo" "tm/pkg/mongo"
"tm/pkg/notification" "tm/pkg/notification"
"tm/pkg/ollama"
"tm/pkg/redis" "tm/pkg/redis"
"github.com/labstack/echo/v4" "github.com/labstack/echo/v4"
@@ -213,3 +214,38 @@ func InitNotificationService(conf config.NotificationConfig, log logger.Logger)
return *notificationSDK return *notificationSDK
} }
// Init Ollama Service
func InitOllamaService(conf config.OllamaConfig, log logger.Logger) ollama.SDK {
ollamaConfig := &ollama.Config{
BaseURL: conf.BaseURL,
Timeout: conf.Timeout,
RetryAttempts: conf.RetryAttempts,
RetryDelay: conf.RetryDelay,
EnableLogging: conf.EnableLogging,
UserAgent: conf.UserAgent,
DefaultModel: conf.DefaultModel,
DefaultTemperature: conf.DefaultTemperature,
DefaultTopP: conf.DefaultTopP,
DefaultMaxTokens: conf.DefaultMaxTokens,
EnableStreaming: conf.EnableStreaming,
}
ollamaSDK, err := ollama.New(ollamaConfig, log)
if err != nil {
log.Error("Failed to initialize Ollama SDK", map[string]interface{}{
"error": err.Error(),
})
}
log.Info("Ollama SDK initialized successfully", map[string]interface{}{
"base_url": conf.BaseURL,
"timeout": conf.Timeout,
"retry_attempts": conf.RetryAttempts,
"retry_delay": conf.RetryDelay,
"enable_logging": conf.EnableLogging,
"user_agent": conf.UserAgent,
})
return *ollamaSDK
}
cmd/web/bootstrap/config.go
+1
View File
@@ -13,6 +13,7 @@ type Config struct {
RateLimit config.RateLimitConfig RateLimit config.RateLimitConfig
Assets config.AssetsConfig Assets config.AssetsConfig
Notification config.NotificationConfig Notification config.NotificationConfig
Ollama config.OllamaConfig
UserAuth AuthConfig UserAuth AuthConfig
CustomerAuth AuthConfig CustomerAuth AuthConfig
} }
cmd/worker/bootstrap/bootstrap.go
+150
View File
@@ -0,0 +1,150 @@
package bootstrap
import (
"fmt"
"time"
"tm/cmd/worker/workers"
"tm/internal/notice"
"tm/internal/tender"
"tm/pkg/config"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/ollama"
"tm/pkg/schedule"
)
// Init Application Configuration
func InitConfig() (*Config, error) {
conf, err := config.LoadConfig(".", &Config{})
if err != nil {
return nil, fmt.Errorf("failed to load config: %v", err)
}
return conf, nil
}
// Init Logger Service
func InitLogger(conf config.LoggingConfig) logger.Logger {
return logger.NewLogger(&logger.Config{
Level: conf.Level,
Format: conf.Format,
Output: conf.Output,
File: logger.FileConfig{
Path: conf.File.Path,
MaxSize: conf.File.MaxSize,
MaxBackups: conf.File.MaxBackups,
MaxAge: conf.File.MaxAge,
Compress: conf.File.Compress,
},
})
}
// Init MongoDB Connection Manager
func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
// Convert infra.MongoConfig to mongo.ConnectionConfig
connectionConfig := mongo.ConnectionConfig{
URI: conf.URI,
Database: conf.Name,
MaxPoolSize: uint64(conf.MaxPoolSize),
MinPoolSize: 5, // Default minimum pool size
MaxConnIdleTime: 30 * time.Minute, // Default idle time
ConnectTimeout: conf.Timeout,
ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
}
// Create MongoDB connection manager
connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
if err != nil {
log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
"error": err.Error(),
"uri": conf.URI,
"db": conf.Name,
})
panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
}
log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
"database": conf.Name,
"uri": conf.URI,
"pool_size": conf.MaxPoolSize,
})
return connectionManager
}
// Init Worker
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, ollamaSDK *ollama.SDK) {
// Initialize repositories
noticeRepo := notice.NewRepository(mongoManager, appLogger)
tenderRepo := tender.NewRepository(mongoManager, appLogger)
worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK)
worker.Run()
// start Worker job
schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
Name: "Worker Job",
Func: func() {
worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK)
worker.Run()
},
Expr: config.Worker.Interval,
})
}
// Init Notification Service
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
notificationConfig := &notification.Config{
BaseURL: conf.BaseURL,
Timeout: conf.Timeout,
RetryAttempts: conf.RetryAttempts,
RetryDelay: conf.RetryDelay,
EnableLogging: conf.EnableLogging,
UserAgent: conf.UserAgent,
}
notificationSDK, err := notification.New(notificationConfig, log)
if err != nil {
log.Error("Failed to initialize Notification SDK", map[string]interface{}{
"error": err.Error(),
})
fmt.Printf("Failed to initialize Notification SDK: %v\n", err.Error())
}
log.Info("Notification SDK initialized successfully", map[string]interface{}{
"base_url": conf.BaseURL,
"timeout": conf.Timeout,
"retry_attempts": conf.RetryAttempts,
"retry_delay": conf.RetryDelay,
"enable_logging": conf.EnableLogging,
"user_agent": conf.UserAgent,
})
return *notificationSDK
}
// Init Ollama Service
func InitOllamaService(conf config.OllamaConfig, log logger.Logger) *ollama.SDK {
ollamaSDK, err := ollama.New(&ollama.Config{
BaseURL: conf.BaseURL,
Timeout: conf.Timeout,
RetryAttempts: conf.RetryAttempts,
RetryDelay: conf.RetryDelay,
EnableLogging: conf.EnableLogging,
UserAgent: conf.UserAgent,
DefaultModel: conf.DefaultModel,
DefaultTemperature: conf.DefaultTemperature,
DefaultTopP: conf.DefaultTopP,
DefaultMaxTokens: conf.DefaultMaxTokens,
EnableStreaming: conf.EnableStreaming,
}, log)
if err != nil {
log.Error("Failed to initialize Ollama SDK", map[string]interface{}{
"error": err.Error(),
})
}
return ollamaSDK
}
cmd/worker/bootstrap/config.go
+19
View File
@@ -0,0 +1,19 @@
package bootstrap
import (
"tm/pkg/config"
)
// Config defines the scraper application configuration
type Config struct {
Database config.DatabaseConfig
Logging config.LoggingConfig
Notification config.NotificationConfig
Ollama config.OllamaConfig
Worker WorkerConfig
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
}
type WorkerConfig struct {
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
}
cmd/worker/main.go
+62
View File
@@ -0,0 +1,62 @@
package main
import (
"context"
"os"
"os/signal"
"syscall"
"time"
"tm/cmd/worker/bootstrap"
)
func main() {
// Load configuration
config, err := bootstrap.InitConfig()
if err != nil {
panic(err)
}
// Initialize logger
appLogger := bootstrap.InitLogger(config.Logging)
appLogger.Info("Starting worker application", map[string]interface{}{
"version": "1.0.0",
})
// Initialize MongoDB connection
mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := mongoManager.Close(ctx); err != nil {
appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
"error": err.Error(),
})
}
}()
// Initialize notification service
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
// Initialize ollama service
ollamaSDK := bootstrap.InitOllamaService(config.Ollama, appLogger)
aiModels, err := ollamaSDK.ListModels(context.Background())
if err != nil {
appLogger.Error("Failed to list ollama models", map[string]interface{}{
"error": err.Error(),
})
}
appLogger.Info("Ollama models", map[string]interface{}{"models": aiModels})
// Initialize Worker
bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, ollamaSDK)
// Set up signal handling for graceful shutdown
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
// Wait for shutdown signal
<-signalChan
appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{})
appLogger.Info("Worker application stopped", map[string]interface{}{})
}
cmd/worker/workers/notice.go
+273
View File
@@ -0,0 +1,273 @@
package workers
import (
"context"
"tm/internal/notice"
"tm/internal/tender"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/ollama"
)
type NoticeWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
Notify notification.SDK
Ollama *ollama.SDK
NoticeRepo notice.Repository
TenderRepo tender.TenderRepository
}
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, ollama *ollama.SDK) *NoticeWorker {
return &NoticeWorker{
Mongo: mongo,
Logger: logger,
Notify: notify,
Ollama: ollama,
NoticeRepo: noticeRepo,
TenderRepo: tenderRepo,
}
}
func (w *NoticeWorker) Run() {
w.Logger.Info("Notice worker started", map[string]interface{}{})
limit := 10
skip := 0
for {
notices, _, err := w.NoticeRepo.GetUnProcessedNotices(context.Background(), limit, skip)
if err != nil {
w.Logger.Error("Failed to get notices", map[string]interface{}{
"error": err.Error(),
})
continue
}
if len(notices) == 0 {
break
}
for _, n := range notices {
w.Logger.Info("Notice", map[string]interface{}{
"notice": n.ID.Hex(),
})
t, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID)
if err == nil {
continue
}
err = w.ToTender(&n, t)
if err != nil {
w.Logger.Error("Failed to create tender", map[string]interface{}{
"error": err.Error(),
})
continue
}
if t.ID.IsZero() {
err = w.TenderRepo.Create(context.Background(), t)
if err != nil {
w.Logger.Error("Failed to create tender", map[string]interface{}{
"error": err.Error(),
})
}
} else {
err = w.TenderRepo.Update(context.Background(), t)
if err != nil {
w.Logger.Error("Failed to update tender", map[string]interface{}{
"error": err.Error(),
})
}
}
}
skip += limit
}
}
func (w *NoticeWorker) ToTender(n *notice.Notice, t *tender.Tender) error {
// Buyer Organization
buyerOrg := &tender.Organization{}
if n.BuyerOrganization != nil {
buyerOrg = &tender.Organization{
Name: n.BuyerOrganization.Name,
CompanyID: n.BuyerOrganization.CompanyID,
WebsiteURI: n.BuyerOrganization.WebsiteURI,
ContactName: n.BuyerOrganization.ContactName,
ContactTelephone: n.BuyerOrganization.ContactTelephone,
ContactEmail: n.BuyerOrganization.ContactEmail,
ContactFax: n.BuyerOrganization.ContactFax,
Role: n.BuyerOrganization.Role,
Address: tender.Address{
StreetName: n.BuyerOrganization.Address.StreetName,
CityName: n.BuyerOrganization.Address.CityName,
PostalZone: n.BuyerOrganization.Address.PostalZone,
CountrySubentityCode: n.BuyerOrganization.Address.CountrySubentityCode,
Department: n.BuyerOrganization.Address.Department,
Region: n.BuyerOrganization.Address.Region,
CountryCode: n.BuyerOrganization.Address.CountryCode,
},
}
}
// Review Organization
reviewOrg := &tender.Organization{}
if n.ReviewOrganization != nil {
reviewOrg = &tender.Organization{
Name: n.ReviewOrganization.Name,
CompanyID: n.ReviewOrganization.CompanyID,
WebsiteURI: n.ReviewOrganization.WebsiteURI,
ContactName: n.ReviewOrganization.ContactName,
ContactTelephone: n.ReviewOrganization.ContactTelephone,
ContactEmail: n.ReviewOrganization.ContactEmail,
ContactFax: n.ReviewOrganization.ContactFax,
Role: n.ReviewOrganization.Role,
Address: tender.Address{
StreetName: n.ReviewOrganization.Address.StreetName,
CityName: n.ReviewOrganization.Address.CityName,
PostalZone: n.ReviewOrganization.Address.PostalZone,
CountrySubentityCode: n.ReviewOrganization.Address.CountrySubentityCode,
Department: n.ReviewOrganization.Address.Department,
Region: n.ReviewOrganization.Address.Region,
CountryCode: n.ReviewOrganization.Address.CountryCode,
},
}
}
// Organizations
organizations := make([]tender.Organization, len(n.Organizations))
for i, org := range n.Organizations {
organizations[i] = tender.Organization{
Name: org.Name,
CompanyID: org.CompanyID,
WebsiteURI: org.WebsiteURI,
ContactName: org.ContactName,
ContactTelephone: org.ContactTelephone,
ContactEmail: org.ContactEmail,
ContactFax: org.ContactFax,
Role: org.Role,
Address: tender.Address{
StreetName: org.Address.StreetName,
CityName: org.Address.CityName,
PostalZone: org.Address.PostalZone,
CountrySubentityCode: org.Address.CountrySubentityCode,
Department: org.Address.Department,
Region: org.Address.Region,
CountryCode: org.Address.CountryCode,
},
}
}
// Selection Criteria
selectionCriteria := make([]tender.SelectionCriterion, len(n.SelectionCriteria))
for i, criterion := range n.SelectionCriteria {
selectionCriteria[i] = tender.SelectionCriterion{
TypeCode: criterion.TypeCode,
Description: criterion.Description,
LanguageID: criterion.LanguageID,
}
}
// Winning Tenderer
winningTenderer := &tender.Organization{}
if n.WinningTenderer != nil {
winningTenderer = &tender.Organization{
Name: n.WinningTenderer.Name,
CompanyID: n.WinningTenderer.CompanyID,
WebsiteURI: n.WinningTenderer.WebsiteURI,
ContactName: n.WinningTenderer.ContactName,
ContactTelephone: n.WinningTenderer.ContactTelephone,
ContactEmail: n.WinningTenderer.ContactEmail,
ContactFax: n.WinningTenderer.ContactFax,
Role: n.WinningTenderer.Role,
Address: tender.Address{
StreetName: n.WinningTenderer.Address.StreetName,
CityName: n.WinningTenderer.Address.CityName,
PostalZone: n.WinningTenderer.Address.PostalZone,
CountrySubentityCode: n.WinningTenderer.Address.CountrySubentityCode,
Department: n.WinningTenderer.Address.Department,
Region: n.WinningTenderer.Address.Region,
CountryCode: n.WinningTenderer.Address.CountryCode,
},
}
}
// Modifications
modifications := make([]tender.TenderModification, len(n.Modifications))
for i, modification := range n.Modifications {
modifications[i] = tender.TenderModification{
ModificationDate: modification.ModificationDate,
ModificationReason: modification.ModificationReason,
Description: modification.Description,
LanguageID: modification.LanguageID,
}
}
// Awarded Entities
awardedEntities := make([]tender.Awarded, len(n.AwardedEntities))
for i, awarded := range n.AwardedEntities {
awardedEntities[i] = tender.Awarded{
Name: awarded.Name,
Address: awarded.Address,
Country: awarded.Country,
Amount: awarded.Amount,
Currency: awarded.Currency,
Share: awarded.Share,
AwardDate: awarded.AwardDate,
ContractID: awarded.ContractID,
TenderID: awarded.TenderID,
LotID: awarded.LotID,
CompanyID: awarded.CompanyID,
OrganizationID: awarded.OrganizationID,
}
}
t.NoticePublicationID = n.NoticePublicationID
t.Title = n.Title
t.Description = n.Description
t.ProcurementTypeCode = n.ProcurementTypeCode
t.ProcedureCode = n.ProcedureCode
t.EstimatedValue = n.EstimatedValue
t.Currency = n.Currency
t.TenderDeadline = n.TenderDeadline
t.SubmissionURL = n.SubmissionURL
t.BuyerOrganization = buyerOrg
t.ReviewOrganization = reviewOrg
t.Organizations = organizations
t.SelectionCriteria = selectionCriteria
t.WinningTenderer = winningTenderer
t.Modifications = modifications
t.AwardedEntities = awardedEntities
t.NoticeTypeCode = n.NoticeTypeCode
t.NoticeSubTypeCode = n.NoticeSubTypeCode
t.NoticeLanguageCode = n.NoticeLanguageCode
t.IssueDate = n.IssueDate
t.IssueTime = n.IssueTime
t.PublicationDate = n.PublicationDate
t.SubmissionDeadline = n.SubmissionDeadline
t.ApplicationDeadline = n.ApplicationDeadline
t.GazetteID = n.GazetteID
t.PlaceOfPerformance = n.PlaceOfPerformance
t.CountryCode = n.CountryCode
t.RegionCode = n.RegionCode
t.CityName = n.CityName
t.PostalCode = n.PostalCode
t.DocumentURI = n.DocumentURI
t.TenderURL = n.TenderURL
t.MainClassification = n.MainClassification
t.AdditionalClassifications = n.AdditionalClassifications
t.OfficialLanguages = n.OfficialLanguages
t.CancellationReason = n.CancellationReason
t.CancellationDate = n.CancellationDate
t.AwardDate = n.AwardDate
t.AwardedValue = n.AwardedValue
t.ContractNumber = n.ContractNumber
t.SuspensionReason = n.SuspensionReason
t.SuspensionDate = n.SuspensionDate
t.WinningTenderer = winningTenderer
t.Modifications = modifications
t.AwardedEntities = awardedEntities
return nil
}
internal/notice/entity.go
+276
View File
@@ -0,0 +1,276 @@
package notice
import (
"fmt"
"strconv"
"strings"
"time"
"tm/pkg/mongo"
)
// Tender represents a tender/contract notice entity
type Notice struct {
mongo.Model `bson:",inline"`
ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"`
NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"`
ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"`
NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"`
NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"`
NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"`
IssueDate int64 `bson:"issue_date" json:"issue_date"`
IssueTime int64 `bson:"issue_time" json:"issue_time"`
TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"`
PublicationDate int64 `bson:"publication_date" json:"publication_date"`
SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"`
ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"`
GazetteID string `bson:"gazette_id" json:"gazette_id"`
Title string `bson:"title" json:"title"`
Description string `bson:"description" json:"description"`
ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"`
ProcedureCode string `bson:"procedure_code" json:"procedure_code"`
MainClassification string `bson:"main_classification" json:"main_classification"`
AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"`
EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"`
Currency string `bson:"currency" json:"currency"`
Duration string `bson:"duration" json:"duration"`
DurationUnit string `bson:"duration_unit" json:"duration_unit"`
PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"`
CountryCode string `bson:"country_code" json:"country_code"`
RegionCode string `bson:"region_code" json:"region_code"`
CityName string `bson:"city_name" json:"city_name"`
PostalCode string `bson:"postal_code" json:"postal_code"`
DocumentURI string `bson:"document_uri" json:"document_uri"`
TenderURL string `bson:"tender_url" json:"tender_url"`
SubmissionURL string `bson:"submission_url" json:"submission_url"`
BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"`
ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"`
Organizations []Organization `bson:"organizations" json:"organizations"`
SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"`
OfficialLanguages []string `bson:"official_languages" json:"official_languages"`
Status TenderStatus `bson:"status" json:"status"`
Source TenderSource `bson:"source" json:"source"`
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
// Status-specific fields
CancellationReason string `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"`
CancellationDate int64 `bson:"cancellation_date,omitempty" json:"cancellation_date,omitempty"`
AwardDate int64 `bson:"award_date,omitempty" json:"award_date,omitempty"`
AwardedValue float64 `bson:"awarded_value,omitempty" json:"awarded_value,omitempty"`
WinningTenderer *Organization `bson:"winning_tenderer,omitempty" json:"winning_tenderer,omitempty"`
ContractNumber string `bson:"contract_number,omitempty" json:"contract_number,omitempty"`
SuspensionReason string `bson:"suspension_reason,omitempty" json:"suspension_reason,omitempty"`
SuspensionDate int64 `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"`
Modifications []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"`
AwardedEntities []Awarded `bson:"awarded,omitempty" json:"awarded,omitempty"`
}
// Organization represents organization information
type Organization struct {
ID string `bson:"id" json:"id"`
Name string `bson:"name" json:"name"`
CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"`
WebsiteURI string `bson:"website_uri,omitempty" json:"website_uri,omitempty"`
ContactName string `bson:"contact_name,omitempty" json:"contact_name,omitempty"`
ContactTelephone string `bson:"contact_telephone,omitempty" json:"contact_telephone,omitempty"`
ContactEmail string `bson:"contact_email,omitempty" json:"contact_email,omitempty"`
ContactFax string `bson:"contact_fax,omitempty" json:"contact_fax,omitempty"`
Address Address `bson:"address" json:"address"`
Role string `bson:"role,omitempty" json:"role,omitempty"`
}
// Address represents address information
type Address struct {
StreetName string `bson:"street_name,omitempty" json:"street_name,omitempty"`
CityName string `bson:"city_name,omitempty" json:"city_name,omitempty"`
PostalZone string `bson:"postal_zone,omitempty" json:"postal_zone,omitempty"`
CountrySubentityCode string `bson:"country_subentity_code,omitempty" json:"country_subentity_code,omitempty"`
Department string `bson:"department,omitempty" json:"department,omitempty"`
Region string `bson:"region,omitempty" json:"region,omitempty"`
CountryCode string `bson:"country_code" json:"country_code"`
}
// SelectionCriterion represents selection criteria
type SelectionCriterion struct {
TypeCode string `bson:"type_code" json:"type_code"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
}
// ProcessingMetadata contains metadata about how the tender was processed
type ProcessingMetadata struct {
ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"`
ProcessedAt int64 `bson:"processed_at" json:"processed_at"`
ProcessingVersion string `bson:"processing_version" json:"processing_version"`
ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"`
ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"`
EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"`
TranslatedData map[string]string `bson:"translated_data,omitempty" json:"translated_data,omitempty"`
TranslatedAt int64 `bson:"translated_at,omitempty" json:"translated_at,omitempty"`
Processed bool `bson:"processed" json:"processed"`
}
// TenderModification represents a modification made to a tender
type TenderModification struct {
ModificationDate int64 `bson:"modification_date" json:"modification_date"` // Unix milliseconds
ModificationReason string `bson:"modification_reason" json:"modification_reason"`
Description string `bson:"description,omitempty" json:"description,omitempty"`
LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"`
}
// AwardedEntity represents an entity that has been awarded a contract
type Awarded struct {
Name string `bson:"name" json:"name"`
Address string `bson:"address,omitempty" json:"address,omitempty"`
Country string `bson:"country,omitempty" json:"country,omitempty"`
Amount float64 `bson:"amount" json:"amount"`
Currency string `bson:"currency,omitempty" json:"currency,omitempty"`
Share float64 `bson:"share,omitempty" json:"share,omitempty"` // Percentage share for multiple winners
AwardDate int64 `bson:"award_date" json:"award_date"` // Unix milliseconds
ContractID string `bson:"contract_id,omitempty" json:"contract_id,omitempty"` // Contract reference ID
TenderID string `bson:"tender_id,omitempty" json:"tender_id,omitempty"` // Tender reference ID
LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` // Lot reference ID
CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"` // Company registration ID
OrganizationID string `bson:"organization_id,omitempty" json:"organization_id,omitempty"` // Organization reference ID
}
// TenderStatus represents the status of a tender
type TenderStatus string
const (
TenderStatusActive TenderStatus = "active"
TenderStatusExpired TenderStatus = "expired"
TenderStatusCancelled TenderStatus = "cancelled"
TenderStatusAwarded TenderStatus = "awarded"
TenderStatusDraft TenderStatus = "draft"
TenderStatusClosed TenderStatus = "closed"
TenderStatusModified TenderStatus = "modified"
TenderStatusSuspended TenderStatus = "suspended"
TenderStatusPublished TenderStatus = "published"
)
// TenderSource represents the source of tender data
type TenderSource string
const (
TenderSourceTEDScraper TenderSource = "ted_scraper"
TenderSourceManual TenderSource = "manual"
TenderSourceAPI TenderSource = "api"
TenderSourceBulkImport TenderSource = "bulk_import"
)
// TenderStatistics represents tender statistics
type TenderStatistics struct {
TotalTenders int64 `json:"total_tenders"`
ActiveTenders int64 `json:"active_tenders"`
ExpiredTenders int64 `json:"expired_tenders"`
TendersByCountry map[string]int64 `json:"tenders_by_country"`
TendersByType map[string]int64 `json:"tenders_by_type"`
TendersByClassification map[string]int64 `json:"tenders_by_classification"`
AverageEstimatedValue float64 `json:"average_estimated_value"`
TotalEstimatedValue float64 `json:"total_estimated_value"`
LastUpdated int64 `json:"last_updated"` // Unix milliseconds
}
// GetID returns the tender ID
func (t *Notice) GetID() string {
return t.ID.Hex()
}
// IsActive returns true if the tender is active
func (t *Notice) IsActive() bool {
return t.Status == TenderStatusActive
}
// IsExpired returns true if the tender deadline has passed
func (t *Notice) IsExpired() bool {
if t.Status == TenderStatusExpired {
return true
}
if t.TenderDeadline == 0 {
return false
}
// Check if deadline (in milliseconds) is in the past
now := time.Now().Unix()
return t.TenderDeadline < now
}
// GetTenderURL returns the TED tender URL
func (t *Notice) GetTenderURL() string {
if t.TenderURL != "" {
return t.TenderURL
}
if t.NoticePublicationID != "" {
return generateTenderURL(t.NoticePublicationID)
}
return ""
}
// generateTenderURL generates the TED tender URL from NoticePublicationID
func generateTenderURL(noticePublicationID string) string {
// Remove leading zeros and format for URL
// Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025
parts := strings.Split(noticePublicationID, "-")
if len(parts) != 2 {
return ""
}
// Convert to int and back to string to remove leading zeros
if num, err := strconv.Atoi(parts[0]); err == nil {
cleanID := fmt.Sprintf("%d-%s", num, parts[1])
return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID)
}
return ""
}
// GetMainOrganization returns the buyer organization
func (t *Notice) GetMainOrganization() *Organization {
return t.BuyerOrganization
}
// HasEstimatedValue returns true if the tender has an estimated value
func (t *Notice) HasEstimatedValue() bool {
return t.EstimatedValue > 0
}
// GetFormattedEstimatedValue returns formatted estimated value with currency
func (t *Notice) GetFormattedEstimatedValue() string {
if !t.HasEstimatedValue() {
return ""
}
if t.Currency != "" {
return fmt.Sprintf("%.2f %s", t.EstimatedValue, t.Currency)
}
return fmt.Sprintf("%.2f", t.EstimatedValue)
}
// UpdateStatus updates the tender status and updated timestamp
func (t *Notice) UpdateStatus(status TenderStatus) {
t.Status = status
t.UpdatedAt = time.Now().Unix()
}
// GetDaysUntilDeadline returns the number of days until the tender deadline
func (t *Notice) GetDaysUntilDeadline() int {
if t.TenderDeadline == 0 {
return 0
}
now := time.Now().Unix()
if t.TenderDeadline <= now {
return 0 // Already expired
}
diffMillis := t.TenderDeadline - now
diffDays := diffMillis / (24 * 60 * 60 * 1000)
return int(diffDays)
}
internal/notice/repository.go
+182
View File
@@ -0,0 +1,182 @@
package notice
import (
"context"
"time"
"tm/pkg/logger"
orm "tm/pkg/mongo"
"go.mongodb.org/mongo-driver/v2/bson"
)
// NoticeRepository interface defines notice data access methods
type Repository interface {
Import(ctx context.Context, notice *Notice) error
BulkImport(ctx context.Context, notices []Notice) error
GetByID(ctx context.Context, id string) (*Notice, error)
GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error)
GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error)
Delete(ctx context.Context, id string) error
}
func collectionName() string {
return "notices"
}
// noticeRepository implements NoticeRepository interface using MongoDB ORM
type noticeRepository struct {
ormRepo orm.Repository[Notice]
mongoManager *orm.ConnectionManager
logger logger.Logger
}
// NewRepository creates a new notice repository
func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Repository {
// Create indexes for notices collection
noticeIndexes := []orm.Index{}
// Create indexes
if err := mongoManager.CreateIndexes(collectionName(), noticeIndexes); err != nil {
logger.Warn("Failed to create notice indexes", map[string]interface{}{
"error": err.Error(),
})
}
// Create ORM repositories
noticeOrmRepo := orm.NewRepository[Notice](mongoManager.GetCollection(collectionName()), logger)
return &noticeRepository{
ormRepo: noticeOrmRepo,
mongoManager: mongoManager,
logger: logger,
}
}
// Import imports a new notice
func (r *noticeRepository) Import(ctx context.Context, notice *Notice) error {
now := time.Now().Unix()
notice.SetCreatedAt(now)
err := r.ormRepo.Create(ctx, notice)
if err != nil {
r.logger.Error("Failed to create notice", map[string]interface{}{
"error": err.Error(),
"notice_publication_id": notice.NoticePublicationID,
})
return err
}
r.logger.Info("Notice created successfully", map[string]interface{}{
"notice_id": notice.ID,
"notice_publication_id": notice.NoticePublicationID,
})
return nil
}
// BulkImport imports a new notice
func (r *noticeRepository) BulkImport(ctx context.Context, notices []Notice) error {
err := r.ormRepo.CreateMany(ctx, notices)
if err != nil {
r.logger.Error("Failed to bulk import notices", map[string]interface{}{
"error": err.Error(),
})
return err
}
return nil
}
// GetByID retrieves a notice by ID
func (r *noticeRepository) GetByID(ctx context.Context, id string) (*Notice, error) {
notice, err := r.ormRepo.FindByID(ctx, id)
if err != nil {
r.logger.Error("Failed to get notice by ID", map[string]interface{}{
"notice_id": id,
"error": err.Error(),
})
return nil, err
}
return notice, nil
}
// GetByContractNoticeID retrieves a notice by contract notice ID
func (r *noticeRepository) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error) {
filter := bson.M{"contract_notice_id": contractNoticeID}
pagination := orm.Pagination{Limit: 1}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to get notice by contract notice ID", map[string]interface{}{
"contract_notice_id": contractNoticeID,
"error": err.Error(),
})
return nil, err
}
if len(result.Items) == 0 {
return nil, orm.ErrDocumentNotFound
}
return &result.Items[0], nil
}
// GetUnProcessedNotices retrieves unprocessed notices
func (r *noticeRepository) GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) {
filter := bson.M{"processing_metadata.processed": false}
pagination := orm.Pagination{
Limit: limit,
Skip: skip,
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to get unprocessed notices", map[string]interface{}{
"error": err.Error(),
})
return nil, 0, err
}
return result.Items, result.TotalCount, nil
}
// Update updates an existing notice
func (r *noticeRepository) Update(ctx context.Context, notice *Notice) error {
notice.UpdatedAt = time.Now().Unix()
err := r.ormRepo.Update(ctx, notice)
if err != nil {
r.logger.Error("Failed to update notice", map[string]interface{}{
"notice_id": notice.ID,
"error": err.Error(),
})
return err
}
r.logger.Info("Notice updated successfully", map[string]interface{}{
"notice_id": notice.ID,
})
return nil
}
// Delete deletes a notice by ID
func (r *noticeRepository) Delete(ctx context.Context, id string) error {
err := r.ormRepo.Delete(ctx, id)
if err != nil {
r.logger.Error("Failed to delete notice", map[string]interface{}{
"notice_id": id,
"error": err.Error(),
})
return err
}
r.logger.Info("Notice deleted successfully", map[string]interface{}{
"notice_id": id,
})
return nil
}
internal/notice/service.go
+36
View File
@@ -0,0 +1,36 @@
package notice
import (
"context"
"tm/pkg/logger"
)
// Service defines business logic for notice operations
type Service interface {
Import(ctx context.Context, form *Notice) error
BulkImport(ctx context.Context, form []Notice) error
}
// noticeService implements the Service interface
type noticeService struct {
repository Repository
logger logger.Logger
}
// NewService creates a new notice service
func NewService(repository Repository, logger logger.Logger) Service {
return &noticeService{
repository: repository,
logger: logger,
}
}
// Import imports a new notice
func (s *noticeService) Import(ctx context.Context, form *Notice) error {
return s.repository.Import(ctx, form)
}
// BulkImport imports a new notice
func (s *noticeService) BulkImport(ctx context.Context, form []Notice) error {
return s.repository.BulkImport(ctx, form)
}
pkg/config/config.go
+15
View File
@@ -74,6 +74,21 @@ type NotificationConfig struct {
UserAgent string `env:"NOTIFICATION_USER_AGENT"` UserAgent string `env:"NOTIFICATION_USER_AGENT"`
} }
// OllamaConfig represents the configuration for the Ollama SDK
type OllamaConfig struct {
BaseURL string `env:"OLLAMA_BASE_URL" default:"http://localhost:11434"`
Timeout time.Duration `env:"OLLAMA_TIMEOUT" default:"300s"`
RetryAttempts int `env:"OLLAMA_RETRY_ATTEMPTS" default:"3"`
RetryDelay time.Duration `env:"OLLAMA_RETRY_DELAY" default:"2s"`
EnableLogging bool `env:"OLLAMA_ENABLE_LOGGING" default:"true"`
UserAgent string `env:"OLLAMA_USER_AGENT" default:"TenderManagement-OllamaSDK/1.0"`
DefaultModel string `env:"OLLAMA_DEFAULT_MODEL" default:"llama2"`
DefaultTemperature float64 `env:"OLLAMA_DEFAULT_TEMPERATURE" default:"0.7"`
DefaultTopP float64 `env:"OLLAMA_DEFAULT_TOP_P" default:"0.9"`
DefaultMaxTokens int `env:"OLLAMA_DEFAULT_MAX_TOKENS" default:"1000"`
EnableStreaming bool `env:"OLLAMA_ENABLE_STREAMING" default:"false"`
}
// LoadConfig loads configuration with priority: OS environment variables > .env file // LoadConfig loads configuration with priority: OS environment variables > .env file
// OS environment variables take precedence over .env file values // OS environment variables take precedence over .env file values
func LoadConfig[T any](path string, config T) (T, error) { func LoadConfig[T any](path string, config T) (T, error) {
pkg/mongo/repository.go
+29
View File
@@ -47,6 +47,9 @@ type Repository[T any] interface {
// Create inserts a new document // Create inserts a new document
Create(ctx context.Context, model *T) error Create(ctx context.Context, model *T) error
// CreateMany inserts multiple documents
CreateMany(ctx context.Context, models []T) error
// Update updates an existing document // Update updates an existing document
Update(ctx context.Context, model *T) error Update(ctx context.Context, model *T) error
@@ -239,6 +242,32 @@ func (r *repository[T]) Create(ctx context.Context, model *T) error {
return nil return nil
} }
// CreateMany inserts multiple documents
func (r *repository[T]) CreateMany(ctx context.Context, models []T) error {
// Set timestamps if the model implements Timestampable
for _, v := range models {
if timestampable, ok := any(v).(Timestampable); ok {
now := time.Now().Unix()
timestampable.SetCreatedAt(now)
timestampable.SetUpdatedAt(now)
}
}
result, err := r.collection.InsertMany(ctx, models)
if err != nil {
r.logger.Error("Failed to create multiple documents", map[string]interface{}{
"error": err.Error(),
})
return fmt.Errorf("failed to create multiple documents: %w", err)
}
r.logger.Info("Documents created successfully", map[string]interface{}{
"ids": result.InsertedIDs,
})
return nil
}
// Update updates an existing document // Update updates an existing document
func (r *repository[T]) Update(ctx context.Context, model *T) error { func (r *repository[T]) Update(ctx context.Context, model *T) error {
// Set updated timestamp if the model implements Timestampable // Set updated timestamp if the model implements Timestampable
pkg/ollama/README.md
+394
View File
@@ -0,0 +1,394 @@
# Ollama SDK
A comprehensive Go SDK for interacting with Ollama AI models. This SDK provides a clean, type-safe interface for text generation, chat conversations, embeddings, and model management.
## Features
- **Text Generation**: Generate text using various AI models
- **Chat Conversations**: Interactive chat with support for system prompts and conversation history
- **Embeddings**: Generate embeddings for text similarity and search
- **Model Management**: Pull, list, delete, and inspect models
- **Streaming Support**: Real-time streaming of generated text and chat responses
- **Automatic Retries**: Configurable retry mechanism with exponential backoff
- **Structured Logging**: Built-in logging with structured fields
- **Fluent API**: Builder pattern for constructing complex chat conversations
- **Type Safety**: Full type safety with comprehensive validation
- **Error Handling**: Detailed error types and proper error wrapping
- **Configuration**: Flexible configuration via environment variables or code
- **Health Checks**: Built-in health checking capabilities
## Installation
The SDK is already part of your `tm` module. Import it as:
```go
import "tm/pkg/ollama"
```
## Quick Start
### Basic Setup
```go
// 1. Configuration
config := ollama.DefaultConfig()
config.BaseURL = "http://localhost:11434" // Your Ollama server URL
config.DefaultModel = "llama2" // Default model to use
// 2. Logger (using project's logger)
loggerConfig := &logger.Config{Level: "info", Format: "json", Output: "stdout"}
projectLogger := logger.NewLogger(loggerConfig)
// 3. Create SDK
sdk, err := ollama.New(config, projectLogger)
if err != nil {
log.Fatal("Failed to create Ollama SDK:", err)
}
```
### Environment Variables
Set environment variables for automatic configuration:
```bash
export OLLAMA_BASE_URL="http://localhost:11434"
export OLLAMA_TIMEOUT="300s"
export OLLAMA_RETRY_ATTEMPTS="3"
export OLLAMA_RETRY_DELAY="2s"
export OLLAMA_ENABLE_LOGGING="true"
export OLLAMA_DEFAULT_MODEL="llama2"
export OLLAMA_DEFAULT_TEMPERATURE="0.7"
export OLLAMA_DEFAULT_TOP_P="0.9"
export OLLAMA_DEFAULT_MAX_TOKENS="1000"
```
## API Reference
### Configuration
```go
type Config struct {
BaseURL string `env:"OLLAMA_BASE_URL"`
Timeout time.Duration `env:"OLLAMA_TIMEOUT"`
RetryAttempts int `env:"OLLAMA_RETRY_ATTEMPTS"`
RetryDelay time.Duration `env:"OLLAMA_RETRY_DELAY"`
EnableLogging bool `env:"OLLAMA_ENABLE_LOGGING"`
UserAgent string `env:"OLLAMA_USER_AGENT"`
DefaultModel string `env:"OLLAMA_DEFAULT_MODEL"`
DefaultTemperature float64 `env:"OLLAMA_DEFAULT_TEMPERATURE"`
DefaultTopP float64 `env:"OLLAMA_DEFAULT_TOP_P"`
DefaultMaxTokens int `env:"OLLAMA_DEFAULT_MAX_TOKENS"`
EnableStreaming bool `env:"OLLAMA_ENABLE_STREAMING"`
}
```
### Creating SDK Instance
```go
// With default configuration
sdk, err := ollama.New(nil, logger)
// With custom configuration
config := &ollama.Config{
BaseURL: "http://localhost:11434",
Timeout: 300 * time.Second,
RetryAttempts: 3,
RetryDelay: 2 * time.Second,
EnableLogging: true,
DefaultModel: "llama2",
DefaultTemperature: 0.7,
DefaultTopP: 0.9,
DefaultMaxTokens: 1000,
}
sdk, err := ollama.New(config, logger)
// With custom HTTP client
httpClient := &http.Client{Timeout: 10 * time.Second}
sdk, err := ollama.NewWithClient(config, httpClient, logger)
```
## Usage Examples
### Simple Text Generation
```go
ctx := context.Background()
// Quick generation with default model
response, err := sdk.QuickGenerate(ctx, "Write a short story about a robot")
if err != nil {
log.Printf("Generation failed: %v", err)
}
fmt.Println(response)
// Generation with specific model
response, err := sdk.QuickGenerateWithModel(ctx, "codellama", "Write a Python function to sort a list")
```
### Advanced Text Generation
```go
req := &ollama.GenerateRequest{
Model: "llama2",
Prompt: "Explain quantum computing in simple terms",
Options: map[string]interface{}{
"temperature": 0.7,
"top_p": 0.9,
"max_tokens": 500,
},
}
resp, err := sdk.Generate(ctx, req)
if err != nil {
log.Printf("Generation failed: %v", err)
}
fmt.Printf("Generated: %s\n", resp.Response)
```
### Chat Conversations
```go
// Simple chat
response, err := sdk.QuickChat(ctx, "What is the capital of France?")
fmt.Println(response)
// Chat with system prompt
response, err := sdk.QuickChatWithSystem(ctx,
"You are a helpful coding assistant.",
"How do I implement a binary search in Go?")
```
### Advanced Chat with Conversation History
```go
req := &ollama.ChatRequest{
Model: "llama2",
Messages: []ollama.Message{
{
Role: ollama.MessageRoleSystem,
Content: "You are a helpful assistant.",
},
{
Role: ollama.MessageRoleUser,
Content: "Hello, how are you?",
},
{
Role: ollama.MessageRoleAssistant,
Content: "I'm doing well, thank you! How can I help you today?",
},
{
Role: ollama.MessageRoleUser,
Content: "Can you help me with a coding problem?",
},
},
}
resp, err := sdk.Chat(ctx, req)
fmt.Printf("Assistant: %s\n", resp.Message.Content)
```
### Fluent Chat Builder
```go
response, err := sdk.NewChatBuilder().
SetModel("llama2").
SetSystem("You are a helpful coding assistant.").
AddMessage(ollama.MessageRoleUser, "How do I create a struct in Go?").
SetTemperature(0.7).
SetMaxTokens(500).
Send(ctx)
fmt.Printf("Assistant: %s\n", response.Message.Content)
// Continue the conversation
response, err = sdk.NewChatBuilder().
SetModel("llama2").
SetSystem("You are a helpful coding assistant.").
AddMessage(ollama.MessageRoleUser, "How do I create a struct in Go?").
SetTemperature(0.7).
SetMaxTokens(500).
Continue(ctx, "Can you show me an example with methods?")
fmt.Printf("Assistant: %s\n", response.Message.Content)
```
### Streaming Responses
```go
// Streaming text generation
err := sdk.StreamGenerate(ctx, &ollama.GenerateRequest{
Model: "llama2",
Prompt: "Write a poem about programming",
}, func(chunk string) error {
fmt.Print(chunk)
return nil
})
// Streaming chat
err := sdk.StreamChat(ctx, &ollama.ChatRequest{
Model: "llama2",
Messages: []ollama.Message{
{
Role: ollama.MessageRoleUser,
Content: "Tell me a story",
},
},
}, func(chunk string) error {
fmt.Print(chunk)
return nil
})
```
### Embeddings
```go
// Generate embeddings
embeddings, err := sdk.QuickEmbed(ctx, "This is some text to embed")
if err != nil {
log.Printf("Embedding failed: %v", err)
}
// Advanced embedding request
req := &ollama.EmbedRequest{
Model: "llama2",
Prompt: "Text to generate embeddings for",
}
resp, err := sdk.Embed(ctx, req)
fmt.Printf("Embedding dimension: %d\n", len(resp.Embedding))
```
### Model Management
```go
// List available models
models, err := sdk.ListModels(ctx)
for _, model := range models.Models {
fmt.Printf("Model: %s, Size: %d bytes\n", model.Name, model.Size)
}
// Pull a new model
pullResp, err := sdk.PullModel(ctx, "codellama")
fmt.Printf("Pull status: %s\n", pullResp.Status)
// Show model information
modelInfo, err := sdk.ShowModel(ctx, "llama2")
fmt.Printf("Model details: %+v\n", modelInfo.Details)
// Delete a model
deleteResp, err := sdk.DeleteModel(ctx, "unwanted-model")
fmt.Printf("Delete status: %s\n", deleteResp.Status)
```
### Health Check
```go
err := sdk.Health(ctx)
if err != nil {
log.Printf("Ollama service is not available: %v", err)
} else {
fmt.Println("Ollama service is healthy")
}
```
## Error Handling
The SDK provides specific error types for different failure scenarios:
```go
resp, err := sdk.Generate(ctx, req)
if err != nil {
switch e := err.(type) {
case *ollama.ConfigError:
fmt.Printf("Configuration error: %s\n", e.Message)
case *ollama.ModelError:
fmt.Printf("Model error for %s: %s\n", e.Model, e.Message)
case *ollama.GenerationError:
fmt.Printf("Generation error: %s\n", e.Message)
case *ollama.ConnectionError:
fmt.Printf("Connection error: %s\n", e.Message)
case *ollama.TimeoutError:
fmt.Printf("Timeout during %s: %s\n", e.Operation, e.Message)
default:
fmt.Printf("Unexpected error: %v\n", err)
}
}
```
## Advanced Usage
### Custom HTTP Client
```go
httpClient := &http.Client{
Timeout: 60 * time.Second,
Transport: &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 30 * time.Second,
DisableCompression: true,
},
}
sdk, err := ollama.NewWithClient(config, httpClient, logger)
```
### Custom Options
```go
req := &ollama.GenerateRequest{
Model: "llama2",
Prompt: "Write a haiku",
}
opts := &ollama.GenerateOptions{
Temperature: &[]float64{0.5}[0],
TopP: &[]float64{0.8}[0],
TopK: &[]int{40}[0],
RepeatPenalty: &[]float64{1.1}[0],
Seed: &[]int{42}[0],
Stop: []string{"</s>", "\n\n"},
Format: "json",
}
resp, err := sdk.GenerateWithOptions(ctx, req, opts)
```
## Best Practices
1. **Always use context**: Pass context for cancellation and timeouts
2. **Handle errors properly**: Use type assertions for specific error handling
3. **Configure timeouts**: Set appropriate timeouts for your use case
4. **Use streaming for long responses**: Enable streaming for better user experience
5. **Monitor model usage**: Keep track of token usage and costs
6. **Validate inputs**: Ensure prompts and parameters are within limits
7. **Use structured logging**: Leverage the built-in logging for debugging
## Troubleshooting
### Common Issues
1. **Connection refused**: Ensure Ollama server is running on the correct port
2. **Model not found**: Pull the model first using `PullModel()`
3. **Timeout errors**: Increase the timeout configuration
4. **Rate limiting**: Implement backoff strategies for high-volume usage
### Debug Mode
Enable debug logging to troubleshoot issues:
```go
config := ollama.DefaultConfig()
config.EnableLogging = true
loggerConfig := &logger.Config{
Level: "debug",
Format: "json",
Output: "stdout",
}
projectLogger := logger.NewLogger(loggerConfig)
sdk, err := ollama.New(config, projectLogger)
```
## License
This SDK is part of the Tender Management system and follows the same licensing terms.
pkg/ollama/USAGE.md
+263
View File
@@ -0,0 +1,263 @@
# Ollama SDK - Quick Usage Guide
## Installation
The SDK is already part of your `tm` module. Import it as:
```go
import "tm/pkg/ollama"
```
## Basic Setup
```go
// 1. Configuration
config := ollama.DefaultConfig()
config.BaseURL = "http://localhost:11434" // Your Ollama server URL
config.DefaultModel = "llama2" // Default model to use
// 2. Logger (using project's logger)
loggerConfig := &logger.Config{Level: "info", Format: "json", Output: "stdout"}
projectLogger := logger.NewLogger(loggerConfig)
// 3. Create SDK
sdk, err := ollama.New(config, projectLogger)
if err != nil {
log.Fatal("Failed to create Ollama SDK:", err)
}
```
## Environment Variables
```bash
export OLLAMA_BASE_URL="http://localhost:11434"
export OLLAMA_TIMEOUT="300s"
export OLLAMA_RETRY_ATTEMPTS="3"
export OLLAMA_RETRY_DELAY="2s"
export OLLAMA_ENABLE_LOGGING="true"
export OLLAMA_DEFAULT_MODEL="llama2"
export OLLAMA_DEFAULT_TEMPERATURE="0.7"
export OLLAMA_DEFAULT_TOP_P="0.9"
export OLLAMA_DEFAULT_MAX_TOKENS="1000"
```
## Quick Examples
### Simple Text Generation
```go
ctx := context.Background()
// Generate text with default model
response, err := sdk.QuickGenerate(ctx, "Write a short story about a robot")
if err != nil {
log.Printf("Generation failed: %v", err)
}
fmt.Println(response)
// Generate with specific model
response, err := sdk.QuickGenerateWithModel(ctx, "codellama", "Write a Python function to sort a list")
```
### Chat Conversations
```go
// Simple chat
response, err := sdk.QuickChat(ctx, "What is the capital of France?")
fmt.Println(response)
// Chat with system prompt
response, err := sdk.QuickChatWithSystem(ctx,
"You are a helpful coding assistant.",
"How do I implement a binary search in Go?")
```
### Advanced Chat with Builder
```go
// Create a chat builder
builder := sdk.NewChatBuilder().
SetModel("llama2").
SetSystem("You are a helpful coding assistant.").
SetTemperature(0.7).
SetMaxTokens(500)
// Start conversation
response, err := builder.
AddMessage(ollama.MessageRoleUser, "How do I create a struct in Go?").
Send(ctx)
fmt.Printf("Assistant: %s\n", response.Message.Content)
// Continue the conversation
response, err = builder.Continue(ctx, "Can you show me an example with methods?")
fmt.Printf("Assistant: %s\n", response.Message.Content)
```
### Streaming Responses
```go
// Streaming text generation
err := sdk.StreamGenerate(ctx, &ollama.GenerateRequest{
Model: "llama2",
Prompt: "Write a poem about programming",
}, func(chunk string) error {
fmt.Print(chunk) // Print each chunk as it arrives
return nil
})
// Streaming chat
err := sdk.StreamChat(ctx, &ollama.ChatRequest{
Model: "llama2",
Messages: []ollama.Message{
{
Role: ollama.MessageRoleUser,
Content: "Tell me a story",
},
},
}, func(chunk string) error {
fmt.Print(chunk)
return nil
})
```
### Embeddings
```go
// Generate embeddings
embeddings, err := sdk.QuickEmbed(ctx, "This is some text to embed")
if err != nil {
log.Printf("Embedding failed: %v", err)
}
fmt.Printf("Embedding dimension: %d\n", len(embeddings))
```
### Model Management
```go
// List available models
models, err := sdk.ListModels(ctx)
for _, model := range models.Models {
fmt.Printf("Model: %s, Size: %d bytes\n", model.Name, model.Size)
}
// Pull a new model
pullResp, err := sdk.PullModel(ctx, "codellama")
fmt.Printf("Pull status: %s\n", pullResp.Status)
// Show model information
modelInfo, err := sdk.ShowModel(ctx, "llama2")
fmt.Printf("Model family: %s\n", modelInfo.Details.Family)
// Delete a model
deleteResp, err := sdk.DeleteModel(ctx, "unwanted-model")
fmt.Printf("Delete status: %s\n", deleteResp.Status)
```
### Health Check
```go
err := sdk.Health(ctx)
if err != nil {
log.Printf("Ollama service is not available: %v", err)
} else {
fmt.Println("Ollama service is healthy")
}
```
## Error Handling
```go
resp, err := sdk.Generate(ctx, req)
if err != nil {
switch e := err.(type) {
case *ollama.ConfigError:
fmt.Printf("Configuration error: %s\n", e.Message)
case *ollama.ModelError:
fmt.Printf("Model error for %s: %s\n", e.Model, e.Message)
case *ollama.GenerationError:
fmt.Printf("Generation error: %s\n", e.Message)
case *ollama.ConnectionError:
fmt.Printf("Connection error: %s\n", e.Message)
case *ollama.TimeoutError:
fmt.Printf("Timeout during %s: %s\n", e.Operation, e.Message)
default:
fmt.Printf("Unexpected error: %v\n", err)
}
}
```
## Advanced Configuration
```go
config := &ollama.Config{
BaseURL: "http://localhost:11434",
Timeout: 300 * time.Second,
RetryAttempts: 3,
RetryDelay: 2 * time.Second,
EnableLogging: true,
UserAgent: "MyApp-OllamaSDK/1.0",
DefaultModel: "llama2",
DefaultTemperature: 0.7,
DefaultTopP: 0.9,
DefaultMaxTokens: 1000,
EnableStreaming: false,
}
sdk, err := ollama.New(config, logger)
```
## Custom HTTP Client
```go
httpClient := &http.Client{
Timeout: 60 * time.Second,
Transport: &http.Transport{
MaxIdleConns: 10,
IdleConnTimeout: 30 * time.Second,
DisableCompression: true,
},
}
sdk, err := ollama.NewWithClient(config, httpClient, logger)
```
## Integration with Project Logger
```go
// Use the project's logger system
loggerConfig := &logger.Config{
Level: "info",
Format: "json",
Output: "stdout",
}
projectLogger := logger.NewLogger(loggerConfig)
sdk, err := ollama.New(config, projectLogger)
```
## Best Practices
1. **Always use context**: Pass context for cancellation and timeouts
2. **Handle errors properly**: Use type assertions for specific error handling
3. **Configure timeouts**: Set appropriate timeouts for your use case
4. **Use streaming for long responses**: Enable streaming for better user experience
5. **Monitor model usage**: Keep track of token usage and costs
6. **Validate inputs**: Ensure prompts and parameters are within limits
7. **Use structured logging**: Leverage the built-in logging for debugging
## Common Use Cases
### Document Summarization
```go
response, err := sdk.QuickChatWithSystem(ctx,
"You are a helpful assistant that summarizes documents concisely.",
"Please summarize this document: [document content]")
```
### Code Review Assistant
```go
response, err := sdk.QuickChatWithSystem(ctx,
"You are a senior software engineer reviewing code. Provide constructive feedback.",
"Please review this Go code: [code snippet]")
```
### Translation Service
```go
response, err := sdk.QuickChatWithSystem(ctx,
"You are a professional translator. Translate the following text accurately.",
"Translate this to French: Hello, how are you?")
```
### Question Answering
```go
response, err := sdk.QuickChatWithSystem(ctx,
"You are a helpful assistant that answers questions accurately and concisely.",
"What is the difference between HTTP and HTTPS?")
```
pkg/ollama/client.go
+290
View File
@@ -0,0 +1,290 @@
package ollama
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"tm/pkg/logger"
)
// Client implements the HTTPClient interface for Ollama API
type Client struct {
config *Config
httpClient *http.Client
logger logger.Logger
}
// NewClient creates a new Ollama HTTP client
func NewClient(config *Config, logger logger.Logger) (*Client, error) {
if config == nil {
config = DefaultConfig()
}
if err := config.Validate(); err != nil {
return nil, err
}
httpClient := &http.Client{
Timeout: config.Timeout,
}
return &Client{
config: config,
httpClient: httpClient,
logger: logger,
}, nil
}
// NewClientWithHTTPClient creates a new client with a custom HTTP client
func NewClientWithHTTPClient(config *Config, httpClient *http.Client, logger logger.Logger) (*Client, error) {
if config == nil {
config = DefaultConfig()
}
if err := config.Validate(); err != nil {
return nil, err
}
return &Client{
config: config,
httpClient: httpClient,
logger: logger,
}, nil
}
// Post performs a POST request to the Ollama API
func (c *Client) Post(ctx context.Context, url string, body interface{}, result interface{}) error {
return c.makeRequest(ctx, "POST", url, body, result)
}
// Get performs a GET request to the Ollama API
func (c *Client) Get(ctx context.Context, url string, result interface{}) error {
return c.makeRequest(ctx, "GET", url, nil, result)
}
// Delete performs a DELETE request to the Ollama API
func (c *Client) Delete(ctx context.Context, url string, body interface{}, result interface{}) error {
return c.makeRequest(ctx, "DELETE", url, body, result)
}
// makeRequest performs an HTTP request with retry logic and logging
func (c *Client) makeRequest(ctx context.Context, method, endpoint string, body interface{}, result interface{}) error {
url := c.config.BaseURL + endpoint
var bodyReader io.Reader
if body != nil {
bodyBytes, err := json.Marshal(body)
if err != nil {
return fmt.Errorf("failed to marshal request body: %w", err)
}
bodyReader = bytes.NewReader(bodyBytes)
}
var lastErr error
for attempt := 0; attempt <= c.config.RetryAttempts; attempt++ {
if attempt > 0 {
// Wait before retry
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(c.config.RetryDelay):
}
}
err := c.doRequest(ctx, method, url, bodyReader, result)
if err == nil {
return nil
}
lastErr = err
// Don't retry on certain errors
if c.shouldNotRetry(err) {
break
}
if c.logger != nil {
c.logger.Warn("Request failed, retrying", map[string]interface{}{
"attempt": attempt + 1,
"max_attempts": c.config.RetryAttempts + 1,
"error": err.Error(),
"url": url,
"method": method,
})
}
}
return fmt.Errorf("request failed after %d attempts: %w", c.config.RetryAttempts+1, lastErr)
}
// doRequest performs a single HTTP request
func (c *Client) doRequest(ctx context.Context, method, url string, body io.Reader, result interface{}) error {
req, err := http.NewRequestWithContext(ctx, method, url, body)
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("User-Agent", c.config.UserAgent)
if c.logger != nil && c.config.EnableLogging {
c.logger.Debug("Making HTTP request", map[string]interface{}{
"method": method,
"url": url,
"user_agent": c.config.UserAgent,
})
}
resp, err := c.httpClient.Do(req)
if err != nil {
return fmt.Errorf("failed to execute request: %w", err)
}
defer resp.Body.Close()
if c.logger != nil && c.config.EnableLogging {
c.logger.Debug("Received HTTP response", map[string]interface{}{
"status_code": resp.StatusCode,
"url": url,
})
}
if resp.StatusCode >= 400 {
bodyBytes, _ := io.ReadAll(resp.Body)
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(bodyBytes))
}
if result != nil {
if err := json.NewDecoder(resp.Body).Decode(result); err != nil {
return fmt.Errorf("failed to decode response: %w", err)
}
}
return nil
}
// shouldNotRetry determines if an error should not be retried
func (c *Client) shouldNotRetry(err error) bool {
// Don't retry on context cancellation
if err == context.Canceled || err == context.DeadlineExceeded {
return true
}
// Don't retry on certain HTTP status codes
if err.Error() == "HTTP 400" || err.Error() == "HTTP 401" || err.Error() == "HTTP 403" || err.Error() == "HTTP 404" {
return true
}
return false
}
// StreamGenerate performs a streaming text generation request
func (c *Client) StreamGenerate(ctx context.Context, req *GenerateRequest, onChunk func(*StreamChunk) error) error {
url := c.config.BaseURL + "/api/generate"
// Force streaming
stream := true
req.Stream = &stream
bodyBytes, err := json.Marshal(req)
if err != nil {
return fmt.Errorf("failed to marshal request body: %w", err)
}
httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("User-Agent", c.config.UserAgent)
resp, err := c.httpClient.Do(httpReq)
if err != nil {
return fmt.Errorf("failed to execute request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
bodyBytes, _ := io.ReadAll(resp.Body)
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(bodyBytes))
}
decoder := json.NewDecoder(resp.Body)
for {
var chunk StreamChunk
if err := decoder.Decode(&chunk); err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("failed to decode stream chunk: %w", err)
}
if err := onChunk(&chunk); err != nil {
return fmt.Errorf("error processing stream chunk: %w", err)
}
if chunk.Done {
break
}
}
return nil
}
// StreamChat performs a streaming chat request
func (c *Client) StreamChat(ctx context.Context, req *ChatRequest, onChunk func(*StreamChunk) error) error {
url := c.config.BaseURL + "/api/chat"
// Force streaming
stream := true
req.Stream = &stream
bodyBytes, err := json.Marshal(req)
if err != nil {
return fmt.Errorf("failed to marshal request body: %w", err)
}
httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes))
if err != nil {
return fmt.Errorf("failed to create request: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
httpReq.Header.Set("User-Agent", c.config.UserAgent)
resp, err := c.httpClient.Do(httpReq)
if err != nil {
return fmt.Errorf("failed to execute request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 400 {
bodyBytes, _ := io.ReadAll(resp.Body)
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(bodyBytes))
}
decoder := json.NewDecoder(resp.Body)
for {
var chunk StreamChunk
if err := decoder.Decode(&chunk); err != nil {
if err == io.EOF {
break
}
return fmt.Errorf("failed to decode stream chunk: %w", err)
}
if err := onChunk(&chunk); err != nil {
return fmt.Errorf("error processing stream chunk: %w", err)
}
if chunk.Done {
break
}
}
return nil
}
pkg/ollama/config.go
+91
View File
@@ -0,0 +1,91 @@
package ollama
import (
"time"
)
// Config represents the configuration for the Ollama SDK
type Config struct {
// BaseURL is the base URL of the Ollama API server
BaseURL string `env:"OLLAMA_BASE_URL" default:"http://localhost:11434"`
// Timeout is the timeout for HTTP requests
Timeout time.Duration `env:"OLLAMA_TIMEOUT" default:"300s"`
// RetryAttempts is the number of retry attempts for failed requests
RetryAttempts int `env:"OLLAMA_RETRY_ATTEMPTS" default:"3"`
// RetryDelay is the delay between retry attempts
RetryDelay time.Duration `env:"OLLAMA_RETRY_DELAY" default:"2s"`
// EnableLogging enables request/response logging
EnableLogging bool `env:"OLLAMA_ENABLE_LOGGING" default:"true"`
// UserAgent is the user agent string for HTTP requests
UserAgent string `env:"OLLAMA_USER_AGENT" default:"TenderManagement-OllamaSDK/1.0"`
// DefaultModel is the default model to use if none is specified
DefaultModel string `env:"OLLAMA_DEFAULT_MODEL" default:"llama2"`
// DefaultTemperature is the default temperature for text generation
DefaultTemperature float64 `env:"OLLAMA_DEFAULT_TEMPERATURE" default:"0.7"`
// DefaultTopP is the default top_p value for text generation
DefaultTopP float64 `env:"OLLAMA_DEFAULT_TOP_P" default:"0.9"`
// DefaultMaxTokens is the default maximum number of tokens to generate
DefaultMaxTokens int `env:"OLLAMA_DEFAULT_MAX_TOKENS" default:"1000"`
// EnableStreaming enables streaming responses by default
EnableStreaming bool `env:"OLLAMA_ENABLE_STREAMING" default:"false"`
}
// DefaultConfig returns a default configuration for the Ollama SDK
func DefaultConfig() *Config {
return &Config{
BaseURL: "http://localhost:11434",
Timeout: 300 * time.Second,
RetryAttempts: 3,
RetryDelay: 2 * time.Second,
EnableLogging: true,
UserAgent: "TenderManagement-OllamaSDK/1.0",
DefaultModel: "llama2",
DefaultTemperature: 0.7,
DefaultTopP: 0.9,
DefaultMaxTokens: 1000,
EnableStreaming: false,
}
}
// Validate validates the configuration
func (c *Config) Validate() error {
if c.BaseURL == "" {
return ErrInvalidConfig("BaseURL cannot be empty")
}
if c.Timeout <= 0 {
return ErrInvalidConfig("Timeout must be positive")
}
if c.RetryAttempts < 0 {
return ErrInvalidConfig("RetryAttempts cannot be negative")
}
if c.RetryDelay < 0 {
return ErrInvalidConfig("RetryDelay cannot be negative")
}
if c.DefaultTemperature < 0 || c.DefaultTemperature > 2 {
return ErrInvalidConfig("DefaultTemperature must be between 0 and 2")
}
if c.DefaultTopP < 0 || c.DefaultTopP > 1 {
return ErrInvalidConfig("DefaultTopP must be between 0 and 1")
}
if c.DefaultMaxTokens < 0 {
return ErrInvalidConfig("DefaultMaxTokens cannot be negative")
}
return nil
}
pkg/ollama/entities.go
+182
View File
@@ -0,0 +1,182 @@
package ollama
import (
"time"
)
// GenerateRequest represents a request to generate text
type GenerateRequest struct {
Model string `json:"model" validate:"required"`
Prompt string `json:"prompt" validate:"required"`
Stream *bool `json:"stream,omitempty"`
Format string `json:"format,omitempty"`
Options map[string]interface{} `json:"options,omitempty"`
System string `json:"system,omitempty"`
Template string `json:"template,omitempty"`
Context []int `json:"context,omitempty"`
Raw bool `json:"raw,omitempty"`
KeepAlive string `json:"keep_alive,omitempty"`
}
// GenerateResponse represents the response from a text generation request
type GenerateResponse struct {
Model string `json:"model"`
CreatedAt time.Time `json:"created_at"`
Response string `json:"response"`
Done bool `json:"done"`
Context []int `json:"context,omitempty"`
TotalDuration int64 `json:"total_duration,omitempty"`
LoadDuration int64 `json:"load_duration,omitempty"`
PromptEvalCount int `json:"prompt_eval_count,omitempty"`
PromptEvalDuration int64 `json:"prompt_eval_duration,omitempty"`
EvalCount int `json:"eval_count,omitempty"`
EvalDuration int64 `json:"eval_duration,omitempty"`
}
// ChatRequest represents a request for a chat conversation
type ChatRequest struct {
Model string `json:"model" validate:"required"`
Messages []Message `json:"messages" validate:"required,min=1"`
Stream *bool `json:"stream,omitempty"`
Format string `json:"format,omitempty"`
Options ChatOptions `json:"options,omitempty"`
KeepAlive string `json:"keep_alive,omitempty"`
}
// Message represents a message in a chat conversation
type Message struct {
Role MessageRole `json:"role" validate:"required,oneof=system user assistant"`
Content string `json:"content" validate:"required"`
Images []string `json:"images,omitempty"`
}
// ChatOptions represents options for chat requests
type ChatOptions struct {
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
TopK *int `json:"top_k,omitempty"`
RepeatPenalty *float64 `json:"repeat_penalty,omitempty"`
Seed *int `json:"seed,omitempty"`
Stop []string `json:"stop,omitempty"`
NumCtx *int `json:"num_ctx,omitempty"`
NumPredict *int `json:"num_predict,omitempty"`
NumKeep *int `json:"num_keep,omitempty"`
NumThread *int `json:"num_thread,omitempty"`
RepeatLastN *int `json:"repeat_last_n,omitempty"`
Options map[string]interface{} `json:"options,omitempty"`
}
// ChatResponse represents the response from a chat request
type ChatResponse struct {
Model string `json:"model"`
CreatedAt time.Time `json:"created_at"`
Message Message `json:"message"`
Done bool `json:"done"`
TotalDuration int64 `json:"total_duration,omitempty"`
LoadDuration int64 `json:"load_duration,omitempty"`
PromptEvalCount int `json:"prompt_eval_count,omitempty"`
PromptEvalDuration int64 `json:"prompt_eval_duration,omitempty"`
EvalCount int `json:"eval_count,omitempty"`
EvalDuration int64 `json:"eval_duration,omitempty"`
}
// EmbedRequest represents a request to generate embeddings
type EmbedRequest struct {
Model string `json:"model" validate:"required"`
Prompt string `json:"prompt" validate:"required"`
Options map[string]interface{} `json:"options,omitempty"`
}
// EmbedResponse represents the response from an embedding request
type EmbedResponse struct {
Embedding []float64 `json:"embedding"`
}
// PullRequest represents a request to pull a model
type PullRequest struct {
Name string `json:"name" validate:"required"`
Insecure bool `json:"insecure,omitempty"`
Stream *bool `json:"stream,omitempty"`
}
// PullResponse represents the response from a pull request
type PullResponse struct {
Status string `json:"status"`
Digest string `json:"digest,omitempty"`
Total int64 `json:"total,omitempty"`
Completed int64 `json:"completed,omitempty"`
}
// ListModelsResponse represents the response from listing models
type ListModelsResponse struct {
Models []ModelInfo `json:"models"`
}
// ModelInfo represents information about a model
type ModelInfo struct {
Name string `json:"name"`
ModifiedAt time.Time `json:"modified_at"`
Size int64 `json:"size"`
Digest string `json:"digest"`
Details ModelDetails `json:"details,omitempty"`
}
// ModelDetails represents detailed information about a model
type ModelDetails struct {
Format string `json:"format,omitempty"`
Family string `json:"family,omitempty"`
Families []string `json:"families,omitempty"`
ParameterSize string `json:"parameter_size,omitempty"`
QuantizationLevel string `json:"quantization_level,omitempty"`
ParentModel string `json:"parent_model,omitempty"`
Templates map[string]interface{} `json:"templates,omitempty"`
System string `json:"system,omitempty"`
Options map[string]interface{} `json:"options,omitempty"`
}
// DeleteRequest represents a request to delete a model
type DeleteRequest struct {
Name string `json:"name" validate:"required"`
}
// DeleteResponse represents the response from a delete request
type DeleteResponse struct {
Status string `json:"status"`
}
// ShowModelRequest represents a request to show model information
type ShowModelRequest struct {
Name string `json:"name" validate:"required"`
Verbose bool `json:"verbose,omitempty"`
}
// ShowModelResponse represents the response from a show model request
type ShowModelResponse struct {
License string `json:"license,omitempty"`
Modelfile string `json:"modelfile,omitempty"`
Parameters string `json:"parameters,omitempty"`
Template string `json:"template,omitempty"`
Details ModelDetails `json:"details,omitempty"`
Options map[string]interface{} `json:"options,omitempty"`
}
// HealthResponse represents the health check response
type HealthResponse struct {
Status string `json:"status"`
}
// StreamChunk represents a chunk in a streaming response
type StreamChunk struct {
Model string `json:"model,omitempty"`
CreatedAt time.Time `json:"created_at,omitempty"`
Response string `json:"response,omitempty"`
Done bool `json:"done"`
Context []int `json:"context,omitempty"`
TotalDuration int64 `json:"total_duration,omitempty"`
LoadDuration int64 `json:"load_duration,omitempty"`
PromptEvalCount int `json:"prompt_eval_count,omitempty"`
PromptEvalDuration int64 `json:"prompt_eval_duration,omitempty"`
EvalCount int `json:"eval_count,omitempty"`
EvalDuration int64 `json:"eval_duration,omitempty"`
Message *Message `json:"message,omitempty"`
}
pkg/ollama/errors.go
+146
View File
@@ -0,0 +1,146 @@
package ollama
import (
"fmt"
)
// Error types for the Ollama SDK
var (
// ErrInvalidConfig is returned when configuration is invalid
ErrInvalidConfig = func(msg string) error {
return &ConfigError{Message: msg}
}
// ErrInvalidRequest is returned when a request is invalid
ErrInvalidRequest = func(msg string) error {
return &RequestError{Message: msg}
}
// ErrModelNotFound is returned when a model is not found
ErrModelNotFound = func(model string) error {
return &ModelError{Message: fmt.Sprintf("model %s not found", model), Model: model}
}
// ErrModelPullFailed is returned when pulling a model fails
ErrModelPullFailed = func(model string, reason string) error {
return &ModelError{Message: fmt.Sprintf("failed to pull model %s: %s", model, reason), Model: model}
}
// ErrModelDeleteFailed is returned when deleting a model fails
ErrModelDeleteFailed = func(model string, reason string) error {
return &ModelError{Message: fmt.Sprintf("failed to delete model %s: %s", model, reason), Model: model}
}
// ErrGenerationFailed is returned when text generation fails
ErrGenerationFailed = func(reason string) error {
return &GenerationError{Message: fmt.Sprintf("generation failed: %s", reason)}
}
// ErrChatFailed is returned when chat fails
ErrChatFailed = func(reason string) error {
return &ChatError{Message: fmt.Sprintf("chat failed: %s", reason)}
}
// ErrEmbeddingFailed is returned when embedding generation fails
ErrEmbeddingFailed = func(reason string) error {
return &EmbeddingError{Message: fmt.Sprintf("embedding failed: %s", reason)}
}
// ErrConnectionFailed is returned when connection to Ollama fails
ErrConnectionFailed = func(reason string) error {
return &ConnectionError{Message: fmt.Sprintf("connection failed: %s", reason)}
}
// ErrTimeout is returned when a request times out
ErrTimeout = func(operation string) error {
return &TimeoutError{Message: fmt.Sprintf("timeout during %s", operation), Operation: operation}
}
// ErrRateLimitExceeded is returned when rate limit is exceeded
ErrRateLimitExceeded = func() error {
return &RateLimitError{Message: "rate limit exceeded"}
}
)
// ConfigError represents a configuration error
type ConfigError struct {
Message string
}
func (e *ConfigError) Error() string {
return fmt.Sprintf("config error: %s", e.Message)
}
// RequestError represents a request validation error
type RequestError struct {
Message string
}
func (e *RequestError) Error() string {
return fmt.Sprintf("request error: %s", e.Message)
}
// ModelError represents a model-related error
type ModelError struct {
Message string
Model string
}
func (e *ModelError) Error() string {
return fmt.Sprintf("model error: %s", e.Message)
}
// GenerationError represents a text generation error
type GenerationError struct {
Message string
}
func (e *GenerationError) Error() string {
return fmt.Sprintf("generation error: %s", e.Message)
}
// ChatError represents a chat error
type ChatError struct {
Message string
}
func (e *ChatError) Error() string {
return fmt.Sprintf("chat error: %s", e.Message)
}
// EmbeddingError represents an embedding generation error
type EmbeddingError struct {
Message string
}
func (e *EmbeddingError) Error() string {
return fmt.Sprintf("embedding error: %s", e.Message)
}
// ConnectionError represents a connection error
type ConnectionError struct {
Message string
}
func (e *ConnectionError) Error() string {
return fmt.Sprintf("connection error: %s", e.Message)
}
// TimeoutError represents a timeout error
type TimeoutError struct {
Message string
Operation string
}
func (e *TimeoutError) Error() string {
return fmt.Sprintf("timeout error: %s", e.Message)
}
// RateLimitError represents a rate limit error
type RateLimitError struct {
Message string
}
func (e *RateLimitError) Error() string {
return fmt.Sprintf("rate limit error: %s", e.Message)
}
pkg/ollama/interfaces.go
+121
View File
@@ -0,0 +1,121 @@
package ollama
import (
"context"
)
// OllamaService defines the interface for Ollama operations
type OllamaService interface {
// Generate generates text using a specified model and prompt
Generate(ctx context.Context, req *GenerateRequest) (*GenerateResponse, error)
// GenerateWithOptions generates text with additional options
GenerateWithOptions(ctx context.Context, req *GenerateRequest, opts *GenerateOptions) (*GenerateResponse, error)
// Chat performs a conversational chat with the model
Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error)
// Embed generates embeddings for the given text
Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error)
// PullModel pulls a model from the registry
PullModel(ctx context.Context, model string) (*PullResponse, error)
// ListModels lists available models
ListModels(ctx context.Context) (*ListModelsResponse, error)
// DeleteModel deletes a model
DeleteModel(ctx context.Context, model string) (*DeleteResponse, error)
// ShowModel shows model information
ShowModel(ctx context.Context, model string) (*ShowModelResponse, error)
// Health checks if the Ollama service is available
Health(ctx context.Context) error
// NewChatBuilder creates a new chat builder for conversational interactions
NewChatBuilder() ChatBuilder
}
// HTTPClient defines the interface for HTTP operations
type HTTPClient interface {
// Post performs a POST request
Post(ctx context.Context, url string, body interface{}, result interface{}) error
// Get performs a GET request
Get(ctx context.Context, url string, result interface{}) error
// Delete performs a DELETE request
Delete(ctx context.Context, url string, result interface{}) error
}
// Logger defines the interface for logging operations
type Logger interface {
Debug(msg string, fields map[string]interface{})
Info(msg string, fields map[string]interface{})
Warn(msg string, fields map[string]interface{})
Error(msg string, fields map[string]interface{})
Fatal(msg string, fields map[string]interface{})
WithFields(fields map[string]interface{}) Logger
Sync() error
}
// ChatBuilder provides a fluent interface for building chat requests
type ChatBuilder interface {
// SetModel sets the model to use
SetModel(model string) ChatBuilder
// SetSystem sets the system prompt
SetSystem(system string) ChatBuilder
// AddMessage adds a message to the conversation
AddMessage(role MessageRole, content string) ChatBuilder
// SetTemperature sets the temperature for generation
SetTemperature(temperature float64) ChatBuilder
// SetTopP sets the top_p parameter
SetTopP(topP float64) ChatBuilder
// SetMaxTokens sets the maximum number of tokens to generate
SetMaxTokens(maxTokens int) ChatBuilder
// SetStream sets whether to stream the response
SetStream(stream bool) ChatBuilder
// SetFormat sets the response format
SetFormat(format string) ChatBuilder
// SetOptions sets custom options
SetOptions(options map[string]interface{}) ChatBuilder
// Build creates the chat request
Build() (*ChatRequest, error)
// Send builds and sends the chat request
Send(ctx context.Context) (*ChatResponse, error)
// Continue continues the conversation with the current context
Continue(ctx context.Context, userMessage string) (*ChatResponse, error)
}
// MessageRole represents the role of a message in a chat
type MessageRole string
const (
MessageRoleSystem MessageRole = "system"
MessageRoleUser MessageRole = "user"
MessageRoleAssistant MessageRole = "assistant"
)
// GenerateOptions represents additional options for text generation
type GenerateOptions struct {
Temperature *float64 `json:"temperature,omitempty"`
TopP *float64 `json:"top_p,omitempty"`
TopK *int `json:"top_k,omitempty"`
RepeatPenalty *float64 `json:"repeat_penalty,omitempty"`
Seed *int `json:"seed,omitempty"`
Stop []string `json:"stop,omitempty"`
Format string `json:"format,omitempty"`
Options map[string]interface{} `json:"options,omitempty"`
}
pkg/ollama/sdk.go
+232
View File
@@ -0,0 +1,232 @@
package ollama
import (
"context"
"fmt"
"net/http"
"tm/pkg/logger"
)
// SDK provides a high-level interface for the Ollama service
type SDK struct {
service OllamaService
client *Client
config *Config
}
// New creates a new Ollama SDK with the provided configuration and logger
func New(config *Config, logger logger.Logger) (*SDK, error) {
if config == nil {
config = DefaultConfig()
}
client, err := NewClient(config, logger)
if err != nil {
return nil, fmt.Errorf("failed to create ollama client: %w", err)
}
service := NewService(client)
return &SDK{
service: service,
client: client,
config: config,
}, nil
}
// NewWithClient creates a new Ollama SDK with a custom HTTP client
func NewWithClient(config *Config, httpClient *http.Client, logger logger.Logger) (*SDK, error) {
if config == nil {
config = DefaultConfig()
}
client, err := NewClientWithHTTPClient(config, httpClient, logger)
if err != nil {
return nil, fmt.Errorf("failed to create ollama client: %w", err)
}
service := NewService(client)
return &SDK{
service: service,
client: client,
config: config,
}, nil
}
// Generate generates text using a specified model and prompt
func (s *SDK) Generate(ctx context.Context, req *GenerateRequest) (*GenerateResponse, error) {
return s.service.Generate(ctx, req)
}
// GenerateWithOptions generates text with additional options
func (s *SDK) GenerateWithOptions(ctx context.Context, req *GenerateRequest, opts *GenerateOptions) (*GenerateResponse, error) {
return s.service.GenerateWithOptions(ctx, req, opts)
}
// Chat performs a conversational chat with the model
func (s *SDK) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
return s.service.Chat(ctx, req)
}
// Embed generates embeddings for the given text
func (s *SDK) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) {
return s.service.Embed(ctx, req)
}
// PullModel pulls a model from the registry
func (s *SDK) PullModel(ctx context.Context, model string) (*PullResponse, error) {
return s.service.PullModel(ctx, model)
}
// ListModels lists available models
func (s *SDK) ListModels(ctx context.Context) (*ListModelsResponse, error) {
return s.service.ListModels(ctx)
}
// DeleteModel deletes a model
func (s *SDK) DeleteModel(ctx context.Context, model string) (*DeleteResponse, error) {
return s.service.DeleteModel(ctx, model)
}
// ShowModel shows model information
func (s *SDK) ShowModel(ctx context.Context, model string) (*ShowModelResponse, error) {
return s.service.ShowModel(ctx, model)
}
// Health checks if the Ollama service is available
func (s *SDK) Health(ctx context.Context) error {
return s.service.Health(ctx)
}
// NewChatBuilder creates a new chat builder for conversational interactions
func (s *SDK) NewChatBuilder() ChatBuilder {
return s.service.NewChatBuilder()
}
// QuickGenerate is a convenience method for simple text generation
func (s *SDK) QuickGenerate(ctx context.Context, prompt string) (string, error) {
req := &GenerateRequest{
Model: s.config.DefaultModel,
Prompt: prompt,
}
resp, err := s.Generate(ctx, req)
if err != nil {
return "", err
}
return resp.Response, nil
}
// QuickGenerateWithModel is a convenience method for text generation with a specific model
func (s *SDK) QuickGenerateWithModel(ctx context.Context, model, prompt string) (string, error) {
req := &GenerateRequest{
Model: model,
Prompt: prompt,
}
resp, err := s.Generate(ctx, req)
if err != nil {
return "", err
}
return resp.Response, nil
}
// QuickChat is a convenience method for simple chat interactions
func (s *SDK) QuickChat(ctx context.Context, userMessage string) (string, error) {
req := &ChatRequest{
Model: s.config.DefaultModel,
Messages: []Message{
{
Role: MessageRoleUser,
Content: userMessage,
},
},
}
resp, err := s.Chat(ctx, req)
if err != nil {
return "", err
}
return resp.Message.Content, nil
}
// QuickChatWithSystem is a convenience method for chat with system prompt
func (s *SDK) QuickChatWithSystem(ctx context.Context, systemPrompt, userMessage string) (string, error) {
req := &ChatRequest{
Model: s.config.DefaultModel,
Messages: []Message{
{
Role: MessageRoleSystem,
Content: systemPrompt,
},
{
Role: MessageRoleUser,
Content: userMessage,
},
},
}
resp, err := s.Chat(ctx, req)
if err != nil {
return "", err
}
return resp.Message.Content, nil
}
// QuickEmbed is a convenience method for generating embeddings
func (s *SDK) QuickEmbed(ctx context.Context, text string) ([]float64, error) {
req := &EmbedRequest{
Model: s.config.DefaultModel,
Prompt: text,
}
resp, err := s.Embed(ctx, req)
if err != nil {
return nil, err
}
return resp.Embedding, nil
}
// StreamGenerate performs streaming text generation
func (s *SDK) StreamGenerate(ctx context.Context, req *GenerateRequest, onChunk func(string) error) error {
// Force streaming
stream := true
req.Stream = &stream
return s.client.StreamGenerate(ctx, req, func(chunk *StreamChunk) error {
if chunk.Response != "" {
return onChunk(chunk.Response)
}
return nil
})
}
// StreamChat performs streaming chat
func (s *SDK) StreamChat(ctx context.Context, req *ChatRequest, onChunk func(string) error) error {
// Force streaming
stream := true
req.Stream = &stream
return s.client.StreamChat(ctx, req, func(chunk *StreamChunk) error {
if chunk.Message != nil && chunk.Message.Content != "" {
return onChunk(chunk.Message.Content)
}
return nil
})
}
// GetService returns the underlying service for advanced usage
func (s *SDK) GetService() OllamaService {
return s.service
}
// GetConfig returns the current configuration
func (s *SDK) GetConfig() *Config {
return s.config
}
pkg/ollama/service.go
+444
View File
@@ -0,0 +1,444 @@
package ollama
import (
"context"
"fmt"
"tm/pkg/logger"
)
// Service implements the OllamaService interface
type Service struct {
client *Client
config *Config
logger logger.Logger
}
// NewService creates a new Ollama service
func NewService(client *Client) OllamaService {
return &Service{
client: client,
config: client.config,
logger: client.logger,
}
}
// Generate generates text using a specified model and prompt
func (s *Service) Generate(ctx context.Context, req *GenerateRequest) (*GenerateResponse, error) {
if s.logger != nil {
s.logger.Info("Generating text", map[string]interface{}{
"model": req.Model,
"prompt_length": len(req.Prompt),
})
}
// Set defaults
if req.Model == "" {
req.Model = s.config.DefaultModel
}
var resp GenerateResponse
err := s.client.Post(ctx, "/api/generate", req, &resp)
if err != nil {
if s.logger != nil {
s.logger.Error("Text generation failed", map[string]interface{}{
"model": req.Model,
"error": err.Error(),
})
}
return nil, ErrGenerationFailed(err.Error())
}
if s.logger != nil {
s.logger.Info("Text generation completed", map[string]interface{}{
"model": req.Model,
"response_length": len(resp.Response),
"eval_count": resp.EvalCount,
})
}
return &resp, nil
}
// GenerateWithOptions generates text with additional options
func (s *Service) GenerateWithOptions(ctx context.Context, req *GenerateRequest, opts *GenerateOptions) (*GenerateResponse, error) {
if s.logger != nil {
s.logger.Info("Generating text with options", map[string]interface{}{
"model": req.Model,
"prompt_length": len(req.Prompt),
})
}
// Apply options to request
if opts != nil {
if req.Options == nil {
req.Options = make(map[string]interface{})
}
if opts.Temperature != nil {
req.Options["temperature"] = *opts.Temperature
}
if opts.TopP != nil {
req.Options["top_p"] = *opts.TopP
}
if opts.TopK != nil {
req.Options["top_k"] = *opts.TopK
}
if opts.RepeatPenalty != nil {
req.Options["repeat_penalty"] = *opts.RepeatPenalty
}
if opts.Seed != nil {
req.Options["seed"] = *opts.Seed
}
if opts.Stop != nil {
req.Options["stop"] = opts.Stop
}
if opts.Format != "" {
req.Format = opts.Format
}
// Merge additional options
for k, v := range opts.Options {
req.Options[k] = v
}
}
return s.Generate(ctx, req)
}
// Chat performs a conversational chat with the model
func (s *Service) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) {
if s.logger != nil {
s.logger.Info("Starting chat", map[string]interface{}{
"model": req.Model,
"message_count": len(req.Messages),
})
}
// Set defaults
if req.Model == "" {
req.Model = s.config.DefaultModel
}
var resp ChatResponse
err := s.client.Post(ctx, "/api/chat", req, &resp)
if err != nil {
if s.logger != nil {
s.logger.Error("Chat failed", map[string]interface{}{
"model": req.Model,
"error": err.Error(),
})
}
return nil, ErrChatFailed(err.Error())
}
if s.logger != nil {
s.logger.Info("Chat completed", map[string]interface{}{
"model": req.Model,
"response_length": len(resp.Message.Content),
"eval_count": resp.EvalCount,
})
}
return &resp, nil
}
// Embed generates embeddings for the given text
func (s *Service) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) {
if s.logger != nil {
s.logger.Info("Generating embeddings", map[string]interface{}{
"model": req.Model,
"text_length": len(req.Prompt),
})
}
var resp EmbedResponse
err := s.client.Post(ctx, "/api/embeddings", req, &resp)
if err != nil {
if s.logger != nil {
s.logger.Error("Embedding generation failed", map[string]interface{}{
"model": req.Model,
"error": err.Error(),
})
}
return nil, ErrEmbeddingFailed(err.Error())
}
if s.logger != nil {
s.logger.Info("Embeddings generated", map[string]interface{}{
"model": req.Model,
"embedding_dimension": len(resp.Embedding),
})
}
return &resp, nil
}
// PullModel pulls a model from the registry
func (s *Service) PullModel(ctx context.Context, model string) (*PullResponse, error) {
if s.logger != nil {
s.logger.Info("Pulling model", map[string]interface{}{
"model": model,
})
}
req := &PullRequest{
Name: model,
}
var resp PullResponse
err := s.client.Post(ctx, "/api/pull", req, &resp)
if err != nil {
if s.logger != nil {
s.logger.Error("Model pull failed", map[string]interface{}{
"model": model,
"error": err.Error(),
})
}
return nil, ErrModelPullFailed(model, err.Error())
}
if s.logger != nil {
s.logger.Info("Model pulled successfully", map[string]interface{}{
"model": model,
"status": resp.Status,
})
}
return &resp, nil
}
// ListModels lists available models
func (s *Service) ListModels(ctx context.Context) (*ListModelsResponse, error) {
if s.logger != nil {
s.logger.Debug("Listing models", map[string]interface{}{})
}
var resp ListModelsResponse
err := s.client.Get(ctx, "/api/tags", &resp)
if err != nil {
if s.logger != nil {
s.logger.Error("Failed to list models", map[string]interface{}{
"error": err.Error(),
})
}
return nil, fmt.Errorf("failed to list models: %w", err)
}
if s.logger != nil {
s.logger.Debug("Models listed", map[string]interface{}{
"model_count": len(resp.Models),
})
}
return &resp, nil
}
// DeleteModel deletes a model
func (s *Service) DeleteModel(ctx context.Context, model string) (*DeleteResponse, error) {
if s.logger != nil {
s.logger.Info("Deleting model", map[string]interface{}{
"model": model,
})
}
req := &DeleteRequest{
Name: model,
}
var resp DeleteResponse
err := s.client.Delete(ctx, "/api/delete", req, &resp)
if err != nil {
if s.logger != nil {
s.logger.Error("Model deletion failed", map[string]interface{}{
"model": model,
"error": err.Error(),
})
}
return nil, ErrModelDeleteFailed(model, err.Error())
}
if s.logger != nil {
s.logger.Info("Model deleted successfully", map[string]interface{}{
"model": model,
"status": resp.Status,
})
}
return &resp, nil
}
// ShowModel shows model information
func (s *Service) ShowModel(ctx context.Context, model string) (*ShowModelResponse, error) {
if s.logger != nil {
s.logger.Debug("Showing model information", map[string]interface{}{
"model": model,
})
}
req := &ShowModelRequest{
Name: model,
}
var resp ShowModelResponse
err := s.client.Post(ctx, "/api/show", req, &resp)
if err != nil {
if s.logger != nil {
s.logger.Error("Failed to show model", map[string]interface{}{
"model": model,
"error": err.Error(),
})
}
return nil, fmt.Errorf("failed to show model %s: %w", model, err)
}
if s.logger != nil {
s.logger.Debug("Model information retrieved", map[string]interface{}{
"model": model,
})
}
return &resp, nil
}
// Health checks if the Ollama service is available
func (s *Service) Health(ctx context.Context) error {
if s.logger != nil {
s.logger.Debug("Checking Ollama health", map[string]interface{}{})
}
var resp HealthResponse
err := s.client.Get(ctx, "/api/version", &resp)
if err != nil {
if s.logger != nil {
s.logger.Error("Health check failed", map[string]interface{}{
"error": err.Error(),
})
}
return ErrConnectionFailed(err.Error())
}
if s.logger != nil {
s.logger.Debug("Health check passed", map[string]interface{}{})
}
return nil
}
// NewChatBuilder creates a new chat builder for conversational interactions
func (s *Service) NewChatBuilder() ChatBuilder {
return &chatBuilder{
service: s,
request: &ChatRequest{
Model: s.config.DefaultModel,
Messages: make([]Message, 0),
Options: ChatOptions{
Temperature: &s.config.DefaultTemperature,
TopP: &s.config.DefaultTopP,
},
},
}
}
// chatBuilder implements the ChatBuilder interface
type chatBuilder struct {
service OllamaService
request *ChatRequest
}
func (b *chatBuilder) SetModel(model string) ChatBuilder {
b.request.Model = model
return b
}
func (b *chatBuilder) SetSystem(system string) ChatBuilder {
// Add or update system message
for i, msg := range b.request.Messages {
if msg.Role == MessageRoleSystem {
b.request.Messages[i].Content = system
return b
}
}
// Add new system message at the beginning
systemMsg := Message{
Role: MessageRoleSystem,
Content: system,
}
b.request.Messages = append([]Message{systemMsg}, b.request.Messages...)
return b
}
func (b *chatBuilder) AddMessage(role MessageRole, content string) ChatBuilder {
msg := Message{
Role: role,
Content: content,
}
b.request.Messages = append(b.request.Messages, msg)
return b
}
func (b *chatBuilder) SetTemperature(temperature float64) ChatBuilder {
b.request.Options.Temperature = &temperature
return b
}
func (b *chatBuilder) SetTopP(topP float64) ChatBuilder {
b.request.Options.TopP = &topP
return b
}
func (b *chatBuilder) SetMaxTokens(maxTokens int) ChatBuilder {
b.request.Options.NumPredict = &maxTokens
return b
}
func (b *chatBuilder) SetStream(stream bool) ChatBuilder {
b.request.Stream = &stream
return b
}
func (b *chatBuilder) SetFormat(format string) ChatBuilder {
b.request.Format = format
return b
}
func (b *chatBuilder) SetOptions(options map[string]interface{}) ChatBuilder {
if b.request.Options.Options == nil {
b.request.Options.Options = make(map[string]interface{})
}
for k, v := range options {
b.request.Options.Options[k] = v
}
return b
}
func (b *chatBuilder) Build() (*ChatRequest, error) {
if len(b.request.Messages) == 0 {
return nil, ErrInvalidRequest("at least one message is required")
}
return b.request, nil
}
func (b *chatBuilder) Send(ctx context.Context) (*ChatResponse, error) {
req, err := b.Build()
if err != nil {
return nil, err
}
return b.service.Chat(ctx, req)
}
func (b *chatBuilder) Continue(ctx context.Context, userMessage string) (*ChatResponse, error) {
// Add user message
b.AddMessage(MessageRoleUser, userMessage)
// Send chat request
resp, err := b.Send(ctx)
if err != nil {
return nil, err
}
// Add assistant response to conversation history
b.AddMessage(MessageRoleAssistant, resp.Message.Content)
return resp, nil
}
pkg/schedule/cron.go
+4 -4
View File
@@ -2,7 +2,7 @@ package schedule
import ( import (
"time" "time"
"tm/internal/tender" "tm/internal/notice"
"tm/pkg/logger" "tm/pkg/logger"
"tm/pkg/mongo" "tm/pkg/mongo"
@@ -13,7 +13,7 @@ import (
type CronScheduler struct { type CronScheduler struct {
cron *cron.Cron cron *cron.Cron
mongoManager *mongo.ConnectionManager mongoManager *mongo.ConnectionManager
tenderRepo tender.TenderRepository noticeRepo notice.Repository
logger logger.Logger logger logger.Logger
} }
@@ -31,14 +31,14 @@ type Job struct {
} }
// NewCronScheduler creates a new cron scheduler instance // NewCronScheduler creates a new cron scheduler instance
func NewCronScheduler(logger logger.Logger, mongoManager *mongo.ConnectionManager, tenderRepo tender.TenderRepository) *CronScheduler { func NewCronScheduler(logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository) *CronScheduler {
// Create cron with timezone support and seconds precision // Create cron with timezone support and seconds precision
c := cron.New(cron.WithLocation(time.UTC), cron.WithSeconds()) c := cron.New(cron.WithLocation(time.UTC), cron.WithSeconds())
return &CronScheduler{ return &CronScheduler{
cron: c, cron: c,
mongoManager: mongoManager, mongoManager: mongoManager,
tenderRepo: tenderRepo, noticeRepo: noticeRepo,
logger: logger, logger: logger,
} }
} }
ted/mapper.go
+21 -21
View File
@@ -4,11 +4,11 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"tm/internal/tender" "tm/internal/notice"
) )
// mapToTenderFromParsedDoc maps a parsed TED document to tender entity // mapToTenderFromParsedDoc maps a parsed TED document to tender entity
func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *tender.Tender { func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice {
switch parsedDoc.Type { switch parsedDoc.Type {
case DocumentTypeContractNotice: case DocumentTypeContractNotice:
if parsedDoc.ContractNotice == nil { if parsedDoc.ContractNotice == nil {
@@ -45,7 +45,7 @@ func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceF
} }
// mapGenericNoticeToTender maps any TED document type to tender entity using the common interface // mapGenericNoticeToTender maps any TED document type to tender entity using the common interface
func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *tender.Tender { func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *notice.Notice {
if doc == nil { if doc == nil {
s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{}) s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{})
return nil return nil
@@ -53,7 +53,7 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so
now := time.Now().Unix() now := time.Now().Unix()
t := &tender.Tender{ t := &notice.Notice{
ContractNoticeID: doc.GetID(), ContractNoticeID: doc.GetID(),
ContractFolderID: "", // Not available in generic interface ContractFolderID: "", // Not available in generic interface
NoticeTypeCode: doc.GetNoticeType(), NoticeTypeCode: doc.GetNoticeType(),
@@ -68,11 +68,11 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so
// Basic tender information // Basic tender information
Title: "Generic Notice " + doc.GetNoticeType(), Title: "Generic Notice " + doc.GetNoticeType(),
Description: "Processed from " + doc.GetNoticeType() + " document", Description: "Processed from " + doc.GetNoticeType() + " document",
Status: tender.TenderStatusActive, Status: notice.TenderStatusActive,
Source: tender.TenderSourceTEDScraper, Source: notice.TenderSourceTEDScraper,
SourceFileURL: sourceFileURL, SourceFileURL: sourceFileURL,
SourceFileName: sourceFileName, SourceFileName: sourceFileName,
ProcessingMetadata: tender.ProcessingMetadata{ ProcessingMetadata: notice.ProcessingMetadata{
ScrapedAt: now, ScrapedAt: now,
ProcessedAt: now, ProcessedAt: now,
ProcessingVersion: "1.0", ProcessingVersion: "1.0",
@@ -86,10 +86,10 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so
} }
// mapToTender maps a TED ContractNotice to tender entity // mapToTender maps a TED ContractNotice to tender entity
func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *tender.Tender { func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *notice.Notice {
now := time.Now().Unix() now := time.Now().Unix()
t := &tender.Tender{ t := &notice.Notice{
ContractNoticeID: cn.ID, ContractNoticeID: cn.ID,
ContractFolderID: cn.ContractFolderID, ContractFolderID: cn.ContractFolderID,
NoticeTypeCode: cn.GetNoticeSubType(), NoticeTypeCode: cn.GetNoticeSubType(),
@@ -105,11 +105,11 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
MainClassification: cn.GetMainClassification(), MainClassification: cn.GetMainClassification(),
PlaceOfPerformance: cn.GetPlaceOfPerformance(), PlaceOfPerformance: cn.GetPlaceOfPerformance(),
DocumentURI: cn.GetDocumentURI(), DocumentURI: cn.GetDocumentURI(),
Status: tender.TenderStatusActive, Status: notice.TenderStatusActive,
Source: tender.TenderSourceTEDScraper, Source: notice.TenderSourceTEDScraper,
SourceFileURL: sourceFileURL, SourceFileURL: sourceFileURL,
SourceFileName: sourceFileName, SourceFileName: sourceFileName,
ProcessingMetadata: tender.ProcessingMetadata{ ProcessingMetadata: notice.ProcessingMetadata{
ScrapedAt: now, ScrapedAt: now,
ProcessedAt: now, ProcessedAt: now,
ProcessingVersion: "1.0", ProcessingVersion: "1.0",
@@ -215,7 +215,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
// Set selection criteria // Set selection criteria
if criteria := cn.GetSelectionCriteria(); criteria != nil { if criteria := cn.GetSelectionCriteria(); criteria != nil {
for _, criterion := range criteria { for _, criterion := range criteria {
t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{ t.SelectionCriteria = append(t.SelectionCriteria, notice.SelectionCriterion{
TypeCode: criterion.CriterionTypeCode, TypeCode: criterion.CriterionTypeCode,
Description: criterion.Description, Description: criterion.Description,
LanguageID: criterion.LanguageID, LanguageID: criterion.LanguageID,
@@ -235,10 +235,10 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa
} }
// mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity // mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity
func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *tender.Tender { func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *notice.Notice {
now := time.Now().Unix() now := time.Now().Unix()
t := &tender.Tender{ t := &notice.Notice{
ContractNoticeID: can.ID, ContractNoticeID: can.ID,
ContractFolderID: can.ContractFolderID, ContractFolderID: can.ContractFolderID,
NoticeTypeCode: can.NoticeTypeCode, NoticeTypeCode: can.NoticeTypeCode,
@@ -252,11 +252,11 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
ProcurementTypeCode: can.GetContractNature(), ProcurementTypeCode: can.GetContractNature(),
ProcedureCode: can.GetProcedureCode(), ProcedureCode: can.GetProcedureCode(),
MainClassification: can.GetMainClassification(), MainClassification: can.GetMainClassification(),
Status: tender.TenderStatusAwarded, // Award notices are for completed tenders Status: notice.TenderStatusAwarded, // Award notices are for completed tenders
Source: tender.TenderSourceTEDScraper, Source: notice.TenderSourceTEDScraper,
SourceFileURL: sourceFileURL, SourceFileURL: sourceFileURL,
SourceFileName: sourceFileName, SourceFileName: sourceFileName,
ProcessingMetadata: tender.ProcessingMetadata{ ProcessingMetadata: notice.ProcessingMetadata{
ScrapedAt: now, ScrapedAt: now,
ProcessedAt: now, ProcessedAt: now,
ProcessingVersion: "1.0", ProcessingVersion: "1.0",
@@ -315,13 +315,13 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so
} }
// mapOrganization maps a TED Organization to tender Organization // mapOrganization maps a TED Organization to tender Organization
func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.Organization { func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.Organization {
if tedOrg == nil || tedOrg.Company == nil { if tedOrg == nil || tedOrg.Company == nil {
return nil return nil
} }
company := tedOrg.Company company := tedOrg.Company
org := &tender.Organization{ org := &notice.Organization{
Role: role, Role: role,
} }
@@ -351,7 +351,7 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.
// Set address // Set address
if company.PostalAddress != nil { if company.PostalAddress != nil {
org.Address = tender.Address{ org.Address = notice.Address{
StreetName: company.PostalAddress.StreetName, StreetName: company.PostalAddress.StreetName,
CityName: company.PostalAddress.CityName, CityName: company.PostalAddress.CityName,
PostalZone: company.PostalAddress.PostalZone, PostalZone: company.PostalAddress.PostalZone,
ted/scraper.go
+39 -35
View File
@@ -10,7 +10,7 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"time" "time"
"tm/internal/tender" "tm/internal/notice"
"tm/pkg/logger" "tm/pkg/logger"
"tm/pkg/mongo" "tm/pkg/mongo"
"tm/pkg/notification" "tm/pkg/notification"
@@ -27,6 +27,7 @@ type Config struct {
RetryDelay time.Duration `mapstructure:"retry_delay"` RetryDelay time.Duration `mapstructure:"retry_delay"`
CleanupAfter time.Duration `mapstructure:"cleanup_after"` CleanupAfter time.Duration `mapstructure:"cleanup_after"`
ScrapingInterval string `mapstructure:"scraping_interval"` ScrapingInterval string `mapstructure:"scraping_interval"`
AlertMail string `mapstructure:"ALERT_MAIL"`
} }
// TEDScraper handles downloading and parsing TED XML files // TEDScraper handles downloading and parsing TED XML files
@@ -37,7 +38,7 @@ type TEDScraper struct {
httpClient *http.Client httpClient *http.Client
xmlParser *TEDParser xmlParser *TEDParser
mongoManager *mongo.ConnectionManager mongoManager *mongo.ConnectionManager
tenderRepo tender.TenderRepository tenderRepo notice.Repository
} }
// NewTEDScraper creates a new TED scraper instance // NewTEDScraper creates a new TED scraper instance
@@ -45,7 +46,7 @@ func NewTEDScraper(
config *Config, config *Config,
logger logger.Logger, logger logger.Logger,
mongoManager *mongo.ConnectionManager, mongoManager *mongo.ConnectionManager,
tenderRepo tender.TenderRepository, tenderRepo notice.Repository,
notify notification.SDK, notify notification.SDK,
) *TEDScraper { ) *TEDScraper {
httpClient := &http.Client{ httpClient := &http.Client{
@@ -251,6 +252,8 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
} }
} }
if s.config.AlertMail != "" {
go s.notify.SendNotification( go s.notify.SendNotification(
context.Background(), context.Background(),
&notification.NotificationRequest{ &notification.NotificationRequest{
@@ -266,16 +269,17 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o
}), }),
Priority: "important", Priority: "important",
Methods: notification.NotificationMethods{ Methods: notification.NotificationMethods{
Email: "nakhostin.nima1998@gmail.com", Email: s.config.AlertMail,
}, },
UserID: "ted-scraper", UserID: "ted-scraper",
Type: "alert", Type: "alert",
}) })
}
return result, nil return result, nil
} }
func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*tender.Tender, error) { func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) {
existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID) existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID)
if err != nil { if err != nil {
if err.Error() == "document not found" { if err.Error() == "document not found" {
@@ -308,37 +312,37 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
// Map to tender entity // Map to tender entity
t := s.mapToTenderFromParsedDoc(notice, "", fileName) t := s.mapToTenderFromParsedDoc(notice, "", fileName)
existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID) // existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID)
if err != nil { // if err != nil {
s.logger.Warn("Error checking for existing tender", map[string]interface{}{ // s.logger.Warn("Error checking for existing tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID, // "contract_notice_id": t.ContractNoticeID,
"error": err.Error(), // "error": err.Error(),
}) // })
} // }
if err == nil && existingTender != nil { // if err == nil && existingTender != nil {
// Update existing tender // // Update existing tender
s.logger.Info("Found existing tender, updating", map[string]interface{}{ // s.logger.Info("Found existing tender, updating", map[string]interface{}{
"existing_tender_id": existingTender.ID.Hex(), // "existing_tender_id": existingTender.ID.Hex(),
"contract_notice_id": t.ContractNoticeID, // "contract_notice_id": t.ContractNoticeID,
}) // })
t.ID = existingTender.ID // t.ID = existingTender.ID
t.UpdatedAt = time.Now().Unix() // t.UpdatedAt = time.Now().Unix()
err = s.tenderRepo.Update(ctx, t) // err = s.tenderRepo.Update(ctx, t)
if err != nil { // if err != nil {
s.logger.Error("Failed to update existing tender", map[string]interface{}{ // s.logger.Error("Failed to update existing tender", map[string]interface{}{
"tender_id": t.ID.Hex(), // "tender_id": t.ID.Hex(),
"error": err.Error(), // "error": err.Error(),
}) // })
return fmt.Errorf("failed to update existing tender: %w", err) // return fmt.Errorf("failed to update existing tender: %w", err)
} // }
s.logger.Info("Successfully updated tender", map[string]interface{}{ // s.logger.Info("Successfully updated tender", map[string]interface{}{
"tender_id": t.ID.Hex(), // "tender_id": t.ID.Hex(),
}) // })
} else { // } else {
// Create new tender // // Create new tender
s.logger.Info("Creating new tender", map[string]interface{}{ s.logger.Info("Creating new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID, "contract_notice_id": t.ContractNoticeID,
"tender_id": t.TenderID, "tender_id": t.TenderID,
@@ -347,7 +351,7 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
t.CreatedAt = time.Now().Unix() t.CreatedAt = time.Now().Unix()
t.UpdatedAt = t.CreatedAt t.UpdatedAt = t.CreatedAt
err = s.tenderRepo.Create(ctx, t) err = s.tenderRepo.Import(ctx, t)
if err != nil { if err != nil {
s.logger.Error("Failed to create new tender", map[string]interface{}{ s.logger.Error("Failed to create new tender", map[string]interface{}{
"contract_notice_id": t.ContractNoticeID, "contract_notice_id": t.ContractNoticeID,
@@ -361,7 +365,7 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file
"tender_id": t.TenderID, "tender_id": t.TenderID,
"contract_notice_id": t.ContractNoticeID, "contract_notice_id": t.ContractNoticeID,
}) })
} // }
// Log detailed parsed information based on notice type // Log detailed parsed information based on notice type
s.logger.Info("Successfully parsed notice", map[string]interface{}{ s.logger.Info("Successfully parsed notice", map[string]interface{}{
ted/service.go
+2 -2
View File
@@ -5,7 +5,7 @@ import (
"strconv" "strconv"
"strings" "strings"
"time" "time"
"tm/internal/tender" "tm/internal/notice"
) )
// parseDate parses various date formats used in TED XML // parseDate parses various date formats used in TED XML
@@ -146,7 +146,7 @@ func GenerateTenderURL(noticePublicationID string) string {
// generateTenderID generates a unique tender ID using the format: // generateTenderID generates a unique tender ID using the format:
// {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} // {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
func GenerateTenderID(t *tender.Tender) string { func GenerateTenderID(t *notice.Notice) string {
var parts []string var parts []string
// Source (TED) // Source (TED)