diff --git a/cmd/scraper/bootstrap/bootstrap.go b/cmd/scraper/bootstrap/bootstrap.go index 2fa2a2c..1f866c4 100644 --- a/cmd/scraper/bootstrap/bootstrap.go +++ b/cmd/scraper/bootstrap/bootstrap.go @@ -7,7 +7,7 @@ import ( "os/signal" "syscall" "time" - "tm/internal/tender" + "tm/internal/notice" "tm/pkg/config" "tm/pkg/logger" "tm/pkg/mongo" @@ -78,7 +78,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa // init TED scraper func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) { // Initialize tender repository - tenderRepo := tender.NewRepository(mongoManager, appLogger) + noticeRepo := notice.NewRepository(mongoManager, appLogger) // Initialize TED scraper tedScraper := ted.NewTEDScraper( @@ -92,15 +92,16 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog DownloadDir: config.TED.DownloadDir, CleanupAfter: config.TED.CleanupAfter, ScrapingInterval: config.TED.ScrapingInterval, + AlertMail: config.AlertMail, }, appLogger, mongoManager, - tenderRepo, + noticeRepo, notify, ) // start TED scraper job - schedule.NewCronScheduler(appLogger, mongoManager, tenderRepo).AddJob(schedule.Job{ + schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ Name: "TED Scraper Job", Func: func() { _ = tedScraper.Run(context.Background()) diff --git a/cmd/scraper/bootstrap/config.go b/cmd/scraper/bootstrap/config.go index 7766ae2..28fc584 100644 --- a/cmd/scraper/bootstrap/config.go +++ b/cmd/scraper/bootstrap/config.go @@ -11,6 +11,7 @@ type Config struct { Logging config.LoggingConfig TED ScraperConfig Notification config.NotificationConfig + AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"` } type ScraperConfig struct { diff --git a/cmd/scraper/main.go b/cmd/scraper/main.go index 4bf7395..ac77140 100644 --- a/cmd/scraper/main.go +++ b/cmd/scraper/main.go @@ -22,8 +22,6 @@ func main() { "version": "1.0.0", }) - // Initialize notification service - // Initialize MongoDB connection mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger) defer func() { @@ -36,7 +34,10 @@ func main() { } }() + // Initialize notification service notificationService := bootstrap.InitNotificationService(config.Notification, appLogger) + + // Initialize TED scraper bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService) // Set up signal handling for graceful shutdown @@ -47,8 +48,5 @@ func main() { <-signalChan appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{}) - // Stop the scheduler - // scheduler.Stop() - appLogger.Info("TED scraper application stopped", map[string]interface{}{}) } diff --git a/cmd/web/bootstrap/bootstrap.go b/cmd/web/bootstrap/bootstrap.go index c42c98c..63402b0 100644 --- a/cmd/web/bootstrap/bootstrap.go +++ b/cmd/web/bootstrap/bootstrap.go @@ -8,6 +8,7 @@ import ( "tm/pkg/logger" "tm/pkg/mongo" "tm/pkg/notification" + "tm/pkg/ollama" "tm/pkg/redis" "github.com/labstack/echo/v4" @@ -213,3 +214,38 @@ func InitNotificationService(conf config.NotificationConfig, log logger.Logger) return *notificationSDK } + +// Init Ollama Service +func InitOllamaService(conf config.OllamaConfig, log logger.Logger) ollama.SDK { + ollamaConfig := &ollama.Config{ + BaseURL: conf.BaseURL, + Timeout: conf.Timeout, + RetryAttempts: conf.RetryAttempts, + RetryDelay: conf.RetryDelay, + EnableLogging: conf.EnableLogging, + UserAgent: conf.UserAgent, + DefaultModel: conf.DefaultModel, + DefaultTemperature: conf.DefaultTemperature, + DefaultTopP: conf.DefaultTopP, + DefaultMaxTokens: conf.DefaultMaxTokens, + EnableStreaming: conf.EnableStreaming, + } + + ollamaSDK, err := ollama.New(ollamaConfig, log) + if err != nil { + log.Error("Failed to initialize Ollama SDK", map[string]interface{}{ + "error": err.Error(), + }) + } + + log.Info("Ollama SDK initialized successfully", map[string]interface{}{ + "base_url": conf.BaseURL, + "timeout": conf.Timeout, + "retry_attempts": conf.RetryAttempts, + "retry_delay": conf.RetryDelay, + "enable_logging": conf.EnableLogging, + "user_agent": conf.UserAgent, + }) + + return *ollamaSDK +} diff --git a/cmd/web/bootstrap/config.go b/cmd/web/bootstrap/config.go index 0f9f9b7..51dbad5 100644 --- a/cmd/web/bootstrap/config.go +++ b/cmd/web/bootstrap/config.go @@ -13,6 +13,7 @@ type Config struct { RateLimit config.RateLimitConfig Assets config.AssetsConfig Notification config.NotificationConfig + Ollama config.OllamaConfig UserAuth AuthConfig CustomerAuth AuthConfig } diff --git a/cmd/worker/bootstrap/bootstrap.go b/cmd/worker/bootstrap/bootstrap.go new file mode 100644 index 0000000..844ac09 --- /dev/null +++ b/cmd/worker/bootstrap/bootstrap.go @@ -0,0 +1,150 @@ +package bootstrap + +import ( + "fmt" + "time" + "tm/cmd/worker/workers" + "tm/internal/notice" + "tm/internal/tender" + "tm/pkg/config" + "tm/pkg/logger" + "tm/pkg/mongo" + "tm/pkg/notification" + "tm/pkg/ollama" + "tm/pkg/schedule" +) + +// Init Application Configuration +func InitConfig() (*Config, error) { + conf, err := config.LoadConfig(".", &Config{}) + if err != nil { + return nil, fmt.Errorf("failed to load config: %v", err) + } + + return conf, nil +} + +// Init Logger Service +func InitLogger(conf config.LoggingConfig) logger.Logger { + return logger.NewLogger(&logger.Config{ + Level: conf.Level, + Format: conf.Format, + Output: conf.Output, + File: logger.FileConfig{ + Path: conf.File.Path, + MaxSize: conf.File.MaxSize, + MaxBackups: conf.File.MaxBackups, + MaxAge: conf.File.MaxAge, + Compress: conf.File.Compress, + }, + }) +} + +// Init MongoDB Connection Manager +func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager { + // Convert infra.MongoConfig to mongo.ConnectionConfig + connectionConfig := mongo.ConnectionConfig{ + URI: conf.URI, + Database: conf.Name, + MaxPoolSize: uint64(conf.MaxPoolSize), + MinPoolSize: 5, // Default minimum pool size + MaxConnIdleTime: 30 * time.Minute, // Default idle time + ConnectTimeout: conf.Timeout, + ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout + } + + // Create MongoDB connection manager + connectionManager, err := mongo.NewConnectionManager(connectionConfig, log) + if err != nil { + log.Error("Failed to initialize MongoDB connection", map[string]interface{}{ + "error": err.Error(), + "uri": conf.URI, + "db": conf.Name, + }) + panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err)) + } + + log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{ + "database": conf.Name, + "uri": conf.URI, + "pool_size": conf.MaxPoolSize, + }) + + return connectionManager +} + +// Init Worker +func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, ollamaSDK *ollama.SDK) { + // Initialize repositories + noticeRepo := notice.NewRepository(mongoManager, appLogger) + tenderRepo := tender.NewRepository(mongoManager, appLogger) + + worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK) + worker.Run() + + // start Worker job + schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{ + Name: "Worker Job", + Func: func() { + worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK) + worker.Run() + }, + Expr: config.Worker.Interval, + }) + +} + +// Init Notification Service +func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK { + notificationConfig := ¬ification.Config{ + BaseURL: conf.BaseURL, + Timeout: conf.Timeout, + RetryAttempts: conf.RetryAttempts, + RetryDelay: conf.RetryDelay, + EnableLogging: conf.EnableLogging, + UserAgent: conf.UserAgent, + } + + notificationSDK, err := notification.New(notificationConfig, log) + if err != nil { + log.Error("Failed to initialize Notification SDK", map[string]interface{}{ + "error": err.Error(), + }) + fmt.Printf("Failed to initialize Notification SDK: %v\n", err.Error()) + } + + log.Info("Notification SDK initialized successfully", map[string]interface{}{ + "base_url": conf.BaseURL, + "timeout": conf.Timeout, + "retry_attempts": conf.RetryAttempts, + "retry_delay": conf.RetryDelay, + "enable_logging": conf.EnableLogging, + "user_agent": conf.UserAgent, + }) + + return *notificationSDK +} + +// Init Ollama Service +func InitOllamaService(conf config.OllamaConfig, log logger.Logger) *ollama.SDK { + ollamaSDK, err := ollama.New(&ollama.Config{ + BaseURL: conf.BaseURL, + Timeout: conf.Timeout, + RetryAttempts: conf.RetryAttempts, + RetryDelay: conf.RetryDelay, + EnableLogging: conf.EnableLogging, + UserAgent: conf.UserAgent, + DefaultModel: conf.DefaultModel, + DefaultTemperature: conf.DefaultTemperature, + DefaultTopP: conf.DefaultTopP, + DefaultMaxTokens: conf.DefaultMaxTokens, + EnableStreaming: conf.EnableStreaming, + }, log) + if err != nil { + log.Error("Failed to initialize Ollama SDK", map[string]interface{}{ + "error": err.Error(), + }) + } + + return ollamaSDK +} diff --git a/cmd/worker/bootstrap/config.go b/cmd/worker/bootstrap/config.go new file mode 100644 index 0000000..8c3485c --- /dev/null +++ b/cmd/worker/bootstrap/config.go @@ -0,0 +1,19 @@ +package bootstrap + +import ( + "tm/pkg/config" +) + +// Config defines the scraper application configuration +type Config struct { + Database config.DatabaseConfig + Logging config.LoggingConfig + Notification config.NotificationConfig + Ollama config.OllamaConfig + Worker WorkerConfig + AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"` +} + +type WorkerConfig struct { + Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"` +} diff --git a/cmd/worker/main.go b/cmd/worker/main.go new file mode 100644 index 0000000..e01131e --- /dev/null +++ b/cmd/worker/main.go @@ -0,0 +1,62 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + "time" + "tm/cmd/worker/bootstrap" +) + +func main() { + // Load configuration + config, err := bootstrap.InitConfig() + if err != nil { + panic(err) + } + + // Initialize logger + appLogger := bootstrap.InitLogger(config.Logging) + appLogger.Info("Starting worker application", map[string]interface{}{ + "version": "1.0.0", + }) + + // Initialize MongoDB connection + mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger) + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := mongoManager.Close(ctx); err != nil { + appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{ + "error": err.Error(), + }) + } + }() + + // Initialize notification service + notificationService := bootstrap.InitNotificationService(config.Notification, appLogger) + + // Initialize ollama service + ollamaSDK := bootstrap.InitOllamaService(config.Ollama, appLogger) + aiModels, err := ollamaSDK.ListModels(context.Background()) + if err != nil { + appLogger.Error("Failed to list ollama models", map[string]interface{}{ + "error": err.Error(), + }) + } + appLogger.Info("Ollama models", map[string]interface{}{"models": aiModels}) + + // Initialize Worker + bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, ollamaSDK) + + // Set up signal handling for graceful shutdown + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM) + + // Wait for shutdown signal + <-signalChan + appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{}) + + appLogger.Info("Worker application stopped", map[string]interface{}{}) +} diff --git a/cmd/worker/workers/notice.go b/cmd/worker/workers/notice.go new file mode 100644 index 0000000..a9e7d82 --- /dev/null +++ b/cmd/worker/workers/notice.go @@ -0,0 +1,273 @@ +package workers + +import ( + "context" + "tm/internal/notice" + "tm/internal/tender" + "tm/pkg/logger" + "tm/pkg/mongo" + "tm/pkg/notification" + "tm/pkg/ollama" +) + +type NoticeWorker struct { + Mongo *mongo.ConnectionManager + Logger logger.Logger + Notify notification.SDK + Ollama *ollama.SDK + NoticeRepo notice.Repository + TenderRepo tender.TenderRepository +} + +func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, ollama *ollama.SDK) *NoticeWorker { + return &NoticeWorker{ + Mongo: mongo, + Logger: logger, + Notify: notify, + Ollama: ollama, + NoticeRepo: noticeRepo, + TenderRepo: tenderRepo, + } +} + +func (w *NoticeWorker) Run() { + w.Logger.Info("Notice worker started", map[string]interface{}{}) + + limit := 10 + skip := 0 + for { + notices, _, err := w.NoticeRepo.GetUnProcessedNotices(context.Background(), limit, skip) + if err != nil { + w.Logger.Error("Failed to get notices", map[string]interface{}{ + "error": err.Error(), + }) + continue + } + + if len(notices) == 0 { + break + } + + for _, n := range notices { + w.Logger.Info("Notice", map[string]interface{}{ + "notice": n.ID.Hex(), + }) + + t, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID) + if err == nil { + continue + } + + err = w.ToTender(&n, t) + if err != nil { + w.Logger.Error("Failed to create tender", map[string]interface{}{ + "error": err.Error(), + }) + continue + } + + if t.ID.IsZero() { + err = w.TenderRepo.Create(context.Background(), t) + if err != nil { + w.Logger.Error("Failed to create tender", map[string]interface{}{ + "error": err.Error(), + }) + } + } else { + err = w.TenderRepo.Update(context.Background(), t) + if err != nil { + w.Logger.Error("Failed to update tender", map[string]interface{}{ + "error": err.Error(), + }) + } + } + } + + skip += limit + } +} + +func (w *NoticeWorker) ToTender(n *notice.Notice, t *tender.Tender) error { + // Buyer Organization + buyerOrg := &tender.Organization{} + if n.BuyerOrganization != nil { + buyerOrg = &tender.Organization{ + Name: n.BuyerOrganization.Name, + CompanyID: n.BuyerOrganization.CompanyID, + WebsiteURI: n.BuyerOrganization.WebsiteURI, + ContactName: n.BuyerOrganization.ContactName, + ContactTelephone: n.BuyerOrganization.ContactTelephone, + ContactEmail: n.BuyerOrganization.ContactEmail, + ContactFax: n.BuyerOrganization.ContactFax, + Role: n.BuyerOrganization.Role, + Address: tender.Address{ + StreetName: n.BuyerOrganization.Address.StreetName, + CityName: n.BuyerOrganization.Address.CityName, + PostalZone: n.BuyerOrganization.Address.PostalZone, + CountrySubentityCode: n.BuyerOrganization.Address.CountrySubentityCode, + Department: n.BuyerOrganization.Address.Department, + Region: n.BuyerOrganization.Address.Region, + CountryCode: n.BuyerOrganization.Address.CountryCode, + }, + } + } + + // Review Organization + reviewOrg := &tender.Organization{} + if n.ReviewOrganization != nil { + reviewOrg = &tender.Organization{ + Name: n.ReviewOrganization.Name, + CompanyID: n.ReviewOrganization.CompanyID, + WebsiteURI: n.ReviewOrganization.WebsiteURI, + ContactName: n.ReviewOrganization.ContactName, + ContactTelephone: n.ReviewOrganization.ContactTelephone, + ContactEmail: n.ReviewOrganization.ContactEmail, + ContactFax: n.ReviewOrganization.ContactFax, + Role: n.ReviewOrganization.Role, + Address: tender.Address{ + StreetName: n.ReviewOrganization.Address.StreetName, + CityName: n.ReviewOrganization.Address.CityName, + PostalZone: n.ReviewOrganization.Address.PostalZone, + CountrySubentityCode: n.ReviewOrganization.Address.CountrySubentityCode, + Department: n.ReviewOrganization.Address.Department, + Region: n.ReviewOrganization.Address.Region, + CountryCode: n.ReviewOrganization.Address.CountryCode, + }, + } + } + + // Organizations + organizations := make([]tender.Organization, len(n.Organizations)) + for i, org := range n.Organizations { + organizations[i] = tender.Organization{ + Name: org.Name, + CompanyID: org.CompanyID, + WebsiteURI: org.WebsiteURI, + ContactName: org.ContactName, + ContactTelephone: org.ContactTelephone, + ContactEmail: org.ContactEmail, + ContactFax: org.ContactFax, + Role: org.Role, + Address: tender.Address{ + StreetName: org.Address.StreetName, + CityName: org.Address.CityName, + PostalZone: org.Address.PostalZone, + CountrySubentityCode: org.Address.CountrySubentityCode, + Department: org.Address.Department, + Region: org.Address.Region, + CountryCode: org.Address.CountryCode, + }, + } + } + + // Selection Criteria + selectionCriteria := make([]tender.SelectionCriterion, len(n.SelectionCriteria)) + for i, criterion := range n.SelectionCriteria { + selectionCriteria[i] = tender.SelectionCriterion{ + TypeCode: criterion.TypeCode, + Description: criterion.Description, + LanguageID: criterion.LanguageID, + } + } + + // Winning Tenderer + winningTenderer := &tender.Organization{} + if n.WinningTenderer != nil { + winningTenderer = &tender.Organization{ + Name: n.WinningTenderer.Name, + CompanyID: n.WinningTenderer.CompanyID, + WebsiteURI: n.WinningTenderer.WebsiteURI, + ContactName: n.WinningTenderer.ContactName, + ContactTelephone: n.WinningTenderer.ContactTelephone, + ContactEmail: n.WinningTenderer.ContactEmail, + ContactFax: n.WinningTenderer.ContactFax, + Role: n.WinningTenderer.Role, + Address: tender.Address{ + StreetName: n.WinningTenderer.Address.StreetName, + CityName: n.WinningTenderer.Address.CityName, + PostalZone: n.WinningTenderer.Address.PostalZone, + CountrySubentityCode: n.WinningTenderer.Address.CountrySubentityCode, + Department: n.WinningTenderer.Address.Department, + Region: n.WinningTenderer.Address.Region, + CountryCode: n.WinningTenderer.Address.CountryCode, + }, + } + } + + // Modifications + modifications := make([]tender.TenderModification, len(n.Modifications)) + for i, modification := range n.Modifications { + modifications[i] = tender.TenderModification{ + ModificationDate: modification.ModificationDate, + ModificationReason: modification.ModificationReason, + Description: modification.Description, + LanguageID: modification.LanguageID, + } + } + + // Awarded Entities + awardedEntities := make([]tender.Awarded, len(n.AwardedEntities)) + for i, awarded := range n.AwardedEntities { + awardedEntities[i] = tender.Awarded{ + Name: awarded.Name, + Address: awarded.Address, + Country: awarded.Country, + Amount: awarded.Amount, + Currency: awarded.Currency, + Share: awarded.Share, + AwardDate: awarded.AwardDate, + ContractID: awarded.ContractID, + TenderID: awarded.TenderID, + LotID: awarded.LotID, + CompanyID: awarded.CompanyID, + OrganizationID: awarded.OrganizationID, + } + } + + t.NoticePublicationID = n.NoticePublicationID + t.Title = n.Title + t.Description = n.Description + t.ProcurementTypeCode = n.ProcurementTypeCode + t.ProcedureCode = n.ProcedureCode + t.EstimatedValue = n.EstimatedValue + t.Currency = n.Currency + t.TenderDeadline = n.TenderDeadline + t.SubmissionURL = n.SubmissionURL + t.BuyerOrganization = buyerOrg + t.ReviewOrganization = reviewOrg + t.Organizations = organizations + t.SelectionCriteria = selectionCriteria + t.WinningTenderer = winningTenderer + t.Modifications = modifications + t.AwardedEntities = awardedEntities + t.NoticeTypeCode = n.NoticeTypeCode + t.NoticeSubTypeCode = n.NoticeSubTypeCode + t.NoticeLanguageCode = n.NoticeLanguageCode + t.IssueDate = n.IssueDate + t.IssueTime = n.IssueTime + t.PublicationDate = n.PublicationDate + t.SubmissionDeadline = n.SubmissionDeadline + t.ApplicationDeadline = n.ApplicationDeadline + t.GazetteID = n.GazetteID + t.PlaceOfPerformance = n.PlaceOfPerformance + t.CountryCode = n.CountryCode + t.RegionCode = n.RegionCode + t.CityName = n.CityName + t.PostalCode = n.PostalCode + t.DocumentURI = n.DocumentURI + t.TenderURL = n.TenderURL + t.MainClassification = n.MainClassification + t.AdditionalClassifications = n.AdditionalClassifications + t.OfficialLanguages = n.OfficialLanguages + t.CancellationReason = n.CancellationReason + t.CancellationDate = n.CancellationDate + t.AwardDate = n.AwardDate + t.AwardedValue = n.AwardedValue + t.ContractNumber = n.ContractNumber + t.SuspensionReason = n.SuspensionReason + t.SuspensionDate = n.SuspensionDate + t.WinningTenderer = winningTenderer + t.Modifications = modifications + t.AwardedEntities = awardedEntities + return nil +} diff --git a/internal/notice/entity.go b/internal/notice/entity.go new file mode 100644 index 0000000..5f8b0bf --- /dev/null +++ b/internal/notice/entity.go @@ -0,0 +1,276 @@ +package notice + +import ( + "fmt" + "strconv" + "strings" + "time" + "tm/pkg/mongo" +) + +// Tender represents a tender/contract notice entity +type Notice struct { + mongo.Model `bson:",inline"` + ContractNoticeID string `bson:"contract_notice_id" json:"contract_notice_id"` + NoticePublicationID string `bson:"notice_publication_id" json:"notice_publication_id"` + ContractFolderID string `bson:"contract_folder_id" json:"contract_folder_id"` + NoticeTypeCode string `bson:"notice_type_code" json:"notice_type_code"` + NoticeSubTypeCode string `bson:"notice_sub_type_code" json:"notice_sub_type_code"` + NoticeLanguageCode string `bson:"notice_language_code" json:"notice_language_code"` + IssueDate int64 `bson:"issue_date" json:"issue_date"` + IssueTime int64 `bson:"issue_time" json:"issue_time"` + TenderDeadline int64 `bson:"tender_deadline" json:"tender_deadline"` + PublicationDate int64 `bson:"publication_date" json:"publication_date"` + SubmissionDeadline int64 `bson:"submission_deadline" json:"submission_deadline"` + ApplicationDeadline int64 `bson:"application_deadline" json:"application_deadline"` + GazetteID string `bson:"gazette_id" json:"gazette_id"` + Title string `bson:"title" json:"title"` + Description string `bson:"description" json:"description"` + ProcurementTypeCode string `bson:"procurement_type_code" json:"procurement_type_code"` + ProcedureCode string `bson:"procedure_code" json:"procedure_code"` + MainClassification string `bson:"main_classification" json:"main_classification"` + AdditionalClassifications []string `bson:"additional_classifications" json:"additional_classifications"` + EstimatedValue float64 `bson:"estimated_value" json:"estimated_value"` + Currency string `bson:"currency" json:"currency"` + Duration string `bson:"duration" json:"duration"` + DurationUnit string `bson:"duration_unit" json:"duration_unit"` + PlaceOfPerformance string `bson:"place_of_performance" json:"place_of_performance"` + CountryCode string `bson:"country_code" json:"country_code"` + RegionCode string `bson:"region_code" json:"region_code"` + CityName string `bson:"city_name" json:"city_name"` + PostalCode string `bson:"postal_code" json:"postal_code"` + DocumentURI string `bson:"document_uri" json:"document_uri"` + TenderURL string `bson:"tender_url" json:"tender_url"` + SubmissionURL string `bson:"submission_url" json:"submission_url"` + BuyerOrganization *Organization `bson:"buyer_organization" json:"buyer_organization"` + ReviewOrganization *Organization `bson:"review_organization,omitempty" json:"review_organization,omitempty"` + Organizations []Organization `bson:"organizations" json:"organizations"` + SelectionCriteria []SelectionCriterion `bson:"selection_criteria" json:"selection_criteria"` + OfficialLanguages []string `bson:"official_languages" json:"official_languages"` + Status TenderStatus `bson:"status" json:"status"` + Source TenderSource `bson:"source" json:"source"` + SourceFileURL string `bson:"source_file_url" json:"source_file_url"` + SourceFileName string `bson:"source_file_name" json:"source_file_name"` + ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"` + TenderID string `bson:"tender_id" json:"tender_id"` // Implement logic to generate a unique tender_id using the following format: {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} + + // Status-specific fields + CancellationReason string `bson:"cancellation_reason,omitempty" json:"cancellation_reason,omitempty"` + CancellationDate int64 `bson:"cancellation_date,omitempty" json:"cancellation_date,omitempty"` + AwardDate int64 `bson:"award_date,omitempty" json:"award_date,omitempty"` + AwardedValue float64 `bson:"awarded_value,omitempty" json:"awarded_value,omitempty"` + WinningTenderer *Organization `bson:"winning_tenderer,omitempty" json:"winning_tenderer,omitempty"` + ContractNumber string `bson:"contract_number,omitempty" json:"contract_number,omitempty"` + SuspensionReason string `bson:"suspension_reason,omitempty" json:"suspension_reason,omitempty"` + SuspensionDate int64 `bson:"suspension_date,omitempty" json:"suspension_date,omitempty"` + Modifications []TenderModification `bson:"modifications,omitempty" json:"modifications,omitempty"` + AwardedEntities []Awarded `bson:"awarded,omitempty" json:"awarded,omitempty"` +} + +// Organization represents organization information +type Organization struct { + ID string `bson:"id" json:"id"` + Name string `bson:"name" json:"name"` + CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"` + WebsiteURI string `bson:"website_uri,omitempty" json:"website_uri,omitempty"` + ContactName string `bson:"contact_name,omitempty" json:"contact_name,omitempty"` + ContactTelephone string `bson:"contact_telephone,omitempty" json:"contact_telephone,omitempty"` + ContactEmail string `bson:"contact_email,omitempty" json:"contact_email,omitempty"` + ContactFax string `bson:"contact_fax,omitempty" json:"contact_fax,omitempty"` + Address Address `bson:"address" json:"address"` + Role string `bson:"role,omitempty" json:"role,omitempty"` +} + +// Address represents address information +type Address struct { + StreetName string `bson:"street_name,omitempty" json:"street_name,omitempty"` + CityName string `bson:"city_name,omitempty" json:"city_name,omitempty"` + PostalZone string `bson:"postal_zone,omitempty" json:"postal_zone,omitempty"` + CountrySubentityCode string `bson:"country_subentity_code,omitempty" json:"country_subentity_code,omitempty"` + Department string `bson:"department,omitempty" json:"department,omitempty"` + Region string `bson:"region,omitempty" json:"region,omitempty"` + CountryCode string `bson:"country_code" json:"country_code"` +} + +// SelectionCriterion represents selection criteria +type SelectionCriterion struct { + TypeCode string `bson:"type_code" json:"type_code"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"` +} + +// ProcessingMetadata contains metadata about how the tender was processed +type ProcessingMetadata struct { + ScrapedAt int64 `bson:"scraped_at" json:"scraped_at"` + ProcessedAt int64 `bson:"processed_at" json:"processed_at"` + ProcessingVersion string `bson:"processing_version" json:"processing_version"` + ParsingErrors []string `bson:"parsing_errors,omitempty" json:"parsing_errors,omitempty"` + ValidationErrors []string `bson:"validation_errors,omitempty" json:"validation_errors,omitempty"` + EnrichmentData map[string]string `bson:"enrichment_data,omitempty" json:"enrichment_data,omitempty"` + TranslatedData map[string]string `bson:"translated_data,omitempty" json:"translated_data,omitempty"` + TranslatedAt int64 `bson:"translated_at,omitempty" json:"translated_at,omitempty"` + Processed bool `bson:"processed" json:"processed"` +} + +// TenderModification represents a modification made to a tender +type TenderModification struct { + ModificationDate int64 `bson:"modification_date" json:"modification_date"` // Unix milliseconds + ModificationReason string `bson:"modification_reason" json:"modification_reason"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + LanguageID string `bson:"language_id,omitempty" json:"language_id,omitempty"` +} + +// AwardedEntity represents an entity that has been awarded a contract +type Awarded struct { + Name string `bson:"name" json:"name"` + Address string `bson:"address,omitempty" json:"address,omitempty"` + Country string `bson:"country,omitempty" json:"country,omitempty"` + Amount float64 `bson:"amount" json:"amount"` + Currency string `bson:"currency,omitempty" json:"currency,omitempty"` + Share float64 `bson:"share,omitempty" json:"share,omitempty"` // Percentage share for multiple winners + AwardDate int64 `bson:"award_date" json:"award_date"` // Unix milliseconds + ContractID string `bson:"contract_id,omitempty" json:"contract_id,omitempty"` // Contract reference ID + TenderID string `bson:"tender_id,omitempty" json:"tender_id,omitempty"` // Tender reference ID + LotID string `bson:"lot_id,omitempty" json:"lot_id,omitempty"` // Lot reference ID + CompanyID string `bson:"company_id,omitempty" json:"company_id,omitempty"` // Company registration ID + OrganizationID string `bson:"organization_id,omitempty" json:"organization_id,omitempty"` // Organization reference ID +} + +// TenderStatus represents the status of a tender +type TenderStatus string + +const ( + TenderStatusActive TenderStatus = "active" + TenderStatusExpired TenderStatus = "expired" + TenderStatusCancelled TenderStatus = "cancelled" + TenderStatusAwarded TenderStatus = "awarded" + TenderStatusDraft TenderStatus = "draft" + TenderStatusClosed TenderStatus = "closed" + TenderStatusModified TenderStatus = "modified" + TenderStatusSuspended TenderStatus = "suspended" + TenderStatusPublished TenderStatus = "published" +) + +// TenderSource represents the source of tender data +type TenderSource string + +const ( + TenderSourceTEDScraper TenderSource = "ted_scraper" + TenderSourceManual TenderSource = "manual" + TenderSourceAPI TenderSource = "api" + TenderSourceBulkImport TenderSource = "bulk_import" +) + +// TenderStatistics represents tender statistics +type TenderStatistics struct { + TotalTenders int64 `json:"total_tenders"` + ActiveTenders int64 `json:"active_tenders"` + ExpiredTenders int64 `json:"expired_tenders"` + TendersByCountry map[string]int64 `json:"tenders_by_country"` + TendersByType map[string]int64 `json:"tenders_by_type"` + TendersByClassification map[string]int64 `json:"tenders_by_classification"` + AverageEstimatedValue float64 `json:"average_estimated_value"` + TotalEstimatedValue float64 `json:"total_estimated_value"` + LastUpdated int64 `json:"last_updated"` // Unix milliseconds +} + +// GetID returns the tender ID +func (t *Notice) GetID() string { + return t.ID.Hex() +} + +// IsActive returns true if the tender is active +func (t *Notice) IsActive() bool { + return t.Status == TenderStatusActive +} + +// IsExpired returns true if the tender deadline has passed +func (t *Notice) IsExpired() bool { + if t.Status == TenderStatusExpired { + return true + } + + if t.TenderDeadline == 0 { + return false + } + + // Check if deadline (in milliseconds) is in the past + now := time.Now().Unix() + return t.TenderDeadline < now +} + +// GetTenderURL returns the TED tender URL +func (t *Notice) GetTenderURL() string { + if t.TenderURL != "" { + return t.TenderURL + } + + if t.NoticePublicationID != "" { + return generateTenderURL(t.NoticePublicationID) + } + + return "" +} + +// generateTenderURL generates the TED tender URL from NoticePublicationID +func generateTenderURL(noticePublicationID string) string { + // Remove leading zeros and format for URL + // Example: 00522942-2025 -> https://ted.europa.eu/en/notice/-/detail/522942-2025 + parts := strings.Split(noticePublicationID, "-") + if len(parts) != 2 { + return "" + } + + // Convert to int and back to string to remove leading zeros + if num, err := strconv.Atoi(parts[0]); err == nil { + cleanID := fmt.Sprintf("%d-%s", num, parts[1]) + return fmt.Sprintf("https://ted.europa.eu/en/notice/-/detail/%s", cleanID) + } + + return "" +} + +// GetMainOrganization returns the buyer organization +func (t *Notice) GetMainOrganization() *Organization { + return t.BuyerOrganization +} + +// HasEstimatedValue returns true if the tender has an estimated value +func (t *Notice) HasEstimatedValue() bool { + return t.EstimatedValue > 0 +} + +// GetFormattedEstimatedValue returns formatted estimated value with currency +func (t *Notice) GetFormattedEstimatedValue() string { + if !t.HasEstimatedValue() { + return "" + } + + if t.Currency != "" { + return fmt.Sprintf("%.2f %s", t.EstimatedValue, t.Currency) + } + + return fmt.Sprintf("%.2f", t.EstimatedValue) +} + +// UpdateStatus updates the tender status and updated timestamp +func (t *Notice) UpdateStatus(status TenderStatus) { + t.Status = status + t.UpdatedAt = time.Now().Unix() +} + +// GetDaysUntilDeadline returns the number of days until the tender deadline +func (t *Notice) GetDaysUntilDeadline() int { + if t.TenderDeadline == 0 { + return 0 + } + + now := time.Now().Unix() + if t.TenderDeadline <= now { + return 0 // Already expired + } + + diffMillis := t.TenderDeadline - now + diffDays := diffMillis / (24 * 60 * 60 * 1000) + return int(diffDays) +} diff --git a/internal/notice/repository.go b/internal/notice/repository.go new file mode 100644 index 0000000..bcf9551 --- /dev/null +++ b/internal/notice/repository.go @@ -0,0 +1,182 @@ +package notice + +import ( + "context" + "time" + "tm/pkg/logger" + orm "tm/pkg/mongo" + + "go.mongodb.org/mongo-driver/v2/bson" +) + +// NoticeRepository interface defines notice data access methods +type Repository interface { + Import(ctx context.Context, notice *Notice) error + BulkImport(ctx context.Context, notices []Notice) error + GetByID(ctx context.Context, id string) (*Notice, error) + GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error) + GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) + Delete(ctx context.Context, id string) error +} + +func collectionName() string { + return "notices" +} + +// noticeRepository implements NoticeRepository interface using MongoDB ORM +type noticeRepository struct { + ormRepo orm.Repository[Notice] + mongoManager *orm.ConnectionManager + logger logger.Logger +} + +// NewRepository creates a new notice repository +func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Repository { + // Create indexes for notices collection + noticeIndexes := []orm.Index{} + + // Create indexes + if err := mongoManager.CreateIndexes(collectionName(), noticeIndexes); err != nil { + logger.Warn("Failed to create notice indexes", map[string]interface{}{ + "error": err.Error(), + }) + } + + // Create ORM repositories + noticeOrmRepo := orm.NewRepository[Notice](mongoManager.GetCollection(collectionName()), logger) + + return ¬iceRepository{ + ormRepo: noticeOrmRepo, + mongoManager: mongoManager, + logger: logger, + } +} + +// Import imports a new notice +func (r *noticeRepository) Import(ctx context.Context, notice *Notice) error { + now := time.Now().Unix() + notice.SetCreatedAt(now) + + err := r.ormRepo.Create(ctx, notice) + if err != nil { + r.logger.Error("Failed to create notice", map[string]interface{}{ + "error": err.Error(), + "notice_publication_id": notice.NoticePublicationID, + }) + return err + } + + r.logger.Info("Notice created successfully", map[string]interface{}{ + "notice_id": notice.ID, + "notice_publication_id": notice.NoticePublicationID, + }) + + return nil +} + +// BulkImport imports a new notice +func (r *noticeRepository) BulkImport(ctx context.Context, notices []Notice) error { + err := r.ormRepo.CreateMany(ctx, notices) + if err != nil { + r.logger.Error("Failed to bulk import notices", map[string]interface{}{ + "error": err.Error(), + }) + return err + } + + return nil +} + +// GetByID retrieves a notice by ID +func (r *noticeRepository) GetByID(ctx context.Context, id string) (*Notice, error) { + notice, err := r.ormRepo.FindByID(ctx, id) + if err != nil { + r.logger.Error("Failed to get notice by ID", map[string]interface{}{ + "notice_id": id, + "error": err.Error(), + }) + return nil, err + } + + return notice, nil +} + +// GetByContractNoticeID retrieves a notice by contract notice ID +func (r *noticeRepository) GetByContractNoticeID(ctx context.Context, contractNoticeID string) (*Notice, error) { + filter := bson.M{"contract_notice_id": contractNoticeID} + pagination := orm.Pagination{Limit: 1} + + result, err := r.ormRepo.FindAll(ctx, filter, pagination) + if err != nil { + r.logger.Error("Failed to get notice by contract notice ID", map[string]interface{}{ + "contract_notice_id": contractNoticeID, + "error": err.Error(), + }) + return nil, err + } + + if len(result.Items) == 0 { + return nil, orm.ErrDocumentNotFound + } + + return &result.Items[0], nil +} + +// GetUnProcessedNotices retrieves unprocessed notices + +func (r *noticeRepository) GetUnProcessedNotices(ctx context.Context, limit int, skip int) ([]Notice, int64, error) { + filter := bson.M{"processing_metadata.processed": false} + + pagination := orm.Pagination{ + Limit: limit, + Skip: skip, + } + + result, err := r.ormRepo.FindAll(ctx, filter, pagination) + if err != nil { + r.logger.Error("Failed to get unprocessed notices", map[string]interface{}{ + "error": err.Error(), + }) + return nil, 0, err + } + + return result.Items, result.TotalCount, nil +} + +// Update updates an existing notice +func (r *noticeRepository) Update(ctx context.Context, notice *Notice) error { + notice.UpdatedAt = time.Now().Unix() + + err := r.ormRepo.Update(ctx, notice) + if err != nil { + r.logger.Error("Failed to update notice", map[string]interface{}{ + "notice_id": notice.ID, + "error": err.Error(), + }) + return err + } + + r.logger.Info("Notice updated successfully", map[string]interface{}{ + "notice_id": notice.ID, + }) + + return nil +} + +// Delete deletes a notice by ID +func (r *noticeRepository) Delete(ctx context.Context, id string) error { + err := r.ormRepo.Delete(ctx, id) + if err != nil { + r.logger.Error("Failed to delete notice", map[string]interface{}{ + "notice_id": id, + "error": err.Error(), + }) + return err + } + + r.logger.Info("Notice deleted successfully", map[string]interface{}{ + "notice_id": id, + }) + + return nil +} diff --git a/internal/notice/service.go b/internal/notice/service.go new file mode 100644 index 0000000..6a8d5c6 --- /dev/null +++ b/internal/notice/service.go @@ -0,0 +1,36 @@ +package notice + +import ( + "context" + "tm/pkg/logger" +) + +// Service defines business logic for notice operations +type Service interface { + Import(ctx context.Context, form *Notice) error + BulkImport(ctx context.Context, form []Notice) error +} + +// noticeService implements the Service interface +type noticeService struct { + repository Repository + logger logger.Logger +} + +// NewService creates a new notice service +func NewService(repository Repository, logger logger.Logger) Service { + return ¬iceService{ + repository: repository, + logger: logger, + } +} + +// Import imports a new notice +func (s *noticeService) Import(ctx context.Context, form *Notice) error { + return s.repository.Import(ctx, form) +} + +// BulkImport imports a new notice +func (s *noticeService) BulkImport(ctx context.Context, form []Notice) error { + return s.repository.BulkImport(ctx, form) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 85ce26f..e785796 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -74,6 +74,21 @@ type NotificationConfig struct { UserAgent string `env:"NOTIFICATION_USER_AGENT"` } +// OllamaConfig represents the configuration for the Ollama SDK +type OllamaConfig struct { + BaseURL string `env:"OLLAMA_BASE_URL" default:"http://localhost:11434"` + Timeout time.Duration `env:"OLLAMA_TIMEOUT" default:"300s"` + RetryAttempts int `env:"OLLAMA_RETRY_ATTEMPTS" default:"3"` + RetryDelay time.Duration `env:"OLLAMA_RETRY_DELAY" default:"2s"` + EnableLogging bool `env:"OLLAMA_ENABLE_LOGGING" default:"true"` + UserAgent string `env:"OLLAMA_USER_AGENT" default:"TenderManagement-OllamaSDK/1.0"` + DefaultModel string `env:"OLLAMA_DEFAULT_MODEL" default:"llama2"` + DefaultTemperature float64 `env:"OLLAMA_DEFAULT_TEMPERATURE" default:"0.7"` + DefaultTopP float64 `env:"OLLAMA_DEFAULT_TOP_P" default:"0.9"` + DefaultMaxTokens int `env:"OLLAMA_DEFAULT_MAX_TOKENS" default:"1000"` + EnableStreaming bool `env:"OLLAMA_ENABLE_STREAMING" default:"false"` +} + // LoadConfig loads configuration with priority: OS environment variables > .env file // OS environment variables take precedence over .env file values func LoadConfig[T any](path string, config T) (T, error) { diff --git a/pkg/mongo/repository.go b/pkg/mongo/repository.go index f9553b3..3ee1538 100644 --- a/pkg/mongo/repository.go +++ b/pkg/mongo/repository.go @@ -47,6 +47,9 @@ type Repository[T any] interface { // Create inserts a new document Create(ctx context.Context, model *T) error + // CreateMany inserts multiple documents + CreateMany(ctx context.Context, models []T) error + // Update updates an existing document Update(ctx context.Context, model *T) error @@ -239,6 +242,32 @@ func (r *repository[T]) Create(ctx context.Context, model *T) error { return nil } +// CreateMany inserts multiple documents +func (r *repository[T]) CreateMany(ctx context.Context, models []T) error { + // Set timestamps if the model implements Timestampable + for _, v := range models { + if timestampable, ok := any(v).(Timestampable); ok { + now := time.Now().Unix() + timestampable.SetCreatedAt(now) + timestampable.SetUpdatedAt(now) + } + } + + result, err := r.collection.InsertMany(ctx, models) + if err != nil { + r.logger.Error("Failed to create multiple documents", map[string]interface{}{ + "error": err.Error(), + }) + return fmt.Errorf("failed to create multiple documents: %w", err) + } + + r.logger.Info("Documents created successfully", map[string]interface{}{ + "ids": result.InsertedIDs, + }) + + return nil +} + // Update updates an existing document func (r *repository[T]) Update(ctx context.Context, model *T) error { // Set updated timestamp if the model implements Timestampable diff --git a/pkg/ollama/README.md b/pkg/ollama/README.md new file mode 100644 index 0000000..ad86132 --- /dev/null +++ b/pkg/ollama/README.md @@ -0,0 +1,394 @@ +# Ollama SDK + +A comprehensive Go SDK for interacting with Ollama AI models. This SDK provides a clean, type-safe interface for text generation, chat conversations, embeddings, and model management. + +## Features + +- **Text Generation**: Generate text using various AI models +- **Chat Conversations**: Interactive chat with support for system prompts and conversation history +- **Embeddings**: Generate embeddings for text similarity and search +- **Model Management**: Pull, list, delete, and inspect models +- **Streaming Support**: Real-time streaming of generated text and chat responses +- **Automatic Retries**: Configurable retry mechanism with exponential backoff +- **Structured Logging**: Built-in logging with structured fields +- **Fluent API**: Builder pattern for constructing complex chat conversations +- **Type Safety**: Full type safety with comprehensive validation +- **Error Handling**: Detailed error types and proper error wrapping +- **Configuration**: Flexible configuration via environment variables or code +- **Health Checks**: Built-in health checking capabilities + +## Installation + +The SDK is already part of your `tm` module. Import it as: +```go +import "tm/pkg/ollama" +``` + +## Quick Start + +### Basic Setup + +```go +// 1. Configuration +config := ollama.DefaultConfig() +config.BaseURL = "http://localhost:11434" // Your Ollama server URL +config.DefaultModel = "llama2" // Default model to use + +// 2. Logger (using project's logger) +loggerConfig := &logger.Config{Level: "info", Format: "json", Output: "stdout"} +projectLogger := logger.NewLogger(loggerConfig) + +// 3. Create SDK +sdk, err := ollama.New(config, projectLogger) +if err != nil { + log.Fatal("Failed to create Ollama SDK:", err) +} +``` + +### Environment Variables + +Set environment variables for automatic configuration: + +```bash +export OLLAMA_BASE_URL="http://localhost:11434" +export OLLAMA_TIMEOUT="300s" +export OLLAMA_RETRY_ATTEMPTS="3" +export OLLAMA_RETRY_DELAY="2s" +export OLLAMA_ENABLE_LOGGING="true" +export OLLAMA_DEFAULT_MODEL="llama2" +export OLLAMA_DEFAULT_TEMPERATURE="0.7" +export OLLAMA_DEFAULT_TOP_P="0.9" +export OLLAMA_DEFAULT_MAX_TOKENS="1000" +``` + +## API Reference + +### Configuration + +```go +type Config struct { + BaseURL string `env:"OLLAMA_BASE_URL"` + Timeout time.Duration `env:"OLLAMA_TIMEOUT"` + RetryAttempts int `env:"OLLAMA_RETRY_ATTEMPTS"` + RetryDelay time.Duration `env:"OLLAMA_RETRY_DELAY"` + EnableLogging bool `env:"OLLAMA_ENABLE_LOGGING"` + UserAgent string `env:"OLLAMA_USER_AGENT"` + DefaultModel string `env:"OLLAMA_DEFAULT_MODEL"` + DefaultTemperature float64 `env:"OLLAMA_DEFAULT_TEMPERATURE"` + DefaultTopP float64 `env:"OLLAMA_DEFAULT_TOP_P"` + DefaultMaxTokens int `env:"OLLAMA_DEFAULT_MAX_TOKENS"` + EnableStreaming bool `env:"OLLAMA_ENABLE_STREAMING"` +} +``` + +### Creating SDK Instance + +```go +// With default configuration +sdk, err := ollama.New(nil, logger) + +// With custom configuration +config := &ollama.Config{ + BaseURL: "http://localhost:11434", + Timeout: 300 * time.Second, + RetryAttempts: 3, + RetryDelay: 2 * time.Second, + EnableLogging: true, + DefaultModel: "llama2", + DefaultTemperature: 0.7, + DefaultTopP: 0.9, + DefaultMaxTokens: 1000, +} +sdk, err := ollama.New(config, logger) + +// With custom HTTP client +httpClient := &http.Client{Timeout: 10 * time.Second} +sdk, err := ollama.NewWithClient(config, httpClient, logger) +``` + +## Usage Examples + +### Simple Text Generation + +```go +ctx := context.Background() + +// Quick generation with default model +response, err := sdk.QuickGenerate(ctx, "Write a short story about a robot") +if err != nil { + log.Printf("Generation failed: %v", err) +} +fmt.Println(response) + +// Generation with specific model +response, err := sdk.QuickGenerateWithModel(ctx, "codellama", "Write a Python function to sort a list") +``` + +### Advanced Text Generation + +```go +req := &ollama.GenerateRequest{ + Model: "llama2", + Prompt: "Explain quantum computing in simple terms", + Options: map[string]interface{}{ + "temperature": 0.7, + "top_p": 0.9, + "max_tokens": 500, + }, +} + +resp, err := sdk.Generate(ctx, req) +if err != nil { + log.Printf("Generation failed: %v", err) +} +fmt.Printf("Generated: %s\n", resp.Response) +``` + +### Chat Conversations + +```go +// Simple chat +response, err := sdk.QuickChat(ctx, "What is the capital of France?") +fmt.Println(response) + +// Chat with system prompt +response, err := sdk.QuickChatWithSystem(ctx, + "You are a helpful coding assistant.", + "How do I implement a binary search in Go?") +``` + +### Advanced Chat with Conversation History + +```go +req := &ollama.ChatRequest{ + Model: "llama2", + Messages: []ollama.Message{ + { + Role: ollama.MessageRoleSystem, + Content: "You are a helpful assistant.", + }, + { + Role: ollama.MessageRoleUser, + Content: "Hello, how are you?", + }, + { + Role: ollama.MessageRoleAssistant, + Content: "I'm doing well, thank you! How can I help you today?", + }, + { + Role: ollama.MessageRoleUser, + Content: "Can you help me with a coding problem?", + }, + }, +} + +resp, err := sdk.Chat(ctx, req) +fmt.Printf("Assistant: %s\n", resp.Message.Content) +``` + +### Fluent Chat Builder + +```go +response, err := sdk.NewChatBuilder(). + SetModel("llama2"). + SetSystem("You are a helpful coding assistant."). + AddMessage(ollama.MessageRoleUser, "How do I create a struct in Go?"). + SetTemperature(0.7). + SetMaxTokens(500). + Send(ctx) + +fmt.Printf("Assistant: %s\n", response.Message.Content) + +// Continue the conversation +response, err = sdk.NewChatBuilder(). + SetModel("llama2"). + SetSystem("You are a helpful coding assistant."). + AddMessage(ollama.MessageRoleUser, "How do I create a struct in Go?"). + SetTemperature(0.7). + SetMaxTokens(500). + Continue(ctx, "Can you show me an example with methods?") + +fmt.Printf("Assistant: %s\n", response.Message.Content) +``` + +### Streaming Responses + +```go +// Streaming text generation +err := sdk.StreamGenerate(ctx, &ollama.GenerateRequest{ + Model: "llama2", + Prompt: "Write a poem about programming", +}, func(chunk string) error { + fmt.Print(chunk) + return nil +}) + +// Streaming chat +err := sdk.StreamChat(ctx, &ollama.ChatRequest{ + Model: "llama2", + Messages: []ollama.Message{ + { + Role: ollama.MessageRoleUser, + Content: "Tell me a story", + }, + }, +}, func(chunk string) error { + fmt.Print(chunk) + return nil +}) +``` + +### Embeddings + +```go +// Generate embeddings +embeddings, err := sdk.QuickEmbed(ctx, "This is some text to embed") +if err != nil { + log.Printf("Embedding failed: %v", err) +} + +// Advanced embedding request +req := &ollama.EmbedRequest{ + Model: "llama2", + Prompt: "Text to generate embeddings for", +} + +resp, err := sdk.Embed(ctx, req) +fmt.Printf("Embedding dimension: %d\n", len(resp.Embedding)) +``` + +### Model Management + +```go +// List available models +models, err := sdk.ListModels(ctx) +for _, model := range models.Models { + fmt.Printf("Model: %s, Size: %d bytes\n", model.Name, model.Size) +} + +// Pull a new model +pullResp, err := sdk.PullModel(ctx, "codellama") +fmt.Printf("Pull status: %s\n", pullResp.Status) + +// Show model information +modelInfo, err := sdk.ShowModel(ctx, "llama2") +fmt.Printf("Model details: %+v\n", modelInfo.Details) + +// Delete a model +deleteResp, err := sdk.DeleteModel(ctx, "unwanted-model") +fmt.Printf("Delete status: %s\n", deleteResp.Status) +``` + +### Health Check + +```go +err := sdk.Health(ctx) +if err != nil { + log.Printf("Ollama service is not available: %v", err) +} else { + fmt.Println("Ollama service is healthy") +} +``` + +## Error Handling + +The SDK provides specific error types for different failure scenarios: + +```go +resp, err := sdk.Generate(ctx, req) +if err != nil { + switch e := err.(type) { + case *ollama.ConfigError: + fmt.Printf("Configuration error: %s\n", e.Message) + case *ollama.ModelError: + fmt.Printf("Model error for %s: %s\n", e.Model, e.Message) + case *ollama.GenerationError: + fmt.Printf("Generation error: %s\n", e.Message) + case *ollama.ConnectionError: + fmt.Printf("Connection error: %s\n", e.Message) + case *ollama.TimeoutError: + fmt.Printf("Timeout during %s: %s\n", e.Operation, e.Message) + default: + fmt.Printf("Unexpected error: %v\n", err) + } +} +``` + +## Advanced Usage + +### Custom HTTP Client + +```go +httpClient := &http.Client{ + Timeout: 60 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + }, +} + +sdk, err := ollama.NewWithClient(config, httpClient, logger) +``` + +### Custom Options + +```go +req := &ollama.GenerateRequest{ + Model: "llama2", + Prompt: "Write a haiku", +} + +opts := &ollama.GenerateOptions{ + Temperature: &[]float64{0.5}[0], + TopP: &[]float64{0.8}[0], + TopK: &[]int{40}[0], + RepeatPenalty: &[]float64{1.1}[0], + Seed: &[]int{42}[0], + Stop: []string{"", "\n\n"}, + Format: "json", +} + +resp, err := sdk.GenerateWithOptions(ctx, req, opts) +``` + +## Best Practices + +1. **Always use context**: Pass context for cancellation and timeouts +2. **Handle errors properly**: Use type assertions for specific error handling +3. **Configure timeouts**: Set appropriate timeouts for your use case +4. **Use streaming for long responses**: Enable streaming for better user experience +5. **Monitor model usage**: Keep track of token usage and costs +6. **Validate inputs**: Ensure prompts and parameters are within limits +7. **Use structured logging**: Leverage the built-in logging for debugging + +## Troubleshooting + +### Common Issues + +1. **Connection refused**: Ensure Ollama server is running on the correct port +2. **Model not found**: Pull the model first using `PullModel()` +3. **Timeout errors**: Increase the timeout configuration +4. **Rate limiting**: Implement backoff strategies for high-volume usage + +### Debug Mode + +Enable debug logging to troubleshoot issues: + +```go +config := ollama.DefaultConfig() +config.EnableLogging = true + +loggerConfig := &logger.Config{ + Level: "debug", + Format: "json", + Output: "stdout", +} +projectLogger := logger.NewLogger(loggerConfig) + +sdk, err := ollama.New(config, projectLogger) +``` + +## License + +This SDK is part of the Tender Management system and follows the same licensing terms. diff --git a/pkg/ollama/USAGE.md b/pkg/ollama/USAGE.md new file mode 100644 index 0000000..ef1dc96 --- /dev/null +++ b/pkg/ollama/USAGE.md @@ -0,0 +1,263 @@ +# Ollama SDK - Quick Usage Guide + +## Installation +The SDK is already part of your `tm` module. Import it as: +```go +import "tm/pkg/ollama" +``` + +## Basic Setup +```go +// 1. Configuration +config := ollama.DefaultConfig() +config.BaseURL = "http://localhost:11434" // Your Ollama server URL +config.DefaultModel = "llama2" // Default model to use + +// 2. Logger (using project's logger) +loggerConfig := &logger.Config{Level: "info", Format: "json", Output: "stdout"} +projectLogger := logger.NewLogger(loggerConfig) + +// 3. Create SDK +sdk, err := ollama.New(config, projectLogger) +if err != nil { + log.Fatal("Failed to create Ollama SDK:", err) +} +``` + +## Environment Variables +```bash +export OLLAMA_BASE_URL="http://localhost:11434" +export OLLAMA_TIMEOUT="300s" +export OLLAMA_RETRY_ATTEMPTS="3" +export OLLAMA_RETRY_DELAY="2s" +export OLLAMA_ENABLE_LOGGING="true" +export OLLAMA_DEFAULT_MODEL="llama2" +export OLLAMA_DEFAULT_TEMPERATURE="0.7" +export OLLAMA_DEFAULT_TOP_P="0.9" +export OLLAMA_DEFAULT_MAX_TOKENS="1000" +``` + +## Quick Examples + +### Simple Text Generation +```go +ctx := context.Background() + +// Generate text with default model +response, err := sdk.QuickGenerate(ctx, "Write a short story about a robot") +if err != nil { + log.Printf("Generation failed: %v", err) +} +fmt.Println(response) + +// Generate with specific model +response, err := sdk.QuickGenerateWithModel(ctx, "codellama", "Write a Python function to sort a list") +``` + +### Chat Conversations +```go +// Simple chat +response, err := sdk.QuickChat(ctx, "What is the capital of France?") +fmt.Println(response) + +// Chat with system prompt +response, err := sdk.QuickChatWithSystem(ctx, + "You are a helpful coding assistant.", + "How do I implement a binary search in Go?") +``` + +### Advanced Chat with Builder +```go +// Create a chat builder +builder := sdk.NewChatBuilder(). + SetModel("llama2"). + SetSystem("You are a helpful coding assistant."). + SetTemperature(0.7). + SetMaxTokens(500) + +// Start conversation +response, err := builder. + AddMessage(ollama.MessageRoleUser, "How do I create a struct in Go?"). + Send(ctx) + +fmt.Printf("Assistant: %s\n", response.Message.Content) + +// Continue the conversation +response, err = builder.Continue(ctx, "Can you show me an example with methods?") +fmt.Printf("Assistant: %s\n", response.Message.Content) +``` + +### Streaming Responses +```go +// Streaming text generation +err := sdk.StreamGenerate(ctx, &ollama.GenerateRequest{ + Model: "llama2", + Prompt: "Write a poem about programming", +}, func(chunk string) error { + fmt.Print(chunk) // Print each chunk as it arrives + return nil +}) + +// Streaming chat +err := sdk.StreamChat(ctx, &ollama.ChatRequest{ + Model: "llama2", + Messages: []ollama.Message{ + { + Role: ollama.MessageRoleUser, + Content: "Tell me a story", + }, + }, +}, func(chunk string) error { + fmt.Print(chunk) + return nil +}) +``` + +### Embeddings +```go +// Generate embeddings +embeddings, err := sdk.QuickEmbed(ctx, "This is some text to embed") +if err != nil { + log.Printf("Embedding failed: %v", err) +} + +fmt.Printf("Embedding dimension: %d\n", len(embeddings)) +``` + +### Model Management +```go +// List available models +models, err := sdk.ListModels(ctx) +for _, model := range models.Models { + fmt.Printf("Model: %s, Size: %d bytes\n", model.Name, model.Size) +} + +// Pull a new model +pullResp, err := sdk.PullModel(ctx, "codellama") +fmt.Printf("Pull status: %s\n", pullResp.Status) + +// Show model information +modelInfo, err := sdk.ShowModel(ctx, "llama2") +fmt.Printf("Model family: %s\n", modelInfo.Details.Family) + +// Delete a model +deleteResp, err := sdk.DeleteModel(ctx, "unwanted-model") +fmt.Printf("Delete status: %s\n", deleteResp.Status) +``` + +### Health Check +```go +err := sdk.Health(ctx) +if err != nil { + log.Printf("Ollama service is not available: %v", err) +} else { + fmt.Println("Ollama service is healthy") +} +``` + +## Error Handling +```go +resp, err := sdk.Generate(ctx, req) +if err != nil { + switch e := err.(type) { + case *ollama.ConfigError: + fmt.Printf("Configuration error: %s\n", e.Message) + case *ollama.ModelError: + fmt.Printf("Model error for %s: %s\n", e.Model, e.Message) + case *ollama.GenerationError: + fmt.Printf("Generation error: %s\n", e.Message) + case *ollama.ConnectionError: + fmt.Printf("Connection error: %s\n", e.Message) + case *ollama.TimeoutError: + fmt.Printf("Timeout during %s: %s\n", e.Operation, e.Message) + default: + fmt.Printf("Unexpected error: %v\n", err) + } +} +``` + +## Advanced Configuration +```go +config := &ollama.Config{ + BaseURL: "http://localhost:11434", + Timeout: 300 * time.Second, + RetryAttempts: 3, + RetryDelay: 2 * time.Second, + EnableLogging: true, + UserAgent: "MyApp-OllamaSDK/1.0", + DefaultModel: "llama2", + DefaultTemperature: 0.7, + DefaultTopP: 0.9, + DefaultMaxTokens: 1000, + EnableStreaming: false, +} + +sdk, err := ollama.New(config, logger) +``` + +## Custom HTTP Client +```go +httpClient := &http.Client{ + Timeout: 60 * time.Second, + Transport: &http.Transport{ + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + DisableCompression: true, + }, +} + +sdk, err := ollama.NewWithClient(config, httpClient, logger) +``` + +## Integration with Project Logger +```go +// Use the project's logger system +loggerConfig := &logger.Config{ + Level: "info", + Format: "json", + Output: "stdout", +} +projectLogger := logger.NewLogger(loggerConfig) + +sdk, err := ollama.New(config, projectLogger) +``` + +## Best Practices + +1. **Always use context**: Pass context for cancellation and timeouts +2. **Handle errors properly**: Use type assertions for specific error handling +3. **Configure timeouts**: Set appropriate timeouts for your use case +4. **Use streaming for long responses**: Enable streaming for better user experience +5. **Monitor model usage**: Keep track of token usage and costs +6. **Validate inputs**: Ensure prompts and parameters are within limits +7. **Use structured logging**: Leverage the built-in logging for debugging + +## Common Use Cases + +### Document Summarization +```go +response, err := sdk.QuickChatWithSystem(ctx, + "You are a helpful assistant that summarizes documents concisely.", + "Please summarize this document: [document content]") +``` + +### Code Review Assistant +```go +response, err := sdk.QuickChatWithSystem(ctx, + "You are a senior software engineer reviewing code. Provide constructive feedback.", + "Please review this Go code: [code snippet]") +``` + +### Translation Service +```go +response, err := sdk.QuickChatWithSystem(ctx, + "You are a professional translator. Translate the following text accurately.", + "Translate this to French: Hello, how are you?") +``` + +### Question Answering +```go +response, err := sdk.QuickChatWithSystem(ctx, + "You are a helpful assistant that answers questions accurately and concisely.", + "What is the difference between HTTP and HTTPS?") +``` diff --git a/pkg/ollama/client.go b/pkg/ollama/client.go new file mode 100644 index 0000000..a070ed6 --- /dev/null +++ b/pkg/ollama/client.go @@ -0,0 +1,290 @@ +package ollama + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" + "tm/pkg/logger" +) + +// Client implements the HTTPClient interface for Ollama API +type Client struct { + config *Config + httpClient *http.Client + logger logger.Logger +} + +// NewClient creates a new Ollama HTTP client +func NewClient(config *Config, logger logger.Logger) (*Client, error) { + if config == nil { + config = DefaultConfig() + } + + if err := config.Validate(); err != nil { + return nil, err + } + + httpClient := &http.Client{ + Timeout: config.Timeout, + } + + return &Client{ + config: config, + httpClient: httpClient, + logger: logger, + }, nil +} + +// NewClientWithHTTPClient creates a new client with a custom HTTP client +func NewClientWithHTTPClient(config *Config, httpClient *http.Client, logger logger.Logger) (*Client, error) { + if config == nil { + config = DefaultConfig() + } + + if err := config.Validate(); err != nil { + return nil, err + } + + return &Client{ + config: config, + httpClient: httpClient, + logger: logger, + }, nil +} + +// Post performs a POST request to the Ollama API +func (c *Client) Post(ctx context.Context, url string, body interface{}, result interface{}) error { + return c.makeRequest(ctx, "POST", url, body, result) +} + +// Get performs a GET request to the Ollama API +func (c *Client) Get(ctx context.Context, url string, result interface{}) error { + return c.makeRequest(ctx, "GET", url, nil, result) +} + +// Delete performs a DELETE request to the Ollama API +func (c *Client) Delete(ctx context.Context, url string, body interface{}, result interface{}) error { + return c.makeRequest(ctx, "DELETE", url, body, result) +} + +// makeRequest performs an HTTP request with retry logic and logging +func (c *Client) makeRequest(ctx context.Context, method, endpoint string, body interface{}, result interface{}) error { + url := c.config.BaseURL + endpoint + + var bodyReader io.Reader + if body != nil { + bodyBytes, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + bodyReader = bytes.NewReader(bodyBytes) + } + + var lastErr error + for attempt := 0; attempt <= c.config.RetryAttempts; attempt++ { + if attempt > 0 { + // Wait before retry + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(c.config.RetryDelay): + } + } + + err := c.doRequest(ctx, method, url, bodyReader, result) + if err == nil { + return nil + } + + lastErr = err + + // Don't retry on certain errors + if c.shouldNotRetry(err) { + break + } + + if c.logger != nil { + c.logger.Warn("Request failed, retrying", map[string]interface{}{ + "attempt": attempt + 1, + "max_attempts": c.config.RetryAttempts + 1, + "error": err.Error(), + "url": url, + "method": method, + }) + } + } + + return fmt.Errorf("request failed after %d attempts: %w", c.config.RetryAttempts+1, lastErr) +} + +// doRequest performs a single HTTP request +func (c *Client) doRequest(ctx context.Context, method, url string, body io.Reader, result interface{}) error { + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", c.config.UserAgent) + + if c.logger != nil && c.config.EnableLogging { + c.logger.Debug("Making HTTP request", map[string]interface{}{ + "method": method, + "url": url, + "user_agent": c.config.UserAgent, + }) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("failed to execute request: %w", err) + } + defer resp.Body.Close() + + if c.logger != nil && c.config.EnableLogging { + c.logger.Debug("Received HTTP response", map[string]interface{}{ + "status_code": resp.StatusCode, + "url": url, + }) + } + + if resp.StatusCode >= 400 { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(bodyBytes)) + } + + if result != nil { + if err := json.NewDecoder(resp.Body).Decode(result); err != nil { + return fmt.Errorf("failed to decode response: %w", err) + } + } + + return nil +} + +// shouldNotRetry determines if an error should not be retried +func (c *Client) shouldNotRetry(err error) bool { + // Don't retry on context cancellation + if err == context.Canceled || err == context.DeadlineExceeded { + return true + } + + // Don't retry on certain HTTP status codes + if err.Error() == "HTTP 400" || err.Error() == "HTTP 401" || err.Error() == "HTTP 403" || err.Error() == "HTTP 404" { + return true + } + + return false +} + +// StreamGenerate performs a streaming text generation request +func (c *Client) StreamGenerate(ctx context.Context, req *GenerateRequest, onChunk func(*StreamChunk) error) error { + url := c.config.BaseURL + "/api/generate" + + // Force streaming + stream := true + req.Stream = &stream + + bodyBytes, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("User-Agent", c.config.UserAgent) + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(bodyBytes)) + } + + decoder := json.NewDecoder(resp.Body) + for { + var chunk StreamChunk + if err := decoder.Decode(&chunk); err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("failed to decode stream chunk: %w", err) + } + + if err := onChunk(&chunk); err != nil { + return fmt.Errorf("error processing stream chunk: %w", err) + } + + if chunk.Done { + break + } + } + + return nil +} + +// StreamChat performs a streaming chat request +func (c *Client) StreamChat(ctx context.Context, req *ChatRequest, onChunk func(*StreamChunk) error) error { + url := c.config.BaseURL + "/api/chat" + + // Force streaming + stream := true + req.Stream = &stream + + bodyBytes, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("User-Agent", c.config.UserAgent) + + resp, err := c.httpClient.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + bodyBytes, _ := io.ReadAll(resp.Body) + return fmt.Errorf("HTTP %d: %s", resp.StatusCode, string(bodyBytes)) + } + + decoder := json.NewDecoder(resp.Body) + for { + var chunk StreamChunk + if err := decoder.Decode(&chunk); err != nil { + if err == io.EOF { + break + } + return fmt.Errorf("failed to decode stream chunk: %w", err) + } + + if err := onChunk(&chunk); err != nil { + return fmt.Errorf("error processing stream chunk: %w", err) + } + + if chunk.Done { + break + } + } + + return nil +} diff --git a/pkg/ollama/config.go b/pkg/ollama/config.go new file mode 100644 index 0000000..2bb1d80 --- /dev/null +++ b/pkg/ollama/config.go @@ -0,0 +1,91 @@ +package ollama + +import ( + "time" +) + +// Config represents the configuration for the Ollama SDK +type Config struct { + // BaseURL is the base URL of the Ollama API server + BaseURL string `env:"OLLAMA_BASE_URL" default:"http://localhost:11434"` + + // Timeout is the timeout for HTTP requests + Timeout time.Duration `env:"OLLAMA_TIMEOUT" default:"300s"` + + // RetryAttempts is the number of retry attempts for failed requests + RetryAttempts int `env:"OLLAMA_RETRY_ATTEMPTS" default:"3"` + + // RetryDelay is the delay between retry attempts + RetryDelay time.Duration `env:"OLLAMA_RETRY_DELAY" default:"2s"` + + // EnableLogging enables request/response logging + EnableLogging bool `env:"OLLAMA_ENABLE_LOGGING" default:"true"` + + // UserAgent is the user agent string for HTTP requests + UserAgent string `env:"OLLAMA_USER_AGENT" default:"TenderManagement-OllamaSDK/1.0"` + + // DefaultModel is the default model to use if none is specified + DefaultModel string `env:"OLLAMA_DEFAULT_MODEL" default:"llama2"` + + // DefaultTemperature is the default temperature for text generation + DefaultTemperature float64 `env:"OLLAMA_DEFAULT_TEMPERATURE" default:"0.7"` + + // DefaultTopP is the default top_p value for text generation + DefaultTopP float64 `env:"OLLAMA_DEFAULT_TOP_P" default:"0.9"` + + // DefaultMaxTokens is the default maximum number of tokens to generate + DefaultMaxTokens int `env:"OLLAMA_DEFAULT_MAX_TOKENS" default:"1000"` + + // EnableStreaming enables streaming responses by default + EnableStreaming bool `env:"OLLAMA_ENABLE_STREAMING" default:"false"` +} + +// DefaultConfig returns a default configuration for the Ollama SDK +func DefaultConfig() *Config { + return &Config{ + BaseURL: "http://localhost:11434", + Timeout: 300 * time.Second, + RetryAttempts: 3, + RetryDelay: 2 * time.Second, + EnableLogging: true, + UserAgent: "TenderManagement-OllamaSDK/1.0", + DefaultModel: "llama2", + DefaultTemperature: 0.7, + DefaultTopP: 0.9, + DefaultMaxTokens: 1000, + EnableStreaming: false, + } +} + +// Validate validates the configuration +func (c *Config) Validate() error { + if c.BaseURL == "" { + return ErrInvalidConfig("BaseURL cannot be empty") + } + + if c.Timeout <= 0 { + return ErrInvalidConfig("Timeout must be positive") + } + + if c.RetryAttempts < 0 { + return ErrInvalidConfig("RetryAttempts cannot be negative") + } + + if c.RetryDelay < 0 { + return ErrInvalidConfig("RetryDelay cannot be negative") + } + + if c.DefaultTemperature < 0 || c.DefaultTemperature > 2 { + return ErrInvalidConfig("DefaultTemperature must be between 0 and 2") + } + + if c.DefaultTopP < 0 || c.DefaultTopP > 1 { + return ErrInvalidConfig("DefaultTopP must be between 0 and 1") + } + + if c.DefaultMaxTokens < 0 { + return ErrInvalidConfig("DefaultMaxTokens cannot be negative") + } + + return nil +} diff --git a/pkg/ollama/entities.go b/pkg/ollama/entities.go new file mode 100644 index 0000000..fb8615e --- /dev/null +++ b/pkg/ollama/entities.go @@ -0,0 +1,182 @@ +package ollama + +import ( + "time" +) + +// GenerateRequest represents a request to generate text +type GenerateRequest struct { + Model string `json:"model" validate:"required"` + Prompt string `json:"prompt" validate:"required"` + Stream *bool `json:"stream,omitempty"` + Format string `json:"format,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` + System string `json:"system,omitempty"` + Template string `json:"template,omitempty"` + Context []int `json:"context,omitempty"` + Raw bool `json:"raw,omitempty"` + KeepAlive string `json:"keep_alive,omitempty"` +} + +// GenerateResponse represents the response from a text generation request +type GenerateResponse struct { + Model string `json:"model"` + CreatedAt time.Time `json:"created_at"` + Response string `json:"response"` + Done bool `json:"done"` + Context []int `json:"context,omitempty"` + TotalDuration int64 `json:"total_duration,omitempty"` + LoadDuration int64 `json:"load_duration,omitempty"` + PromptEvalCount int `json:"prompt_eval_count,omitempty"` + PromptEvalDuration int64 `json:"prompt_eval_duration,omitempty"` + EvalCount int `json:"eval_count,omitempty"` + EvalDuration int64 `json:"eval_duration,omitempty"` +} + +// ChatRequest represents a request for a chat conversation +type ChatRequest struct { + Model string `json:"model" validate:"required"` + Messages []Message `json:"messages" validate:"required,min=1"` + Stream *bool `json:"stream,omitempty"` + Format string `json:"format,omitempty"` + Options ChatOptions `json:"options,omitempty"` + KeepAlive string `json:"keep_alive,omitempty"` +} + +// Message represents a message in a chat conversation +type Message struct { + Role MessageRole `json:"role" validate:"required,oneof=system user assistant"` + Content string `json:"content" validate:"required"` + Images []string `json:"images,omitempty"` +} + +// ChatOptions represents options for chat requests +type ChatOptions struct { + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + TopK *int `json:"top_k,omitempty"` + RepeatPenalty *float64 `json:"repeat_penalty,omitempty"` + Seed *int `json:"seed,omitempty"` + Stop []string `json:"stop,omitempty"` + NumCtx *int `json:"num_ctx,omitempty"` + NumPredict *int `json:"num_predict,omitempty"` + NumKeep *int `json:"num_keep,omitempty"` + NumThread *int `json:"num_thread,omitempty"` + RepeatLastN *int `json:"repeat_last_n,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` +} + +// ChatResponse represents the response from a chat request +type ChatResponse struct { + Model string `json:"model"` + CreatedAt time.Time `json:"created_at"` + Message Message `json:"message"` + Done bool `json:"done"` + TotalDuration int64 `json:"total_duration,omitempty"` + LoadDuration int64 `json:"load_duration,omitempty"` + PromptEvalCount int `json:"prompt_eval_count,omitempty"` + PromptEvalDuration int64 `json:"prompt_eval_duration,omitempty"` + EvalCount int `json:"eval_count,omitempty"` + EvalDuration int64 `json:"eval_duration,omitempty"` +} + +// EmbedRequest represents a request to generate embeddings +type EmbedRequest struct { + Model string `json:"model" validate:"required"` + Prompt string `json:"prompt" validate:"required"` + Options map[string]interface{} `json:"options,omitempty"` +} + +// EmbedResponse represents the response from an embedding request +type EmbedResponse struct { + Embedding []float64 `json:"embedding"` +} + +// PullRequest represents a request to pull a model +type PullRequest struct { + Name string `json:"name" validate:"required"` + Insecure bool `json:"insecure,omitempty"` + Stream *bool `json:"stream,omitempty"` +} + +// PullResponse represents the response from a pull request +type PullResponse struct { + Status string `json:"status"` + Digest string `json:"digest,omitempty"` + Total int64 `json:"total,omitempty"` + Completed int64 `json:"completed,omitempty"` +} + +// ListModelsResponse represents the response from listing models +type ListModelsResponse struct { + Models []ModelInfo `json:"models"` +} + +// ModelInfo represents information about a model +type ModelInfo struct { + Name string `json:"name"` + ModifiedAt time.Time `json:"modified_at"` + Size int64 `json:"size"` + Digest string `json:"digest"` + Details ModelDetails `json:"details,omitempty"` +} + +// ModelDetails represents detailed information about a model +type ModelDetails struct { + Format string `json:"format,omitempty"` + Family string `json:"family,omitempty"` + Families []string `json:"families,omitempty"` + ParameterSize string `json:"parameter_size,omitempty"` + QuantizationLevel string `json:"quantization_level,omitempty"` + ParentModel string `json:"parent_model,omitempty"` + Templates map[string]interface{} `json:"templates,omitempty"` + System string `json:"system,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` +} + +// DeleteRequest represents a request to delete a model +type DeleteRequest struct { + Name string `json:"name" validate:"required"` +} + +// DeleteResponse represents the response from a delete request +type DeleteResponse struct { + Status string `json:"status"` +} + +// ShowModelRequest represents a request to show model information +type ShowModelRequest struct { + Name string `json:"name" validate:"required"` + Verbose bool `json:"verbose,omitempty"` +} + +// ShowModelResponse represents the response from a show model request +type ShowModelResponse struct { + License string `json:"license,omitempty"` + Modelfile string `json:"modelfile,omitempty"` + Parameters string `json:"parameters,omitempty"` + Template string `json:"template,omitempty"` + Details ModelDetails `json:"details,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` +} + +// HealthResponse represents the health check response +type HealthResponse struct { + Status string `json:"status"` +} + +// StreamChunk represents a chunk in a streaming response +type StreamChunk struct { + Model string `json:"model,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + Response string `json:"response,omitempty"` + Done bool `json:"done"` + Context []int `json:"context,omitempty"` + TotalDuration int64 `json:"total_duration,omitempty"` + LoadDuration int64 `json:"load_duration,omitempty"` + PromptEvalCount int `json:"prompt_eval_count,omitempty"` + PromptEvalDuration int64 `json:"prompt_eval_duration,omitempty"` + EvalCount int `json:"eval_count,omitempty"` + EvalDuration int64 `json:"eval_duration,omitempty"` + Message *Message `json:"message,omitempty"` +} diff --git a/pkg/ollama/errors.go b/pkg/ollama/errors.go new file mode 100644 index 0000000..e121c07 --- /dev/null +++ b/pkg/ollama/errors.go @@ -0,0 +1,146 @@ +package ollama + +import ( + "fmt" +) + +// Error types for the Ollama SDK +var ( + // ErrInvalidConfig is returned when configuration is invalid + ErrInvalidConfig = func(msg string) error { + return &ConfigError{Message: msg} + } + + // ErrInvalidRequest is returned when a request is invalid + ErrInvalidRequest = func(msg string) error { + return &RequestError{Message: msg} + } + + // ErrModelNotFound is returned when a model is not found + ErrModelNotFound = func(model string) error { + return &ModelError{Message: fmt.Sprintf("model %s not found", model), Model: model} + } + + // ErrModelPullFailed is returned when pulling a model fails + ErrModelPullFailed = func(model string, reason string) error { + return &ModelError{Message: fmt.Sprintf("failed to pull model %s: %s", model, reason), Model: model} + } + + // ErrModelDeleteFailed is returned when deleting a model fails + ErrModelDeleteFailed = func(model string, reason string) error { + return &ModelError{Message: fmt.Sprintf("failed to delete model %s: %s", model, reason), Model: model} + } + + // ErrGenerationFailed is returned when text generation fails + ErrGenerationFailed = func(reason string) error { + return &GenerationError{Message: fmt.Sprintf("generation failed: %s", reason)} + } + + // ErrChatFailed is returned when chat fails + ErrChatFailed = func(reason string) error { + return &ChatError{Message: fmt.Sprintf("chat failed: %s", reason)} + } + + // ErrEmbeddingFailed is returned when embedding generation fails + ErrEmbeddingFailed = func(reason string) error { + return &EmbeddingError{Message: fmt.Sprintf("embedding failed: %s", reason)} + } + + // ErrConnectionFailed is returned when connection to Ollama fails + ErrConnectionFailed = func(reason string) error { + return &ConnectionError{Message: fmt.Sprintf("connection failed: %s", reason)} + } + + // ErrTimeout is returned when a request times out + ErrTimeout = func(operation string) error { + return &TimeoutError{Message: fmt.Sprintf("timeout during %s", operation), Operation: operation} + } + + // ErrRateLimitExceeded is returned when rate limit is exceeded + ErrRateLimitExceeded = func() error { + return &RateLimitError{Message: "rate limit exceeded"} + } +) + +// ConfigError represents a configuration error +type ConfigError struct { + Message string +} + +func (e *ConfigError) Error() string { + return fmt.Sprintf("config error: %s", e.Message) +} + +// RequestError represents a request validation error +type RequestError struct { + Message string +} + +func (e *RequestError) Error() string { + return fmt.Sprintf("request error: %s", e.Message) +} + +// ModelError represents a model-related error +type ModelError struct { + Message string + Model string +} + +func (e *ModelError) Error() string { + return fmt.Sprintf("model error: %s", e.Message) +} + +// GenerationError represents a text generation error +type GenerationError struct { + Message string +} + +func (e *GenerationError) Error() string { + return fmt.Sprintf("generation error: %s", e.Message) +} + +// ChatError represents a chat error +type ChatError struct { + Message string +} + +func (e *ChatError) Error() string { + return fmt.Sprintf("chat error: %s", e.Message) +} + +// EmbeddingError represents an embedding generation error +type EmbeddingError struct { + Message string +} + +func (e *EmbeddingError) Error() string { + return fmt.Sprintf("embedding error: %s", e.Message) +} + +// ConnectionError represents a connection error +type ConnectionError struct { + Message string +} + +func (e *ConnectionError) Error() string { + return fmt.Sprintf("connection error: %s", e.Message) +} + +// TimeoutError represents a timeout error +type TimeoutError struct { + Message string + Operation string +} + +func (e *TimeoutError) Error() string { + return fmt.Sprintf("timeout error: %s", e.Message) +} + +// RateLimitError represents a rate limit error +type RateLimitError struct { + Message string +} + +func (e *RateLimitError) Error() string { + return fmt.Sprintf("rate limit error: %s", e.Message) +} diff --git a/pkg/ollama/interfaces.go b/pkg/ollama/interfaces.go new file mode 100644 index 0000000..b9695e1 --- /dev/null +++ b/pkg/ollama/interfaces.go @@ -0,0 +1,121 @@ +package ollama + +import ( + "context" +) + +// OllamaService defines the interface for Ollama operations +type OllamaService interface { + // Generate generates text using a specified model and prompt + Generate(ctx context.Context, req *GenerateRequest) (*GenerateResponse, error) + + // GenerateWithOptions generates text with additional options + GenerateWithOptions(ctx context.Context, req *GenerateRequest, opts *GenerateOptions) (*GenerateResponse, error) + + // Chat performs a conversational chat with the model + Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) + + // Embed generates embeddings for the given text + Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) + + // PullModel pulls a model from the registry + PullModel(ctx context.Context, model string) (*PullResponse, error) + + // ListModels lists available models + ListModels(ctx context.Context) (*ListModelsResponse, error) + + // DeleteModel deletes a model + DeleteModel(ctx context.Context, model string) (*DeleteResponse, error) + + // ShowModel shows model information + ShowModel(ctx context.Context, model string) (*ShowModelResponse, error) + + // Health checks if the Ollama service is available + Health(ctx context.Context) error + + // NewChatBuilder creates a new chat builder for conversational interactions + NewChatBuilder() ChatBuilder +} + +// HTTPClient defines the interface for HTTP operations +type HTTPClient interface { + // Post performs a POST request + Post(ctx context.Context, url string, body interface{}, result interface{}) error + + // Get performs a GET request + Get(ctx context.Context, url string, result interface{}) error + + // Delete performs a DELETE request + Delete(ctx context.Context, url string, result interface{}) error +} + +// Logger defines the interface for logging operations +type Logger interface { + Debug(msg string, fields map[string]interface{}) + Info(msg string, fields map[string]interface{}) + Warn(msg string, fields map[string]interface{}) + Error(msg string, fields map[string]interface{}) + Fatal(msg string, fields map[string]interface{}) + WithFields(fields map[string]interface{}) Logger + Sync() error +} + +// ChatBuilder provides a fluent interface for building chat requests +type ChatBuilder interface { + // SetModel sets the model to use + SetModel(model string) ChatBuilder + + // SetSystem sets the system prompt + SetSystem(system string) ChatBuilder + + // AddMessage adds a message to the conversation + AddMessage(role MessageRole, content string) ChatBuilder + + // SetTemperature sets the temperature for generation + SetTemperature(temperature float64) ChatBuilder + + // SetTopP sets the top_p parameter + SetTopP(topP float64) ChatBuilder + + // SetMaxTokens sets the maximum number of tokens to generate + SetMaxTokens(maxTokens int) ChatBuilder + + // SetStream sets whether to stream the response + SetStream(stream bool) ChatBuilder + + // SetFormat sets the response format + SetFormat(format string) ChatBuilder + + // SetOptions sets custom options + SetOptions(options map[string]interface{}) ChatBuilder + + // Build creates the chat request + Build() (*ChatRequest, error) + + // Send builds and sends the chat request + Send(ctx context.Context) (*ChatResponse, error) + + // Continue continues the conversation with the current context + Continue(ctx context.Context, userMessage string) (*ChatResponse, error) +} + +// MessageRole represents the role of a message in a chat +type MessageRole string + +const ( + MessageRoleSystem MessageRole = "system" + MessageRoleUser MessageRole = "user" + MessageRoleAssistant MessageRole = "assistant" +) + +// GenerateOptions represents additional options for text generation +type GenerateOptions struct { + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + TopK *int `json:"top_k,omitempty"` + RepeatPenalty *float64 `json:"repeat_penalty,omitempty"` + Seed *int `json:"seed,omitempty"` + Stop []string `json:"stop,omitempty"` + Format string `json:"format,omitempty"` + Options map[string]interface{} `json:"options,omitempty"` +} diff --git a/pkg/ollama/sdk.go b/pkg/ollama/sdk.go new file mode 100644 index 0000000..eab9b0c --- /dev/null +++ b/pkg/ollama/sdk.go @@ -0,0 +1,232 @@ +package ollama + +import ( + "context" + "fmt" + "net/http" + "tm/pkg/logger" +) + +// SDK provides a high-level interface for the Ollama service +type SDK struct { + service OllamaService + client *Client + config *Config +} + +// New creates a new Ollama SDK with the provided configuration and logger +func New(config *Config, logger logger.Logger) (*SDK, error) { + if config == nil { + config = DefaultConfig() + } + + client, err := NewClient(config, logger) + if err != nil { + return nil, fmt.Errorf("failed to create ollama client: %w", err) + } + + service := NewService(client) + + return &SDK{ + service: service, + client: client, + config: config, + }, nil +} + +// NewWithClient creates a new Ollama SDK with a custom HTTP client +func NewWithClient(config *Config, httpClient *http.Client, logger logger.Logger) (*SDK, error) { + if config == nil { + config = DefaultConfig() + } + + client, err := NewClientWithHTTPClient(config, httpClient, logger) + if err != nil { + return nil, fmt.Errorf("failed to create ollama client: %w", err) + } + + service := NewService(client) + + return &SDK{ + service: service, + client: client, + config: config, + }, nil +} + +// Generate generates text using a specified model and prompt +func (s *SDK) Generate(ctx context.Context, req *GenerateRequest) (*GenerateResponse, error) { + return s.service.Generate(ctx, req) +} + +// GenerateWithOptions generates text with additional options +func (s *SDK) GenerateWithOptions(ctx context.Context, req *GenerateRequest, opts *GenerateOptions) (*GenerateResponse, error) { + return s.service.GenerateWithOptions(ctx, req, opts) +} + +// Chat performs a conversational chat with the model +func (s *SDK) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) { + return s.service.Chat(ctx, req) +} + +// Embed generates embeddings for the given text +func (s *SDK) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) { + return s.service.Embed(ctx, req) +} + +// PullModel pulls a model from the registry +func (s *SDK) PullModel(ctx context.Context, model string) (*PullResponse, error) { + return s.service.PullModel(ctx, model) +} + +// ListModels lists available models +func (s *SDK) ListModels(ctx context.Context) (*ListModelsResponse, error) { + return s.service.ListModels(ctx) +} + +// DeleteModel deletes a model +func (s *SDK) DeleteModel(ctx context.Context, model string) (*DeleteResponse, error) { + return s.service.DeleteModel(ctx, model) +} + +// ShowModel shows model information +func (s *SDK) ShowModel(ctx context.Context, model string) (*ShowModelResponse, error) { + return s.service.ShowModel(ctx, model) +} + +// Health checks if the Ollama service is available +func (s *SDK) Health(ctx context.Context) error { + return s.service.Health(ctx) +} + +// NewChatBuilder creates a new chat builder for conversational interactions +func (s *SDK) NewChatBuilder() ChatBuilder { + return s.service.NewChatBuilder() +} + +// QuickGenerate is a convenience method for simple text generation +func (s *SDK) QuickGenerate(ctx context.Context, prompt string) (string, error) { + req := &GenerateRequest{ + Model: s.config.DefaultModel, + Prompt: prompt, + } + + resp, err := s.Generate(ctx, req) + if err != nil { + return "", err + } + + return resp.Response, nil +} + +// QuickGenerateWithModel is a convenience method for text generation with a specific model +func (s *SDK) QuickGenerateWithModel(ctx context.Context, model, prompt string) (string, error) { + req := &GenerateRequest{ + Model: model, + Prompt: prompt, + } + + resp, err := s.Generate(ctx, req) + if err != nil { + return "", err + } + + return resp.Response, nil +} + +// QuickChat is a convenience method for simple chat interactions +func (s *SDK) QuickChat(ctx context.Context, userMessage string) (string, error) { + req := &ChatRequest{ + Model: s.config.DefaultModel, + Messages: []Message{ + { + Role: MessageRoleUser, + Content: userMessage, + }, + }, + } + + resp, err := s.Chat(ctx, req) + if err != nil { + return "", err + } + + return resp.Message.Content, nil +} + +// QuickChatWithSystem is a convenience method for chat with system prompt +func (s *SDK) QuickChatWithSystem(ctx context.Context, systemPrompt, userMessage string) (string, error) { + req := &ChatRequest{ + Model: s.config.DefaultModel, + Messages: []Message{ + { + Role: MessageRoleSystem, + Content: systemPrompt, + }, + { + Role: MessageRoleUser, + Content: userMessage, + }, + }, + } + + resp, err := s.Chat(ctx, req) + if err != nil { + return "", err + } + + return resp.Message.Content, nil +} + +// QuickEmbed is a convenience method for generating embeddings +func (s *SDK) QuickEmbed(ctx context.Context, text string) ([]float64, error) { + req := &EmbedRequest{ + Model: s.config.DefaultModel, + Prompt: text, + } + + resp, err := s.Embed(ctx, req) + if err != nil { + return nil, err + } + + return resp.Embedding, nil +} + +// StreamGenerate performs streaming text generation +func (s *SDK) StreamGenerate(ctx context.Context, req *GenerateRequest, onChunk func(string) error) error { + // Force streaming + stream := true + req.Stream = &stream + + return s.client.StreamGenerate(ctx, req, func(chunk *StreamChunk) error { + if chunk.Response != "" { + return onChunk(chunk.Response) + } + return nil + }) +} + +// StreamChat performs streaming chat +func (s *SDK) StreamChat(ctx context.Context, req *ChatRequest, onChunk func(string) error) error { + // Force streaming + stream := true + req.Stream = &stream + + return s.client.StreamChat(ctx, req, func(chunk *StreamChunk) error { + if chunk.Message != nil && chunk.Message.Content != "" { + return onChunk(chunk.Message.Content) + } + return nil + }) +} + +// GetService returns the underlying service for advanced usage +func (s *SDK) GetService() OllamaService { + return s.service +} + +// GetConfig returns the current configuration +func (s *SDK) GetConfig() *Config { + return s.config +} diff --git a/pkg/ollama/service.go b/pkg/ollama/service.go new file mode 100644 index 0000000..53e4bd4 --- /dev/null +++ b/pkg/ollama/service.go @@ -0,0 +1,444 @@ +package ollama + +import ( + "context" + "fmt" + "tm/pkg/logger" +) + +// Service implements the OllamaService interface +type Service struct { + client *Client + config *Config + logger logger.Logger +} + +// NewService creates a new Ollama service +func NewService(client *Client) OllamaService { + return &Service{ + client: client, + config: client.config, + logger: client.logger, + } +} + +// Generate generates text using a specified model and prompt +func (s *Service) Generate(ctx context.Context, req *GenerateRequest) (*GenerateResponse, error) { + if s.logger != nil { + s.logger.Info("Generating text", map[string]interface{}{ + "model": req.Model, + "prompt_length": len(req.Prompt), + }) + } + + // Set defaults + if req.Model == "" { + req.Model = s.config.DefaultModel + } + + var resp GenerateResponse + err := s.client.Post(ctx, "/api/generate", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Text generation failed", map[string]interface{}{ + "model": req.Model, + "error": err.Error(), + }) + } + return nil, ErrGenerationFailed(err.Error()) + } + + if s.logger != nil { + s.logger.Info("Text generation completed", map[string]interface{}{ + "model": req.Model, + "response_length": len(resp.Response), + "eval_count": resp.EvalCount, + }) + } + + return &resp, nil +} + +// GenerateWithOptions generates text with additional options +func (s *Service) GenerateWithOptions(ctx context.Context, req *GenerateRequest, opts *GenerateOptions) (*GenerateResponse, error) { + if s.logger != nil { + s.logger.Info("Generating text with options", map[string]interface{}{ + "model": req.Model, + "prompt_length": len(req.Prompt), + }) + } + + // Apply options to request + if opts != nil { + if req.Options == nil { + req.Options = make(map[string]interface{}) + } + + if opts.Temperature != nil { + req.Options["temperature"] = *opts.Temperature + } + if opts.TopP != nil { + req.Options["top_p"] = *opts.TopP + } + if opts.TopK != nil { + req.Options["top_k"] = *opts.TopK + } + if opts.RepeatPenalty != nil { + req.Options["repeat_penalty"] = *opts.RepeatPenalty + } + if opts.Seed != nil { + req.Options["seed"] = *opts.Seed + } + if opts.Stop != nil { + req.Options["stop"] = opts.Stop + } + if opts.Format != "" { + req.Format = opts.Format + } + + // Merge additional options + for k, v := range opts.Options { + req.Options[k] = v + } + } + + return s.Generate(ctx, req) +} + +// Chat performs a conversational chat with the model +func (s *Service) Chat(ctx context.Context, req *ChatRequest) (*ChatResponse, error) { + if s.logger != nil { + s.logger.Info("Starting chat", map[string]interface{}{ + "model": req.Model, + "message_count": len(req.Messages), + }) + } + + // Set defaults + if req.Model == "" { + req.Model = s.config.DefaultModel + } + + var resp ChatResponse + err := s.client.Post(ctx, "/api/chat", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Chat failed", map[string]interface{}{ + "model": req.Model, + "error": err.Error(), + }) + } + return nil, ErrChatFailed(err.Error()) + } + + if s.logger != nil { + s.logger.Info("Chat completed", map[string]interface{}{ + "model": req.Model, + "response_length": len(resp.Message.Content), + "eval_count": resp.EvalCount, + }) + } + + return &resp, nil +} + +// Embed generates embeddings for the given text +func (s *Service) Embed(ctx context.Context, req *EmbedRequest) (*EmbedResponse, error) { + if s.logger != nil { + s.logger.Info("Generating embeddings", map[string]interface{}{ + "model": req.Model, + "text_length": len(req.Prompt), + }) + } + + var resp EmbedResponse + err := s.client.Post(ctx, "/api/embeddings", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Embedding generation failed", map[string]interface{}{ + "model": req.Model, + "error": err.Error(), + }) + } + return nil, ErrEmbeddingFailed(err.Error()) + } + + if s.logger != nil { + s.logger.Info("Embeddings generated", map[string]interface{}{ + "model": req.Model, + "embedding_dimension": len(resp.Embedding), + }) + } + + return &resp, nil +} + +// PullModel pulls a model from the registry +func (s *Service) PullModel(ctx context.Context, model string) (*PullResponse, error) { + if s.logger != nil { + s.logger.Info("Pulling model", map[string]interface{}{ + "model": model, + }) + } + + req := &PullRequest{ + Name: model, + } + + var resp PullResponse + err := s.client.Post(ctx, "/api/pull", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Model pull failed", map[string]interface{}{ + "model": model, + "error": err.Error(), + }) + } + return nil, ErrModelPullFailed(model, err.Error()) + } + + if s.logger != nil { + s.logger.Info("Model pulled successfully", map[string]interface{}{ + "model": model, + "status": resp.Status, + }) + } + + return &resp, nil +} + +// ListModels lists available models +func (s *Service) ListModels(ctx context.Context) (*ListModelsResponse, error) { + if s.logger != nil { + s.logger.Debug("Listing models", map[string]interface{}{}) + } + + var resp ListModelsResponse + err := s.client.Get(ctx, "/api/tags", &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Failed to list models", map[string]interface{}{ + "error": err.Error(), + }) + } + return nil, fmt.Errorf("failed to list models: %w", err) + } + + if s.logger != nil { + s.logger.Debug("Models listed", map[string]interface{}{ + "model_count": len(resp.Models), + }) + } + + return &resp, nil +} + +// DeleteModel deletes a model +func (s *Service) DeleteModel(ctx context.Context, model string) (*DeleteResponse, error) { + if s.logger != nil { + s.logger.Info("Deleting model", map[string]interface{}{ + "model": model, + }) + } + + req := &DeleteRequest{ + Name: model, + } + + var resp DeleteResponse + err := s.client.Delete(ctx, "/api/delete", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Model deletion failed", map[string]interface{}{ + "model": model, + "error": err.Error(), + }) + } + return nil, ErrModelDeleteFailed(model, err.Error()) + } + + if s.logger != nil { + s.logger.Info("Model deleted successfully", map[string]interface{}{ + "model": model, + "status": resp.Status, + }) + } + + return &resp, nil +} + +// ShowModel shows model information +func (s *Service) ShowModel(ctx context.Context, model string) (*ShowModelResponse, error) { + if s.logger != nil { + s.logger.Debug("Showing model information", map[string]interface{}{ + "model": model, + }) + } + + req := &ShowModelRequest{ + Name: model, + } + + var resp ShowModelResponse + err := s.client.Post(ctx, "/api/show", req, &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Failed to show model", map[string]interface{}{ + "model": model, + "error": err.Error(), + }) + } + return nil, fmt.Errorf("failed to show model %s: %w", model, err) + } + + if s.logger != nil { + s.logger.Debug("Model information retrieved", map[string]interface{}{ + "model": model, + }) + } + + return &resp, nil +} + +// Health checks if the Ollama service is available +func (s *Service) Health(ctx context.Context) error { + if s.logger != nil { + s.logger.Debug("Checking Ollama health", map[string]interface{}{}) + } + + var resp HealthResponse + err := s.client.Get(ctx, "/api/version", &resp) + if err != nil { + if s.logger != nil { + s.logger.Error("Health check failed", map[string]interface{}{ + "error": err.Error(), + }) + } + return ErrConnectionFailed(err.Error()) + } + + if s.logger != nil { + s.logger.Debug("Health check passed", map[string]interface{}{}) + } + + return nil +} + +// NewChatBuilder creates a new chat builder for conversational interactions +func (s *Service) NewChatBuilder() ChatBuilder { + return &chatBuilder{ + service: s, + request: &ChatRequest{ + Model: s.config.DefaultModel, + Messages: make([]Message, 0), + Options: ChatOptions{ + Temperature: &s.config.DefaultTemperature, + TopP: &s.config.DefaultTopP, + }, + }, + } +} + +// chatBuilder implements the ChatBuilder interface +type chatBuilder struct { + service OllamaService + request *ChatRequest +} + +func (b *chatBuilder) SetModel(model string) ChatBuilder { + b.request.Model = model + return b +} + +func (b *chatBuilder) SetSystem(system string) ChatBuilder { + // Add or update system message + for i, msg := range b.request.Messages { + if msg.Role == MessageRoleSystem { + b.request.Messages[i].Content = system + return b + } + } + + // Add new system message at the beginning + systemMsg := Message{ + Role: MessageRoleSystem, + Content: system, + } + b.request.Messages = append([]Message{systemMsg}, b.request.Messages...) + return b +} + +func (b *chatBuilder) AddMessage(role MessageRole, content string) ChatBuilder { + msg := Message{ + Role: role, + Content: content, + } + b.request.Messages = append(b.request.Messages, msg) + return b +} + +func (b *chatBuilder) SetTemperature(temperature float64) ChatBuilder { + b.request.Options.Temperature = &temperature + return b +} + +func (b *chatBuilder) SetTopP(topP float64) ChatBuilder { + b.request.Options.TopP = &topP + return b +} + +func (b *chatBuilder) SetMaxTokens(maxTokens int) ChatBuilder { + b.request.Options.NumPredict = &maxTokens + return b +} + +func (b *chatBuilder) SetStream(stream bool) ChatBuilder { + b.request.Stream = &stream + return b +} + +func (b *chatBuilder) SetFormat(format string) ChatBuilder { + b.request.Format = format + return b +} + +func (b *chatBuilder) SetOptions(options map[string]interface{}) ChatBuilder { + if b.request.Options.Options == nil { + b.request.Options.Options = make(map[string]interface{}) + } + for k, v := range options { + b.request.Options.Options[k] = v + } + return b +} + +func (b *chatBuilder) Build() (*ChatRequest, error) { + if len(b.request.Messages) == 0 { + return nil, ErrInvalidRequest("at least one message is required") + } + return b.request, nil +} + +func (b *chatBuilder) Send(ctx context.Context) (*ChatResponse, error) { + req, err := b.Build() + if err != nil { + return nil, err + } + return b.service.Chat(ctx, req) +} + +func (b *chatBuilder) Continue(ctx context.Context, userMessage string) (*ChatResponse, error) { + // Add user message + b.AddMessage(MessageRoleUser, userMessage) + + // Send chat request + resp, err := b.Send(ctx) + if err != nil { + return nil, err + } + + // Add assistant response to conversation history + b.AddMessage(MessageRoleAssistant, resp.Message.Content) + + return resp, nil +} diff --git a/pkg/schedule/cron.go b/pkg/schedule/cron.go index 2f9220d..ab9ac46 100644 --- a/pkg/schedule/cron.go +++ b/pkg/schedule/cron.go @@ -2,7 +2,7 @@ package schedule import ( "time" - "tm/internal/tender" + "tm/internal/notice" "tm/pkg/logger" "tm/pkg/mongo" @@ -13,7 +13,7 @@ import ( type CronScheduler struct { cron *cron.Cron mongoManager *mongo.ConnectionManager - tenderRepo tender.TenderRepository + noticeRepo notice.Repository logger logger.Logger } @@ -31,14 +31,14 @@ type Job struct { } // NewCronScheduler creates a new cron scheduler instance -func NewCronScheduler(logger logger.Logger, mongoManager *mongo.ConnectionManager, tenderRepo tender.TenderRepository) *CronScheduler { +func NewCronScheduler(logger logger.Logger, mongoManager *mongo.ConnectionManager, noticeRepo notice.Repository) *CronScheduler { // Create cron with timezone support and seconds precision c := cron.New(cron.WithLocation(time.UTC), cron.WithSeconds()) return &CronScheduler{ cron: c, mongoManager: mongoManager, - tenderRepo: tenderRepo, + noticeRepo: noticeRepo, logger: logger, } } diff --git a/ted/mapper.go b/ted/mapper.go index 2134d86..2d254c4 100644 --- a/ted/mapper.go +++ b/ted/mapper.go @@ -4,11 +4,11 @@ import ( "strconv" "strings" "time" - "tm/internal/tender" + "tm/internal/notice" ) // mapToTenderFromParsedDoc maps a parsed TED document to tender entity -func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *tender.Tender { +func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceFileURL, sourceFileName string) *notice.Notice { switch parsedDoc.Type { case DocumentTypeContractNotice: if parsedDoc.ContractNotice == nil { @@ -45,7 +45,7 @@ func (s *TEDScraper) mapToTenderFromParsedDoc(parsedDoc *ParsedDocument, sourceF } // mapGenericNoticeToTender maps any TED document type to tender entity using the common interface -func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *tender.Tender { +func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, sourceFileName string) *notice.Notice { if doc == nil { s.logger.Error("Document is nil in mapGenericNoticeToTender", map[string]interface{}{}) return nil @@ -53,7 +53,7 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so now := time.Now().Unix() - t := &tender.Tender{ + t := ¬ice.Notice{ ContractNoticeID: doc.GetID(), ContractFolderID: "", // Not available in generic interface NoticeTypeCode: doc.GetNoticeType(), @@ -68,11 +68,11 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so // Basic tender information Title: "Generic Notice " + doc.GetNoticeType(), Description: "Processed from " + doc.GetNoticeType() + " document", - Status: tender.TenderStatusActive, - Source: tender.TenderSourceTEDScraper, + Status: notice.TenderStatusActive, + Source: notice.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, - ProcessingMetadata: tender.ProcessingMetadata{ + ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", @@ -86,10 +86,10 @@ func (s *TEDScraper) mapGenericNoticeToTender(doc TEDDocument, sourceFileURL, so } // mapToTender maps a TED ContractNotice to tender entity -func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *tender.Tender { +func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileName string) *notice.Notice { now := time.Now().Unix() - t := &tender.Tender{ + t := ¬ice.Notice{ ContractNoticeID: cn.ID, ContractFolderID: cn.ContractFolderID, NoticeTypeCode: cn.GetNoticeSubType(), @@ -105,11 +105,11 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa MainClassification: cn.GetMainClassification(), PlaceOfPerformance: cn.GetPlaceOfPerformance(), DocumentURI: cn.GetDocumentURI(), - Status: tender.TenderStatusActive, - Source: tender.TenderSourceTEDScraper, + Status: notice.TenderStatusActive, + Source: notice.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, - ProcessingMetadata: tender.ProcessingMetadata{ + ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", @@ -215,7 +215,7 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa // Set selection criteria if criteria := cn.GetSelectionCriteria(); criteria != nil { for _, criterion := range criteria { - t.SelectionCriteria = append(t.SelectionCriteria, tender.SelectionCriterion{ + t.SelectionCriteria = append(t.SelectionCriteria, notice.SelectionCriterion{ TypeCode: criterion.CriterionTypeCode, Description: criterion.Description, LanguageID: criterion.LanguageID, @@ -235,10 +235,10 @@ func (s *TEDScraper) mapToTender(cn *ContractNotice, sourceFileURL, sourceFileNa } // mapContractAwardNoticeToTender maps a TED ContractAwardNotice to tender entity -func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *tender.Tender { +func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, sourceFileURL, sourceFileName string) *notice.Notice { now := time.Now().Unix() - t := &tender.Tender{ + t := ¬ice.Notice{ ContractNoticeID: can.ID, ContractFolderID: can.ContractFolderID, NoticeTypeCode: can.NoticeTypeCode, @@ -252,11 +252,11 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so ProcurementTypeCode: can.GetContractNature(), ProcedureCode: can.GetProcedureCode(), MainClassification: can.GetMainClassification(), - Status: tender.TenderStatusAwarded, // Award notices are for completed tenders - Source: tender.TenderSourceTEDScraper, + Status: notice.TenderStatusAwarded, // Award notices are for completed tenders + Source: notice.TenderSourceTEDScraper, SourceFileURL: sourceFileURL, SourceFileName: sourceFileName, - ProcessingMetadata: tender.ProcessingMetadata{ + ProcessingMetadata: notice.ProcessingMetadata{ ScrapedAt: now, ProcessedAt: now, ProcessingVersion: "1.0", @@ -315,13 +315,13 @@ func (s *TEDScraper) mapContractAwardNoticeToTender(can *ContractAwardNotice, so } // mapOrganization maps a TED Organization to tender Organization -func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender.Organization { +func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *notice.Organization { if tedOrg == nil || tedOrg.Company == nil { return nil } company := tedOrg.Company - org := &tender.Organization{ + org := ¬ice.Organization{ Role: role, } @@ -351,7 +351,7 @@ func (s *TEDScraper) mapOrganization(tedOrg *Organization, role string) *tender. // Set address if company.PostalAddress != nil { - org.Address = tender.Address{ + org.Address = notice.Address{ StreetName: company.PostalAddress.StreetName, CityName: company.PostalAddress.CityName, PostalZone: company.PostalAddress.PostalZone, diff --git a/ted/scraper.go b/ted/scraper.go index 8f5a18a..3f28d15 100644 --- a/ted/scraper.go +++ b/ted/scraper.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" "time" - "tm/internal/tender" + "tm/internal/notice" "tm/pkg/logger" "tm/pkg/mongo" "tm/pkg/notification" @@ -27,6 +27,7 @@ type Config struct { RetryDelay time.Duration `mapstructure:"retry_delay"` CleanupAfter time.Duration `mapstructure:"cleanup_after"` ScrapingInterval string `mapstructure:"scraping_interval"` + AlertMail string `mapstructure:"ALERT_MAIL"` } // TEDScraper handles downloading and parsing TED XML files @@ -37,7 +38,7 @@ type TEDScraper struct { httpClient *http.Client xmlParser *TEDParser mongoManager *mongo.ConnectionManager - tenderRepo tender.TenderRepository + tenderRepo notice.Repository } // NewTEDScraper creates a new TED scraper instance @@ -45,7 +46,7 @@ func NewTEDScraper( config *Config, logger logger.Logger, mongoManager *mongo.ConnectionManager, - tenderRepo tender.TenderRepository, + tenderRepo notice.Repository, notify notification.SDK, ) *TEDScraper { httpClient := &http.Client{ @@ -251,31 +252,34 @@ func (s *TEDScraper) tarGzFileProcessor(ctx context.Context, reader io.Reader, o } } - go s.notify.SendNotification( - context.Background(), - ¬ification.NotificationRequest{ - EventType: notification.EventTypeEmail, - Title: "TED scraper completed", - Message: GenerateFileProcessingEmailTemplate( - &FileProcessingEmailTemplateData{ - OJS: ojs, - ProcessedCount: result.ProcessedCount, - SuccessCount: result.SuccessCount, - Errors: result.Errors, - ProcessedAt: result.ProcessedAt, - }), - Priority: "important", - Methods: notification.NotificationMethods{ - Email: "nakhostin.nima1998@gmail.com", - }, - UserID: "ted-scraper", - Type: "alert", - }) + if s.config.AlertMail != "" { + + go s.notify.SendNotification( + context.Background(), + ¬ification.NotificationRequest{ + EventType: notification.EventTypeEmail, + Title: "TED scraper completed", + Message: GenerateFileProcessingEmailTemplate( + &FileProcessingEmailTemplateData{ + OJS: ojs, + ProcessedCount: result.ProcessedCount, + SuccessCount: result.SuccessCount, + Errors: result.Errors, + ProcessedAt: result.ProcessedAt, + }), + Priority: "important", + Methods: notification.NotificationMethods{ + Email: s.config.AlertMail, + }, + UserID: "ted-scraper", + Type: "alert", + }) + } return result, nil } -func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*tender.Tender, error) { +func (s *TEDScraper) findTenderByContractNoticeID(ctx context.Context, contractNoticeID string) (*notice.Notice, error) { existingTender, err := s.tenderRepo.GetByContractNoticeID(ctx, contractNoticeID) if err != nil { if err.Error() == "document not found" { @@ -308,60 +312,60 @@ func (s *TEDScraper) processXMLContent(ctx context.Context, content []byte, file // Map to tender entity t := s.mapToTenderFromParsedDoc(notice, "", fileName) - existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID) + // existingTender, err := s.findTenderByContractNoticeID(ctx, t.ContractNoticeID) + // if err != nil { + // s.logger.Warn("Error checking for existing tender", map[string]interface{}{ + // "contract_notice_id": t.ContractNoticeID, + // "error": err.Error(), + // }) + // } + + // if err == nil && existingTender != nil { + // // Update existing tender + // s.logger.Info("Found existing tender, updating", map[string]interface{}{ + // "existing_tender_id": existingTender.ID.Hex(), + // "contract_notice_id": t.ContractNoticeID, + // }) + + // t.ID = existingTender.ID + // t.UpdatedAt = time.Now().Unix() + // err = s.tenderRepo.Update(ctx, t) + // if err != nil { + // s.logger.Error("Failed to update existing tender", map[string]interface{}{ + // "tender_id": t.ID.Hex(), + // "error": err.Error(), + // }) + // return fmt.Errorf("failed to update existing tender: %w", err) + // } + + // s.logger.Info("Successfully updated tender", map[string]interface{}{ + // "tender_id": t.ID.Hex(), + // }) + // } else { + // // Create new tender + s.logger.Info("Creating new tender", map[string]interface{}{ + "contract_notice_id": t.ContractNoticeID, + "tender_id": t.TenderID, + }) + + t.CreatedAt = time.Now().Unix() + t.UpdatedAt = t.CreatedAt + + err = s.tenderRepo.Import(ctx, t) if err != nil { - s.logger.Warn("Error checking for existing tender", map[string]interface{}{ + s.logger.Error("Failed to create new tender", map[string]interface{}{ "contract_notice_id": t.ContractNoticeID, + "tender_id": t.TenderID, "error": err.Error(), }) + return fmt.Errorf("failed to create tender: %w", err) } - if err == nil && existingTender != nil { - // Update existing tender - s.logger.Info("Found existing tender, updating", map[string]interface{}{ - "existing_tender_id": existingTender.ID.Hex(), - "contract_notice_id": t.ContractNoticeID, - }) - - t.ID = existingTender.ID - t.UpdatedAt = time.Now().Unix() - err = s.tenderRepo.Update(ctx, t) - if err != nil { - s.logger.Error("Failed to update existing tender", map[string]interface{}{ - "tender_id": t.ID.Hex(), - "error": err.Error(), - }) - return fmt.Errorf("failed to update existing tender: %w", err) - } - - s.logger.Info("Successfully updated tender", map[string]interface{}{ - "tender_id": t.ID.Hex(), - }) - } else { - // Create new tender - s.logger.Info("Creating new tender", map[string]interface{}{ - "contract_notice_id": t.ContractNoticeID, - "tender_id": t.TenderID, - }) - - t.CreatedAt = time.Now().Unix() - t.UpdatedAt = t.CreatedAt - - err = s.tenderRepo.Create(ctx, t) - if err != nil { - s.logger.Error("Failed to create new tender", map[string]interface{}{ - "contract_notice_id": t.ContractNoticeID, - "tender_id": t.TenderID, - "error": err.Error(), - }) - return fmt.Errorf("failed to create tender: %w", err) - } - - s.logger.Info("Successfully created new tender", map[string]interface{}{ - "tender_id": t.TenderID, - "contract_notice_id": t.ContractNoticeID, - }) - } + s.logger.Info("Successfully created new tender", map[string]interface{}{ + "tender_id": t.TenderID, + "contract_notice_id": t.ContractNoticeID, + }) + // } // Log detailed parsed information based on notice type s.logger.Info("Successfully parsed notice", map[string]interface{}{ diff --git a/ted/service.go b/ted/service.go index 622b16e..ef691d0 100644 --- a/ted/service.go +++ b/ted/service.go @@ -5,7 +5,7 @@ import ( "strconv" "strings" "time" - "tm/internal/tender" + "tm/internal/notice" ) // parseDate parses various date formats used in TED XML @@ -146,7 +146,7 @@ func GenerateTenderURL(noticePublicationID string) string { // generateTenderID generates a unique tender ID using the format: // {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE} -func GenerateTenderID(t *tender.Tender) string { +func GenerateTenderID(t *notice.Notice) string { var parts []string // Source (TED)