mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor Worker Initialization to Integrate GLM SDK

Browse Source 
- Updated the worker initialization process to include the new GLM SDK, enhancing the worker's capabilities for translation tasks.
- Modified the InitWorker function and NewNoticeWorker constructor to accept the GLM service, ensuring a cohesive integration.
- Implemented the GLM service initialization and logging for successful setup, improving maintainability and usability.
- Updated the NoticeWorker to utilize the GLM SDK for translating notice titles and descriptions, enhancing functionality and user experience.
This commit is contained in:
n.nakhostin
2025-11-04 16:51:45 +03:30
parent a5d3a94c97
commit fafccd0d74
10 changed files with 379 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/worker/bootstrap/bootstrap.go
+49 -3
View File
@@ -7,6 +7,7 @@ import (
"tm/internal/notice"
"tm/internal/tender"
"tm/pkg/config"
"tm/pkg/glm"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
@@ -74,19 +75,19 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
}
// Init Worker
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) {
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK) {
// Initialize repositories
noticeRepo := notice.NewRepository(mongoManager, appLogger)
tenderRepo := tender.NewRepository(mongoManager, appLogger)
worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo)
worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService)
worker.Run()
// start Worker job
schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
Name: "Worker Job",
Func: func() {
worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo)
worker := workers.NewNoticeWorker(mongoManager, appLogger, &notify, noticeRepo, tenderRepo, glmService)
worker.Run()
},
Expr: config.Worker.Interval,
@@ -148,3 +149,48 @@ func InitOllamaService(conf config.OllamaConfig, log logger.Logger) *ollama.SDK
return ollamaSDK
}
// Init GLM Service
func InitGLMService(conf GLMConfig, log logger.Logger) *glm.SDK {
glmSDK, err := glm.New(
&glm.Config{
BaseURL: conf.BaseURL,
APIKey: conf.APIKey,
Timeout: conf.Timeout,
RetryAttempts: conf.RetryAttempts,
RetryDelay: conf.RetryDelay,
EnableLogging: conf.EnableLogging,
UserAgent: conf.UserAgent,
DefaultModel: conf.DefaultModel,
DefaultMaxTokens: conf.DefaultMaxTokens,
DefaultTopP: conf.DefaultTopP,
DefaultTemperature: conf.DefaultTemperature,
DefaultFrequencyPenalty: conf.DefaultFrequencyPenalty,
DefaultPresencePenalty: conf.DefaultPresencePenalty,
EnableStreaming: conf.EnableStreaming,
EnableThinking: conf.EnableThinking,
TranslationOptions: glm.TranslationOption{
Model: conf.TranslateOptions.Model,
MaxTokens: conf.TranslateOptions.MaxTokens,
Temperature: conf.TranslateOptions.Temperature,
FrequencyPenalty: conf.TranslateOptions.FrequencyPenalty,
PresencePenalty: conf.TranslateOptions.PresencePenalty,
TopP: conf.TranslateOptions.TopP,
EnableThinking: conf.TranslateOptions.EnableThinking,
},
}, log)
if err != nil {
log.Error("Failed to initialize GLM SDK", map[string]interface{}{
"error": err.Error(),
})
panic(fmt.Sprintf("Failed to initialize GLM SDK: %v", err))
}
log.Info("GLM SDK initialized successfully", map[string]interface{}{
"base_url": conf.BaseURL,
"timeout": conf.Timeout,
})
return glmSDK
}
cmd/worker/bootstrap/config.go
+31
View File
@@ -1,6 +1,7 @@
package bootstrap
import (
"time"
"tm/pkg/config"
)
@@ -11,9 +12,39 @@ type Config struct {
Notification config.NotificationConfig
Ollama config.OllamaConfig
Worker WorkerConfig
GLM GLMConfig
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
}
type WorkerConfig struct {
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
}
type GLMConfig struct {
BaseURL string `env:"GLM_BASE_URL" envDefault:"https://api.z.ai"`
APIKey string `env:"GLM_API_KEY" envDefault:""`
Timeout time.Duration `env:"GLM_TIMEOUT" envDefault:"300s"`
RetryAttempts int `env:"GLM_RETRY_ATTEMPTS" envDefault:"3"`
RetryDelay time.Duration `env:"GLM_RETRY_DELAY" envDefault:"2s"`
EnableLogging bool `env:"GLM_ENABLE_LOGGING" envDefault:"true"`
UserAgent string `env:"GLM_USER_AGENT" envDefault:"Opplens-GLMSDK/1.0"`
DefaultModel string `env:"GLM_DEFAULT_MODEL" envDefault:"glm-4.5"`
DefaultMaxTokens int `env:"GLM_DEFAULT_MAX_TOKENS" envDefault:"4096"`
DefaultTopP float64 `env:"GLM_DEFAULT_TOP_P" envDefault:"1.0"`
DefaultTemperature float64 `env:"GLM_DEFAULT_TEMPERATURE" envDefault:"0.7"`
DefaultFrequencyPenalty float64 `env:"GLM_DEFAULT_FREQUENCY_PENALTY" envDefault:"0"`
DefaultPresencePenalty float64 `env:"GLM_DEFAULT_PRESENCE_PENALTY" envDefault:"0"`
EnableStreaming bool `env:"GLM_ENABLE_STREAMING" envDefault:"false"`
EnableThinking bool `env:"GLM_ENABLE_THINKING" envDefault:"false"`
TranslateOptions TranslateOptions
}
type TranslateOptions struct {
Model string `env:"GLM_TRANSLATION_MODEL" envDefault:"glm-4.5"`
MaxTokens int `env:"GLM_TRANSLATION_MAX_TOKENS" envDefault:"2048"`
Temperature float64 `env:"GLM_TRANSLATION_TEMPERATURE" envDefault:"0.3"`
FrequencyPenalty float64 `env:"GLM_TRANSLATION_FREQUENCY_PENALTY" envDefault:"0.1"`
PresencePenalty float64 `env:"GLM_TRANSLATION_PRESENCE_PENALTY" envDefault:"0.1"`
TopP float64 `env:"GLM_TRANSLATION_TOP_P" envDefault:"1.0"`
EnableThinking bool `env:"GLM_TRANSLATION_ENABLE_THINKING" envDefault:"false"`
}
cmd/worker/main.go
+3 -10
View File
@@ -37,18 +37,11 @@ func main() {
// Initialize notification service
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
// Initialize ollama service
// ollamaSDK := bootstrap.InitOllamaService(config.Ollama, appLogger)
// aiModels, err := ollamaSDK.ListModels(context.Background())
// if err != nil {
// appLogger.Error("Failed to list ollama models", map[string]interface{}{
// "error": err.Error(),
// })
// }
// appLogger.Info("Ollama models", map[string]interface{}{"models": aiModels})
// Initialize GLM service
glmService := bootstrap.InitGLMService(config.GLM, appLogger)
// Initialize Worker
bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService)
bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, glmService)
// Set up signal handling for graceful shutdown
signalChan := make(chan os.Signal, 1)
cmd/worker/workers/notice.go
+115 -14
View File
@@ -2,30 +2,33 @@ package workers
import (
"context"
"strings"
"time"
"tm/internal/notice"
"tm/internal/tender"
"tm/pkg/glm"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/ollama"
)
type NoticeWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
Notify notification.SDK
Ollama *ollama.SDK
Notify *notification.SDK
NoticeRepo notice.Repository
TenderRepo tender.TenderRepository
GLM *glm.SDK
}
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository) *NoticeWorker {
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify *notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, glmService *glm.SDK) *NoticeWorker {
return &NoticeWorker{
Mongo: mongo,
Logger: logger,
Notify: notify,
NoticeRepo: noticeRepo,
TenderRepo: tenderRepo,
GLM: glmService,
}
}
@@ -52,12 +55,7 @@ func (w *NoticeWorker) Run() {
"notice": n.ID.Hex(),
})
t, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID)
if err == nil {
continue
}
err = w.ToTender(&n, t)
t, err := w.ToTender(&n)
if err != nil {
w.Logger.Error("Failed to create tender", map[string]interface{}{
"error": err.Error(),
@@ -80,13 +78,33 @@ func (w *NoticeWorker) Run() {
})
}
}
n.ProcessingMetadata.Processed = true
n.ProcessingMetadata.ProcessedAt = time.Now().Unix()
err = w.NoticeRepo.Update(context.Background(), &n)
if err != nil {
w.Logger.Error("Failed to update notice", map[string]interface{}{
"error": err.Error(),
})
}
w.Logger.Info("Notice processed", map[string]interface{}{
"notice": n.ID.Hex(),
})
}
skip += limit
}
}
func (w *NoticeWorker) ToTender(n *notice.Notice, t *tender.Tender) error {
func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
t, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID)
if err != nil {
w.Logger.Info("New tender raised", map[string]interface{}{
"notice_publication_id": n.NoticePublicationID,
"country_code": n.CountryCode,
})
t = new(tender.Tender)
}
// Buyer Organization
buyerOrg := &tender.Organization{}
if n.BuyerOrganization != nil {
@@ -223,9 +241,32 @@ func (w *NoticeWorker) ToTender(n *notice.Notice, t *tender.Tender) error {
}
}
title, err := w.GLM.Translate(context.Background(), n.Title, n.NoticeLanguageCode, "en")
if err != nil {
w.Logger.Error("Failed to translate title", map[string]interface{}{
"notice_publication_id": n.NoticePublicationID,
"country_code": n.CountryCode,
"content": n.Title,
"error": err.Error(),
})
title = n.Title
// return err
}
description, err := w.GLM.Translate(context.Background(), n.Description, n.NoticeLanguageCode, "en")
if err != nil {
w.Logger.Error("Failed to translate description", map[string]interface{}{
"notice_publication_id": n.NoticePublicationID,
"country_code": n.CountryCode,
"content": n.Description,
"error": err.Error(),
})
description = n.Description
// return err
}
t.Title = title
t.Description = description
t.NoticePublicationID = n.NoticePublicationID
t.Title = n.Title
t.Description = n.Description
t.ProcurementTypeCode = n.ProcurementTypeCode
t.ProcedureCode = n.ProcedureCode
t.EstimatedValue = n.EstimatedValue
@@ -268,5 +309,65 @@ func (w *NoticeWorker) ToTender(n *notice.Notice, t *tender.Tender) error {
t.WinningTenderer = winningTenderer
t.Modifications = modifications
t.AwardedEntities = awardedEntities
return nil
t.TenderID = GenerateTenderID(n)
return t, nil
}
// generateTenderID generates a unique tender ID using the format:
// {Source}{BUYER_CODE}{TED_CODE}{DATE}{LOCATION_CODE}
func GenerateTenderID(t *notice.Notice) string {
var parts []string
// Source (TED)
parts = append(parts, "1")
// TED code (Notice Publication ID, first 8 chars)
tedCode := "00000000"
if t.NoticePublicationID != "" {
tedCode = PadOrTruncate(t.NoticePublicationID, 8)
}
parts = append(parts, tedCode)
// Buyer code (first 8 chars of buyer organization ID, or 8 zeros if not available)
// buyerCode := "00000000"
// if t.BuyerOrganization != nil && t.BuyerOrganization.ID != "" {
// buyerCode = padOrTruncate(t.BuyerOrganization.ID, 8)
// }
// parts = append(parts, buyerCode)
// Date (YYYYMMDD format from issue date)
parts = append(parts, PadOrTruncate(UnixToDateString(t.IssueDate), 4))
// Location code (country code + region code, padded to 6 chars)
locationCode := "EU"
if t.CountryCode != "" {
locationCode = t.CountryCode
}
parts = append(parts, locationCode)
return strings.Join(parts, "")
}
// padOrTruncate pads a string with zeros or truncates it to the specified length
func PadOrTruncate(str string, length int) string {
// Remove any non-alphanumeric characters and convert to uppercase
clean := strings.ToUpper(strings.ReplaceAll(str, "-", ""))
clean = strings.ReplaceAll(clean, " ", "")
if len(clean) >= length {
return clean[:length]
}
// Pad with zeros
return clean + strings.Repeat("0", length-len(clean))
}
// unixToDateString converts Unix to date string for TenderID generation
func UnixToDateString(unix int64) string {
if unix == 0 {
return "00000000"
}
return time.Unix(unix, 0).Format("20060102")
}