mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature/ai-translator' into develop

Browse Source
This commit is contained in:
n.nakhostin
2025-10-18 09:04:32 +03:30
parent 952986331a 8e5308b70b
commit 69d201479f
27 changed files with 3354 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/scraper/bootstrap/bootstrap.go
+5 -4
View File
@@ -7,7 +7,7 @@ import (
"os/signal"
"syscall"
"time"
"tm/internal/tender"
"tm/internal/notice"
"tm/pkg/config"
"tm/pkg/logger"
"tm/pkg/mongo"
@@ -78,7 +78,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
// init TED scraper
func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK) {
// Initialize tender repository
tenderRepo := tender.NewRepository(mongoManager, appLogger)
noticeRepo := notice.NewRepository(mongoManager, appLogger)
// Initialize TED scraper
tedScraper := ted.NewTEDScraper(
@@ -92,15 +92,16 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
DownloadDir: config.TED.DownloadDir,
CleanupAfter: config.TED.CleanupAfter,
ScrapingInterval: config.TED.ScrapingInterval,
AlertMail: config.AlertMail,
},
appLogger,
mongoManager,
tenderRepo,
noticeRepo,
notify,
)
// start TED scraper job
schedule.NewCronScheduler(appLogger, mongoManager, tenderRepo).AddJob(schedule.Job{
schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
Name: "TED Scraper Job",
Func: func() {
_ = tedScraper.Run(context.Background())
cmd/scraper/bootstrap/config.go
+1
View File
@@ -11,6 +11,7 @@ type Config struct {
Logging config.LoggingConfig
TED ScraperConfig
Notification config.NotificationConfig
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
}
type ScraperConfig struct {
cmd/scraper/main.go
+3 -5
View File
@@ -22,8 +22,6 @@ func main() {
"version": "1.0.0",
})
// Initialize notification service
// Initialize MongoDB connection
mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
defer func() {
@@ -36,7 +34,10 @@ func main() {
}
}()
// Initialize notification service
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
// Initialize TED scraper
bootstrap.InitTEDScraper(*config, mongoManager, appLogger, notificationService)
// Set up signal handling for graceful shutdown
@@ -47,8 +48,5 @@ func main() {
<-signalChan
appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{})
// Stop the scheduler
// scheduler.Stop()
appLogger.Info("TED scraper application stopped", map[string]interface{}{})
}
cmd/web/bootstrap/bootstrap.go
+36
View File
@@ -8,6 +8,7 @@ import (
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/ollama"
"tm/pkg/redis"
"github.com/labstack/echo/v4"
@@ -213,3 +214,38 @@ func InitNotificationService(conf config.NotificationConfig, log logger.Logger)
return *notificationSDK
}
// Init Ollama Service
func InitOllamaService(conf config.OllamaConfig, log logger.Logger) ollama.SDK {
ollamaConfig := &ollama.Config{
BaseURL: conf.BaseURL,
Timeout: conf.Timeout,
RetryAttempts: conf.RetryAttempts,
RetryDelay: conf.RetryDelay,
EnableLogging: conf.EnableLogging,
UserAgent: conf.UserAgent,
DefaultModel: conf.DefaultModel,
DefaultTemperature: conf.DefaultTemperature,
DefaultTopP: conf.DefaultTopP,
DefaultMaxTokens: conf.DefaultMaxTokens,
EnableStreaming: conf.EnableStreaming,
}
ollamaSDK, err := ollama.New(ollamaConfig, log)
if err != nil {
log.Error("Failed to initialize Ollama SDK", map[string]interface{}{
"error": err.Error(),
})
}
log.Info("Ollama SDK initialized successfully", map[string]interface{}{
"base_url": conf.BaseURL,
"timeout": conf.Timeout,
"retry_attempts": conf.RetryAttempts,
"retry_delay": conf.RetryDelay,
"enable_logging": conf.EnableLogging,
"user_agent": conf.UserAgent,
})
return *ollamaSDK
}
cmd/web/bootstrap/config.go
+1
View File
@@ -13,6 +13,7 @@ type Config struct {
RateLimit config.RateLimitConfig
Assets config.AssetsConfig
Notification config.NotificationConfig
Ollama config.OllamaConfig
UserAuth AuthConfig
CustomerAuth AuthConfig
}
cmd/worker/bootstrap/bootstrap.go
+150
View File
@@ -0,0 +1,150 @@
package bootstrap
import (
"fmt"
"time"
"tm/cmd/worker/workers"
"tm/internal/notice"
"tm/internal/tender"
"tm/pkg/config"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/ollama"
"tm/pkg/schedule"
)
// Init Application Configuration
func InitConfig() (*Config, error) {
conf, err := config.LoadConfig(".", &Config{})
if err != nil {
return nil, fmt.Errorf("failed to load config: %v", err)
}
return conf, nil
}
// Init Logger Service
func InitLogger(conf config.LoggingConfig) logger.Logger {
return logger.NewLogger(&logger.Config{
Level: conf.Level,
Format: conf.Format,
Output: conf.Output,
File: logger.FileConfig{
Path: conf.File.Path,
MaxSize: conf.File.MaxSize,
MaxBackups: conf.File.MaxBackups,
MaxAge: conf.File.MaxAge,
Compress: conf.File.Compress,
},
})
}
// Init MongoDB Connection Manager
func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionManager {
// Convert infra.MongoConfig to mongo.ConnectionConfig
connectionConfig := mongo.ConnectionConfig{
URI: conf.URI,
Database: conf.Name,
MaxPoolSize: uint64(conf.MaxPoolSize),
MinPoolSize: 5, // Default minimum pool size
MaxConnIdleTime: 30 * time.Minute, // Default idle time
ConnectTimeout: conf.Timeout,
ServerSelectionTimeout: 5 * time.Second, // Default server selection timeout
}
// Create MongoDB connection manager
connectionManager, err := mongo.NewConnectionManager(connectionConfig, log)
if err != nil {
log.Error("Failed to initialize MongoDB connection", map[string]interface{}{
"error": err.Error(),
"uri": conf.URI,
"db": conf.Name,
})
panic(fmt.Sprintf("Failed to initialize MongoDB connection: %v", err))
}
log.Info("MongoDB connection manager initialized successfully", map[string]interface{}{
"database": conf.Name,
"uri": conf.URI,
"pool_size": conf.MaxPoolSize,
})
return connectionManager
}
// Init Worker
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, ollamaSDK *ollama.SDK) {
// Initialize repositories
noticeRepo := notice.NewRepository(mongoManager, appLogger)
tenderRepo := tender.NewRepository(mongoManager, appLogger)
worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK)
worker.Run()
// start Worker job
schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo).AddJob(schedule.Job{
Name: "Worker Job",
Func: func() {
worker := workers.NewNoticeWorker(mongoManager, appLogger, notify, noticeRepo, tenderRepo, ollamaSDK)
worker.Run()
},
Expr: config.Worker.Interval,
})
}
// Init Notification Service
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
notificationConfig := &notification.Config{
BaseURL: conf.BaseURL,
Timeout: conf.Timeout,
RetryAttempts: conf.RetryAttempts,
RetryDelay: conf.RetryDelay,
EnableLogging: conf.EnableLogging,
UserAgent: conf.UserAgent,
}
notificationSDK, err := notification.New(notificationConfig, log)
if err != nil {
log.Error("Failed to initialize Notification SDK", map[string]interface{}{
"error": err.Error(),
})
fmt.Printf("Failed to initialize Notification SDK: %v\n", err.Error())
}
log.Info("Notification SDK initialized successfully", map[string]interface{}{
"base_url": conf.BaseURL,
"timeout": conf.Timeout,
"retry_attempts": conf.RetryAttempts,
"retry_delay": conf.RetryDelay,
"enable_logging": conf.EnableLogging,
"user_agent": conf.UserAgent,
})
return *notificationSDK
}
// Init Ollama Service
func InitOllamaService(conf config.OllamaConfig, log logger.Logger) *ollama.SDK {
ollamaSDK, err := ollama.New(&ollama.Config{
BaseURL: conf.BaseURL,
Timeout: conf.Timeout,
RetryAttempts: conf.RetryAttempts,
RetryDelay: conf.RetryDelay,
EnableLogging: conf.EnableLogging,
UserAgent: conf.UserAgent,
DefaultModel: conf.DefaultModel,
DefaultTemperature: conf.DefaultTemperature,
DefaultTopP: conf.DefaultTopP,
DefaultMaxTokens: conf.DefaultMaxTokens,
EnableStreaming: conf.EnableStreaming,
}, log)
if err != nil {
log.Error("Failed to initialize Ollama SDK", map[string]interface{}{
"error": err.Error(),
})
}
return ollamaSDK
}
cmd/worker/bootstrap/config.go
+19
View File
@@ -0,0 +1,19 @@
package bootstrap
import (
"tm/pkg/config"
)
// Config defines the scraper application configuration
type Config struct {
Database config.DatabaseConfig
Logging config.LoggingConfig
Notification config.NotificationConfig
Ollama config.OllamaConfig
Worker WorkerConfig
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
}
type WorkerConfig struct {
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
}
cmd/worker/main.go
+62
View File
@@ -0,0 +1,62 @@
package main
import (
"context"
"os"
"os/signal"
"syscall"
"time"
"tm/cmd/worker/bootstrap"
)
func main() {
// Load configuration
config, err := bootstrap.InitConfig()
if err != nil {
panic(err)
}
// Initialize logger
appLogger := bootstrap.InitLogger(config.Logging)
appLogger.Info("Starting worker application", map[string]interface{}{
"version": "1.0.0",
})
// Initialize MongoDB connection
mongoManager := bootstrap.InitMongoDB(config.Database.MongoDB, appLogger)
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
if err := mongoManager.Close(ctx); err != nil {
appLogger.Error("Failed to close MongoDB connection", map[string]interface{}{
"error": err.Error(),
})
}
}()
// Initialize notification service
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
// Initialize ollama service
ollamaSDK := bootstrap.InitOllamaService(config.Ollama, appLogger)
aiModels, err := ollamaSDK.ListModels(context.Background())
if err != nil {
appLogger.Error("Failed to list ollama models", map[string]interface{}{
"error": err.Error(),
})
}
appLogger.Info("Ollama models", map[string]interface{}{"models": aiModels})
// Initialize Worker
bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, ollamaSDK)
// Set up signal handling for graceful shutdown
signalChan := make(chan os.Signal, 1)
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
// Wait for shutdown signal
<-signalChan
appLogger.Info("Received shutdown signal, stopping scheduler...", map[string]interface{}{})
appLogger.Info("Worker application stopped", map[string]interface{}{})
}
cmd/worker/workers/notice.go
+273
View File
@@ -0,0 +1,273 @@
package workers
import (
"context"
"tm/internal/notice"
"tm/internal/tender"
"tm/pkg/logger"
"tm/pkg/mongo"
"tm/pkg/notification"
"tm/pkg/ollama"
)
type NoticeWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
Notify notification.SDK
Ollama *ollama.SDK
NoticeRepo notice.Repository
TenderRepo tender.TenderRepository
}
func NewNoticeWorker(mongo *mongo.ConnectionManager, logger logger.Logger, notify notification.SDK, noticeRepo notice.Repository, tenderRepo tender.TenderRepository, ollama *ollama.SDK) *NoticeWorker {
return &NoticeWorker{
Mongo: mongo,
Logger: logger,
Notify: notify,
Ollama: ollama,
NoticeRepo: noticeRepo,
TenderRepo: tenderRepo,
}
}
func (w *NoticeWorker) Run() {
w.Logger.Info("Notice worker started", map[string]interface{}{})
limit := 10
skip := 0
for {
notices, _, err := w.NoticeRepo.GetUnProcessedNotices(context.Background(), limit, skip)
if err != nil {
w.Logger.Error("Failed to get notices", map[string]interface{}{
"error": err.Error(),
})
continue
}
if len(notices) == 0 {
break
}
for _, n := range notices {
w.Logger.Info("Notice", map[string]interface{}{
"notice": n.ID.Hex(),
})
t, err := w.TenderRepo.GetByNoticePublicationID(context.Background(), n.NoticePublicationID)
if err == nil {
continue
}
err = w.ToTender(&n, t)
if err != nil {
w.Logger.Error("Failed to create tender", map[string]interface{}{
"error": err.Error(),
})
continue
}
if t.ID.IsZero() {
err = w.TenderRepo.Create(context.Background(), t)
if err != nil {
w.Logger.Error("Failed to create tender", map[string]interface{}{
"error": err.Error(),
})
}
} else {
err = w.TenderRepo.Update(context.Background(), t)
if err != nil {
w.Logger.Error("Failed to update tender", map[string]interface{}{
"error": err.Error(),
})
}
}
}
skip += limit
}
}
func (w *NoticeWorker) ToTender(n *notice.Notice, t *tender.Tender) error {
// Buyer Organization
buyerOrg := &tender.Organization{}
if n.BuyerOrganization != nil {
buyerOrg = &tender.Organization{
Name: n.BuyerOrganization.Name,
CompanyID: n.BuyerOrganization.CompanyID,
WebsiteURI: n.BuyerOrganization.WebsiteURI,
ContactName: n.BuyerOrganization.ContactName,
ContactTelephone: n.BuyerOrganization.ContactTelephone,
ContactEmail: n.BuyerOrganization.ContactEmail,
ContactFax: n.BuyerOrganization.ContactFax,
Role: n.BuyerOrganization.Role,
Address: tender.Address{
StreetName: n.BuyerOrganization.Address.StreetName,
CityName: n.BuyerOrganization.Address.CityName,
PostalZone: n.BuyerOrganization.Address.PostalZone,
CountrySubentityCode: n.BuyerOrganization.Address.CountrySubentityCode,
Department: n.BuyerOrganization.Address.Department,
Region: n.BuyerOrganization.Address.Region,
CountryCode: n.BuyerOrganization.Address.CountryCode,
},
}
}
// Review Organization
reviewOrg := &tender.Organization{}
if n.ReviewOrganization != nil {
reviewOrg = &tender.Organization{
Name: n.ReviewOrganization.Name,
CompanyID: n.ReviewOrganization.CompanyID,
WebsiteURI: n.ReviewOrganization.WebsiteURI,
ContactName: n.ReviewOrganization.ContactName,
ContactTelephone: n.ReviewOrganization.ContactTelephone,
ContactEmail: n.ReviewOrganization.ContactEmail,
ContactFax: n.ReviewOrganization.ContactFax,
Role: n.ReviewOrganization.Role,
Address: tender.Address{
StreetName: n.ReviewOrganization.Address.StreetName,
CityName: n.ReviewOrganization.Address.CityName,
PostalZone: n.ReviewOrganization.Address.PostalZone,
CountrySubentityCode: n.ReviewOrganization.Address.CountrySubentityCode,
Department: n.ReviewOrganization.Address.Department,
Region: n.ReviewOrganization.Address.Region,
CountryCode: n.ReviewOrganization.Address.CountryCode,
},
}
}
// Organizations
organizations := make([]tender.Organization, len(n.Organizations))
for i, org := range n.Organizations {
organizations[i] = tender.Organization{
Name: org.Name,
CompanyID: org.CompanyID,
WebsiteURI: org.WebsiteURI,
ContactName: org.ContactName,
ContactTelephone: org.ContactTelephone,
ContactEmail: org.ContactEmail,
ContactFax: org.ContactFax,
Role: org.Role,
Address: tender.Address{
StreetName: org.Address.StreetName,
CityName: org.Address.CityName,
PostalZone: org.Address.PostalZone,
CountrySubentityCode: org.Address.CountrySubentityCode,
Department: org.Address.Department,
Region: org.Address.Region,
CountryCode: org.Address.CountryCode,
},
}
}
// Selection Criteria
selectionCriteria := make([]tender.SelectionCriterion, len(n.SelectionCriteria))
for i, criterion := range n.SelectionCriteria {
selectionCriteria[i] = tender.SelectionCriterion{
TypeCode: criterion.TypeCode,
Description: criterion.Description,
LanguageID: criterion.LanguageID,
}
}
// Winning Tenderer
winningTenderer := &tender.Organization{}
if n.WinningTenderer != nil {
winningTenderer = &tender.Organization{
Name: n.WinningTenderer.Name,
CompanyID: n.WinningTenderer.CompanyID,
WebsiteURI: n.WinningTenderer.WebsiteURI,
ContactName: n.WinningTenderer.ContactName,
ContactTelephone: n.WinningTenderer.ContactTelephone,
ContactEmail: n.WinningTenderer.ContactEmail,
ContactFax: n.WinningTenderer.ContactFax,
Role: n.WinningTenderer.Role,
Address: tender.Address{
StreetName: n.WinningTenderer.Address.StreetName,
CityName: n.WinningTenderer.Address.CityName,
PostalZone: n.WinningTenderer.Address.PostalZone,
CountrySubentityCode: n.WinningTenderer.Address.CountrySubentityCode,
Department: n.WinningTenderer.Address.Department,
Region: n.WinningTenderer.Address.Region,
CountryCode: n.WinningTenderer.Address.CountryCode,
},
}
}
// Modifications
modifications := make([]tender.TenderModification, len(n.Modifications))
for i, modification := range n.Modifications {
modifications[i] = tender.TenderModification{
ModificationDate: modification.ModificationDate,
ModificationReason: modification.ModificationReason,
Description: modification.Description,
LanguageID: modification.LanguageID,
}
}
// Awarded Entities
awardedEntities := make([]tender.Awarded, len(n.AwardedEntities))
for i, awarded := range n.AwardedEntities {
awardedEntities[i] = tender.Awarded{
Name: awarded.Name,
Address: awarded.Address,
Country: awarded.Country,
Amount: awarded.Amount,
Currency: awarded.Currency,
Share: awarded.Share,
AwardDate: awarded.AwardDate,
ContractID: awarded.ContractID,
TenderID: awarded.TenderID,
LotID: awarded.LotID,
CompanyID: awarded.CompanyID,
OrganizationID: awarded.OrganizationID,
}
}
t.NoticePublicationID = n.NoticePublicationID
t.Title = n.Title
t.Description = n.Description
t.ProcurementTypeCode = n.ProcurementTypeCode
t.ProcedureCode = n.ProcedureCode
t.EstimatedValue = n.EstimatedValue
t.Currency = n.Currency
t.TenderDeadline = n.TenderDeadline
t.SubmissionURL = n.SubmissionURL
t.BuyerOrganization = buyerOrg
t.ReviewOrganization = reviewOrg
t.Organizations = organizations
t.SelectionCriteria = selectionCriteria
t.WinningTenderer = winningTenderer
t.Modifications = modifications
t.AwardedEntities = awardedEntities
t.NoticeTypeCode = n.NoticeTypeCode
t.NoticeSubTypeCode = n.NoticeSubTypeCode
t.NoticeLanguageCode = n.NoticeLanguageCode
t.IssueDate = n.IssueDate
t.IssueTime = n.IssueTime
t.PublicationDate = n.PublicationDate
t.SubmissionDeadline = n.SubmissionDeadline
t.ApplicationDeadline = n.ApplicationDeadline
t.GazetteID = n.GazetteID
t.PlaceOfPerformance = n.PlaceOfPerformance
t.CountryCode = n.CountryCode
t.RegionCode = n.RegionCode
t.CityName = n.CityName
t.PostalCode = n.PostalCode
t.DocumentURI = n.DocumentURI
t.TenderURL = n.TenderURL
t.MainClassification = n.MainClassification
t.AdditionalClassifications = n.AdditionalClassifications
t.OfficialLanguages = n.OfficialLanguages
t.CancellationReason = n.CancellationReason
t.CancellationDate = n.CancellationDate
t.AwardDate = n.AwardDate
t.AwardedValue = n.AwardedValue
t.ContractNumber = n.ContractNumber
t.SuspensionReason = n.SuspensionReason
t.SuspensionDate = n.SuspensionDate
t.WinningTenderer = winningTenderer
t.Modifications = modifications
t.AwardedEntities = awardedEntities
return nil
}