Compare commits
122 Commits
147be11973
..
dev
| Author | SHA1 | Date | |
|---|---|---|---|
| 1ec1557a9a | |||
| 974ba8d638 | |||
| fa16d4080a | |||
| 6ba964275e | |||
| e4f7c4a04c | |||
| 51a1a6aa82 | |||
| 6ee81c3581 | |||
| d55b2685ca | |||
| 04c4102170 | |||
| 0fde5772e2 | |||
| c35fa331f9 | |||
| c8616940ff | |||
| fa3d466579 | |||
| 0cce9ef1b5 | |||
| 5c93e0f01b | |||
| 10385e997b | |||
| adfd291245 | |||
| 26a593306d | |||
| c5e62a4061 | |||
| 81b0d94ba3 | |||
| 3e2700bc36 | |||
| aafae2a26f | |||
| 9d8ce12816 | |||
| b388af3518 | |||
| 932b0cf24e | |||
| 1df44ec8ca | |||
| 843e1508df | |||
| 2fbf329182 | |||
| 81a94b4879 | |||
| 11e0b44ebe | |||
| f68b6d7787 | |||
| 784c3d6563 | |||
| 0b74e9ad23 | |||
| b28bc23975 | |||
| f108733c2a | |||
| 2980a3beb4 | |||
| adaae4c4bc | |||
| 107469c328 | |||
| 0e4fadaf29 | |||
| 4e5296d5dd | |||
| 1ad0206e61 | |||
| 89faa08b1c | |||
| 3aeb8630e3 | |||
| a762430bca | |||
| 3221a31ac8 | |||
| 4c48e0bb3b | |||
| 467090e5d2 | |||
| 1edf42187d | |||
| f8c98da132 | |||
| fe3a71ba2b | |||
| ec7db8b9f0 | |||
| 541d08a014 | |||
| eeafe2a625 | |||
| 506ac01cda | |||
| 492f9ba3c8 | |||
| c46a8d54f4 | |||
| 7a9de273bb | |||
| 50c018af62 | |||
| addd616d59 | |||
| 12d1cabf7e | |||
| 8dbd9927b0 | |||
| 3002935b76 | |||
| 7aacb7dfc9 | |||
| 20ce9c53ff | |||
| ff6bdfcb09 | |||
| 86789dcd20 | |||
| 92c6c7d99a | |||
| 241e3b5f2d | |||
| 534213f10e | |||
| 6d5694977b | |||
| 39ac76e7b0 | |||
| 582f8b5c02 | |||
| 69445130ce | |||
| 20518e7b64 | |||
| 326e49886b | |||
| a2661651c9 | |||
| a7a49fc411 | |||
| 8fad771b7a | |||
| 992d41374f | |||
| 3ba33459e8 | |||
| 5fbfcd4149 | |||
| 3bfed0dc74 | |||
| 2a9682aea2 | |||
| ce8a18aa8b | |||
| d486a5e44f | |||
| db14bfe270 | |||
| fa258020f1 | |||
| 2b6e25f979 | |||
| b671dc3fd8 | |||
| 2538747768 | |||
| e04cdd1d10 | |||
| e31bccced6 | |||
| 45cfa24a72 | |||
| 550f11a77e | |||
| ee5414bc10 | |||
| a04b3fe1ec | |||
| dfab3e17d2 | |||
| 4cfca5aa55 | |||
| 6fb57c41c1 | |||
| e5fa0dfe47 | |||
| 2f19ecae55 | |||
| 8035118f44 | |||
| 9676f99304 | |||
| 7120a954b6 | |||
| 243f667005 | |||
| 58345ac4f3 | |||
| 902cc686a0 | |||
| 7f84746400 | |||
| 31d8efef0b | |||
| 2e112fe08c | |||
| ca2a1b4425 | |||
| ca35eb5f15 | |||
| 42d64c4ec6 | |||
| dcf19b91cd | |||
| 6af5cf3c4e | |||
| 96ec0f8a8f | |||
| a825c4a271 | |||
| 416b040c2b | |||
| 24c57a3ec5 | |||
| 06e663b691 | |||
| f16f9fb5a9 | |||
| 3b26c4f5e1 |
@@ -0,0 +1 @@
|
|||||||
|
* text=auto
|
||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"tm/internal/notice"
|
"tm/internal/notice"
|
||||||
"tm/pkg/config"
|
"tm/pkg/config"
|
||||||
@@ -14,6 +15,9 @@ import (
|
|||||||
"tm/ted"
|
"tm/ted"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// tedScraperRunMu ensures only one TED scraper run executes at a time (startup catch-up and cron).
|
||||||
|
var tedScraperRunMu sync.Mutex
|
||||||
|
|
||||||
// Init Application Configuration
|
// Init Application Configuration
|
||||||
func InitConfig() (*Config, error) {
|
func InitConfig() (*Config, error) {
|
||||||
conf, err := config.LoadConfig(".", &Config{})
|
conf, err := config.LoadConfig(".", &Config{})
|
||||||
@@ -101,22 +105,64 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
|
|||||||
|
|
||||||
// start TED scraper job with error handling
|
// start TED scraper job with error handling
|
||||||
scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
|
scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
|
||||||
|
dailyJobTracker := schedule.NewDailyJobTracker(mongoManager, appLogger)
|
||||||
|
|
||||||
|
runDailyScrape := func(trigger string) {
|
||||||
|
tedScraperRunMu.Lock()
|
||||||
|
defer tedScraperRunMu.Unlock()
|
||||||
|
|
||||||
function := func() {
|
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
|
today := time.Now().Local()
|
||||||
|
|
||||||
|
if trigger == "startup" {
|
||||||
|
completed, checkErr := dailyJobTracker.IsCompleted(ctx, schedule.TEDScraperJobName, today)
|
||||||
|
if checkErr != nil {
|
||||||
|
appLogger.Error("Failed to check daily scrape completion status", map[string]interface{}{
|
||||||
|
"error": checkErr.Error(),
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
} else if completed {
|
||||||
|
appLogger.Info("Startup catch-up skipped: today's TED scrape already completed", map[string]interface{}{
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
appLogger.Info("Running startup catch-up for today's TED scrape", map[string]interface{}{
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
appLogger.Info("Running scheduled TED scrape", map[string]interface{}{
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
err := tedScraper.Run(ctx, nil, nil)
|
err := tedScraper.Run(ctx, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
appLogger.Error("Scheduled scraper run failed", map[string]interface{}{
|
appLogger.Error("TED scraper run failed", map[string]interface{}{
|
||||||
|
"trigger": trigger,
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
// Don't exit - continue running for next scheduled execution
|
return
|
||||||
} else {
|
|
||||||
appLogger.Info("Scheduled scraper run completed successfully", map[string]interface{}{})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if markErr := dailyJobTracker.MarkCompleted(ctx, schedule.TEDScraperJobName, today); markErr != nil {
|
||||||
|
appLogger.Error("Failed to mark daily scrape as completed", map[string]interface{}{
|
||||||
|
"trigger": trigger,
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
"error": markErr.Error(),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
appLogger.Info("TED scraper run completed successfully", map[string]interface{}{
|
||||||
|
"trigger": trigger,
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
err := scheduler.AddJob(schedule.Job{
|
err := scheduler.AddJob(schedule.Job{
|
||||||
Name: "TED Scraper Job",
|
Name: "TED Scraper Job",
|
||||||
Func: function,
|
Func: func() { runDailyScrape("scheduled") },
|
||||||
Expr: config.TED.ScrapingInterval,
|
Expr: config.TED.ScrapingInterval,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -126,6 +172,11 @@ func InitTEDScraper(config Config, mongoManager *mongo.ConnectionManager, appLog
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// After a server restart, resume today's scrape if it was interrupted or never ran.
|
||||||
|
go func() {
|
||||||
|
runDailyScrape("startup")
|
||||||
|
}()
|
||||||
|
|
||||||
// Start the scheduler
|
// Start the scheduler
|
||||||
scheduler.Start()
|
scheduler.Start()
|
||||||
|
|
||||||
|
|||||||
@@ -4,8 +4,10 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
ai_summarizer "tm/pkg/ai_summarizer"
|
ai_summarizer "tm/pkg/ai_summarizer"
|
||||||
|
"tm/pkg/audit"
|
||||||
"tm/pkg/authorization"
|
"tm/pkg/authorization"
|
||||||
"tm/pkg/config"
|
"tm/pkg/config"
|
||||||
|
"tm/pkg/elasticsearch"
|
||||||
"tm/pkg/filestore"
|
"tm/pkg/filestore"
|
||||||
"tm/pkg/gorules"
|
"tm/pkg/gorules"
|
||||||
"tm/pkg/hcaptcha"
|
"tm/pkg/hcaptcha"
|
||||||
@@ -90,6 +92,7 @@ func InitHTTPServer(conf Config, log logger.Logger) *echo.Echo {
|
|||||||
// Add middleware
|
// Add middleware
|
||||||
e.Use(middleware.Recover())
|
e.Use(middleware.Recover())
|
||||||
e.Use(middleware.RequestID())
|
e.Use(middleware.RequestID())
|
||||||
|
e.Use(audit.RequestMetaMiddleware())
|
||||||
e.Use(middleware.Logger())
|
e.Use(middleware.Logger())
|
||||||
e.Use(middleware.CORSWithConfig(middleware.CORSConfig{
|
e.Use(middleware.CORSWithConfig(middleware.CORSConfig{
|
||||||
AllowOrigins: []string{"*"},
|
AllowOrigins: []string{"*"},
|
||||||
@@ -132,7 +135,7 @@ func InitHTTPServer(conf Config, log logger.Logger) *echo.Echo {
|
|||||||
e.GET("/swagger*", echoSwagger.WrapHandler)
|
e.GET("/swagger*", echoSwagger.WrapHandler)
|
||||||
|
|
||||||
log.Info("HTTP server initialized successfully", map[string]interface{}{
|
log.Info("HTTP server initialized successfully", map[string]interface{}{
|
||||||
"middleware": []string{"recover", "request_id", "logger", "cors", "custom_logging"},
|
"middleware": []string{"recover", "request_id", "audit_meta", "logger", "cors", "custom_logging"},
|
||||||
})
|
})
|
||||||
|
|
||||||
return e
|
return e
|
||||||
@@ -402,3 +405,34 @@ func InitGoRulesClient(_ GoRulesConfig, log logger.Logger) gorules.Client {
|
|||||||
return client
|
return client
|
||||||
*/
|
*/
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InitElasticsearch creates an Elasticsearch client for audit log storage.
|
||||||
|
func InitElasticsearch(conf config.ElasticsearchConfig, log logger.Logger) elasticsearch.Client {
|
||||||
|
esConfig := elasticsearch.Config{
|
||||||
|
URL: conf.URL,
|
||||||
|
Username: conf.Username,
|
||||||
|
Password: conf.Password,
|
||||||
|
IndexPrefix: conf.IndexPrefix,
|
||||||
|
Enabled: conf.Enabled,
|
||||||
|
RequestTimeout: conf.RequestTimeout,
|
||||||
|
BulkFlushPeriod: conf.BulkFlushPeriod,
|
||||||
|
QueueSize: conf.QueueSize,
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := elasticsearch.NewClient(esConfig, log)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("Elasticsearch unavailable, audit logs will be file-only", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"url": conf.URL,
|
||||||
|
})
|
||||||
|
return elasticsearch.NewNoopClient()
|
||||||
|
}
|
||||||
|
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
|
||||||
|
// InitAuditLogger creates a composite audit logger with optional Elasticsearch persistence.
|
||||||
|
func InitAuditLogger(log logger.Logger, esClient elasticsearch.Client) (audit.Logger, audit.Store) {
|
||||||
|
store := elasticsearch.NewAuditStore(esClient)
|
||||||
|
return audit.NewCompositeLogger(log, store), store
|
||||||
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ type Config struct {
|
|||||||
DocumentScraper DocumentScraperConfig
|
DocumentScraper DocumentScraperConfig
|
||||||
AISummarizer AISummarizerConfig
|
AISummarizer AISummarizerConfig
|
||||||
GoRules GoRulesConfig
|
GoRules GoRulesConfig
|
||||||
|
Elasticsearch config.ElasticsearchConfig
|
||||||
}
|
}
|
||||||
|
|
||||||
type ScraperConfig struct {
|
type ScraperConfig struct {
|
||||||
@@ -69,6 +70,9 @@ type AISummarizerConfig struct {
|
|||||||
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
|
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
|
||||||
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
|
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
|
||||||
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
|
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
|
||||||
|
// RecommendationCacheTTL is an optional max Redis TTL for cached recommendations.
|
||||||
|
// When 0 (default), entries persist until the company is created, updated, or onboarded.
|
||||||
|
RecommendationCacheTTL time.Duration `env:"AI_RECOMMENDATION_CACHE_TTL" envDefault:"0"`
|
||||||
|
|
||||||
// MinIO storage settings
|
// MinIO storage settings
|
||||||
MinioEndpoint string `env:"AI_SUMMARIZER_MINIO_ENDPOINT" envDefault:""`
|
MinioEndpoint string `env:"AI_SUMMARIZER_MINIO_ENDPOINT" envDefault:""`
|
||||||
@@ -76,7 +80,7 @@ type AISummarizerConfig struct {
|
|||||||
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
|
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
|
||||||
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
|
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
|
||||||
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
|
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
|
||||||
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens-documents"`
|
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// GoRulesConfig holds configuration for the GoRules status engine.
|
// GoRulesConfig holds configuration for the GoRules status engine.
|
||||||
|
|||||||
+71
-14
@@ -59,6 +59,12 @@ package main
|
|||||||
// @tag.name Admin-CMS
|
// @tag.name Admin-CMS
|
||||||
// @tag.description Administrative CMS management operations for web panel including CRUD operations, search, statistics, and comprehensive filtering with pagination
|
// @tag.description Administrative CMS management operations for web panel including CRUD operations, search, statistics, and comprehensive filtering with pagination
|
||||||
|
|
||||||
|
// @tag.name Admin-AI-Pipeline
|
||||||
|
// @tag.description Administrative Opplens AI pipeline operations including scrape, summarize, translate, sync, and background pipeline runs
|
||||||
|
|
||||||
|
// @tag.name Admin-AuditLogs
|
||||||
|
// @tag.description Administrative user action audit log search with filtering by actor, action, target, and time range
|
||||||
|
|
||||||
// @tag.name Authorization
|
// @tag.name Authorization
|
||||||
// @tag.description Customer authentication and authorization operations for mobile application including login, logout, token refresh, and profile access
|
// @tag.description Customer authentication and authorization operations for mobile application including login, logout, token refresh, and profile access
|
||||||
|
|
||||||
@@ -97,7 +103,9 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"time"
|
"time"
|
||||||
|
"tm/internal/ai_pipeline"
|
||||||
"tm/internal/assets"
|
"tm/internal/assets"
|
||||||
|
"tm/internal/auditlog"
|
||||||
"tm/internal/cms"
|
"tm/internal/cms"
|
||||||
"tm/internal/company"
|
"tm/internal/company"
|
||||||
"tm/internal/company_category"
|
"tm/internal/company_category"
|
||||||
@@ -111,8 +119,8 @@ import (
|
|||||||
"tm/internal/notification"
|
"tm/internal/notification"
|
||||||
"tm/internal/tender"
|
"tm/internal/tender"
|
||||||
"tm/internal/tender_approval"
|
"tm/internal/tender_approval"
|
||||||
|
"tm/internal/tender_submission"
|
||||||
"tm/internal/user"
|
"tm/internal/user"
|
||||||
"tm/pkg/audit"
|
|
||||||
"tm/pkg/filestore"
|
"tm/pkg/filestore"
|
||||||
"tm/pkg/security"
|
"tm/pkg/security"
|
||||||
|
|
||||||
@@ -167,12 +175,18 @@ func main() {
|
|||||||
// Initialize AI Summarizer service
|
// Initialize AI Summarizer service
|
||||||
var aiSummarizerClient tender.AISummarizerClient
|
var aiSummarizerClient tender.AISummarizerClient
|
||||||
var aiSummarizerStorage tender.AISummarizerStorage
|
var aiSummarizerStorage tender.AISummarizerStorage
|
||||||
|
var procedureDocumentsLister dashboard.ProcedureDocumentsLister
|
||||||
|
var aiRecommendationClient company.AIRecommendationClient
|
||||||
|
var aiPipelineClient ai_pipeline.Client
|
||||||
|
|
||||||
if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, mongoManager, logger); c != nil {
|
if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, mongoManager, logger); c != nil {
|
||||||
aiSummarizerClient = c
|
aiSummarizerClient = c
|
||||||
|
aiPipelineClient = c
|
||||||
|
aiRecommendationClient = c
|
||||||
}
|
}
|
||||||
if s := bootstrap.InitAISummarizerStorage(conf.AISummarizer, logger); s != nil {
|
if s := bootstrap.InitAISummarizerStorage(conf.AISummarizer, logger); s != nil {
|
||||||
aiSummarizerStorage = s
|
aiSummarizerStorage = s
|
||||||
|
procedureDocumentsLister = s
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize authorization service
|
// Initialize authorization service
|
||||||
@@ -188,6 +202,7 @@ func main() {
|
|||||||
tenderRepository := tender.NewRepository(mongoManager, logger)
|
tenderRepository := tender.NewRepository(mongoManager, logger)
|
||||||
feedbackRepo := feedback.NewRepository(mongoManager, logger)
|
feedbackRepo := feedback.NewRepository(mongoManager, logger)
|
||||||
tenderApprovalRepository := tender_approval.NewRepository(mongoManager, logger)
|
tenderApprovalRepository := tender_approval.NewRepository(mongoManager, logger)
|
||||||
|
tenderSubmissionRepository := tender_submission.NewRepository(mongoManager, logger)
|
||||||
inquiryRepository := inquiry.NewRepository(mongoManager, logger)
|
inquiryRepository := inquiry.NewRepository(mongoManager, logger)
|
||||||
contactRepository := contact.NewContactRepository(mongoManager, logger)
|
contactRepository := contact.NewContactRepository(mongoManager, logger)
|
||||||
cmsRepository := cms.NewRepository(mongoManager, logger)
|
cmsRepository := cms.NewRepository(mongoManager, logger)
|
||||||
@@ -197,7 +212,7 @@ func main() {
|
|||||||
cardRepository := kanban.NewCardRepository(mongoManager, logger)
|
cardRepository := kanban.NewCardRepository(mongoManager, logger)
|
||||||
notificationRepository := notification.NewRepository(mongoManager, logger)
|
notificationRepository := notification.NewRepository(mongoManager, logger)
|
||||||
logger.Info("Repositories initialized successfully", map[string]interface{}{
|
logger.Info("Repositories initialized successfully", map[string]interface{}{
|
||||||
"repositories": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "contact", "cms", "notice", "kanban_board", "kanban_column", "kanban_card", "notification"},
|
"repositories": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "tender_submission", "inquiry", "contact", "cms", "notice", "kanban_board", "kanban_column", "kanban_card", "notification"},
|
||||||
})
|
})
|
||||||
|
|
||||||
// Initialize validation services
|
// Initialize validation services
|
||||||
@@ -207,25 +222,64 @@ func main() {
|
|||||||
_ = cms.NewValidationService() // Register mediaRef validator
|
_ = cms.NewValidationService() // Register mediaRef validator
|
||||||
|
|
||||||
// Initialize services with repositories
|
// Initialize services with repositories
|
||||||
auditLogger := audit.NewLogger(logger)
|
esClient := bootstrap.InitElasticsearch(conf.Elasticsearch, logger)
|
||||||
|
defer func() {
|
||||||
|
if err := esClient.Close(); err != nil {
|
||||||
|
logger.Error("Failed to close Elasticsearch client", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
auditLogger, auditStore := bootstrap.InitAuditLogger(logger, esClient)
|
||||||
|
defer func() {
|
||||||
|
if err := auditStore.Close(); err != nil {
|
||||||
|
logger.Error("Failed to close audit store", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
userService := user.NewService(userRepository, logger, userAuthService, userValidator, notificationSDK, redisClient, auditLogger)
|
userService := user.NewService(userRepository, logger, userAuthService, userValidator, notificationSDK, redisClient, auditLogger)
|
||||||
categoryService := company_category.NewService(categoryRepository, logger)
|
categoryService := company_category.NewService(categoryRepository, logger)
|
||||||
companyService := company.NewService(companyRepository, categoryService, fileStoreService, logger)
|
companyService := company.NewService(companyRepository, categoryService, fileStoreService, aiRecommendationClient, aiPipelineClient, redisClient, conf.AISummarizer.RecommendationCacheTTL, logger)
|
||||||
customerService := customer.NewService(customerRepository, logger, customerAuthService, companyService, redisClient, notificationSDK, customerValidator, auditLogger)
|
customerService := customer.NewService(customerRepository, logger, customerAuthService, companyService, redisClient, notificationSDK, customerValidator, auditLogger)
|
||||||
tenderService := tender.NewService(tenderRepository, companyService, customerService, logger, aiSummarizerClient, aiSummarizerStorage, conf.AISummarizer.DefaultLanguage)
|
tenderService := tender.NewService(
|
||||||
|
tenderRepository,
|
||||||
|
companyService,
|
||||||
|
customerService,
|
||||||
|
tenderApprovalRepository,
|
||||||
|
logger,
|
||||||
|
aiSummarizerClient,
|
||||||
|
aiSummarizerStorage,
|
||||||
|
redisClient,
|
||||||
|
conf.AISummarizer.RecommendationCacheTTL,
|
||||||
|
conf.AISummarizer.DefaultLanguage,
|
||||||
|
)
|
||||||
|
if configurer, ok := companyService.(interface {
|
||||||
|
SetRecommendedTendersPageCacheRefresher(company.RecommendedTendersPageCacheRefresher)
|
||||||
|
}); ok {
|
||||||
|
if refresher, ok := tenderService.(company.RecommendedTendersPageCacheRefresher); ok {
|
||||||
|
configurer.SetRecommendedTendersPageCacheRefresher(refresher)
|
||||||
|
}
|
||||||
|
}
|
||||||
feedbackService := feedback.NewService(feedbackRepo, tenderService, companyService, logger)
|
feedbackService := feedback.NewService(feedbackRepo, tenderService, companyService, logger)
|
||||||
tenderApprovalService := tender_approval.NewService(tenderApprovalRepository, tenderService, logger)
|
tenderSubmissionService := tender_submission.NewService(tenderSubmissionRepository, tenderApprovalRepository, tenderService, logger)
|
||||||
|
tenderApprovalService := tender_approval.NewService(tenderApprovalRepository, tenderService, logger, tenderSubmissionService)
|
||||||
inquiryService := inquiry.NewService(inquiryRepository, notificationSDK, logger, xssPolicy)
|
inquiryService := inquiry.NewService(inquiryRepository, notificationSDK, logger, xssPolicy)
|
||||||
flagService := assets.NewService(logger, conf.Assets.FlagsPath)
|
flagService := assets.NewService(logger, conf.Assets.FlagsPath)
|
||||||
notificationService := notification.NewService(notificationSDK, notificationRepository, userService, customerService, logger)
|
notificationService := notification.NewService(notificationSDK, notificationRepository, userService, customerService, logger)
|
||||||
contactService := contact.NewService(contactRepository, logger, notificationSDK)
|
contactService := contact.NewService(contactRepository, logger, notificationSDK)
|
||||||
cmsService := cms.NewService(cmsRepository, logger)
|
cmsService := cms.NewService(cmsRepository, logger)
|
||||||
kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, goRulesClient, conf.GoRules.RuleID, logger)
|
kanbanService := kanban.NewService(boardRepository, columnRepository, cardRepository, goRulesClient, conf.GoRules.RuleID, logger)
|
||||||
documentScraperService := document_scraper.NewService(tenderRepository, logger)
|
documentScraperService := document_scraper.NewService(tenderRepository, aiPipelineClient, logger)
|
||||||
dashboardRepository := dashboard.NewRepository(mongoManager, logger)
|
dashboardRepository := dashboard.NewRepository(mongoManager, logger, procedureDocumentsLister)
|
||||||
dashboardService := dashboard.NewService(dashboardRepository, logger)
|
dashboardService := dashboard.NewService(dashboardRepository, logger, redisClient)
|
||||||
|
aiPipelineService := ai_pipeline.NewService(aiPipelineClient, logger, tenderService, companyService)
|
||||||
|
auditLogRepository := auditlog.NewRepository(auditStore)
|
||||||
|
auditLogService := auditlog.NewService(auditLogRepository, logger)
|
||||||
logger.Info("Services initialized successfully", map[string]interface{}{
|
logger.Info("Services initialized successfully", map[string]interface{}{
|
||||||
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper", "dashboard"},
|
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "tender_submission", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper", "dashboard", "ai_pipeline", "auditlog"},
|
||||||
})
|
})
|
||||||
|
|
||||||
// Initialize handlers with services
|
// Initialize handlers with services
|
||||||
@@ -236,17 +290,20 @@ func main() {
|
|||||||
tenderHandler := tender.NewHandler(tenderService, logger)
|
tenderHandler := tender.NewHandler(tenderService, logger)
|
||||||
feedbackHandler := feedback.NewHandler(feedbackService, userHandler, customerHandler, logger)
|
feedbackHandler := feedback.NewHandler(feedbackService, userHandler, customerHandler, logger)
|
||||||
tenderApprovalHandler := tender_approval.NewHandler(tenderApprovalService, logger)
|
tenderApprovalHandler := tender_approval.NewHandler(tenderApprovalService, logger)
|
||||||
|
tenderSubmissionHandler := tender_submission.NewHandler(tenderSubmissionService, logger)
|
||||||
inquiryHandler := inquiry.NewHandler(inquiryService, logger, hcaptchaVerifier, xssPolicy)
|
inquiryHandler := inquiry.NewHandler(inquiryService, logger, hcaptchaVerifier, xssPolicy)
|
||||||
flagHandler := assets.NewHandler(flagService, logger)
|
flagHandler := assets.NewHandler(flagService, logger)
|
||||||
notificationHandler := notification.NewHandler(notificationService)
|
notificationHandler := notification.NewHandler(notificationService)
|
||||||
contactHandler := contact.NewHandler(contactService, hcaptchaVerifier)
|
contactHandler := contact.NewHandler(contactService, hcaptchaVerifier)
|
||||||
cmsHandler := cms.NewHandler(cmsService)
|
cmsHandler := cms.NewHandler(cmsService)
|
||||||
kanbanHandler := kanban.NewHandler(kanbanService)
|
kanbanHandler := kanban.NewHandler(kanbanService)
|
||||||
fileStoreHandler := filestore.NewHandler(fileStoreService, logger)
|
fileStoreHandler := filestore.NewHandler(fileStoreService, logger, companyService)
|
||||||
documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger)
|
documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger)
|
||||||
dashboardHandler := dashboard.NewHandler(dashboardService)
|
dashboardHandler := dashboard.NewHandler(dashboardService)
|
||||||
|
aiPipelineHandler := ai_pipeline.NewHandler(aiPipelineService)
|
||||||
|
auditLogHandler := auditlog.NewHandler(auditLogService)
|
||||||
logger.Info("Handlers initialized successfully", map[string]interface{}{
|
logger.Info("Handlers initialized successfully", map[string]interface{}{
|
||||||
"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper", "dashboard"},
|
"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "tender_submission", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper", "dashboard", "ai_pipeline", "auditlog"},
|
||||||
})
|
})
|
||||||
|
|
||||||
// Initialize HTTP server
|
// Initialize HTTP server
|
||||||
@@ -255,8 +312,8 @@ func main() {
|
|||||||
router.SetupCSPSecurity(e)
|
router.SetupCSPSecurity(e)
|
||||||
|
|
||||||
// Register routes
|
// Register routes
|
||||||
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, dashboardHandler, xssPolicy)
|
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, tenderSubmissionHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, dashboardHandler, aiPipelineHandler, auditLogHandler, xssPolicy)
|
||||||
router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, xssPolicy)
|
router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, tenderSubmissionHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, xssPolicy)
|
||||||
router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey)
|
router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey)
|
||||||
|
|
||||||
// Start server
|
// Start server
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ package router
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"tm/internal/assets"
|
"tm/internal/assets"
|
||||||
|
"tm/internal/ai_pipeline"
|
||||||
|
"tm/internal/auditlog"
|
||||||
"tm/internal/cms"
|
"tm/internal/cms"
|
||||||
"tm/internal/company"
|
"tm/internal/company"
|
||||||
"tm/internal/company_category"
|
"tm/internal/company_category"
|
||||||
@@ -15,6 +17,7 @@ import (
|
|||||||
"tm/internal/notification"
|
"tm/internal/notification"
|
||||||
"tm/internal/tender"
|
"tm/internal/tender"
|
||||||
"tm/internal/tender_approval"
|
"tm/internal/tender_approval"
|
||||||
|
"tm/internal/tender_submission"
|
||||||
"tm/internal/user"
|
"tm/internal/user"
|
||||||
"tm/pkg/filestore"
|
"tm/pkg/filestore"
|
||||||
"tm/pkg/security"
|
"tm/pkg/security"
|
||||||
@@ -34,7 +37,7 @@ func SetupCSPSecurity(e *echo.Echo) {
|
|||||||
e.POST("/api/v1/csp-report", security.CSPReportHandler)
|
e.POST("/api/v1/csp-report", security.CSPReportHandler)
|
||||||
}
|
}
|
||||||
|
|
||||||
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, dashboardHandler *dashboard.Handler, xssPolicy *security.XSSPolicy) {
|
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, tenderSubmissionHandler *tender_submission.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, dashboardHandler *dashboard.Handler, aiPipelineHandler *ai_pipeline.Handler, auditLogHandler *auditlog.Handler, xssPolicy *security.XSSPolicy) {
|
||||||
adminV1 := e.Group("/admin/v1")
|
adminV1 := e.Group("/admin/v1")
|
||||||
|
|
||||||
// Admin Users Routes
|
// Admin Users Routes
|
||||||
@@ -50,6 +53,13 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
|
|||||||
adminUsersGP.POST("/:id/reset-password", userHandler.ResetUserPassword, userHandler.AdminMiddleware())
|
adminUsersGP.POST("/:id/reset-password", userHandler.ResetUserPassword, userHandler.AdminMiddleware())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Admin Audit Logs Routes
|
||||||
|
auditLogsGP := adminV1.Group("/audit-logs")
|
||||||
|
{
|
||||||
|
auditLogsGP.Use(userHandler.AuthMiddleware(), userHandler.AdminMiddleware())
|
||||||
|
auditLogsGP.GET("", auditLogHandler.Search)
|
||||||
|
}
|
||||||
|
|
||||||
// Admin user profile
|
// Admin user profile
|
||||||
profileGP := adminV1.Group("/profile")
|
profileGP := adminV1.Group("/profile")
|
||||||
{
|
{
|
||||||
@@ -72,12 +82,14 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
|
|||||||
companiesGP.Use(userHandler.AuthMiddleware())
|
companiesGP.Use(userHandler.AuthMiddleware())
|
||||||
companiesGP.POST("", companyHandler.Create)
|
companiesGP.POST("", companyHandler.Create)
|
||||||
companiesGP.GET("", companyHandler.Search)
|
companiesGP.GET("", companyHandler.Search)
|
||||||
|
companiesGP.GET("/:id/recommended-tenders", tenderHandler.AdminRecommendTenders)
|
||||||
companiesGP.GET("/:id", companyHandler.Get)
|
companiesGP.GET("/:id", companyHandler.Get)
|
||||||
companiesGP.PUT("/:id", companyHandler.Update)
|
companiesGP.PUT("/:id", companyHandler.Update)
|
||||||
companiesGP.DELETE("/:id", companyHandler.Delete)
|
companiesGP.DELETE("/:id", companyHandler.Delete)
|
||||||
companiesGP.PATCH("/:id/status", companyHandler.UpdateStatus)
|
companiesGP.PATCH("/:id/status", companyHandler.UpdateStatus)
|
||||||
companiesGP.PATCH("/:id/verification", companyHandler.UpdateVerification)
|
companiesGP.PATCH("/:id/verification", companyHandler.UpdateVerification)
|
||||||
companiesGP.POST("/:id/documents", companyHandler.UploadDocuments)
|
companiesGP.POST("/:id/documents", companyHandler.UploadDocuments)
|
||||||
|
companiesGP.POST("/:id/links", companyHandler.AddLinks)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Admin Categories Routes
|
// Admin Categories Routes
|
||||||
@@ -145,6 +157,15 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
|
|||||||
tenderApprovalGP.GET("/status/:status", tenderApprovalHandler.GetTenderApprovalsByStatus)
|
tenderApprovalGP.GET("/status/:status", tenderApprovalHandler.GetTenderApprovalsByStatus)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Admin Tender-Submissions Routes
|
||||||
|
tenderSubmissionGP := adminV1.Group("/tender-submissions")
|
||||||
|
{
|
||||||
|
tenderSubmissionGP.Use(userHandler.AuthMiddleware())
|
||||||
|
tenderSubmissionGP.GET("", tenderSubmissionHandler.AdminListSubmissions)
|
||||||
|
tenderSubmissionGP.GET("/stats", tenderSubmissionHandler.AdminGetSubmissionStats)
|
||||||
|
tenderSubmissionGP.GET("/:id", tenderSubmissionHandler.AdminGetSubmissionByID)
|
||||||
|
}
|
||||||
|
|
||||||
// Admin Inquiry Routes
|
// Admin Inquiry Routes
|
||||||
inquiryGP := adminV1.Group("/inquiries")
|
inquiryGP := adminV1.Group("/inquiries")
|
||||||
{
|
{
|
||||||
@@ -225,9 +246,31 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
|
|||||||
dashboardGP.GET("/recent", dashboardHandler.Recent)
|
dashboardGP.GET("/recent", dashboardHandler.Recent)
|
||||||
dashboardGP.GET("/statistics", dashboardHandler.Statistics)
|
dashboardGP.GET("/statistics", dashboardHandler.Statistics)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AI Pipeline Routes (proxy to Opplens AI service)
|
||||||
|
aiPipelineGP := adminV1.Group("/ai-pipeline")
|
||||||
|
{
|
||||||
|
aiPipelineGP.Use(userHandler.AuthMiddleware())
|
||||||
|
aiPipelineGP.POST("/scrape/documents", aiPipelineHandler.ScrapeDocuments)
|
||||||
|
aiPipelineGP.POST("/scrape/documents/batch", aiPipelineHandler.ScrapeDocumentsBatch)
|
||||||
|
aiPipelineGP.GET("/scrape/portals", aiPipelineHandler.GetScrapePortals)
|
||||||
|
aiPipelineGP.POST("/summarize/batch", aiPipelineHandler.SummarizeBatch)
|
||||||
|
aiPipelineGP.POST("/translate/batch", aiPipelineHandler.TranslateBatch)
|
||||||
|
aiPipelineGP.POST("/sync", aiPipelineHandler.Sync)
|
||||||
|
aiPipelineGP.POST("/run", aiPipelineHandler.Run)
|
||||||
|
aiPipelineGP.POST("/run/batch", aiPipelineHandler.RunBatch)
|
||||||
|
aiPipelineGP.POST("/analyze", aiPipelineHandler.Analyze)
|
||||||
|
aiPipelineGP.POST("/daily-run", aiPipelineHandler.DailyRun)
|
||||||
|
aiPipelineGP.POST("/auto", aiPipelineHandler.Auto)
|
||||||
|
aiPipelineGP.GET("/last-run", aiPipelineHandler.GetLastRun)
|
||||||
|
aiPipelineGP.GET("/last-auto-run", aiPipelineHandler.GetLastAutoRun)
|
||||||
|
aiPipelineGP.GET("/report", aiPipelineHandler.GetReport)
|
||||||
|
aiPipelineGP.GET("/stats", aiPipelineHandler.GetStats)
|
||||||
|
aiPipelineGP.GET("/minio-stats", aiPipelineHandler.GetMinioStats)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) {
|
func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, tenderSubmissionHandler *tender_submission.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) {
|
||||||
v1 := e.Group("/api/v1")
|
v1 := e.Group("/api/v1")
|
||||||
|
|
||||||
customerGP := v1.Group("/profile")
|
customerGP := v1.Group("/profile")
|
||||||
@@ -249,7 +292,7 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende
|
|||||||
|
|
||||||
// Public Tenders Routes
|
// Public Tenders Routes
|
||||||
tendersGP := v1.Group("/tenders")
|
tendersGP := v1.Group("/tenders")
|
||||||
tendersGP.Use(customerHandler.AuthMiddleware())
|
tendersGP.Use(customerHandler.AuthMiddleware(), customerHandler.CompanyContextMiddleware())
|
||||||
{
|
{
|
||||||
tendersGP.GET("", tenderHandler.GetPublicTenders)
|
tendersGP.GET("", tenderHandler.GetPublicTenders)
|
||||||
tendersGP.GET("/recommend", tenderHandler.RecommendTenders)
|
tendersGP.GET("/recommend", tenderHandler.RecommendTenders)
|
||||||
@@ -276,7 +319,7 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende
|
|||||||
// Public Tender Approvals Routes
|
// Public Tender Approvals Routes
|
||||||
tenderApprovalGP := v1.Group("/tender-approvals")
|
tenderApprovalGP := v1.Group("/tender-approvals")
|
||||||
{
|
{
|
||||||
tenderApprovalGP.Use(customerHandler.AuthMiddleware())
|
tenderApprovalGP.Use(customerHandler.AuthMiddleware(), customerHandler.CompanyContextMiddleware())
|
||||||
tenderApprovalGP.POST("", tenderApprovalHandler.ToggleTenderApproval)
|
tenderApprovalGP.POST("", tenderApprovalHandler.ToggleTenderApproval)
|
||||||
tenderApprovalGP.GET("", tenderApprovalHandler.PublicGetTenderApprovals)
|
tenderApprovalGP.GET("", tenderApprovalHandler.PublicGetTenderApprovals)
|
||||||
tenderApprovalGP.GET("/:id", tenderApprovalHandler.GetPublicTenderApproval)
|
tenderApprovalGP.GET("/:id", tenderApprovalHandler.GetPublicTenderApproval)
|
||||||
@@ -284,13 +327,33 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende
|
|||||||
tenderApprovalGP.GET("/stats", tenderApprovalHandler.GetCompanyTenderApprovalStats)
|
tenderApprovalGP.GET("/stats", tenderApprovalHandler.GetCompanyTenderApprovalStats)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Public Tender Submissions Routes
|
||||||
|
tenderSubmissionGP := v1.Group("/tender-submissions")
|
||||||
|
{
|
||||||
|
tenderSubmissionGP.Use(customerHandler.AuthMiddleware(), customerHandler.CompanyContextMiddleware())
|
||||||
|
tenderSubmissionGP.GET("", tenderSubmissionHandler.ListCompanySubmissions)
|
||||||
|
tenderSubmissionGP.GET("/stats", tenderSubmissionHandler.GetCompanySubmissionStats)
|
||||||
|
tenderSubmissionGP.POST("/tender/:tender_id/ensure", tenderSubmissionHandler.EnsureSubmission)
|
||||||
|
tenderSubmissionGP.GET("/tender/:tender_id", tenderSubmissionHandler.GetSubmissionByTender)
|
||||||
|
tenderSubmissionGP.PATCH("/:id/status", tenderSubmissionHandler.UpdateSubmissionStatus)
|
||||||
|
tenderSubmissionGP.GET("/:id", tenderSubmissionHandler.GetSubmissionByID)
|
||||||
|
}
|
||||||
|
|
||||||
// Public Company Routes
|
// Public Company Routes
|
||||||
companiesGP := v1.Group("/companies")
|
companiesGP := v1.Group("/companies")
|
||||||
{
|
{
|
||||||
companiesGP.Use(customerHandler.AuthMiddleware())
|
companiesGP.Use(customerHandler.AuthMiddleware(), customerHandler.CompanyContextMiddleware())
|
||||||
companiesGP.GET("", companyHandler.MyCompany)
|
companiesGP.GET("", companyHandler.MyCompany)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AI tender recommendation routes
|
||||||
|
recommendationGP := v1.Group("")
|
||||||
|
recommendationGP.Use(customerHandler.AuthMiddleware(), customerHandler.CompanyContextMiddleware())
|
||||||
|
{
|
||||||
|
recommendationGP.POST("/onboarding", companyHandler.StartOnboarding)
|
||||||
|
recommendationGP.POST("/recommend", companyHandler.RecommendTenders)
|
||||||
|
}
|
||||||
|
|
||||||
// Public Inquiry Routes
|
// Public Inquiry Routes
|
||||||
inquiryGP := v1.Group("/inquiries")
|
inquiryGP := v1.Group("/inquiries")
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,11 +1,17 @@
|
|||||||
package bootstrap
|
package bootstrap
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"tm/cmd/worker/workers"
|
"tm/cmd/worker/workers"
|
||||||
|
"tm/internal/company"
|
||||||
|
"tm/internal/company_category"
|
||||||
"tm/internal/notice"
|
"tm/internal/notice"
|
||||||
|
notificationDomain "tm/internal/notification"
|
||||||
"tm/internal/tender"
|
"tm/internal/tender"
|
||||||
ai_summarizer "tm/pkg/ai_summarizer"
|
ai_summarizer "tm/pkg/ai_summarizer"
|
||||||
"tm/pkg/config"
|
"tm/pkg/config"
|
||||||
@@ -13,9 +19,13 @@ import (
|
|||||||
"tm/pkg/mongo"
|
"tm/pkg/mongo"
|
||||||
"tm/pkg/notification"
|
"tm/pkg/notification"
|
||||||
"tm/pkg/ollama"
|
"tm/pkg/ollama"
|
||||||
|
"tm/pkg/redis"
|
||||||
"tm/pkg/schedule"
|
"tm/pkg/schedule"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// aiPipelineDailyRunMu ensures only one AI pipeline auto run executes at a time (startup catch-up and cron).
|
||||||
|
var aiPipelineDailyRunMu sync.Mutex
|
||||||
|
|
||||||
// Init Application Configuration
|
// Init Application Configuration
|
||||||
func InitConfig() (*Config, error) {
|
func InitConfig() (*Config, error) {
|
||||||
conf, err := config.LoadConfig(".", &Config{})
|
conf, err := config.LoadConfig(".", &Config{})
|
||||||
@@ -76,7 +86,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
|
|||||||
}
|
}
|
||||||
|
|
||||||
// InitWorker
|
// InitWorker
|
||||||
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient) *schedule.CronScheduler {
|
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, aiClient *ai_summarizer.Client, aiStorage *ai_summarizer.StorageClient, redisClient redis.Client) *schedule.CronScheduler {
|
||||||
// Debug: Log worker config
|
// Debug: Log worker config
|
||||||
appLogger.Info("Worker configuration", map[string]interface{}{
|
appLogger.Info("Worker configuration", map[string]interface{}{
|
||||||
"worker_interval": config.Worker.Interval,
|
"worker_interval": config.Worker.Interval,
|
||||||
@@ -85,28 +95,40 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
|||||||
"notice_processing_limit": config.Worker.NoticeProcessingLimit,
|
"notice_processing_limit": config.Worker.NoticeProcessingLimit,
|
||||||
"notice_batch_pause": config.Worker.NoticeBatchPause.String(),
|
"notice_batch_pause": config.Worker.NoticeBatchPause.String(),
|
||||||
"notice_fetch_error_backoff": config.Worker.NoticeFetchErrorBackoff.String(),
|
"notice_fetch_error_backoff": config.Worker.NoticeFetchErrorBackoff.String(),
|
||||||
|
"translation_enabled": config.Worker.TranslationEnabled,
|
||||||
|
"translation_interval": config.Worker.TranslationInterval,
|
||||||
|
"notification_interval": config.Worker.NotificationInterval,
|
||||||
|
"ai_pipeline_auto_enabled": config.Worker.AIPipelineAutoEnabled,
|
||||||
|
"ai_pipeline_auto_interval": config.Worker.AIPipelineAutoInterval,
|
||||||
})
|
})
|
||||||
// Initialize repositories
|
// Initialize repositories
|
||||||
noticeRepo := notice.NewRepository(mongoManager, appLogger)
|
noticeRepo := notice.NewRepository(mongoManager, appLogger)
|
||||||
tenderRepo := tender.NewRepository(mongoManager, appLogger)
|
tenderRepo := tender.NewRepository(mongoManager, appLogger)
|
||||||
|
notificationRepo := notificationDomain.NewRepository(mongoManager, appLogger)
|
||||||
|
|
||||||
|
var recommendationRefresher company.Service
|
||||||
|
if config.Worker.RecommendationRefreshAfterPipelineEnabled && aiClient != nil && redisClient != nil {
|
||||||
|
companyRepo := company.NewRepository(mongoManager, appLogger)
|
||||||
|
categoryRepo := company_category.NewRepository(mongoManager, appLogger)
|
||||||
|
categoryService := company_category.NewService(categoryRepo, appLogger)
|
||||||
|
recommendationRefresher = company.NewService(
|
||||||
|
companyRepo,
|
||||||
|
categoryService,
|
||||||
|
nil,
|
||||||
|
aiClient,
|
||||||
|
aiClient,
|
||||||
|
redisClient,
|
||||||
|
config.AISummarizer.RecommendationCacheTTL,
|
||||||
|
appLogger,
|
||||||
|
)
|
||||||
|
appLogger.Info("Company recommendation refresh after pipeline enabled", map[string]interface{}{})
|
||||||
|
} else if config.Worker.RecommendationRefreshAfterPipelineEnabled {
|
||||||
|
appLogger.Warn("Company recommendation refresh after pipeline disabled: AI client or Redis unavailable", map[string]interface{}{})
|
||||||
|
}
|
||||||
|
|
||||||
// Create a single shared cron scheduler for all recurring jobs
|
// Create a single shared cron scheduler for all recurring jobs
|
||||||
scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
|
scheduler := schedule.NewCronScheduler(appLogger, mongoManager, noticeRepo)
|
||||||
|
|
||||||
// Document summarization via external AI service (requires AI_SUMMARIZER_API_BASE_URL)
|
|
||||||
if aiClient != nil {
|
|
||||||
scheduler.AddJob(schedule.Job{
|
|
||||||
Name: "Document Summarization Worker Job",
|
|
||||||
Func: func() {
|
|
||||||
worker := workers.NewDocumentSummarizationWorker(mongoManager, appLogger, tenderRepo, aiClient, aiStorage)
|
|
||||||
worker.Run()
|
|
||||||
},
|
|
||||||
Expr: "0 11 * * * *", // Run at 11 AM every day (after AI pipeline scrape)
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
appLogger.Warn("Document summarization worker not scheduled: AI summarizer API is not configured", map[string]interface{}{})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Schedule Notice Worker job (no blocking run at startup; first tick runs on schedule)
|
// Schedule Notice Worker job (no blocking run at startup; first tick runs on schedule)
|
||||||
workerInterval := config.Worker.Interval
|
workerInterval := config.Worker.Interval
|
||||||
if workerInterval == "" {
|
if workerInterval == "" {
|
||||||
@@ -138,7 +160,11 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
|||||||
})
|
})
|
||||||
|
|
||||||
// Initialize translation backfill worker for configured languages
|
// Initialize translation backfill worker for configured languages
|
||||||
if aiClient != nil {
|
if !config.Worker.TranslationEnabled {
|
||||||
|
appLogger.Info("Tender translation worker disabled by configuration", map[string]interface{}{
|
||||||
|
"translation_enabled": false,
|
||||||
|
})
|
||||||
|
} else if aiClient != nil {
|
||||||
targetLanguages := parseTranslationLanguages(config.AISummarizer)
|
targetLanguages := parseTranslationLanguages(config.AISummarizer)
|
||||||
translationSuccessCounter := mongo.NewCounter(mongoManager)
|
translationSuccessCounter := mongo.NewCounter(mongoManager)
|
||||||
scheduler.AddJob(schedule.Job{
|
scheduler.AddJob(schedule.Job{
|
||||||
@@ -163,12 +189,140 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
|||||||
"interval": config.Worker.TranslationInterval,
|
"interval": config.Worker.TranslationInterval,
|
||||||
"batch_size": config.Worker.TranslationBatchSize,
|
"batch_size": config.Worker.TranslationBatchSize,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
appLogger.Info("Running startup catch-up for tender translations", map[string]interface{}{
|
||||||
|
"target_languages": targetLanguages,
|
||||||
|
})
|
||||||
|
worker := workers.NewTranslationWorker(
|
||||||
|
mongoManager,
|
||||||
|
appLogger,
|
||||||
|
tenderRepo,
|
||||||
|
aiClient,
|
||||||
|
aiStorage,
|
||||||
|
translationSuccessCounter,
|
||||||
|
targetLanguages,
|
||||||
|
config.Worker.TranslationBatchSize,
|
||||||
|
)
|
||||||
|
worker.Run()
|
||||||
|
}()
|
||||||
} else {
|
} else {
|
||||||
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
|
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
|
||||||
}
|
}
|
||||||
|
|
||||||
// Kick off one notice-processing pass without blocking startup (cron continues on schedule)
|
notificationInterval := config.Worker.NotificationInterval
|
||||||
|
if notificationInterval == "" {
|
||||||
|
notificationInterval = "0 * * * * *"
|
||||||
|
appLogger.Warn("WORKER_NOTIFICATION_INTERVAL not set, using default schedule", map[string]interface{}{
|
||||||
|
"interval": notificationInterval,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduler.AddJob(schedule.Job{
|
||||||
|
Name: "Scheduled Notification Worker Job",
|
||||||
|
Func: func() {
|
||||||
|
worker := workers.NewNotificationWorker(notificationRepo, appLogger)
|
||||||
|
worker.Run()
|
||||||
|
},
|
||||||
|
Expr: notificationInterval,
|
||||||
|
})
|
||||||
|
appLogger.Info("Scheduled notification delivery worker", map[string]interface{}{
|
||||||
|
"interval": notificationInterval,
|
||||||
|
})
|
||||||
|
|
||||||
|
if !config.Worker.AIPipelineAutoEnabled {
|
||||||
|
appLogger.Info("AI pipeline auto worker disabled by configuration", map[string]interface{}{
|
||||||
|
"ai_pipeline_auto_enabled": false,
|
||||||
|
})
|
||||||
|
} else if aiClient != nil {
|
||||||
|
dailyJobTracker := schedule.NewDailyJobTracker(mongoManager, appLogger)
|
||||||
|
|
||||||
|
runAIPipelineDaily := func(trigger string) {
|
||||||
|
aiPipelineDailyRunMu.Lock()
|
||||||
|
defer aiPipelineDailyRunMu.Unlock()
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
today := time.Now().Local()
|
||||||
|
|
||||||
|
completed, checkErr := dailyJobTracker.IsCompleted(ctx, schedule.AIPipelineDailyJobName, today)
|
||||||
|
if checkErr != nil {
|
||||||
|
appLogger.Error("Failed to check AI pipeline auto completion status", map[string]interface{}{
|
||||||
|
"trigger": trigger,
|
||||||
|
"error": checkErr.Error(),
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
} else if completed {
|
||||||
|
appLogger.Info("AI pipeline auto skipped: already completed today", map[string]interface{}{
|
||||||
|
"trigger": trigger,
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if trigger == "startup" {
|
||||||
|
appLogger.Info("Running startup catch-up for today's AI pipeline auto run", map[string]interface{}{
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
appLogger.Info("Running scheduled AI pipeline auto run", map[string]interface{}{
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
worker := workers.NewAIPipelineDailyWorker(appLogger, aiClient)
|
||||||
|
if err := worker.Run(); err != nil {
|
||||||
|
if errors.Is(err, ai_summarizer.ErrPipelineJobRunning) {
|
||||||
|
appLogger.Info("AI pipeline auto deferred: job already running", map[string]interface{}{
|
||||||
|
"trigger": trigger,
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
appLogger.Error("AI pipeline auto failed", map[string]interface{}{
|
||||||
|
"trigger": trigger,
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if markErr := dailyJobTracker.MarkCompleted(ctx, schedule.AIPipelineDailyJobName, today); markErr != nil {
|
||||||
|
appLogger.Error("Failed to mark AI pipeline auto as completed", map[string]interface{}{
|
||||||
|
"trigger": trigger,
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
"error": markErr.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
appLogger.Info("AI pipeline auto completed successfully", map[string]interface{}{
|
||||||
|
"trigger": trigger,
|
||||||
|
"date": today.Format("02/01/2006"),
|
||||||
|
})
|
||||||
|
|
||||||
|
if recommendationRefresher != nil {
|
||||||
|
recommendationRefresher.ScheduleRefreshCachedAIRecommendationsAfterPipeline()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
scheduler.AddJob(schedule.Job{
|
||||||
|
Name: "AI Pipeline Auto Job",
|
||||||
|
Func: func() { runAIPipelineDaily("scheduled") },
|
||||||
|
Expr: config.Worker.AIPipelineAutoInterval,
|
||||||
|
})
|
||||||
|
appLogger.Info("Scheduled AI pipeline auto worker", map[string]interface{}{
|
||||||
|
"interval": config.Worker.AIPipelineAutoInterval,
|
||||||
|
})
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
|
runAIPipelineDaily("startup")
|
||||||
|
}()
|
||||||
|
} else {
|
||||||
|
appLogger.Warn("AI summarizer client not available, AI pipeline auto worker is disabled", map[string]interface{}{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// After a server restart, process any unprocessed notices (including today's) without blocking startup.
|
||||||
|
go func() {
|
||||||
|
appLogger.Info("Running startup catch-up for unprocessed notices", map[string]interface{}{})
|
||||||
w := workers.NewNoticeWorker(
|
w := workers.NewNoticeWorker(
|
||||||
mongoManager,
|
mongoManager,
|
||||||
appLogger,
|
appLogger,
|
||||||
@@ -260,6 +414,35 @@ func InitAISummarizerStorage(conf AISummarizerConfig, log logger.Logger) *ai_sum
|
|||||||
return storage
|
return storage
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InitRedis initializes the Redis client used by worker jobs.
|
||||||
|
func InitRedis(conf config.RedisConfig, log logger.Logger) redis.Client {
|
||||||
|
connectionConfig := &redis.Config{
|
||||||
|
Host: conf.Host,
|
||||||
|
Port: conf.Port,
|
||||||
|
Password: conf.Password,
|
||||||
|
DB: conf.DB,
|
||||||
|
PoolSize: conf.PoolSize,
|
||||||
|
}
|
||||||
|
|
||||||
|
redisClient, err := redis.NewClient(connectionConfig, log)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Failed to initialize Redis connection for worker", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"host": conf.Host,
|
||||||
|
"port": conf.Port,
|
||||||
|
})
|
||||||
|
panic(fmt.Sprintf("Failed to initialize Redis connection: %v", err))
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("Redis connection initialized for worker", map[string]interface{}{
|
||||||
|
"host": conf.Host,
|
||||||
|
"port": conf.Port,
|
||||||
|
"pool_size": conf.PoolSize,
|
||||||
|
})
|
||||||
|
|
||||||
|
return redisClient
|
||||||
|
}
|
||||||
|
|
||||||
func parseTranslationLanguages(conf AISummarizerConfig) []string {
|
func parseTranslationLanguages(conf AISummarizerConfig) []string {
|
||||||
raw := strings.TrimSpace(conf.TranslationLanguages)
|
raw := strings.TrimSpace(conf.TranslationLanguages)
|
||||||
if raw == "" {
|
if raw == "" {
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
// Config defines the worker application configuration
|
// Config defines the worker application configuration
|
||||||
type Config struct {
|
type Config struct {
|
||||||
Database config.DatabaseConfig
|
Database config.DatabaseConfig
|
||||||
|
Redis config.RedisConfig
|
||||||
Logging config.LoggingConfig
|
Logging config.LoggingConfig
|
||||||
Notification config.NotificationConfig
|
Notification config.NotificationConfig
|
||||||
Ollama config.OllamaConfig
|
Ollama config.OllamaConfig
|
||||||
@@ -30,6 +31,18 @@ type WorkerConfig struct {
|
|||||||
NoticeFetchErrorBackoff time.Duration `env:"WORKER_NOTICE_FETCH_ERROR_BACKOFF" envDefault:"2s"`
|
NoticeFetchErrorBackoff time.Duration `env:"WORKER_NOTICE_FETCH_ERROR_BACKOFF" envDefault:"2s"`
|
||||||
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
|
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
|
||||||
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
|
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
|
||||||
|
// TranslationEnabled schedules the automatic batch translation cron job on the worker.
|
||||||
|
// On-demand translation (admin/public tender endpoints and AI pipeline routes on web) is unaffected.
|
||||||
|
TranslationEnabled bool `env:"WORKER_TRANSLATION_ENABLED" envDefault:"true"`
|
||||||
|
// NotificationInterval schedules promotion of due scheduled notifications from pending to sent.
|
||||||
|
NotificationInterval string `env:"WORKER_NOTIFICATION_INTERVAL" envDefault:"0 * * * * *"`
|
||||||
|
// AIPipelineAutoEnabled schedules the daily AI pipeline auto run on the worker.
|
||||||
|
// On-demand pipeline routes on web are unaffected.
|
||||||
|
AIPipelineAutoEnabled bool `env:"WORKER_AI_PIPELINE_AUTO_ENABLED" envDefault:"true"`
|
||||||
|
// AIPipelineAutoInterval runs pipeline auto once per day by default at 11:00 UTC (after TED scrape at 10:00).
|
||||||
|
AIPipelineAutoInterval string `env:"WORKER_AI_PIPELINE_AUTO_INTERVAL" envDefault:"0 0 11 * * *"`
|
||||||
|
// RecommendationRefreshAfterPipelineEnabled refreshes cached company recommendations after the daily AI pipeline auto run.
|
||||||
|
RecommendationRefreshAfterPipelineEnabled bool `env:"WORKER_RECOMMENDATION_REFRESH" envDefault:"true"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// AISummarizerConfig holds configuration for the external AI summarizer service.
|
// AISummarizerConfig holds configuration for the external AI summarizer service.
|
||||||
@@ -45,5 +58,6 @@ type AISummarizerConfig struct {
|
|||||||
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
|
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
|
||||||
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
|
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
|
||||||
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
|
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
|
||||||
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens-documents"`
|
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens"`
|
||||||
|
RecommendationCacheTTL time.Duration `env:"AI_RECOMMENDATION_CACHE_TTL" envDefault:"0"`
|
||||||
}
|
}
|
||||||
|
|||||||
+12
-2
@@ -37,12 +37,22 @@ func main() {
|
|||||||
// Initialize notification service
|
// Initialize notification service
|
||||||
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
|
notificationService := bootstrap.InitNotificationService(config.Notification, appLogger)
|
||||||
|
|
||||||
// Initialize AI summarizer client (translation + document summarization workers)
|
// Initialize AI summarizer client (translation worker)
|
||||||
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, mongoManager, appLogger)
|
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, mongoManager, appLogger)
|
||||||
aiSummarizerStorage := bootstrap.InitAISummarizerStorage(config.AISummarizer, appLogger)
|
aiSummarizerStorage := bootstrap.InitAISummarizerStorage(config.AISummarizer, appLogger)
|
||||||
|
redisClient := bootstrap.InitRedis(config.Redis, appLogger)
|
||||||
|
defer func() {
|
||||||
|
if redisClient != nil {
|
||||||
|
if err := redisClient.Close(); err != nil {
|
||||||
|
appLogger.Error("Failed to close Redis connection", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
// Initialize Worker
|
// Initialize Worker
|
||||||
scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, aiSummarizerClient, aiSummarizerStorage)
|
scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, aiSummarizerClient, aiSummarizerStorage, redisClient)
|
||||||
|
|
||||||
// Set up signal handling for graceful shutdown
|
// Set up signal handling for graceful shutdown
|
||||||
signalChan := make(chan os.Signal, 1)
|
signalChan := make(chan os.Signal, 1)
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package workers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
"tm/pkg/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AIPipelineDailyWorker triggers the scheduled Opplens AI pipeline auto run.
|
||||||
|
type AIPipelineDailyWorker struct {
|
||||||
|
Logger logger.Logger
|
||||||
|
AIClient *ai_summarizer.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewAIPipelineDailyWorker creates a worker that calls POST /pipeline/auto on the AI service.
|
||||||
|
func NewAIPipelineDailyWorker(log logger.Logger, aiClient *ai_summarizer.Client) *AIPipelineDailyWorker {
|
||||||
|
return &AIPipelineDailyWorker{
|
||||||
|
Logger: log,
|
||||||
|
AIClient: aiClient,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run starts the pipeline auto job. A 409 response means a run is already in progress and is returned as an error
|
||||||
|
// so the daily job tracker does not mark today as completed before the run actually starts.
|
||||||
|
func (w *AIPipelineDailyWorker) Run() error {
|
||||||
|
if w.AIClient == nil {
|
||||||
|
w.Logger.Warn("AI pipeline auto worker skipped: AI client is not configured", map[string]interface{}{})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
w.Logger.Info("AI pipeline auto worker started", map[string]interface{}{})
|
||||||
|
|
||||||
|
resp, err := w.AIClient.PipelineAuto(ctx)
|
||||||
|
if err != nil {
|
||||||
|
w.Logger.Error("AI pipeline auto worker failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Logger.Info("AI pipeline auto worker triggered successfully", map[string]interface{}{
|
||||||
|
"status": resp.Status,
|
||||||
|
"report": resp.Report,
|
||||||
|
})
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -476,6 +476,16 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
|
|||||||
awardedEntities = mergeAwardedEntitiesByLotID(t.AwardedEntities, awardedEntities)
|
awardedEntities = mergeAwardedEntitiesByLotID(t.AwardedEntities, awardedEntities)
|
||||||
title, description = mergeTitleAndDescriptionForMultiLot(t.Title, t.Description, title, description)
|
title, description = mergeTitleAndDescriptionForMultiLot(t.Title, t.Description, title, description)
|
||||||
|
|
||||||
|
if estimatedVal == 0 && t.EstimatedValue > 0 {
|
||||||
|
estimatedVal = t.EstimatedValue
|
||||||
|
}
|
||||||
|
if awardedVal == 0 && t.AwardedValue > 0 {
|
||||||
|
awardedVal = t.AwardedValue
|
||||||
|
}
|
||||||
|
if currency == "" && t.Currency != "" {
|
||||||
|
currency = t.Currency
|
||||||
|
}
|
||||||
|
|
||||||
if strings.TrimSpace(previousNoticePublicationID) != "" && previousNoticePublicationID != n.NoticePublicationID {
|
if strings.TrimSpace(previousNoticePublicationID) != "" && previousNoticePublicationID != n.NoticePublicationID {
|
||||||
if n.EstimatedValue > 0 {
|
if n.EstimatedValue > 0 {
|
||||||
estimatedVal = t.EstimatedValue + n.EstimatedValue
|
estimatedVal = t.EstimatedValue + n.EstimatedValue
|
||||||
@@ -495,6 +505,15 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if estimatedVal == 0 {
|
||||||
|
if lotValue, lotCurrency := tender.AggregateProcurementLotEstimatedValue(procurementLots); lotValue > 0 {
|
||||||
|
estimatedVal = lotValue
|
||||||
|
if currency == "" {
|
||||||
|
currency = lotCurrency
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Title and description stay in the notice language; English translation is handled by the AI service (translation worker).
|
// Title and description stay in the notice language; English translation is handled by the AI service (translation worker).
|
||||||
t.Title = title
|
t.Title = title
|
||||||
t.Description = description
|
t.Description = description
|
||||||
@@ -546,7 +565,6 @@ func (w *NoticeWorker) ToTender(n *notice.Notice) (*tender.Tender, error) {
|
|||||||
t.ProcurementProjectID = n.ProcurementProjectID
|
t.ProcurementProjectID = n.ProcurementProjectID
|
||||||
t.SourceFileURL = n.SourceFileURL
|
t.SourceFileURL = n.SourceFileURL
|
||||||
t.SourceFileName = n.SourceFileName
|
t.SourceFileName = n.SourceFileName
|
||||||
t.ContentXML = n.ContentXML
|
|
||||||
// Preserve document-scraping/summarization flags on the tender (managed by
|
// Preserve document-scraping/summarization flags on the tender (managed by
|
||||||
// downstream workers), but refresh the notice-side processing metadata.
|
// downstream workers), but refresh the notice-side processing metadata.
|
||||||
t.ProcessingMetadata.ScrapedAt = n.ProcessingMetadata.ScrapedAt
|
t.ProcessingMetadata.ScrapedAt = n.ProcessingMetadata.ScrapedAt
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
package workers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"tm/internal/notification"
|
||||||
|
"tm/pkg/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NotificationWorker promotes due scheduled notifications from pending to sent.
|
||||||
|
type NotificationWorker struct {
|
||||||
|
Repository notification.Repository
|
||||||
|
Logger logger.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewNotificationWorker creates a notification delivery worker.
|
||||||
|
func NewNotificationWorker(repository notification.Repository, logger logger.Logger) *NotificationWorker {
|
||||||
|
return &NotificationWorker{
|
||||||
|
Repository: repository,
|
||||||
|
Logger: logger,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run marks pending scheduled notifications as sent once their delivery time has passed.
|
||||||
|
func (w *NotificationWorker) Run() {
|
||||||
|
count, err := w.Repository.MarkDueScheduledAsSent(context.Background(), time.Now().Unix())
|
||||||
|
if err != nil {
|
||||||
|
w.Logger.Error("Scheduled notification worker failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if count > 0 {
|
||||||
|
w.Logger.Info("Scheduled notification worker completed", map[string]interface{}{
|
||||||
|
"count": count,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,173 +0,0 @@
|
|||||||
package workers
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
"tm/internal/tender"
|
|
||||||
ai_summarizer "tm/pkg/ai_summarizer"
|
|
||||||
"tm/pkg/logger"
|
|
||||||
"tm/pkg/mongo"
|
|
||||||
)
|
|
||||||
|
|
||||||
// DocumentSummarizationWorker triggers AI summarization for scraped tenders.
|
|
||||||
// The AI service owns MinIO artefacts; this worker marks Mongo processing flags
|
|
||||||
// after the pipeline or on-demand summarize API completes.
|
|
||||||
type DocumentSummarizationWorker struct {
|
|
||||||
Mongo *mongo.ConnectionManager
|
|
||||||
Logger logger.Logger
|
|
||||||
TenderRepo tender.TenderRepository
|
|
||||||
AIClient *ai_summarizer.Client
|
|
||||||
AIStorage *ai_summarizer.StorageClient
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewDocumentSummarizationWorker creates a document summarization worker.
|
|
||||||
func NewDocumentSummarizationWorker(
|
|
||||||
mongo *mongo.ConnectionManager,
|
|
||||||
logger logger.Logger,
|
|
||||||
tenderRepo tender.TenderRepository,
|
|
||||||
aiClient *ai_summarizer.Client,
|
|
||||||
aiStorage *ai_summarizer.StorageClient,
|
|
||||||
) *DocumentSummarizationWorker {
|
|
||||||
return &DocumentSummarizationWorker{
|
|
||||||
Mongo: mongo,
|
|
||||||
Logger: logger,
|
|
||||||
TenderRepo: tenderRepo,
|
|
||||||
AIClient: aiClient,
|
|
||||||
AIStorage: aiStorage,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run starts the document summarization process.
|
|
||||||
func (w *DocumentSummarizationWorker) Run() {
|
|
||||||
w.Logger.Info("Document summarization worker started", map[string]interface{}{})
|
|
||||||
|
|
||||||
if w.AIClient == nil {
|
|
||||||
w.Logger.Warn("AI summarizer client not configured, skipping document summarization run", map[string]interface{}{})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if _, err := w.AIClient.TriggerPipelineSummarize(context.Background()); err != nil {
|
|
||||||
w.Logger.Warn("Failed to trigger AI pipeline summarize (continuing with per-tender sync)", map[string]interface{}{
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
limit := 5
|
|
||||||
skip := 0
|
|
||||||
for {
|
|
||||||
tenders, totalCount, err := w.TenderRepo.GetUnSummarizedTenders(context.Background(), limit, skip)
|
|
||||||
if err != nil {
|
|
||||||
w.Logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Logger.Info("Found tenders for document summarization", map[string]interface{}{
|
|
||||||
"count": len(tenders),
|
|
||||||
"total_count": totalCount,
|
|
||||||
"skip": skip,
|
|
||||||
})
|
|
||||||
|
|
||||||
if len(tenders) == 0 {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := range tenders {
|
|
||||||
t := &tenders[i]
|
|
||||||
w.Logger.Info("Processing tender for document summarization", map[string]interface{}{
|
|
||||||
"tender_id": t.ID.Hex(),
|
|
||||||
"notice_publication_id": t.NoticePublicationID,
|
|
||||||
})
|
|
||||||
w.summarizeTender(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
skip += len(tenders)
|
|
||||||
if int64(skip) >= totalCount {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Logger.Info("Document summarization worker completed", map[string]interface{}{})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *DocumentSummarizationWorker) summarizeTender(t *tender.Tender) {
|
|
||||||
if strings.TrimSpace(t.NoticePublicationID) == "" {
|
|
||||||
w.Logger.Warn("Skipping tender summarization: missing notice_publication_id", map[string]interface{}{
|
|
||||||
"tender_id": t.ID.Hex(),
|
|
||||||
})
|
|
||||||
w.markTenderSummarized(t)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(t.ContractFolderID) == "" {
|
|
||||||
w.Logger.Warn("Skipping tender summarization: missing contract_folder_id", map[string]interface{}{
|
|
||||||
"tender_id": t.ID.Hex(),
|
|
||||||
})
|
|
||||||
w.markTenderSummarized(t)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
if w.isSummaryReadyInStorage(ctx, t) {
|
|
||||||
w.markTenderSummarized(t)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
req := ai_summarizer.NewSummarizeRequest(t.ContractFolderID, t.NoticePublicationID)
|
|
||||||
if _, err := w.AIClient.FetchSummaryOnDemand(ctx, req); err != nil {
|
|
||||||
w.Logger.Error("Failed to summarize tender via AI service", map[string]interface{}{
|
|
||||||
"tender_id": t.ID.Hex(),
|
|
||||||
"notice_publication_id": t.NoticePublicationID,
|
|
||||||
"contract_folder_id": t.ContractFolderID,
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
if !w.isSummaryReadyInStorage(ctx, t) {
|
|
||||||
w.Logger.Warn("AI summarize completed but storage is not ready yet", map[string]interface{}{
|
|
||||||
"tender_id": t.ID.Hex(),
|
|
||||||
"notice_publication_id": t.NoticePublicationID,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
w.markTenderSummarized(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *DocumentSummarizationWorker) isSummaryReadyInStorage(ctx context.Context, t *tender.Tender) bool {
|
|
||||||
if w.AIStorage == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
ready, err := w.AIStorage.IsSummaryReady(ctx, t.ContractFolderID, t.NoticePublicationID)
|
|
||||||
if err != nil {
|
|
||||||
w.Logger.Debug("Could not check summarization status in storage", map[string]interface{}{
|
|
||||||
"tender_id": t.ID.Hex(),
|
|
||||||
"notice_publication_id": t.NoticePublicationID,
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return ready
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *DocumentSummarizationWorker) markTenderSummarized(t *tender.Tender) {
|
|
||||||
now := time.Now().Unix()
|
|
||||||
t.ProcessingMetadata.DocumentsSummarized = true
|
|
||||||
t.ProcessingMetadata.DocumentsSummarizedAt = now
|
|
||||||
t.UpdatedAt = now
|
|
||||||
|
|
||||||
if err := w.TenderRepo.Update(context.Background(), t); err != nil {
|
|
||||||
w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{
|
|
||||||
"tender_id": t.ID.Hex(),
|
|
||||||
"notice_publication_id": t.NoticePublicationID,
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
w.Logger.Info("Tender marked as summarized", map[string]interface{}{
|
|
||||||
"tender_id": t.ID.Hex(),
|
|
||||||
"notice_publication_id": t.NoticePublicationID,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
@@ -10,7 +10,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// TranslationWorker syncs translation completion from the AI pipeline MinIO storage.
|
// TranslationWorker syncs translation completion from the AI pipeline MinIO storage.
|
||||||
// Batch translation is triggered via POST /pipeline/translate; this worker only
|
// Batch translation is triggered via POST /ai/translate/batch; this worker only
|
||||||
// verifies which tenders now have translations available in storage.
|
// verifies which tenders now have translations available in storage.
|
||||||
type TranslationWorker struct {
|
type TranslationWorker struct {
|
||||||
Mongo *mongo.ConnectionManager
|
Mongo *mongo.ConnectionManager
|
||||||
@@ -82,12 +82,15 @@ func (w *TranslationWorker) Run() {
|
|||||||
|
|
||||||
startCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
|
startCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
|
||||||
|
|
||||||
if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
|
if refs := w.collectUnTranslatedTenderRefs(context.Background()); len(refs) > 0 {
|
||||||
w.Logger.Warn("Failed to trigger AI pipeline translate", map[string]interface{}{
|
if _, err := w.AIClient.TriggerTranslateBatch(context.Background(), refs, w.TargetLanguages); err != nil {
|
||||||
|
w.Logger.Warn("Failed to trigger AI translate batch", map[string]interface{}{
|
||||||
|
"tender_count": len(refs),
|
||||||
"target_languages": w.TargetLanguages,
|
"target_languages": w.TargetLanguages,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
readyCount := 0
|
readyCount := 0
|
||||||
for _, language := range w.TargetLanguages {
|
for _, language := range w.TargetLanguages {
|
||||||
@@ -186,3 +189,44 @@ func (w *TranslationWorker) hasTranslationInStorage(t *tender.Tender, language s
|
|||||||
)
|
)
|
||||||
return err == nil && strings.TrimSpace(stored.Title) != ""
|
return err == nil && strings.TrimSpace(stored.Title) != ""
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (w *TranslationWorker) collectUnTranslatedTenderRefs(ctx context.Context) []ai_summarizer.TenderRef {
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
refs := make([]ai_summarizer.TenderRef, 0)
|
||||||
|
|
||||||
|
for _, language := range w.TargetLanguages {
|
||||||
|
skip := 0
|
||||||
|
for {
|
||||||
|
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(ctx, language, w.BatchSize, skip)
|
||||||
|
if err != nil {
|
||||||
|
w.Logger.Warn("Failed to collect tenders for translate batch", map[string]interface{}{
|
||||||
|
"target_language": language,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if len(tenders) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, t := range tenders {
|
||||||
|
if strings.TrimSpace(t.ContractFolderID) == "" || strings.TrimSpace(t.NoticePublicationID) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
key := t.ContractFolderID + "|" + t.NoticePublicationID
|
||||||
|
if _, dup := seen[key]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
refs = append(refs, ai_summarizer.NewTenderRef(t.ContractFolderID, t.NoticePublicationID))
|
||||||
|
}
|
||||||
|
|
||||||
|
skip += len(tenders)
|
||||||
|
if int64(skip) >= totalCount {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return refs
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,6 +19,24 @@ Welcome to the Tender Management System documentation. This directory contains a
|
|||||||
### 📡 API Documentation
|
### 📡 API Documentation
|
||||||
- **[examples/API_EXAMPLES.md](./examples/API_EXAMPLES.md)** - API usage examples and endpoints documentation
|
- **[examples/API_EXAMPLES.md](./examples/API_EXAMPLES.md)** - API usage examples and endpoints documentation
|
||||||
|
|
||||||
|
### 🔒 Security & Compliance (ISMS)
|
||||||
|
- **[security/ISMS_FOUNDATION.md](./security/ISMS_FOUNDATION.md)** - ISMS scope, asset inventory, roles, and control baseline
|
||||||
|
- **[security/RISK_ASSESSMENT_MATRIX.md](./security/RISK_ASSESSMENT_MATRIX.md)** - Initial risk register and treatment plans
|
||||||
|
- **[security/ISO27001_ROADMAP.md](./security/ISO27001_ROADMAP.md)** - ISO/IEC 27001 certification roadmap and phases
|
||||||
|
- **[security/GAP_ANALYSIS_REPORT.md](./security/GAP_ANALYSIS_REPORT.md)** - ISO 27001 Clauses 4–10 and Annex A gap analysis
|
||||||
|
- **[security/STATEMENT_OF_APPLICABILITY.md](./security/STATEMENT_OF_APPLICABILITY.md)** - Annex A control applicability (93 controls)
|
||||||
|
- **[security/ISMS_SCOPE_APPROVAL.md](./security/ISMS_SCOPE_APPROVAL.md)** - Formal ISMS scope sign-off (Clause 4.3)
|
||||||
|
|
||||||
|
#### Policies (Phase 1)
|
||||||
|
- **[security/policies/information-security-policy.md](./security/policies/information-security-policy.md)** - Top-level ISMS policy (A.5.1)
|
||||||
|
- **[security/policies/access-control-policy.md](./security/policies/access-control-policy.md)** - Authentication, RBAC, access reviews (A.5.15–A.5.18)
|
||||||
|
- **[security/policies/incident-response-plan.md](./security/policies/incident-response-plan.md)** - Incident classification, IRT, playbooks (A.5.24–A.5.28)
|
||||||
|
- **[security/policies/backup-and-recovery-policy.md](./security/policies/backup-and-recovery-policy.md)** - RTO/RPO, backup schedule, restore testing (A.8.13)
|
||||||
|
- **[security/policies/acceptable-use-policy.md](./security/policies/acceptable-use-policy.md)** - Personnel acceptable use rules (A.6.2)
|
||||||
|
|
||||||
|
#### Procedures
|
||||||
|
- **[security/procedures/risk-assessment-procedure.md](./security/procedures/risk-assessment-procedure.md)** - Risk identification, analysis, and treatment (Clause 6.1.2)
|
||||||
|
|
||||||
## 🏗️ Project Architecture
|
## 🏗️ Project Architecture
|
||||||
|
|
||||||
### Clean Architecture Layers
|
### Clean Architecture Layers
|
||||||
|
|||||||
@@ -0,0 +1,282 @@
|
|||||||
|
# ISO/IEC 27001 Gap Analysis Report — Tender Management System
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | ISMS-004 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Draft — Pending review |
|
||||||
|
| **Owner** | Information Security Officer (ISO) |
|
||||||
|
| **Last Updated** | 2026-06-11 |
|
||||||
|
| **Assessment Date** | 2026-06-11 |
|
||||||
|
| **Standard** | ISO/IEC 27001:2022 |
|
||||||
|
| **Scope** | [ISMS Foundation §3](./ISMS_FOUNDATION.md#3-isms-scope) |
|
||||||
|
|
||||||
|
Related: [Statement of Applicability](./STATEMENT_OF_APPLICABILITY.md) | [ISO 27001 Roadmap](./ISO27001_ROADMAP.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Executive Summary
|
||||||
|
|
||||||
|
This report assesses the current state of the Tender Management (TM) ISMS against **ISO/IEC 27001:2022** mandatory clauses (4–10) and Annex A controls. The assessment is based on documentation review, codebase analysis, and existing operational practices.
|
||||||
|
|
||||||
|
### Overall Maturity
|
||||||
|
|
||||||
|
| Area | Maturity | Summary |
|
||||||
|
|------|----------|---------|
|
||||||
|
| **Clauses 4–6** (Context, Leadership, Planning) | 🟡 Partial | Foundation docs exist; formal leadership approval pending |
|
||||||
|
| **Clause 7** (Support) | 🟡 Partial | Competence and awareness programs not yet formalized |
|
||||||
|
| **Clause 8** (Operation) | 🟡 Partial | Technical controls strong; operational procedures incomplete |
|
||||||
|
| **Clause 9** (Performance evaluation) | 🔴 Minimal | No internal audit program or management review yet |
|
||||||
|
| **Clause 10** (Improvement) | 🔴 Minimal | Corrective action process not formalized |
|
||||||
|
| **Annex A** (93 controls) | 🟡 Partial | 6 implemented (7% of 81 applicable), 56 partial (69%), 19 not implemented (23%), 12 N/A (13%) — per [SoA §7](./STATEMENT_OF_APPLICABILITY.md#7-summary-statistics) |
|
||||||
|
|
||||||
|
### Key Strengths
|
||||||
|
|
||||||
|
- Application-layer security controls (JWT, RBAC, validation, XSS, audit logging)
|
||||||
|
- Clean Architecture with dependency injection and structured logging
|
||||||
|
- Initial ISMS documentation suite (foundation, risk matrix, policies)
|
||||||
|
- Configurable rate limiting and environment-based secret management
|
||||||
|
|
||||||
|
### Critical Gaps (Must Address Before Certification)
|
||||||
|
|
||||||
|
1. No internal audit program or management review records (Clauses 9.2, 9.3)
|
||||||
|
2. No SIEM/centralized monitoring (A.8.16)
|
||||||
|
3. No formal vulnerability management in CI (A.8.8)
|
||||||
|
4. Secrets in repository / no vault (A.8.9, R-013)
|
||||||
|
5. No security awareness training program (A.6.3)
|
||||||
|
6. No vendor security assessments (A.5.19–A.5.22)
|
||||||
|
7. Encryption at rest not confirmed (A.8.24)
|
||||||
|
8. Executive sign-off on ISMS scope and policies pending
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Assessment Methodology
|
||||||
|
|
||||||
|
| Step | Activity |
|
||||||
|
|------|----------|
|
||||||
|
| 1 | Review ISMS scope and asset inventory |
|
||||||
|
| 2 | Map existing controls to ISO 27001:2022 clauses and Annex A |
|
||||||
|
| 3 | Interview technical leads (informal / codebase-based) |
|
||||||
|
| 4 | Classify each requirement: **Implemented**, **Partial**, **Not Implemented**, **N/A** |
|
||||||
|
| 5 | Prioritize gaps by risk linkage ([Risk Assessment Matrix](./RISK_ASSESSMENT_MATRIX.md)) |
|
||||||
|
| 6 | Define remediation actions in [ISO27001_ROADMAP.md](./ISO27001_ROADMAP.md) Phase 2 |
|
||||||
|
|
||||||
|
### Maturity Scale
|
||||||
|
|
||||||
|
| Rating | Definition |
|
||||||
|
|--------|------------|
|
||||||
|
| **Implemented** | Control documented, implemented, and operating effectively |
|
||||||
|
| **Partial** | Control exists informally or is incomplete |
|
||||||
|
| **Not Implemented** | No control or documentation |
|
||||||
|
| **N/A** | Not applicable to TM scope (with justification) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Clause 4 — Context of the Organization
|
||||||
|
|
||||||
|
| Ref | Requirement | Status | Evidence | Gap / Remediation |
|
||||||
|
|-----|-------------|--------|----------|-------------------|
|
||||||
|
| 4.1 | Understanding the organization and its context | **Partial** | [ISMS Foundation §2](./ISMS_FOUNDATION.md#2-organizational-context) | Formalize interested parties register |
|
||||||
|
| 4.2 | Understanding needs of interested parties | **Partial** | Business objectives in ISMS Foundation | Document customers, regulators, partners explicitly |
|
||||||
|
| 4.3 | Determining ISMS scope | **Partial** | [ISMS Foundation §3](./ISMS_FOUNDATION.md#3-isms-scope); [Scope Approval](./ISMS_SCOPE_APPROVAL.md) draft | Executive sign-off required |
|
||||||
|
| 4.4 | Information security management system | **Partial** | ISMS doc structure defined | Complete PDCA operating records |
|
||||||
|
|
||||||
|
**Gap actions:**
|
||||||
|
- Create interested parties register (customers, EU DPA, certification body, cloud providers)
|
||||||
|
- Obtain signed scope approval from Executive Sponsor
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Clause 5 — Leadership
|
||||||
|
|
||||||
|
| Ref | Requirement | Status | Evidence | Gap / Remediation |
|
||||||
|
|-----|-------------|--------|----------|-------------------|
|
||||||
|
| 5.1 | Leadership and commitment | **Partial** | Roadmap exists; sponsor not yet assigned | Appoint Executive Sponsor; first management review |
|
||||||
|
| 5.2 | Policy | **Partial** | [Information Security Policy](./policies/information-security-policy.md) draft | Approve and communicate policy |
|
||||||
|
| 5.3 | Organizational roles, responsibilities, authorities | **Partial** | [ISMS Foundation §5](./ISMS_FOUNDATION.md#5-roles-and-responsibilities) | Assign named individuals to ISO, DPO roles |
|
||||||
|
|
||||||
|
**Gap actions:**
|
||||||
|
- Name and document ISO and DPO (can be interim)
|
||||||
|
- Communicate approved Information Security Policy to all staff
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Clause 6 — Planning
|
||||||
|
|
||||||
|
| Ref | Requirement | Status | Evidence | Gap / Remediation |
|
||||||
|
|-----|-------------|--------|----------|-------------------|
|
||||||
|
| 6.1.1 | Actions to address risks and opportunities | **Partial** | Risk matrix with 23 risks | Link opportunities (e.g. certification as market differentiator) |
|
||||||
|
| 6.1.2 | Information security risk assessment | **Implemented** | [Risk Assessment Matrix](./RISK_ASSESSMENT_MATRIX.md), [Procedure](./procedures/risk-assessment-procedure.md) | Maintain quarterly reviews |
|
||||||
|
| 6.1.3 | Information security risk treatment | **Partial** | Treatment plans in risk matrix | Execute Phase 2 technical remediations |
|
||||||
|
| 6.2 | Information security objectives | **Partial** | [Information Security Policy §4](./policies/information-security-policy.md#4-information-security-objectives) | Track KPIs monthly |
|
||||||
|
| 6.3 | Planning of changes | **Not Implemented** | — | Document change planning in change management procedure (Phase 2) |
|
||||||
|
|
||||||
|
**Gap actions:**
|
||||||
|
- Begin executing risk treatment plans (R-001, R-010, R-013, R-015, R-023 priority)
|
||||||
|
- Draft change management procedure
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Clause 7 — Support
|
||||||
|
|
||||||
|
| Ref | Requirement | Status | Evidence | Gap / Remediation |
|
||||||
|
|-----|-------------|--------|----------|-------------------|
|
||||||
|
| 7.1 | Resources | **Partial** | Roadmap resource estimate | Allocate part-time ISO capacity |
|
||||||
|
| 7.2 | Competence | **Not Implemented** | — | Define security competency requirements per role |
|
||||||
|
| 7.3 | Awareness | **Not Implemented** | Acceptable Use Policy draft | Launch onboarding + annual training |
|
||||||
|
| 7.4 | Communication | **Partial** | Docs in `docs/security/` | Define internal/external comms plan for ISMS |
|
||||||
|
| 7.5 | Documented information | **Partial** | ISMS doc suite | Document control procedure; version approval workflow |
|
||||||
|
|
||||||
|
**Gap actions:**
|
||||||
|
- Security awareness program (A.6.3) — target Q3 2026
|
||||||
|
- Document control: naming, approval, retention standards
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Clause 8 — Operation
|
||||||
|
|
||||||
|
| Ref | Requirement | Status | Evidence | Gap / Remediation |
|
||||||
|
|-----|-------------|--------|----------|-------------------|
|
||||||
|
| 8.1 | Operational planning and control | **Partial** | DevOps practices informal | Formalize change and deploy procedures |
|
||||||
|
| 8.2 | Information security risk assessment | **Implemented** | Quarterly procedure defined | Execute first scheduled review Sep 2026 |
|
||||||
|
| 8.3 | Information security risk treatment | **Partial** | Phase 2 roadmap | Implement prioritized controls |
|
||||||
|
|
||||||
|
**Gap actions:**
|
||||||
|
- Operational procedures for backup, incident response (drafted — need testing records)
|
||||||
|
- Change management procedure (Phase 2)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Clause 9 — Performance Evaluation
|
||||||
|
|
||||||
|
| Ref | Requirement | Status | Evidence | Gap / Remediation |
|
||||||
|
|-----|-------------|--------|----------|-------------------|
|
||||||
|
| 9.1 | Monitoring, measurement, analysis, evaluation | **Partial** | KPIs defined in roadmap | Implement SIEM; monthly KPI reporting |
|
||||||
|
| 9.2 | Internal audit | **Not Implemented** | — | Schedule first internal audit Q1 2027 |
|
||||||
|
| 9.3 | Management review | **Not Implemented** | — | First review within 90 days of scope approval |
|
||||||
|
|
||||||
|
**Gap actions:**
|
||||||
|
- **Critical:** Establish internal audit program before Stage 2 audit
|
||||||
|
- Schedule first management review meeting
|
||||||
|
- Implement monitoring (A.8.16) for measurable KPIs
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Clause 10 — Improvement
|
||||||
|
|
||||||
|
| Ref | Requirement | Status | Evidence | Gap / Remediation |
|
||||||
|
|-----|-------------|--------|----------|-------------------|
|
||||||
|
| 10.1 | Continual improvement | **Partial** | PDCA in roadmap | Track improvement actions from audits/incidents |
|
||||||
|
| 10.2 | Nonconformity and corrective action | **Not Implemented** | — | Define CAPA (Corrective and Preventive Action) procedure |
|
||||||
|
|
||||||
|
**Gap actions:**
|
||||||
|
- CAPA procedure linked to incident response and audit findings
|
||||||
|
- Nonconformity register template
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Annex A Summary by Theme
|
||||||
|
|
||||||
|
Full control-level detail: [Statement of Applicability](./STATEMENT_OF_APPLICABILITY.md)
|
||||||
|
|
||||||
|
| Theme | Total Controls | Implemented | Partial | Not Impl. | N/A |
|
||||||
|
|-------|----------------|-------------|---------|-----------|-----|
|
||||||
|
| **A.5** Organizational | 37 | 2 | 25 | 10 | 0 |
|
||||||
|
| **A.6** People | 8 | 0 | 6 | 2 | 0 |
|
||||||
|
| **A.7** Physical | 14 | 0 | 3 | 0 | 11 |
|
||||||
|
| **A.8** Technological | 34 | 4 | 22 | 7 | 1 |
|
||||||
|
| **Total** | **93** | **6** | **56** | **19** | **12** |
|
||||||
|
|
||||||
|
*Counts match [Statement of Applicability §7.1](./STATEMENT_OF_APPLICABILITY.md#71-by-theme). "Partial" indicates the control exists but lacks full documentation, testing, or operating evidence.*
|
||||||
|
|
||||||
|
### A.5 — Top Organizational Gaps
|
||||||
|
|
||||||
|
| Control | Title | Status | Priority |
|
||||||
|
|---------|-------|--------|----------|
|
||||||
|
| A.5.19–A.5.22 | Supplier relationships | Not Implemented | High |
|
||||||
|
| A.5.29–A.5.30 | Business continuity | Partial | High |
|
||||||
|
| A.5.35 | Independent review | Not Implemented | Medium |
|
||||||
|
| A.5.7 | Threat intelligence | Not Implemented | Medium |
|
||||||
|
| A.5.32 | Intellectual property | Partial | Low |
|
||||||
|
|
||||||
|
### A.6 — Top People Gaps
|
||||||
|
|
||||||
|
| Control | Title | Status | Priority |
|
||||||
|
|---------|-------|--------|----------|
|
||||||
|
| A.6.3 | Security awareness training | Not Implemented | High |
|
||||||
|
| A.6.1 | Screening | Not Implemented | Medium |
|
||||||
|
| A.6.6 | NDAs | Partial | Medium |
|
||||||
|
| A.6.8 | Event reporting | Partial | Medium |
|
||||||
|
|
||||||
|
### A.7 — Physical Controls
|
||||||
|
|
||||||
|
**11 N/A** — cloud-hosted infrastructure; physical security transferred to cloud provider per shared responsibility model (A.7.1–A.7.6, A.7.8, A.7.10–A.7.13). Provider compliance evidence required (A.5.23).
|
||||||
|
|
||||||
|
**3 Partial** — A.7.7, A.7.9, A.7.14 remain applicable for personnel devices accessing TM systems (screen lock, off-premises asset security, equipment disposal).
|
||||||
|
|
||||||
|
### A.8 — Top Technological Gaps
|
||||||
|
|
||||||
|
| Control | Title | Status | Priority |
|
||||||
|
|---------|-------|--------|----------|
|
||||||
|
| A.8.8 | Vulnerability management | Not Implemented | Critical |
|
||||||
|
| A.8.16 | Monitoring activities | Partial | Critical |
|
||||||
|
| A.8.24 | Cryptography | Partial | High |
|
||||||
|
| A.8.2 | Privileged access rights | Partial | High |
|
||||||
|
| A.8.32 | Change management | Partial | High |
|
||||||
|
| A.8.29 | Security testing | Not Implemented | High |
|
||||||
|
| A.8.12 | Data leakage prevention | Not Implemented | Medium |
|
||||||
|
|
||||||
|
### A.8 — Fully Implemented Technical Controls (Status = Yes)
|
||||||
|
|
||||||
|
| Control | Implementation |
|
||||||
|
|---------|----------------|
|
||||||
|
| A.8.3 | RBAC + company-scoped middleware (`pkg/authorization`) |
|
||||||
|
| A.8.4 | Git-based source control with PR workflow |
|
||||||
|
| A.8.15 | Structured logging + `pkg/audit` |
|
||||||
|
| A.8.26 | Govalidator, XSS policy, Swagger |
|
||||||
|
|
||||||
|
Additional controls (e.g. A.8.5 authentication, A.8.9 configuration, A.8.28 secure coding) are **Partial** in the SoA — implemented in code but lacking full operating evidence or completeness (MFA, vault, SAST).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. Gap Remediation Roadmap
|
||||||
|
|
||||||
|
Aligned with [ISO27001_ROADMAP.md](./ISO27001_ROADMAP.md):
|
||||||
|
|
||||||
|
| Priority | Gap | Remediation | Phase | Target |
|
||||||
|
|----------|-----|-------------|-------|--------|
|
||||||
|
| P0 | No incident response testing | Tabletop exercise | 2 | Q2 2026 |
|
||||||
|
| P0 | Secrets in repo | Vault + secret scanning | 2 | Q2 2026 |
|
||||||
|
| P0 | No vulnerability scanning | govulncheck in CI | 2 | Q2 2026 |
|
||||||
|
| P0 | No management review | First review meeting | 1 | Q3 2026 |
|
||||||
|
| P1 | No SIEM | Centralized logging + alerts | 2 | Q3 2026 |
|
||||||
|
| P1 | No encryption at rest | MongoDB/MinIO encryption | 2 | Q3 2026 |
|
||||||
|
| P1 | No vendor assessments | Vendor questionnaire | 2 | Q4 2026 |
|
||||||
|
| P1 | No security training | Awareness program | 2 | Q3 2026 |
|
||||||
|
| P1 | No internal audit | Audit program + first audit | 3 | Q1 2027 |
|
||||||
|
| P2 | No CAPA procedure | Document CAPA process | 2 | Q3 2026 |
|
||||||
|
| P2 | No penetration test | External pentest | 4 | Q2 2027 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. Conclusion
|
||||||
|
|
||||||
|
The TM platform has a **solid technical security foundation** but lacks the **operational and governance evidence** required for ISO 27001 certification. Phase 1 documentation (foundation, risk assessment, policies, SoA, gap analysis) addresses Clauses 4.3, 6.1.2, and 5.2 at a draft level.
|
||||||
|
|
||||||
|
**Recommendation:** Proceed to Phase 2 control implementation while pursuing executive approval of scope and policies. Target first internal audit no later than **Q1 2027** to allow 3+ months of operating evidence before Stage 2 audit.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 13. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial gap analysis |
|
||||||
|
| 1.1 | 2026-06-11 | Engineering | Annex A theme table and executive summary aligned with SoA §7 |
|
||||||
|
|
||||||
|
**Review**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
|
| Executive Sponsor | _Pending_ | | |
|
||||||
@@ -0,0 +1,235 @@
|
|||||||
|
# ISMS Foundation — Tender Management System
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | ISMS-001 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Draft — Initial foundation |
|
||||||
|
| **Owner** | Information Security Officer (ISO) |
|
||||||
|
| **Last Updated** | 2026-06-11 |
|
||||||
|
| **Review Cycle** | Annual (or after major architecture change) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This document establishes the **Information Security Management System (ISMS)** foundation for the Tender Management (TM) platform. It defines security scope, information assets, roles, and the baseline control environment required to support ISO/IEC 27001 certification and GDPR-aligned data protection.
|
||||||
|
|
||||||
|
Related documents:
|
||||||
|
|
||||||
|
- [ISO 27001 Roadmap](./ISO27001_ROADMAP.md)
|
||||||
|
- [Risk Assessment Matrix](./RISK_ASSESSMENT_MATRIX.md)
|
||||||
|
- [Gap Analysis Report](./GAP_ANALYSIS_REPORT.md)
|
||||||
|
- [Statement of Applicability](./STATEMENT_OF_APPLICABILITY.md)
|
||||||
|
- [ISMS Scope Approval](./ISMS_SCOPE_APPROVAL.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Organizational Context
|
||||||
|
|
||||||
|
### 2.1 System Overview
|
||||||
|
|
||||||
|
The Tender Management System is a Go-based backend API that:
|
||||||
|
|
||||||
|
- Ingests public tender data from TED (Tenders Electronic Daily) and related sources
|
||||||
|
- Matches tenders to registered companies based on CPV codes, categories, and keywords
|
||||||
|
- Provides admin panel APIs for user, company, and tender management
|
||||||
|
- Provides public/mobile APIs for company customers to browse, filter, and interact with tenders
|
||||||
|
- Sends notifications (email, push via FCM) and supports AI-assisted translation/summarization
|
||||||
|
- Stores documents and tender artifacts in MinIO object storage
|
||||||
|
|
||||||
|
### 2.2 Business Objectives Relevant to Security
|
||||||
|
|
||||||
|
| Objective | Security Implication |
|
||||||
|
|-----------|---------------------|
|
||||||
|
| Reliable tender data for business decisions | Integrity and availability of tender records |
|
||||||
|
| Company and customer onboarding | Protection of PII and business credentials |
|
||||||
|
| Multi-tenant company access | Strong authentication, authorization, and data isolation |
|
||||||
|
| Regulatory and procurement compliance | Audit trails, data retention, and access control |
|
||||||
|
| Platform availability for mobile and admin clients | Resilience, monitoring, and incident response |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. ISMS Scope
|
||||||
|
|
||||||
|
### 3.1 In Scope
|
||||||
|
|
||||||
|
| Layer | Components |
|
||||||
|
|-------|------------|
|
||||||
|
| **Applications** | `cmd/web` (HTTP API), `cmd/worker` (background jobs), `cmd/scraper` (TED XML ingestion) |
|
||||||
|
| **Domain services** | User, customer, company, tender, feedback, notification, inquiry, contact, CMS, kanban, dashboard, document scraper, tender approval |
|
||||||
|
| **Shared packages** | Authorization (JWT), security (XSS/CSP), audit logging, file store, MinIO client, config |
|
||||||
|
| **Data stores** | MongoDB (primary DB), Redis (cache/sessions/token blacklist), MinIO (files, tender JSON/translations) |
|
||||||
|
| **External integrations** | Notification service, hCaptcha, AI summarizer/translation service, GoRules (kanban), TED public data sources, FCM (push) |
|
||||||
|
| **Environments** | Development, staging, production (and associated CI/CD pipelines) |
|
||||||
|
| **People & processes** | Developers, DevOps, QA, product owners with access to TM infrastructure or production data |
|
||||||
|
|
||||||
|
### 3.2 Out of Scope (Initial Phase)
|
||||||
|
|
||||||
|
| Item | Rationale |
|
||||||
|
|------|-----------|
|
||||||
|
| End-user mobile app codebase | Separate repository; interface governed by API contracts |
|
||||||
|
| Admin panel frontend | Separate repository; authenticated via same backend |
|
||||||
|
| Third-party TED infrastructure | Public data source; risk managed via ingestion validation |
|
||||||
|
| Customer on-premise deployments | SaaS model assumed; revisit if offering self-hosted |
|
||||||
|
|
||||||
|
### 3.3 Scope Statement
|
||||||
|
|
||||||
|
> The ISMS covers the design, development, deployment, and operation of the Tender Management backend platform, including all information assets processed, stored, or transmitted by `cmd/web`, `cmd/worker`, and `cmd/scraper`, and the supporting MongoDB, Redis, and MinIO infrastructure in scoped environments.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Information Asset Inventory
|
||||||
|
|
||||||
|
### 4.1 Classification Levels
|
||||||
|
|
||||||
|
| Level | Label | Description | Handling |
|
||||||
|
|-------|-------|-------------|----------|
|
||||||
|
| **C1** | Public | Intended for public disclosure | No restrictions |
|
||||||
|
| **C2** | Internal | Operational data not for public release | Access limited to staff |
|
||||||
|
| **C3** | Confidential | Business or personal data requiring protection | Encrypted in transit; access on need-to-know |
|
||||||
|
| **C4** | Restricted | Credentials, secrets, authentication factors | Vault/KMS; never logged; rotation required |
|
||||||
|
|
||||||
|
### 4.2 Asset Register
|
||||||
|
|
||||||
|
| Asset ID | Asset Name | Type | Owner | Classification | Location | Notes |
|
||||||
|
|----------|------------|------|-------|----------------|----------|-------|
|
||||||
|
| A-001 | MongoDB database (`tm`) | Data store | DevOps / DBA | C3 | Managed MongoDB cluster | Companies, customers, tenders, users, feedback, notifications |
|
||||||
|
| A-002 | Redis cache | Data store | DevOps | C3 | Managed Redis | Sessions, rate limits, token blacklist |
|
||||||
|
| A-003 | MinIO object storage | Data store | DevOps | C2–C3 | MinIO cluster | Company documents, tender JSON, translations |
|
||||||
|
| A-004 | Customer PII | Data | Product / DPO | C3 | MongoDB `customers` | Email, name, phone, device tokens |
|
||||||
|
| A-005 | Company business data | Data | Product | C3 | MongoDB `companies` | Registration, tax ID, address, documents |
|
||||||
|
| A-006 | User (admin) accounts | Data | Product | C3 | MongoDB `users` | Admin panel operators |
|
||||||
|
| A-007 | Authentication credentials | Data | ISO | C4 | MongoDB (hashed passwords), env secrets | bcrypt-hashed passwords; JWT signing keys |
|
||||||
|
| A-008 | JWT access/refresh tokens | Data | ISO | C4 | Client devices, Redis blacklist | Short-lived access tokens |
|
||||||
|
| A-009 | Application source code | Software | Engineering | C2 | Git repository | Go monorepo `tm_back` |
|
||||||
|
| A-010 | Configuration & secrets | Config | DevOps | C4 | `.env`, OS env vars, secret manager | DB URIs, API keys, MinIO keys, hCaptcha secret |
|
||||||
|
| A-011 | Application logs | Data | DevOps | C2–C3 | Log files / log aggregator | Structured logs; must exclude passwords/tokens |
|
||||||
|
| A-012 | Audit logs | Data | ISO | C3 | Application log pipeline | Login, password reset, admin actions via `pkg/audit` |
|
||||||
|
| A-013 | TED tender data | Data | Product | C2 | MongoDB `tenders`, `notices` | Public procurement notices |
|
||||||
|
| A-014 | API endpoints | Service | Engineering | C2 | `cmd/web` Echo server | `/api/v1`, `/admin/v1` |
|
||||||
|
| A-015 | Worker & scraper jobs | Service | Engineering | C2 | `cmd/worker`, `cmd/scraper` | Scheduled ingestion and translation |
|
||||||
|
| A-016 | Notification service integration | Service | DevOps | C3 | External HTTP API | Email and messaging |
|
||||||
|
| A-017 | AI summarizer service | Service | DevOps | C3 | External HTTP API | Tender text sent for translation |
|
||||||
|
| A-018 | FCM credentials | Config | DevOps | C4 | `docs/fcm/` (must not be committed in prod) | Push notification keys |
|
||||||
|
| A-019 | Backup snapshots | Data | DevOps | C3 | Backup storage | MongoDB and MinIO backups |
|
||||||
|
| A-020 | CI/CD pipeline | Process | DevOps | C2 | GitHub / CI runner | Build, test, deploy automation |
|
||||||
|
|
||||||
|
### 4.3 Data Flow Summary
|
||||||
|
|
||||||
|
```
|
||||||
|
[TED XML] → Scraper → MongoDB (tenders/notices)
|
||||||
|
↓
|
||||||
|
[Admin Panel / Mobile App] → Web API → MongoDB / Redis / MinIO
|
||||||
|
↓
|
||||||
|
Worker → AI Service → MinIO (translations)
|
||||||
|
↓
|
||||||
|
Notification Service → Email / FCM
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Roles and Responsibilities
|
||||||
|
|
||||||
|
| Role | Responsibilities |
|
||||||
|
|------|------------------|
|
||||||
|
| **Executive Sponsor** | Approves ISMS budget, scope, and risk acceptance |
|
||||||
|
| **Information Security Officer (ISO)** | Owns ISMS, risk register, policies, audit coordination |
|
||||||
|
| **Data Protection Officer (DPO)** | GDPR/privacy compliance, DPIA, data subject requests |
|
||||||
|
| **Engineering Lead** | Secure SDLC, code review, dependency management |
|
||||||
|
| **DevOps Lead** | Infrastructure hardening, secrets, backups, monitoring |
|
||||||
|
| **Product Owner** | Data classification decisions, retention requirements |
|
||||||
|
| **All personnel** | Acceptable use, incident reporting, security training |
|
||||||
|
|
||||||
|
*Note: One person may hold multiple roles in smaller teams; responsibilities must still be documented.*
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Current Security Control Baseline
|
||||||
|
|
||||||
|
Controls already implemented in the codebase (to be formalized and verified):
|
||||||
|
|
||||||
|
| Control Area | Implementation | ISO 27001 Annex A Reference |
|
||||||
|
|--------------|----------------|----------------------------|
|
||||||
|
| Authentication | JWT access/refresh tokens (`pkg/authorization`) | A.8.5 |
|
||||||
|
| Authorization | Role-based and company-scoped middleware | A.8.3 |
|
||||||
|
| Input validation | Govalidator on all API forms | A.8.26 |
|
||||||
|
| XSS prevention | Bluemonday HTML sanitization (`pkg/security`) | A.8.26 |
|
||||||
|
| Password storage | bcrypt hashing (customer/user services) | A.8.5 |
|
||||||
|
| Bot protection | hCaptcha integration | A.8.6 |
|
||||||
|
| Structured logging | Service-layer logging with context fields | A.8.15 |
|
||||||
|
| Audit events | `pkg/audit` for security-relevant actions | A.8.15 |
|
||||||
|
| Config secrets | OS env overrides `.env` (`pkg/config`) | A.8.9 |
|
||||||
|
| Rate limiting | Configurable rate limit (`RateLimitConfig`) | A.8.6 |
|
||||||
|
| API documentation | Swagger/OpenAPI (`cmd/web/docs`) | A.8.32 |
|
||||||
|
| File storage | MinIO with access control via file store service | A.8.11 |
|
||||||
|
|
||||||
|
### 6.1 Known Gaps (To Address in Roadmap)
|
||||||
|
|
||||||
|
- Formal written policies (access control, incident response, backup, change management)
|
||||||
|
- Documented vulnerability management and penetration testing schedule
|
||||||
|
- Centralized secrets management (vault) instead of flat env files
|
||||||
|
- SIEM/alerting for security events
|
||||||
|
- Formal data retention and deletion procedures
|
||||||
|
- Business continuity and disaster recovery testing
|
||||||
|
- Security awareness training records
|
||||||
|
- Supplier security assessments for AI and notification services
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Legal and Regulatory Requirements
|
||||||
|
|
||||||
|
| Requirement | Applicability | ISMS Response |
|
||||||
|
|-------------|---------------|---------------|
|
||||||
|
| **GDPR** | EU customer/company PII | Lawful basis documentation, DPIA, DSR process, encryption in transit |
|
||||||
|
| **ISO/IEC 27001** | Certification target | This ISMS foundation and roadmap |
|
||||||
|
| **Procurement data licensing** | TED public data terms | Source attribution, ingestion compliance review |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. ISMS Documentation Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
docs/security/
|
||||||
|
├── ISMS_FOUNDATION.md ← This document
|
||||||
|
├── ISO27001_ROADMAP.md ← Certification phases and timeline
|
||||||
|
├── RISK_ASSESSMENT_MATRIX.md ← Initial risk register
|
||||||
|
├── GAP_ANALYSIS_REPORT.md ← Clause 4–10 gap analysis
|
||||||
|
├── STATEMENT_OF_APPLICABILITY.md ← Annex A SoA (93 controls)
|
||||||
|
├── ISMS_SCOPE_APPROVAL.md ← Scope sign-off (Clause 4.3)
|
||||||
|
├── policies/ ← Phase 1 deliverables
|
||||||
|
│ ├── information-security-policy.md
|
||||||
|
│ ├── access-control-policy.md
|
||||||
|
│ ├── incident-response-plan.md
|
||||||
|
│ ├── backup-and-recovery-policy.md
|
||||||
|
│ └── acceptable-use-policy.md
|
||||||
|
└── procedures/
|
||||||
|
├── risk-assessment-procedure.md
|
||||||
|
├── change-management-procedure.md ← (Phase 2)
|
||||||
|
└── vendor-management-procedure.md ← (Phase 2)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial ISMS foundation, scope, asset inventory |
|
||||||
|
|
||||||
|
**Approval**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
|
| Executive Sponsor | _Pending_ | | |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Next Steps
|
||||||
|
|
||||||
|
1. Review and approve this foundation document with executive sponsor
|
||||||
|
2. Complete initial risk assessment ([RISK_ASSESSMENT_MATRIX.md](./RISK_ASSESSMENT_MATRIX.md))
|
||||||
|
3. Execute Phase 1 of the [ISO 27001 Roadmap](./ISO27001_ROADMAP.md)
|
||||||
|
4. Assign ISO/DPO roles (can be interim)
|
||||||
|
5. Schedule first management review within 90 days
|
||||||
@@ -0,0 +1,168 @@
|
|||||||
|
# ISMS Scope Approval
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | ISMS-006 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Pending signature |
|
||||||
|
| **Owner** | Executive Sponsor |
|
||||||
|
| **Prepared By** | Information Security Officer (ISO) |
|
||||||
|
| **Date Prepared** | 2026-06-11 |
|
||||||
|
| **ISO 27001 Reference** | Clause 4.3 — Determining the scope of the information security management system |
|
||||||
|
|
||||||
|
Related: [ISMS Foundation](./ISMS_FOUNDATION.md) | [Gap Analysis Report](./GAP_ANALYSIS_REPORT.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This document records formal management approval of the **Information Security Management System (ISMS) scope** for the Tender Management platform, as required by ISO/IEC 27001:2022 Clause 4.3.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Organization
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Organization** | _[Company legal name]_ |
|
||||||
|
| **Product / Service** | Tender Management System (TM) |
|
||||||
|
| **Repository** | `tm_back` (Go backend API) |
|
||||||
|
| **Business Unit** | Engineering / Product |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Approved ISMS Scope
|
||||||
|
|
||||||
|
### 3.1 Scope Statement
|
||||||
|
|
||||||
|
> The ISMS covers the design, development, deployment, and operation of the **Tender Management backend platform**, including all information assets processed, stored, or transmitted by the web API (`cmd/web`), background worker (`cmd/worker`), and TED scraper (`cmd/scraper`), together with supporting **MongoDB**, **Redis**, and **MinIO** infrastructure in development, staging, and production environments.
|
||||||
|
|
||||||
|
### 3.2 In Scope
|
||||||
|
|
||||||
|
| Category | Items |
|
||||||
|
|----------|-------|
|
||||||
|
| **Applications** | `cmd/web`, `cmd/worker`, `cmd/scraper` |
|
||||||
|
| **Domain services** | User, customer, company, tender, feedback, notification, inquiry, contact, CMS, kanban, dashboard, document scraper, tender approval |
|
||||||
|
| **Shared packages** | Authorization (JWT), security (XSS/CSP), audit logging, file store, MinIO client, configuration |
|
||||||
|
| **Data stores** | MongoDB (`tm` database), Redis, MinIO (`files`, `documents` buckets) |
|
||||||
|
| **External integrations** | Notification service, hCaptcha, AI summarizer/translation, GoRules, TED data sources, FCM push |
|
||||||
|
| **Environments** | Development, staging, production, CI/CD pipelines |
|
||||||
|
| **Personnel** | Developers, DevOps, QA, product owners with access to TM infrastructure or production data |
|
||||||
|
| **Locations** | Cloud-hosted infrastructure (provider TBD / per deployment); remote work for engineering staff |
|
||||||
|
|
||||||
|
### 3.3 Explicitly Out of Scope
|
||||||
|
|
||||||
|
| Item | Rationale | Interface Control |
|
||||||
|
|------|-----------|-----------------|
|
||||||
|
| Mobile application codebase | Separate repository and release cycle | API contract (`/api/v1`); JWT authentication |
|
||||||
|
| Admin panel frontend | Separate repository | API contract (`/admin/v1`); admin JWT + RBAC |
|
||||||
|
| TED / EU publication infrastructure | Third-party public data source | XML ingestion validation in scraper |
|
||||||
|
| Customer on-premise deployments | SaaS delivery model | N/A — revisit if product offering changes |
|
||||||
|
| End-user devices (phones, laptops) | Covered by Acceptable Use Policy, not ISMS technical scope | AUP acknowledgment |
|
||||||
|
| Physical data center facilities | Cloud shared responsibility model | Provider compliance evidence (A.5.23) |
|
||||||
|
|
||||||
|
### 3.4 Boundaries and Interfaces
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────┐
|
||||||
|
│ APPROVED ISMS SCOPE │
|
||||||
|
│ ┌──────────┐ ┌──────────┐ ┌──────────┐ │
|
||||||
|
│ │ cmd/web │ │cmd/worker│ │cmd/scraper│ │
|
||||||
|
│ └────┬─────┘ └────┬─────┘ └────┬─────┘ │
|
||||||
|
│ └─────────────┼─────────────┘ │
|
||||||
|
│ ▼ │
|
||||||
|
│ ┌─────────────────────────────┐ │
|
||||||
|
│ │ MongoDB │ Redis │ MinIO │ │
|
||||||
|
│ └─────────────────────────────┘ │
|
||||||
|
└──────────────┬──────────────────────────────────────────────┘
|
||||||
|
│ API boundaries (out of scope beyond interface)
|
||||||
|
┌──────────┼──────────┬─────────────┬──────────────┐
|
||||||
|
▼ ▼ ▼ ▼ ▼
|
||||||
|
Mobile Admin Notification AI Service TED (public)
|
||||||
|
App Panel Service XML feed
|
||||||
|
(OOS) (OOS) (interface) (interface) (OOS)
|
||||||
|
```
|
||||||
|
|
||||||
|
**OOS** = Out of Scope (interface governed by contracts and vendor assessment)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Interested Parties (Summary)
|
||||||
|
|
||||||
|
| Party | Interest | ISMS Response |
|
||||||
|
|-------|----------|---------------|
|
||||||
|
| Company customers | PII protection, service availability | Access control, encryption, backup |
|
||||||
|
| EU data subjects | GDPR rights | DPO oversight, DSR process |
|
||||||
|
| Engineering team | Secure development environment | Policies, training, tooling |
|
||||||
|
| Executive management | Certification, risk management | This approval, management review |
|
||||||
|
| Cloud / SaaS providers | Shared security responsibility | A.5.23 cloud controls |
|
||||||
|
| Certification body | Conformity evidence | Audit program |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Applicable Requirements
|
||||||
|
|
||||||
|
| Requirement | Applicability |
|
||||||
|
|-------------|---------------|
|
||||||
|
| ISO/IEC 27001:2022 | Full ISMS certification target |
|
||||||
|
| GDPR (EU 2016/679) | Customer and company PII processing |
|
||||||
|
| TED data terms of use | Public procurement data ingestion |
|
||||||
|
| Customer contracts | Per agreement (security addenda as applicable) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Asset Summary Reference
|
||||||
|
|
||||||
|
Full inventory: [ISMS Foundation §4](./ISMS_FOUNDATION.md#4-information-asset-inventory) — **20 assets** registered (A-001 through A-020).
|
||||||
|
|
||||||
|
Key data categories in scope:
|
||||||
|
|
||||||
|
- Customer PII (C3)
|
||||||
|
- Company business data (C3)
|
||||||
|
- Authentication credentials (C4)
|
||||||
|
- Tender/procurement data (C2)
|
||||||
|
- Application logs and audit records (C2–C3)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Scope Change Process
|
||||||
|
|
||||||
|
Changes to this scope require:
|
||||||
|
|
||||||
|
1. Written justification (new service, architecture change, new data type)
|
||||||
|
2. Risk assessment per [Risk Assessment Procedure](./procedures/risk-assessment-procedure.md)
|
||||||
|
3. Update to [ISMS Foundation](./ISMS_FOUNDATION.md), [SoA](./STATEMENT_OF_APPLICABILITY.md), and this document
|
||||||
|
4. Re-approval by Executive Sponsor
|
||||||
|
5. Notification to certification body if already certified
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Approval
|
||||||
|
|
||||||
|
By signing below, management confirms:
|
||||||
|
|
||||||
|
1. The scope defined in Section 3 accurately reflects the boundaries of the TM ISMS
|
||||||
|
2. Resources will be made available to implement and maintain the ISMS per [ISO27001_ROADMAP.md](./ISO27001_ROADMAP.md)
|
||||||
|
3. Information security objectives in the [Information Security Policy](./policies/information-security-policy.md) are endorsed
|
||||||
|
4. An Information Security Officer will be designated (named or interim)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Signatures
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| **Executive Sponsor** | _________________________ | _________________________ | __________ |
|
||||||
|
| **Information Security Officer** | _________________________ | _________________________ | __________ |
|
||||||
|
| **Engineering Lead** | _________________________ | _________________________ | __________ |
|
||||||
|
| **DevOps Lead** | _________________________ | _________________________ | __________ |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial scope approval document |
|
||||||
|
|
||||||
|
**Next review:** 2027-06-11 (annual) or upon material architecture change
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
# ISO/IEC 27001 Certification Roadmap — Tender Management System
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | ISMS-003 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Active roadmap |
|
||||||
|
| **Owner** | Information Security Officer (ISO) |
|
||||||
|
| **Last Updated** | 2026-06-11 |
|
||||||
|
| **Target Certification** | ISO/IEC 27001:2022 |
|
||||||
|
|
||||||
|
Related: [ISMS Foundation](./ISMS_FOUNDATION.md) | [Risk Assessment Matrix](./RISK_ASSESSMENT_MATRIX.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Executive Summary
|
||||||
|
|
||||||
|
This roadmap defines a phased path from the current informal security practices to a certifiable ISMS for the Tender Management backend platform. The plan spans approximately **12–18 months** from foundation to Stage 2 certification audit, assuming dedicated part-time security ownership (0.25–0.5 FTE).
|
||||||
|
|
||||||
|
### Current State
|
||||||
|
|
||||||
|
- Strong technical controls in application layer (JWT, RBAC, validation, XSS, audit logging)
|
||||||
|
- No formal ISMS documentation, policies, or certified processes
|
||||||
|
- Risk assessment and asset inventory now documented (initial versions)
|
||||||
|
|
||||||
|
### Target State
|
||||||
|
|
||||||
|
- Documented and operating ISMS aligned with ISO/IEC 27001:2022
|
||||||
|
- Risk treatment plans executed for High and Critical risks
|
||||||
|
- Internal audit program and management review cycle established
|
||||||
|
- Successful Stage 1 and Stage 2 certification audits
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Certification Timeline Overview
|
||||||
|
|
||||||
|
```
|
||||||
|
Phase 0 ──► Phase 1 ──► Phase 2 ──► Phase 3 ──► Phase 4 ──► Phase 5
|
||||||
|
Foundation Gap & Implement Operate Pre-Audit Certification
|
||||||
|
(Complete) Policies Controls ISMS Readiness Audit
|
||||||
|
│ │ │ │ │ │
|
||||||
|
Jun 2026 Jul-Aug Sep-Nov Dec-Mar Apr-Jun Jul-Sep 2027
|
||||||
|
```
|
||||||
|
|
||||||
|
| Phase | Name | Duration | Key Deliverable |
|
||||||
|
|-------|------|----------|-----------------|
|
||||||
|
| **0** | Foundation | 2 weeks | ISMS scope, asset inventory, risk matrix *(this release)* |
|
||||||
|
| **1** | Gap Analysis & Policies | 6–8 weeks | Policy suite, SoA draft, gap report |
|
||||||
|
| **2** | Control Implementation | 10–12 weeks | Technical and procedural controls |
|
||||||
|
| **3** | ISMS Operation | 12+ weeks | Internal audits, KPIs, management review |
|
||||||
|
| **4** | Pre-Audit Readiness | 4–6 weeks | Mock audit, evidence pack, corrective actions |
|
||||||
|
| **5** | Certification Audit | 4–8 weeks | Stage 1 + Stage 2 with accredited CB |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Phase 0 — Foundation (Complete)
|
||||||
|
|
||||||
|
**Status:** ✅ Complete as of 2026-06-11
|
||||||
|
|
||||||
|
| Task | Deliverable | Status |
|
||||||
|
|------|-------------|--------|
|
||||||
|
| Define ISMS scope | [ISMS_FOUNDATION.md §3](./ISMS_FOUNDATION.md#3-isms-scope) | ✅ |
|
||||||
|
| Create asset inventory | [ISMS_FOUNDATION.md §4](./ISMS_FOUNDATION.md#4-information-asset-inventory) | ✅ |
|
||||||
|
| Initial risk assessment | [RISK_ASSESSMENT_MATRIX.md](./RISK_ASSESSMENT_MATRIX.md) | ✅ |
|
||||||
|
| Assign interim ISO role | Pending management approval | ⬜ |
|
||||||
|
| Executive briefing on ISMS scope | Presentation / sign-off | ⬜ |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Phase 1 — Gap Analysis & Policy Framework
|
||||||
|
|
||||||
|
**Target:** Jul–Aug 2026 (6–8 weeks)
|
||||||
|
|
||||||
|
### 4.1 Objectives
|
||||||
|
|
||||||
|
- Map existing controls to ISO 27001:2022 Annex A
|
||||||
|
- Identify gaps and produce Statement of Applicability (SoA)
|
||||||
|
- Publish core ISMS policies
|
||||||
|
|
||||||
|
### 4.2 Deliverables
|
||||||
|
|
||||||
|
| # | Deliverable | ISO 27001 Reference | Owner | Status |
|
||||||
|
|---|-------------|---------------------|-------|--------|
|
||||||
|
| 1.1 | Gap analysis report | Clauses 4–10 | ISO | ✅ Draft |
|
||||||
|
| 1.2 | Statement of Applicability (SoA) | Annex A | ISO | ✅ Draft |
|
||||||
|
| 1.3 | Information Security Policy | A.5.1 | Executive Sponsor | ✅ Draft |
|
||||||
|
| 1.4 | Access Control Policy | A.5.15–A.5.18 | ISO | ✅ Draft |
|
||||||
|
| 1.5 | Incident Response Plan | A.5.24–A.5.28 | ISO | ✅ Draft |
|
||||||
|
| 1.6 | Backup & Recovery Policy | A.8.13 | DevOps | ✅ Draft |
|
||||||
|
| 1.7 | Acceptable Use Policy | A.6.2 | HR / ISO | ✅ Draft |
|
||||||
|
| 1.8 | Risk Assessment Procedure | Clause 6.1.2 | ISO | ✅ Draft |
|
||||||
|
| 1.9 | ISMS scope approval (signed) | Clause 4.3 | Executive Sponsor | ✅ Draft — pending signature |
|
||||||
|
|
||||||
|
### 4.3 Annex A Gap Snapshot (Initial)
|
||||||
|
|
||||||
|
Based on codebase review and [Risk Assessment Matrix](./RISK_ASSESSMENT_MATRIX.md):
|
||||||
|
|
||||||
|
| Annex A Theme | Current Maturity | Gap Priority |
|
||||||
|
|---------------|------------------|--------------|
|
||||||
|
| A.5 Organizational controls | 2 Yes / 25 Partial / 10 No (of 37) | High |
|
||||||
|
| A.6 People controls | 0 Yes / 6 Partial / 2 No (of 8) | High |
|
||||||
|
| A.7 Physical controls | 11 N/A (cloud); 3 Partial (personnel devices) | Transfer to cloud provider |
|
||||||
|
| A.8 Technological controls | 4 Yes / 22 Partial / 7 No / 1 N/A (of 34) | Medium |
|
||||||
|
|
||||||
|
*Per-theme counts: [SoA §7.1](./STATEMENT_OF_APPLICABILITY.md#71-by-theme).*
|
||||||
|
|
||||||
|
**Controls with Status = Yes in SoA (fully evidenced):**
|
||||||
|
|
||||||
|
| Control | Evidence in Codebase |
|
||||||
|
|---------|---------------------|
|
||||||
|
| A.5.9, A.5.12 | Asset inventory; information classification |
|
||||||
|
| A.8.3 Access restriction | RBAC + company-scoped middleware |
|
||||||
|
| A.8.4 Access to source code | Git repo with PR review |
|
||||||
|
| A.8.15 Logging | Structured service logging, `pkg/audit` |
|
||||||
|
| A.8.26 Application security requirements | Govalidator, XSS policy, Swagger |
|
||||||
|
|
||||||
|
**Additional partial in-app controls** (e.g. A.8.5 authentication, A.8.9 configuration, A.8.6 rate limiting) are implemented in code but not yet fully evidenced in the SoA.
|
||||||
|
|
||||||
|
**Priority gaps to close in Phase 2:**
|
||||||
|
|
||||||
|
| Control | Gap | Planned Action |
|
||||||
|
|---------|-----|----------------|
|
||||||
|
| A.8.24 Cryptography | No documented crypto policy; at-rest encryption TBD | Encryption policy; MongoDB/MinIO at-rest |
|
||||||
|
| A.8.8 Vulnerability management | No formal scanning | CI: govulncheck, SAST, container scan |
|
||||||
|
| A.8.16 Monitoring | Logs only, no SIEM | Centralized logging + alerts |
|
||||||
|
| A.5.19–A.5.23 Supplier relationships | No vendor assessments | Vendor questionnaire for AI, notification |
|
||||||
|
| A.5.29–A.5.30 Business continuity | No DR plan | BCP/DRP with RTO/RPO |
|
||||||
|
| A.6.3 Security awareness | No training program | Annual training + onboarding module |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Phase 2 — Control Implementation
|
||||||
|
|
||||||
|
**Target:** Sep–Nov 2026 (10–12 weeks)
|
||||||
|
|
||||||
|
### 5.1 Technical Controls
|
||||||
|
|
||||||
|
| Priority | Control | Tasks | Risk IDs | Owner |
|
||||||
|
|----------|---------|-------|----------|-------|
|
||||||
|
| P0 | Secrets management | Migrate to vault; remove FCM keys from repo; rotate credentials | R-013 | DevOps |
|
||||||
|
| P0 | Auth hardening | Rate limit all auth endpoints; MFA for admin users | R-001, R-021 | Engineering |
|
||||||
|
| P0 | Vulnerability scanning | govulncheck + Dependabot in CI; monthly review | R-010 | Engineering |
|
||||||
|
| P1 | Encryption at rest | Enable MongoDB and MinIO encryption | R-005, R-006 | DevOps |
|
||||||
|
| P1 | Monitoring & alerting | Centralize logs; alert on auth failures, 5xx spikes | R-015 | DevOps |
|
||||||
|
| P1 | RBAC audit | Automated tests for all admin route authorization | R-003 | Engineering |
|
||||||
|
| P2 | WAF / CDN | DDoS protection for public API | R-012 | DevOps |
|
||||||
|
| P2 | File upload security | MIME validation, size limits, malware scan | R-011 | Engineering |
|
||||||
|
| P2 | Refresh token rotation | Complete token rotation implementation | R-002 | Engineering |
|
||||||
|
|
||||||
|
### 5.2 Procedural Controls
|
||||||
|
|
||||||
|
| Priority | Control | Tasks | Owner |
|
||||||
|
|----------|---------|-------|-------|
|
||||||
|
| P0 | Incident response | IR plan, on-call rotation, tabletop exercise | ISO |
|
||||||
|
| P0 | Change management | PR approval rules, prod deployment checklist | Engineering |
|
||||||
|
| P1 | Access reviews | Quarterly review of admin and infra access | ISO |
|
||||||
|
| P1 | Backup verification | Monthly restore test; documented RTO/RPO | DevOps |
|
||||||
|
| P1 | Vendor management | Security questionnaires for AI, notification, cloud | ISO |
|
||||||
|
| P2 | DSR process | GDPR data subject request workflow | DPO |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Phase 3 — ISMS Operation
|
||||||
|
|
||||||
|
**Target:** Dec 2026 – Mar 2027 (minimum 3 months operation required)
|
||||||
|
|
||||||
|
### 6.1 Operating Activities
|
||||||
|
|
||||||
|
| Activity | Frequency | Owner | ISO 27001 Reference |
|
||||||
|
|----------|-----------|-------|---------------------|
|
||||||
|
| Risk register review | Quarterly | ISO | 6.1.2 |
|
||||||
|
| Internal audit | Semi-annual | Internal auditor | 9.2 |
|
||||||
|
| Management review | Quarterly | Executive Sponsor | 9.3 |
|
||||||
|
| Vulnerability scan review | Monthly | Engineering | A.8.8 |
|
||||||
|
| Access review | Quarterly | ISO | A.5.18 |
|
||||||
|
| Backup restore test | Monthly | DevOps | A.8.13 |
|
||||||
|
| Security awareness training | Annual + onboarding | ISO | A.6.3 |
|
||||||
|
| Supplier review | Annual | ISO | A.5.19 |
|
||||||
|
| KPI reporting | Monthly | ISO | 9.1 |
|
||||||
|
|
||||||
|
### 6.2 Key Performance Indicators (KPIs)
|
||||||
|
|
||||||
|
| KPI | Target | Measurement |
|
||||||
|
|-----|--------|-------------|
|
||||||
|
| Critical/High open risks | 0 Critical; ≤3 High | Risk register |
|
||||||
|
| Mean time to patch (Critical CVE) | ≤ 7 days | Vulnerability tracker |
|
||||||
|
| Failed login rate | Baseline + alert threshold | SIEM |
|
||||||
|
| Backup restore success | 100% monthly test | DevOps report |
|
||||||
|
| Security training completion | 100% staff | HR records |
|
||||||
|
| Incident response time (P1) | ≤ 1 hour acknowledgment | IR log |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Phase 4 — Pre-Audit Readiness
|
||||||
|
|
||||||
|
**Target:** Apr–Jun 2027 (4–6 weeks)
|
||||||
|
|
||||||
|
| Task | Description |
|
||||||
|
|------|-------------|
|
||||||
|
| Select certification body (CB) | Accredited ISO 27001 registrar |
|
||||||
|
| Mock Stage 1 audit | Documentation review against Clauses 4–10 |
|
||||||
|
| Evidence pack assembly | Policies, risk register, audit reports, training records, change logs |
|
||||||
|
| Corrective actions | Close all findings from mock audit |
|
||||||
|
| Employee interviews prep | Staff briefed on ISMS policies and their roles |
|
||||||
|
| SoA finalization | All Annex A controls marked with implementation status |
|
||||||
|
|
||||||
|
### 7.1 Evidence Checklist
|
||||||
|
|
||||||
|
- [ ] Signed Information Security Policy
|
||||||
|
- [ ] Approved ISMS scope document
|
||||||
|
- [ ] Risk assessment and treatment plan (current version)
|
||||||
|
- [ ] Statement of Applicability
|
||||||
|
- [ ] Internal audit reports (≥1 cycle)
|
||||||
|
- [ ] Management review minutes (≥1 cycle)
|
||||||
|
- [ ] Incident response plan and test record
|
||||||
|
- [ ] Access review records
|
||||||
|
- [ ] Vulnerability scan reports
|
||||||
|
- [ ] Backup/restore test records
|
||||||
|
- [ ] Training completion records
|
||||||
|
- [ ] Vendor assessment records
|
||||||
|
- [ ] Change management records (sample)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Phase 5 — Certification Audit
|
||||||
|
|
||||||
|
**Target:** Jul–Sep 2027
|
||||||
|
|
||||||
|
### 8.1 Stage 1 — Documentation Review
|
||||||
|
|
||||||
|
- CB reviews ISMS documentation for conformity with ISO 27001:2022
|
||||||
|
- Scope, SoA, risk assessment, and policy completeness verified
|
||||||
|
- Findings documented; corrective actions before Stage 2
|
||||||
|
|
||||||
|
### 8.2 Stage 2 — Implementation Audit
|
||||||
|
|
||||||
|
- CB verifies controls are implemented and operating effectively
|
||||||
|
- Interviews with key personnel
|
||||||
|
- Sampling of evidence (logs, tickets, access records)
|
||||||
|
- Nonconformities classified as Minor or Major
|
||||||
|
|
||||||
|
### 8.3 Post-Certification
|
||||||
|
|
||||||
|
- Surveillance audits annually
|
||||||
|
- Full recertification every 3 years
|
||||||
|
- Continuous improvement via PDCA cycle
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Resource Estimate
|
||||||
|
|
||||||
|
| Role | Effort | Phase(s) |
|
||||||
|
|------|--------|----------|
|
||||||
|
| ISO (part-time) | 0.25–0.5 FTE ongoing | All |
|
||||||
|
| Engineering Lead | 0.1 FTE | 2, 3 |
|
||||||
|
| DevOps Lead | 0.15 FTE | 2, 3 |
|
||||||
|
| DPO (part-time) | 0.05 FTE | 1, 2 |
|
||||||
|
| External consultant (optional) | 5–10 days | 1, 4 |
|
||||||
|
| Certification body | Fixed fee | 5 |
|
||||||
|
| Security tooling (SIEM, scanning) | Budget item | 2 |
|
||||||
|
|
||||||
|
**Estimated total cost range:** €15,000–€40,000 (highly dependent on org size, tooling, and consultant use)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. PDCA Cycle Integration
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────┐
|
||||||
|
│ PLAN │ Scope, risk assessment, policies, SoA
|
||||||
|
└──────┬──────┘
|
||||||
|
▼
|
||||||
|
┌─────────────┐
|
||||||
|
│ DO │ Implement controls, training, procedures
|
||||||
|
└──────┬──────┘
|
||||||
|
▼
|
||||||
|
┌─────────────┐
|
||||||
|
│ CHECK │ Internal audit, KPIs, management review
|
||||||
|
└──────┬──────┘
|
||||||
|
▼
|
||||||
|
┌─────────────┐
|
||||||
|
│ ACT │ Corrective actions, risk updates, improve
|
||||||
|
└──────┬──────┘
|
||||||
|
└──────────► (back to PLAN)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. Milestones & Decision Gates
|
||||||
|
|
||||||
|
| Milestone | Target Date | Gate Criteria |
|
||||||
|
|-----------|-------------|---------------|
|
||||||
|
| M0: Foundation approved | 2026-06-30 | Executive sign-off on scope and asset inventory |
|
||||||
|
| M1: Policy suite published | 2026-08-31 | 5 core policies + risk procedure drafted; pending executive approval |
|
||||||
|
| M2: Critical risks mitigated | 2026-11-30 | No Critical residual risks |
|
||||||
|
| M3: ISMS operational | 2027-03-31 | 1 internal audit + 1 management review complete |
|
||||||
|
| M4: Pre-audit passed | 2027-06-30 | Mock audit with no Major findings |
|
||||||
|
| M5: ISO 27001 certified | 2027-09-30 | Stage 2 certificate issued |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 12. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial certification roadmap |
|
||||||
|
|
||||||
|
**Approval**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
|
| Executive Sponsor | _Pending_ | | |
|
||||||
@@ -0,0 +1,166 @@
|
|||||||
|
# Risk Assessment Matrix — Tender Management System
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | ISMS-002 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Initial assessment |
|
||||||
|
| **Owner** | Information Security Officer (ISO) |
|
||||||
|
| **Last Updated** | 2026-06-11 |
|
||||||
|
| **Review Cycle** | Quarterly (minimum) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This document records the initial information security risk assessment for the Tender Management System ISMS. It identifies threats, evaluates inherent and residual risk, and defines treatment plans aligned with ISO/IEC 27001 Clause 6.1.2 and Annex A controls.
|
||||||
|
|
||||||
|
Related: [ISMS Foundation](./ISMS_FOUNDATION.md) | [ISO 27001 Roadmap](./ISO27001_ROADMAP.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Methodology
|
||||||
|
|
||||||
|
### 2.1 Risk Formula
|
||||||
|
|
||||||
|
```
|
||||||
|
Risk Score = Likelihood (1–5) × Impact (1–5)
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.2 Likelihood Scale
|
||||||
|
|
||||||
|
| Score | Label | Description |
|
||||||
|
|-------|-------|-------------|
|
||||||
|
| 1 | Rare | Unlikely in 3+ years |
|
||||||
|
| 2 | Unlikely | Possible but not expected annually |
|
||||||
|
| 3 | Possible | May occur once per year |
|
||||||
|
| 4 | Likely | Expected several times per year |
|
||||||
|
| 5 | Almost certain | Expected frequently or already observed |
|
||||||
|
|
||||||
|
### 2.3 Impact Scale
|
||||||
|
|
||||||
|
| Score | Label | Description |
|
||||||
|
|-------|-------|-------------|
|
||||||
|
| 1 | Negligible | No data exposure; minimal downtime (<1 h) |
|
||||||
|
| 2 | Minor | Limited internal data; downtime 1–4 h |
|
||||||
|
| 3 | Moderate | PII exposure for subset of users; downtime 4–24 h |
|
||||||
|
| 4 | Major | Large-scale PII breach; regulatory notification; downtime 1–3 days |
|
||||||
|
| 5 | Critical | Systemic breach; legal liability; prolonged outage (>3 days) |
|
||||||
|
|
||||||
|
### 2.4 Risk Level Matrix
|
||||||
|
|
||||||
|
| Score Range | Level | Action Required |
|
||||||
|
|-------------|-------|-----------------|
|
||||||
|
| 1–4 | **Low** | Monitor; accept with documented justification |
|
||||||
|
| 5–9 | **Medium** | Mitigate within 6 months |
|
||||||
|
| 10–15 | **High** | Mitigate within 3 months; management awareness |
|
||||||
|
| 16–25 | **Critical** | Immediate treatment; executive escalation |
|
||||||
|
|
||||||
|
### 2.5 Treatment Options
|
||||||
|
|
||||||
|
- **Mitigate** — Apply controls to reduce likelihood or impact
|
||||||
|
- **Transfer** — Insurance or contractual liability shift
|
||||||
|
- **Avoid** — Discontinue the activity
|
||||||
|
- **Accept** — Documented acceptance by authorized role (residual risk only)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Risk Register
|
||||||
|
|
||||||
|
### 3.1 Authentication & Access
|
||||||
|
|
||||||
|
| Risk ID | Threat | Vulnerability | Asset(s) | L | I | Inherent | Existing Controls | Residual L | Residual I | Residual | Treatment | Owner | Target Date | Status |
|
||||||
|
|---------|--------|---------------|----------|---|---|----------|-------------------|------------|------------|----------|-----------|-------|-------------|--------|
|
||||||
|
| R-001 | Credential stuffing / brute force | Login endpoints without sufficient throttling | A-004, A-006, A-007 | 4 | 4 | **16 Critical** | JWT auth, hCaptcha on selected flows, rate limit config | 3 | 3 | **9 Medium** | Enforce rate limiting on all auth endpoints; account lockout; MFA for admin | Engineering | Q3 2026 | Open |
|
||||||
|
| R-002 | JWT token theft (XSS, MITM) | Tokens in client storage | A-008 | 3 | 4 | **12 High** | HTTPS required in prod; short token TTL; refresh rotation partial | 2 | 3 | **6 Medium** | Complete refresh token rotation; HttpOnly cookies where applicable; CSP headers | Engineering | Q3 2026 | Open |
|
||||||
|
| R-003 | Privilege escalation | Insufficient RBAC checks on admin routes | A-006, A-014 | 2 | 5 | **10 High** | Role middleware in `pkg/authorization` | 2 | 4 | **8 Medium** | RBAC audit of all admin endpoints; automated authorization tests | Engineering | Q2 2026 | Open |
|
||||||
|
| R-004 | Orphaned / excessive access | No formal access review process | A-006, A-010 | 3 | 3 | **9 Medium** | Manual provisioning | 2 | 2 | **4 Low** | Quarterly access review procedure; offboarding checklist | ISO | Q3 2026 | Open |
|
||||||
|
|
||||||
|
### 3.2 Data Protection & Privacy
|
||||||
|
|
||||||
|
| Risk ID | Threat | Vulnerability | Asset(s) | L | I | Inherent | Existing Controls | Residual L | Residual I | Residual | Treatment | Owner | Target Date | Status |
|
||||||
|
|---------|--------|---------------|----------|---|---|----------|-------------------|------------|------------|----------|-----------|-------|-------------|--------|
|
||||||
|
| R-005 | Unauthorized PII access | Missing encryption at rest | A-001, A-004, A-005 | 3 | 5 | **15 High** | Network isolation; app-level auth | 2 | 4 | **8 Medium** | Enable MongoDB encryption at rest; field-level encryption for sensitive fields | DevOps | Q3 2026 | Open |
|
||||||
|
| R-006 | Data breach via backup exposure | Unencrypted or overly permissive backups | A-019 | 2 | 5 | **10 High** | Ad-hoc backups | 2 | 3 | **6 Medium** | Encrypted backups; restricted access; backup retention policy | DevOps | Q3 2026 | Open |
|
||||||
|
| R-007 | GDPR non-compliance (DSR) | No documented data subject request process | A-004, A-005 | 3 | 4 | **12 High** | — | 3 | 3 | **9 Medium** | DSR procedure; data inventory mapping; deletion API/workflow | DPO | Q4 2026 | Open |
|
||||||
|
| R-008 | Sensitive data in logs | Accidental logging of passwords/tokens | A-011 | 3 | 4 | **12 High** | Coding standards prohibit password logging | 2 | 3 | **6 Medium** | Log scrubbing; automated secret detection in CI | Engineering | Q2 2026 | Open |
|
||||||
|
|
||||||
|
### 3.3 Application Security
|
||||||
|
|
||||||
|
| Risk ID | Threat | Vulnerability | Asset(s) | L | I | Inherent | Existing Controls | Residual L | Residual I | Residual | Treatment | Owner | Target Date | Status |
|
||||||
|
|---------|--------|---------------|----------|---|---|----------|-------------------|------------|------------|----------|-----------|-------|-------------|--------|
|
||||||
|
| R-009 | Injection attacks (NoSQL, XSS) | Unvalidated user input | A-014 | 3 | 4 | **12 High** | Govalidator; Bluemonday XSS policy | 2 | 3 | **6 Medium** | SAST/DAST in CI; security test cases for injection | Engineering | Q2 2026 | Open |
|
||||||
|
| R-010 | Dependency vulnerability exploit | Outdated Go modules | A-009 | 4 | 4 | **16 Critical** | go.mod pinned versions | 3 | 3 | **9 Medium** | Dependabot/Renovate; monthly dependency review; `govulncheck` in CI | Engineering | Q2 2026 | Open |
|
||||||
|
| R-011 | Insecure file upload | Malicious document upload | A-003, A-005 | 2 | 4 | **8 Medium** | File store service; GridFS/MinIO | 2 | 3 | **6 Medium** | MIME validation; size limits; malware scanning | Engineering | Q4 2026 | Open |
|
||||||
|
| R-012 | API abuse / DoS | High-volume unauthenticated requests | A-014 | 4 | 3 | **12 High** | Rate limit config exists | 2 | 2 | **4 Low** | Enable rate limiting in production; WAF/CDN | DevOps | Q2 2026 | Open |
|
||||||
|
|
||||||
|
### 3.4 Infrastructure & Operations
|
||||||
|
|
||||||
|
| Risk ID | Threat | Vulnerability | Asset(s) | L | I | Inherent | Existing Controls | Residual L | Residual I | Residual | Treatment | Owner | Target Date | Status |
|
||||||
|
|---------|--------|---------------|----------|---|---|----------|-------------------|------------|------------|----------|-----------|-------|-------------|--------|
|
||||||
|
| R-013 | Secret leakage | Secrets in repo, logs, or `.env` files | A-010, A-018 | 3 | 5 | **15 High** | `.gitignore`; env priority system | 2 | 4 | **8 Medium** | Secret manager (Vault/AWS SM); secret scanning in CI; rotate FCM keys out of repo | DevOps | Q2 2026 | Open |
|
||||||
|
| R-014 | Service outage (MongoDB/Redis/MinIO) | Single points of failure | A-001, A-002, A-003 | 3 | 4 | **12 High** | Managed services assumed | 2 | 3 | **6 Medium** | HA deployment; DR plan; RTO/RPO definitions | DevOps | Q4 2026 | Open |
|
||||||
|
| R-015 | Insufficient monitoring | Delayed incident detection | A-011, A-012 | 4 | 4 | **16 Critical** | Application logging only | 3 | 3 | **9 Medium** | SIEM integration; alerting on auth failures, error spikes | DevOps | Q3 2026 | Open |
|
||||||
|
| R-016 | Unpatched OS/container images | Delayed security patching | A-014, A-015 | 3 | 4 | **12 High** | — | 2 | 3 | **6 Medium** | Patch management procedure; automated image scanning | DevOps | Q3 2026 | Open |
|
||||||
|
|
||||||
|
### 3.5 Third-Party & Supply Chain
|
||||||
|
|
||||||
|
| Risk ID | Threat | Vulnerability | Asset(s) | L | I | Inherent | Existing Controls | Residual L | Residual I | Residual | Treatment | Owner | Target Date | Status |
|
||||||
|
|---------|--------|---------------|----------|---|---|----------|-------------------|------------|------------|----------|-----------|-------|-------------|--------|
|
||||||
|
| R-017 | AI service data leakage | Tender text sent to external AI | A-017, A-013 | 3 | 4 | **12 High** | Configurable AI endpoint | 2 | 3 | **6 Medium** | DPA with AI vendor; data minimization; on-prem option evaluation | DPO / DevOps | Q4 2026 | Open |
|
||||||
|
| R-018 | Notification service compromise | Email content interception | A-016 | 2 | 3 | **6 Medium** | TLS to notification API | 2 | 2 | **4 Low** | Vendor security questionnaire; TLS enforcement | ISO | Q4 2026 | Open |
|
||||||
|
| R-019 | Compromised TED source data | Malicious XML in scraper pipeline | A-013, A-015 | 2 | 3 | **6 Medium** | XML parser validation | 2 | 2 | **4 Low** | Schema validation; sandboxed parsing; input size limits | Engineering | Q3 2026 | Open |
|
||||||
|
|
||||||
|
### 3.6 Human & Process
|
||||||
|
|
||||||
|
| Risk ID | Threat | Vulnerability | Asset(s) | L | I | Inherent | Existing Controls | Residual L | Residual I | Residual | Treatment | Owner | Target Date | Status |
|
||||||
|
|---------|--------|---------------|----------|---|---|----------|-------------------|------------|------------|----------|-----------|-------|-------------|--------|
|
||||||
|
| R-020 | Insider threat | Broad production DB access | A-001, A-004 | 2 | 5 | **10 High** | — | 2 | 4 | **8 Medium** | Least privilege; break-glass access; audit DB queries | DevOps | Q3 2026 | Open |
|
||||||
|
| R-021 | Phishing / social engineering | Staff credential compromise | A-006, A-010 | 3 | 4 | **12 High** | — | 2 | 3 | **6 Medium** | Security awareness training; MFA for admin and infra | ISO | Q3 2026 | Open |
|
||||||
|
| R-022 | Undocumented change deployment | No formal change management | A-009, A-014 | 3 | 3 | **9 Medium** | Git + PR workflow informal | 2 | 2 | **4 Low** | Change management procedure; prod deployment approval | Engineering | Q3 2026 | Open |
|
||||||
|
| R-023 | Delayed incident response | No IR plan or runbooks | All | 3 | 5 | **15 High** | — | 2 | 4 | **8 Medium** | Incident response plan; tabletop exercise | ISO | Q2 2026 | Open |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Risk Summary Dashboard
|
||||||
|
|
||||||
|
| Risk Level | Count | Risk IDs |
|
||||||
|
|------------|-------|----------|
|
||||||
|
| **Critical** (16–25) | 0 (after treatment planning) | R-001, R-010, R-015 were Critical inherent — all have mitigation plans |
|
||||||
|
| **High** (10–15) | 8 residual | R-003, R-005, R-007, R-009, R-013, R-014, R-020, R-023 |
|
||||||
|
| **Medium** (5–9) | 12 residual | R-001, R-002, R-004, R-006, R-008, R-010, R-011, R-015, R-016, R-017, R-021, R-022 |
|
||||||
|
| **Low** (1–4) | 3 residual | R-012, R-018, R-019 |
|
||||||
|
|
||||||
|
### 4.1 Top Priority Actions (Next 90 Days)
|
||||||
|
|
||||||
|
1. **R-023** — Draft and approve incident response plan
|
||||||
|
2. **R-013** — Remove secrets from repository; implement secret scanning
|
||||||
|
3. **R-010** — Add `govulncheck` and dependency scanning to CI
|
||||||
|
4. **R-003** — Complete RBAC audit of admin API routes
|
||||||
|
5. **R-001** — Enforce authentication rate limiting in production
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Risk Acceptance Log
|
||||||
|
|
||||||
|
Use this section to record risks explicitly accepted after treatment.
|
||||||
|
|
||||||
|
| Risk ID | Residual Score | Accepted By | Role | Date | Justification | Review Date |
|
||||||
|
|---------|----------------|-------------|------|------|---------------|-------------|
|
||||||
|
| _None yet_ | | | | | | |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Review History
|
||||||
|
|
||||||
|
| Version | Date | Reviewer | Changes |
|
||||||
|
|---------|------|----------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial risk identification for TM platform |
|
||||||
|
|
||||||
|
**Next scheduled review:** 2026-09-11
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Annex — Asset Reference
|
||||||
|
|
||||||
|
See [ISMS Foundation §4](./ISMS_FOUNDATION.md#4-information-asset-inventory) for full asset inventory (A-001 through A-020).
|
||||||
@@ -0,0 +1,255 @@
|
|||||||
|
# Statement of Applicability (SoA) — Tender Management System
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | ISMS-005 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Draft — Pending approval |
|
||||||
|
| **Owner** | Information Security Officer (ISO) |
|
||||||
|
| **Last Updated** | 2026-06-11 |
|
||||||
|
| **Standard** | ISO/IEC 27001:2022 Annex A |
|
||||||
|
| **Scope** | [ISMS Foundation §3](./ISMS_FOUNDATION.md#3-isms-scope) |
|
||||||
|
|
||||||
|
Related: [Gap Analysis Report](./GAP_ANALYSIS_REPORT.md) | [Risk Assessment Matrix](./RISK_ASSESSMENT_MATRIX.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
The Statement of Applicability (SoA) documents:
|
||||||
|
|
||||||
|
- Which Annex A controls are **applicable** to the TM ISMS
|
||||||
|
- Whether each control is **implemented**, **partially implemented**, or **not implemented**
|
||||||
|
- Justification for exclusions
|
||||||
|
- Reference to implementation evidence or planned remediation
|
||||||
|
|
||||||
|
This document satisfies ISO/IEC 27001:2022 Clause 6.1.3(d) and is a mandatory input for certification audits.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Status Legend
|
||||||
|
|
||||||
|
| Status | Meaning |
|
||||||
|
|--------|---------|
|
||||||
|
| **Yes** | Applicable and implemented with evidence |
|
||||||
|
| **Partial** | Applicable; control exists but incomplete or not fully evidenced |
|
||||||
|
| **No** | Applicable but not yet implemented |
|
||||||
|
| **N/A** | Not applicable to scope (justification required) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. A.5 — Organizational Controls
|
||||||
|
|
||||||
|
| ID | Control | Applicable | Status | Implementation / Justification | Risk Ref |
|
||||||
|
|----|---------|------------|--------|----------------------------------|----------|
|
||||||
|
| A.5.1 | Policies for information security | Yes | Partial | [Information Security Policy](./policies/information-security-policy.md) draft; pending approval | — |
|
||||||
|
| A.5.2 | Information security roles and responsibilities | Yes | Partial | [ISMS Foundation §5](./ISMS_FOUNDATION.md#5-roles-and-responsibilities); named assignments pending | — |
|
||||||
|
| A.5.3 | Segregation of duties | Yes | Partial | Dev vs DevOps separation informal; no single prod DB write without break-glass | R-020 |
|
||||||
|
| A.5.4 | Management responsibilities | Yes | Partial | Roadmap and objectives defined; management review not yet held | — |
|
||||||
|
| A.5.5 | Contact with authorities | Yes | No | No documented contacts with regulators/law enforcement | R-023 |
|
||||||
|
| A.5.6 | Contact with special interest groups | Yes | No | No ISAC or security forum membership | — |
|
||||||
|
| A.5.7 | Threat intelligence | Yes | No | No formal threat intel feed or process | — |
|
||||||
|
| A.5.8 | Information security in project management | Yes | Partial | Major change checklist in risk procedure; informal for small changes | — |
|
||||||
|
| A.5.9 | Inventory of information and other associated assets | Yes | Yes | [ISMS Foundation §4](./ISMS_FOUNDATION.md#4-information-asset-inventory) — 20 assets | — |
|
||||||
|
| A.5.10 | Acceptable use of information and other associated assets | Yes | Partial | [Acceptable Use Policy](./policies/acceptable-use-policy.md) draft | — |
|
||||||
|
| A.5.11 | Return of assets | Yes | No | No formal offboarding asset return checklist | R-004 |
|
||||||
|
| A.5.12 | Classification of information | Yes | Yes | C1–C4 classification in ISMS Foundation | — |
|
||||||
|
| A.5.13 | Labelling of information | Yes | No | Classification defined but no labelling in apps/repos | — |
|
||||||
|
| A.5.14 | Information transfer | Yes | Partial | HTTPS for APIs; email via notification service; no DLP | R-017 |
|
||||||
|
| A.5.15 | Access control | Yes | Partial | [Access Control Policy](./policies/access-control-policy.md); JWT + RBAC | R-003 |
|
||||||
|
| A.5.16 | Identity management | Yes | Partial | Unique accounts; provisioning/deprovisioning procedure in policy | R-004 |
|
||||||
|
| A.5.17 | Authentication information | Yes | Partial | bcrypt passwords; secrets in env; MFA planned | R-001, R-013 |
|
||||||
|
| A.5.18 | Access rights | Yes | Partial | Quarterly review defined; not yet executed | R-004 |
|
||||||
|
| A.5.19 | Information security in supplier relationships | Yes | No | No vendor security assessments | R-017, R-018 |
|
||||||
|
| A.5.20 | Addressing information security within supplier agreements | Yes | No | Contracts lack security clauses review | R-017 |
|
||||||
|
| A.5.21 | Managing information security in the ICT supply chain | Yes | Partial | Go modules pinned; no formal supply chain policy | R-010 |
|
||||||
|
| A.5.22 | Monitoring, review and change management of supplier services | Yes | No | No annual vendor review process | R-018 |
|
||||||
|
| A.5.23 | Information security for use of cloud services | Yes | Partial | Cloud-hosted MongoDB/Redis/MinIO; provider evidence not collected | R-014 |
|
||||||
|
| A.5.24 | Information security incident management planning and preparation | Yes | Partial | [Incident Response Plan](./policies/incident-response-plan.md) draft | R-023 |
|
||||||
|
| A.5.25 | Assessment and decision on information security events | Yes | Partial | Severity classification in IR plan; not yet tested | R-023 |
|
||||||
|
| A.5.26 | Response to information security incidents | Yes | Partial | Playbooks defined; IRT contacts not finalized | R-023 |
|
||||||
|
| A.5.27 | Learning from information security incidents | Yes | Partial | Post-incident review required in IR plan; no incidents recorded yet | — |
|
||||||
|
| A.5.28 | Collection of evidence | Yes | Partial | Evidence handling in IR plan; chain of custody untested | — |
|
||||||
|
| A.5.29 | Information security during disruption | Yes | Partial | RTO/RPO in backup policy; no full BCP document | R-014 |
|
||||||
|
| A.5.30 | ICT readiness for business continuity | Yes | Partial | Backup policy; DR test not yet performed | R-014 |
|
||||||
|
| A.5.31 | Legal, statutory, regulatory and contractual requirements | Yes | Partial | GDPR referenced; no requirements register | R-007 |
|
||||||
|
| A.5.32 | Intellectual property rights | Yes | Partial | Open-source licenses in go.mod; no IP policy | — |
|
||||||
|
| A.5.33 | Protection of records | Yes | Partial | Audit logs; retention periods not fully defined | — |
|
||||||
|
| A.5.34 | Privacy and protection of PII | Yes | Partial | PII inventory; DSR process not implemented | R-007 |
|
||||||
|
| A.5.35 | Independent review of information security | Yes | No | No internal audit or external review yet | — |
|
||||||
|
| A.5.36 | Compliance with policies, rules and standards | Yes | No | No compliance checks or audit program | — |
|
||||||
|
| A.5.37 | Documented operating procedures | Yes | Partial | ISMS procedures started; ops runbooks incomplete | — |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. A.6 — People Controls
|
||||||
|
|
||||||
|
| ID | Control | Applicable | Status | Implementation / Justification | Risk Ref |
|
||||||
|
|----|---------|------------|--------|----------------------------------|----------|
|
||||||
|
| A.6.1 | Screening | Yes | No | No background check policy documented | R-021 |
|
||||||
|
| A.6.2 | Terms and conditions of employment | Yes | Partial | Acceptable Use Policy; HR terms not linked | — |
|
||||||
|
| A.6.3 | Information security awareness, education and training | Yes | No | No training program or records | R-021 |
|
||||||
|
| A.6.4 | Disciplinary process | Yes | Partial | Referenced in AUP; HR process not documented | — |
|
||||||
|
| A.6.5 | Responsibilities after termination or change of employment | Yes | Partial | Deprovisioning in Access Control Policy (24 h) | R-004 |
|
||||||
|
| A.6.6 | Confidentiality or non-disclosure agreements | Yes | Partial | Assumed for contractors; no central register | — |
|
||||||
|
| A.6.7 | Remote working | Yes | Partial | VPN/bastion requirement in Access Control Policy | — |
|
||||||
|
| A.6.8 | Information security event reporting | Yes | Partial | Reporting channels in IR plan; not communicated to staff | R-023 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. A.7 — Physical Controls
|
||||||
|
|
||||||
|
| ID | Control | Applicable | Status | Implementation / Justification | Risk Ref |
|
||||||
|
|----|---------|------------|--------|----------------------------------|----------|
|
||||||
|
| A.7.1 | Physical security perimeters | N/A | N/A | Cloud provider responsibility (A.5.23) | — |
|
||||||
|
| A.7.2 | Physical entry | N/A | N/A | Cloud provider responsibility | — |
|
||||||
|
| A.7.3 | Securing offices, rooms and facilities | N/A | N/A | No on-premise TM infrastructure | — |
|
||||||
|
| A.7.4 | Physical security monitoring | N/A | N/A | Cloud provider responsibility | — |
|
||||||
|
| A.7.5 | Protecting against physical and environmental threats | N/A | N/A | Cloud provider responsibility | — |
|
||||||
|
| A.7.6 | Working in secure areas | N/A | N/A | No secure areas for TM systems | — |
|
||||||
|
| A.7.7 | Clear desk and clear screen | Yes | Partial | AUP references screen lock; not enforced | — |
|
||||||
|
| A.7.8 | Equipment siting and protection | N/A | N/A | Cloud-hosted servers | — |
|
||||||
|
| A.7.9 | Security of assets off-premises | Yes | Partial | AUP device encryption requirement for prod access | — |
|
||||||
|
| A.7.10 | Storage media | N/A | N/A | No removable media for TM data stores | — |
|
||||||
|
| A.7.11 | Supporting utilities | N/A | N/A | Cloud provider responsibility | — |
|
||||||
|
| A.7.12 | Cabling security | N/A | N/A | Cloud provider responsibility | — |
|
||||||
|
| A.7.13 | Equipment maintenance | N/A | N/A | Cloud provider responsibility | — |
|
||||||
|
| A.7.14 | Secure disposal or re-use of equipment | Yes | Partial | AUP prohibits local PII storage; no disposal procedure | — |
|
||||||
|
|
||||||
|
*Note: A.7.7, A.7.9, A.7.14 remain applicable for personnel devices accessing TM systems.*
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. A.8 — Technological Controls
|
||||||
|
|
||||||
|
| ID | Control | Applicable | Status | Implementation / Justification | Risk Ref |
|
||||||
|
|----|---------|------------|--------|----------------------------------|----------|
|
||||||
|
| A.8.1 | User endpoint devices | Yes | Partial | AUP device requirements; no MDM | — |
|
||||||
|
| A.8.2 | Privileged access rights | Yes | Partial | Break-glass procedure; no PAM tool | R-020 |
|
||||||
|
| A.8.3 | Information access restriction | Yes | Yes | RBAC + company-scoped middleware (`pkg/authorization`) | R-003 |
|
||||||
|
| A.8.4 | Access to source code | Yes | Yes | Git repo with PR review; branch protection (assumed) | — |
|
||||||
|
| A.8.5 | Secure authentication | Yes | Partial | JWT, bcrypt, hCaptcha; MFA not yet enabled | R-001, R-002 |
|
||||||
|
| A.8.6 | Capacity management | Yes | Partial | Rate limit config; no capacity monitoring | R-012 |
|
||||||
|
| A.8.7 | Protection against malware | Yes | No | No malware scanning on uploads or endpoints | R-011 |
|
||||||
|
| A.8.8 | Management of technical vulnerabilities | Yes | No | No govulncheck/SAST in CI | R-010 |
|
||||||
|
| A.8.9 | Configuration management | Yes | Partial | `pkg/config` env priority; secrets in flat env files | R-013 |
|
||||||
|
| A.8.10 | Information deletion | Yes | Partial | Domain delete operations; no retention/deletion policy | R-007 |
|
||||||
|
| A.8.11 | Data masking | Yes | No | Passwords excluded from API responses; no masking elsewhere | — |
|
||||||
|
| A.8.12 | Data leakage prevention | Yes | No | No DLP tooling | R-008 |
|
||||||
|
| A.8.13 | Information backup | Yes | Partial | [Backup & Recovery Policy](./policies/backup-and-recovery-policy.md); tests not yet run | R-006, R-014 |
|
||||||
|
| A.8.14 | Redundancy of information processing facilities | Yes | Partial | Depends on cloud provider HA; not documented | R-014 |
|
||||||
|
| A.8.15 | Logging | Yes | Yes | Structured service logging; `pkg/audit` for security events | R-008, R-015 |
|
||||||
|
| A.8.16 | Monitoring activities | Yes | Partial | Application logs only; no SIEM or alerting | R-015 |
|
||||||
|
| A.8.17 | Clock synchronization | Yes | Partial | NTP assumed on cloud hosts; not verified | — |
|
||||||
|
| A.8.18 | Use of privileged utility programs | Yes | Partial | DB admin tools restricted; no formal policy | R-020 |
|
||||||
|
| A.8.19 | Installation of software on operational systems | Yes | Partial | Container/deploy pipeline; no formal whitelist | R-016 |
|
||||||
|
| A.8.20 | Networks security | Yes | Partial | HTTPS in prod; DB not public; no documented network diagram | — |
|
||||||
|
| A.8.21 | Security of network services | Yes | Partial | TLS to external APIs; no network service inventory | — |
|
||||||
|
| A.8.22 | Segregation of networks | Yes | Partial | Dev/staging/prod separated; informal | — |
|
||||||
|
| A.8.23 | Web filtering | Yes | No | No web filtering for staff endpoints | — |
|
||||||
|
| A.8.24 | Use of cryptography | Yes | Partial | TLS in transit; at-rest encryption TBD | R-005 |
|
||||||
|
| A.8.25 | Secure development life cycle | Yes | Partial | `.cursorrules`, code review, Clean Architecture | — |
|
||||||
|
| A.8.26 | Application security requirements | Yes | Yes | Govalidator, XSS policy, Swagger, response standards | R-009 |
|
||||||
|
| A.8.27 | Secure system architecture and engineering principles | Yes | Partial | Clean Architecture, DI, no global state | — |
|
||||||
|
| A.8.28 | Secure coding | Yes | Partial | Coding standards documented; no SAST | R-009, R-010 |
|
||||||
|
| A.8.29 | Security testing in development and acceptance | Yes | No | Unit tests exist; no security test suite or DAST | R-009 |
|
||||||
|
| A.8.30 | Outsourced development | N/A | N/A | All TM backend development is in-house; no outsourced development of scoped applications | — |
|
||||||
|
| A.8.31 | Separation of development, test and production environments | Yes | Partial | Separate env configs; staging data may contain real data | — |
|
||||||
|
| A.8.32 | Change management | Yes | Partial | Git PR workflow; no formal change procedure | R-022 |
|
||||||
|
| A.8.33 | Test information | Yes | Partial | Staging should use anonymized data; not enforced | — |
|
||||||
|
| A.8.34 | Protection of information systems during audit testing | Yes | No | No audit testing procedure | — |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Summary Statistics
|
||||||
|
|
||||||
|
Counts derived from the control rows in §3–§6 (authoritative source for cross-document reporting).
|
||||||
|
|
||||||
|
### 7.1 By Theme
|
||||||
|
|
||||||
|
| Theme | Total | Implemented (Yes) | Partial | Not impl. (No) | N/A |
|
||||||
|
|-------|-------|-------------------|---------|----------------|-----|
|
||||||
|
| **A.5** Organizational | 37 | 2 | 25 | 10 | 0 |
|
||||||
|
| **A.6** People | 8 | 0 | 6 | 2 | 0 |
|
||||||
|
| **A.7** Physical | 14 | 0 | 3 | 0 | 11 |
|
||||||
|
| **A.8** Technological | 34 | 4 | 22 | 7 | 1 |
|
||||||
|
| **Total** | **93** | **6** | **56** | **19** | **12** |
|
||||||
|
|
||||||
|
### 7.2 Overall
|
||||||
|
|
||||||
|
| Category | Count | % of all 93 |
|
||||||
|
|----------|-------|-------------|
|
||||||
|
| Total Annex A controls | 93 | 100% |
|
||||||
|
| Not applicable (N/A) | 12 | 13% |
|
||||||
|
| **Applicable** | **81** | **87%** |
|
||||||
|
| Implemented (Yes) | 6 | 6% |
|
||||||
|
| Partial | 56 | 60% |
|
||||||
|
| Not implemented (No) | 19 | 20% |
|
||||||
|
|
||||||
|
*Applicable = Total − N/A. Yes/Partial/No counts apply only to applicable controls (6 + 56 + 19 = 81).*
|
||||||
|
|
||||||
|
### 7.3 Applicable Controls by Status
|
||||||
|
|
||||||
|
Percentages below are of **81 applicable** controls (excluding 12 N/A).
|
||||||
|
|
||||||
|
```
|
||||||
|
Implemented ██░░░░░░░░░░░░░░░░░░░░ 7% (6/81)
|
||||||
|
Partial █████████████████░░░░░ 69% (56/81)
|
||||||
|
Not impl. █████░░░░░░░░░░░░░░░░░ 23% (19/81)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Priority Remediation (Applicable "No" and Critical "Partial")
|
||||||
|
|
||||||
|
| Priority | Control(s) | Action | Target | Owner |
|
||||||
|
|----------|------------|--------|--------|-------|
|
||||||
|
| P0 | A.8.8 | Add govulncheck + dependency scanning to CI | Q2 2026 | Engineering |
|
||||||
|
| P0 | A.8.16 | Deploy centralized logging and alerting | Q3 2026 | DevOps |
|
||||||
|
| P0 | A.5.35, A.5.36 | Establish internal audit program | Q1 2027 | ISO |
|
||||||
|
| P1 | A.6.3 | Launch security awareness training | Q3 2026 | ISO |
|
||||||
|
| P1 | A.5.19–A.5.22 | Vendor security assessments | Q4 2026 | ISO |
|
||||||
|
| P1 | A.8.24 | Enable encryption at rest | Q3 2026 | DevOps |
|
||||||
|
| P1 | A.8.5 | Enable MFA for admin users | Q3 2026 | Engineering |
|
||||||
|
| P1 | A.8.29 | Add security testing to CI/CD | Q3 2026 | Engineering |
|
||||||
|
| P2 | A.5.5 | Document authority contacts (GDPR DPA) | Q3 2026 | DPO |
|
||||||
|
| P2 | A.8.7 | Malware scan on file uploads | Q4 2026 | Engineering |
|
||||||
|
| P2 | A.8.12 | Evaluate DLP for log pipeline | Q4 2026 | DevOps |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Exclusion Justifications
|
||||||
|
|
||||||
|
| Control(s) | Justification |
|
||||||
|
|------------|---------------|
|
||||||
|
| A.7.1–A.7.6, A.7.8, A.7.10–A.7.13 | TM infrastructure is fully cloud-hosted; physical controls are the responsibility of the cloud/IaaS provider per shared responsibility model. Evidence collected via A.5.23. |
|
||||||
|
| A.8.30 | All TM backend development is in-house; no outsourced development of scoped applications. |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial SoA for 93 Annex A controls |
|
||||||
|
| 1.1 | 2026-06-11 | Engineering | Reconciled summary counts with control rows; fixed A.8.30 applicability |
|
||||||
|
|
||||||
|
**Approval**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
|
| Executive Sponsor | _Pending_ | | |
|
||||||
|
|
||||||
|
**Next review:** Quarterly or upon significant scope/control change
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. Cross-Reference Index
|
||||||
|
|
||||||
|
| Document | Purpose |
|
||||||
|
|----------|---------|
|
||||||
|
| [GAP_ANALYSIS_REPORT.md](./GAP_ANALYSIS_REPORT.md) | Clause-level gap detail |
|
||||||
|
| [RISK_ASSESSMENT_MATRIX.md](./RISK_ASSESSMENT_MATRIX.md) | Risk IDs linked in SoA |
|
||||||
|
| [ISO27001_ROADMAP.md](./ISO27001_ROADMAP.md) | Remediation timeline |
|
||||||
|
| [policies/](./policies/) | Policy evidence for A.5.x controls |
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
# Acceptable Use Policy
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | POL-005 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Draft — Pending approval |
|
||||||
|
| **Owner** | Information Security Officer (ISO) |
|
||||||
|
| **Effective Date** | _Pending approval_ |
|
||||||
|
| **Review Cycle** | Annual |
|
||||||
|
| **ISO 27001 Reference** | A.6.2 — Terms and conditions of employment; A.5.10 — Acceptable use |
|
||||||
|
|
||||||
|
Related: [Information Security Policy](./information-security-policy.md) | [Access Control Policy](./access-control-policy.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This policy defines acceptable and unacceptable use of Tender Management System resources, data, and infrastructure by employees, contractors, and third parties.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Scope
|
||||||
|
|
||||||
|
Applies to anyone who:
|
||||||
|
|
||||||
|
- Accesses TM production, staging, or development environments
|
||||||
|
- Handles TM source code, configuration, or customer/company data
|
||||||
|
- Uses company-issued or personal devices to perform TM-related work
|
||||||
|
- Integrates third-party services with TM APIs
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Acceptable Use
|
||||||
|
|
||||||
|
Personnel **may**:
|
||||||
|
|
||||||
|
- Access systems and data required for their assigned role
|
||||||
|
- Use company-approved tools for development, testing, and deployment
|
||||||
|
- Store TM source code only in authorized Git repositories
|
||||||
|
- Use staging environments for testing with synthetic or anonymized data
|
||||||
|
- Report security concerns or suspected incidents without retaliation
|
||||||
|
- Work remotely using VPN/bastion access as configured by DevOps
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Unacceptable Use
|
||||||
|
|
||||||
|
Personnel **must not**:
|
||||||
|
|
||||||
|
### 4.1 Data Handling
|
||||||
|
|
||||||
|
- Copy production PII to personal devices, unauthorized cloud storage, or local unencrypted files
|
||||||
|
- Share customer or company data with unauthorized persons
|
||||||
|
- Use production data for development/testing without anonymization approval
|
||||||
|
- Export bulk data without business justification and ISO approval
|
||||||
|
|
||||||
|
### 4.2 Credentials & Access
|
||||||
|
|
||||||
|
- Share passwords, API keys, JWT tokens, or SSH keys
|
||||||
|
- Commit secrets, credentials, or `.env` files with production values to Git
|
||||||
|
- Use personal accounts for production system access
|
||||||
|
- Bypass authentication, authorization, or audit controls
|
||||||
|
- Leave privileged sessions unattended
|
||||||
|
|
||||||
|
### 4.3 Systems & Network
|
||||||
|
|
||||||
|
- Install unauthorized software on production or staging servers
|
||||||
|
- Run penetration tests against production without written ISO approval
|
||||||
|
- Introduce malware, unauthorized scripts, or unvetted dependencies
|
||||||
|
- Disable security controls (logging, rate limiting, firewalls) without approval
|
||||||
|
- Mine cryptocurrency or use TM infrastructure for non-business purposes
|
||||||
|
|
||||||
|
### 4.4 Development Practices
|
||||||
|
|
||||||
|
- Push directly to production branches without peer review
|
||||||
|
- Deploy untested code to production
|
||||||
|
- Ignore critical security scan findings without documented risk acceptance
|
||||||
|
- Log passwords, tokens, or full PII in application logs
|
||||||
|
|
||||||
|
### 4.5 Third Parties
|
||||||
|
|
||||||
|
- Grant vendor access beyond minimum required scope
|
||||||
|
- Share API credentials with unauthorized integrators
|
||||||
|
- Use unapproved AI/LLM tools to process production customer data without DPO review
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Device and Remote Work Requirements
|
||||||
|
|
||||||
|
| Requirement | Detail |
|
||||||
|
|-------------|--------|
|
||||||
|
| Screen lock | Enabled when unattended (≤ 5 min) |
|
||||||
|
| Full-disk encryption | Required on devices accessing production |
|
||||||
|
| OS updates | Security patches applied within 30 days |
|
||||||
|
| Antivirus | Company-approved endpoint protection where applicable |
|
||||||
|
| Public Wi-Fi | VPN required for production access |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Email and Communication
|
||||||
|
|
||||||
|
- Do not transmit passwords or API keys via email or chat
|
||||||
|
- Use approved secure channels for credential delivery
|
||||||
|
- Report phishing attempts to ISO immediately
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Monitoring and Privacy
|
||||||
|
|
||||||
|
The organization reserves the right to monitor use of TM systems and company equipment to:
|
||||||
|
|
||||||
|
- Detect security incidents
|
||||||
|
- Ensure policy compliance
|
||||||
|
- Protect information assets
|
||||||
|
|
||||||
|
Monitoring is conducted in accordance with applicable privacy laws and employment agreements.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Consequences
|
||||||
|
|
||||||
|
Violations may result in:
|
||||||
|
|
||||||
|
| Severity | Consequence |
|
||||||
|
|----------|-------------|
|
||||||
|
| Minor / first offense | Written warning; mandatory retraining |
|
||||||
|
| Moderate | Suspension of access; formal disciplinary action |
|
||||||
|
| Severe (data breach, intentional misuse) | Termination; legal action; regulatory reporting |
|
||||||
|
|
||||||
|
All violations involving data exposure follow the [Incident Response Plan](./incident-response-plan.md).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Acknowledgment
|
||||||
|
|
||||||
|
All personnel must sign or electronically acknowledge this policy:
|
||||||
|
|
||||||
|
- Upon onboarding (before production access is granted)
|
||||||
|
- Annually thereafter
|
||||||
|
- When materially updated
|
||||||
|
|
||||||
|
Records retained by HR/ISO for **3 years**.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial policy |
|
||||||
|
|
||||||
|
**Approval**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
|
| Executive Sponsor | _Pending_ | | |
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
# Access Control Policy
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | POL-002 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Draft — Pending approval |
|
||||||
|
| **Owner** | Information Security Officer (ISO) |
|
||||||
|
| **Effective Date** | _Pending approval_ |
|
||||||
|
| **Review Cycle** | Annual |
|
||||||
|
| **ISO 27001 Reference** | A.5.15–A.5.18 — Access control |
|
||||||
|
|
||||||
|
Related: [Information Security Policy](./information-security-policy.md) | [ISMS Foundation §4](../ISMS_FOUNDATION.md#4-information-asset-inventory)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This policy defines requirements for granting, reviewing, modifying, and revoking access to Tender Management System resources, ensuring that only authorized individuals and services can access information based on business need.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Scope
|
||||||
|
|
||||||
|
Applies to access for:
|
||||||
|
|
||||||
|
- **Admin users** — internal operators of the admin panel (`/admin/v1`)
|
||||||
|
- **Customer users** — company representatives using the mobile/public API (`/api/v1`)
|
||||||
|
- **Service accounts** — worker, scraper, CI/CD, and integration credentials
|
||||||
|
- **Infrastructure access** — MongoDB, Redis, MinIO, deployment environments
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Access Control Principles
|
||||||
|
|
||||||
|
1. **Least privilege** — minimum access required for the role
|
||||||
|
2. **Need-to-know** — access limited to data required for the task
|
||||||
|
3. **Separation of duties** — no single person holds unrestricted production access without oversight
|
||||||
|
4. **Default deny** — access is denied unless explicitly granted
|
||||||
|
5. **Accountability** — all access is attributable to an individual or service identity
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. User Access Management
|
||||||
|
|
||||||
|
### 4.1 Account Provisioning
|
||||||
|
|
||||||
|
| Step | Requirement |
|
||||||
|
|------|-------------|
|
||||||
|
| Request | Access request submitted by manager with role justification |
|
||||||
|
| Approval | ISO or Engineering Lead approves based on role matrix |
|
||||||
|
| Creation | Account created with default-deny; only required permissions assigned |
|
||||||
|
| Notification | User receives credentials via secure channel (never email plaintext passwords) |
|
||||||
|
| Recording | Access grant logged in access register |
|
||||||
|
|
||||||
|
### 4.2 Role Matrix — Application Users
|
||||||
|
|
||||||
|
| Role | System | Permissions |
|
||||||
|
|------|--------|-------------|
|
||||||
|
| **Admin** | Admin panel API | Full CRUD on users, companies, tenders, CMS, notifications |
|
||||||
|
| **Customer Admin** | Public API | Manage company profile, customers within company, view matched tenders |
|
||||||
|
| **Customer Analyst** | Public API | View tenders, submit feedback; no user management |
|
||||||
|
| **Service: Web API** | MongoDB, Redis, MinIO | Read/write per domain; no direct admin DB access |
|
||||||
|
| **Service: Worker** | MongoDB, MinIO, AI API | Read/write tenders; invoke translation jobs |
|
||||||
|
| **Service: Scraper** | MongoDB, TED source | Write tenders/notices only |
|
||||||
|
|
||||||
|
Technical enforcement: JWT role claims and middleware in `pkg/authorization`; company-scoped access for customer roles.
|
||||||
|
|
||||||
|
### 4.3 Role Matrix — Infrastructure
|
||||||
|
|
||||||
|
| Role | MongoDB | Redis | MinIO | Production Deploy |
|
||||||
|
|------|---------|-------|-------|-------------------|
|
||||||
|
| Developer | Staging read/write | Staging | Staging | No |
|
||||||
|
| DevOps | Prod read (break-glass write) | Prod | Prod | Yes (with approval) |
|
||||||
|
| ISO | Audit read-only | No | No | No |
|
||||||
|
|
||||||
|
Production database write access requires break-glass procedure (§7).
|
||||||
|
|
||||||
|
### 4.4 Account Modification
|
||||||
|
|
||||||
|
- Role changes require re-approval by manager and ISO
|
||||||
|
- Privilege elevation is temporary where possible (time-limited tokens)
|
||||||
|
|
||||||
|
### 4.5 Account Deprovisioning
|
||||||
|
|
||||||
|
Access must be revoked **within 24 hours** of:
|
||||||
|
|
||||||
|
- Employment or contract termination
|
||||||
|
- Role change removing need for access
|
||||||
|
- Extended leave (>30 days) for privileged accounts
|
||||||
|
|
||||||
|
Checklist: disable admin user account, revoke JWT/refresh tokens (Redis blacklist), remove SSH/infra access, rotate shared secrets if exposed.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Authentication Requirements
|
||||||
|
|
||||||
|
### 5.1 Human Users
|
||||||
|
|
||||||
|
| Requirement | Admin Users | Customer Users |
|
||||||
|
|-------------|-------------|----------------|
|
||||||
|
| Unique account | Mandatory | Mandatory |
|
||||||
|
| Password complexity | Min 12 chars; mixed case, number, symbol | Min 8 chars; mixed case, number |
|
||||||
|
| Password storage | bcrypt hash (never plaintext) | bcrypt hash (never plaintext) |
|
||||||
|
| MFA | Required (target: Q3 2026) | Recommended |
|
||||||
|
| Session/token TTL | Access token ≤ 15 min; refresh rotation | Access token ≤ 1 h; refresh rotation |
|
||||||
|
| Failed login lockout | 5 attempts / 15 min lockout | 5 attempts / 15 min lockout |
|
||||||
|
| hCaptcha | On registration and password reset | On registration and password reset |
|
||||||
|
|
||||||
|
### 5.2 Service Accounts
|
||||||
|
|
||||||
|
- Unique credentials per service and environment
|
||||||
|
- Stored in secret manager (not `.env` in production)
|
||||||
|
- Rotated at least annually or on personnel change
|
||||||
|
- No shared passwords between services
|
||||||
|
|
||||||
|
### 5.3 API Authentication
|
||||||
|
|
||||||
|
- All protected endpoints require valid JWT in `Authorization: Bearer` header
|
||||||
|
- Public endpoints limited to explicitly whitelisted routes (health, public tender list, contact form with hCaptcha)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Access Reviews
|
||||||
|
|
||||||
|
| Review Type | Frequency | Reviewer | Scope |
|
||||||
|
|-------------|-----------|----------|-------|
|
||||||
|
| Admin user accounts | Quarterly | ISO + Engineering Lead | All active admin users |
|
||||||
|
| Customer admin accounts | Semi-annual | Product Owner | Accounts inactive >90 days |
|
||||||
|
| Infrastructure access | Quarterly | DevOps Lead | SSH, DB, MinIO, CI/CD |
|
||||||
|
| Service account credentials | Semi-annual | DevOps Lead | All non-human identities |
|
||||||
|
|
||||||
|
Findings are documented and remediated within 30 days.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Break-Glass Access
|
||||||
|
|
||||||
|
Emergency production access when normal procedures cannot meet operational need:
|
||||||
|
|
||||||
|
1. Request logged in incident/ticket system with justification
|
||||||
|
2. Approved by DevOps Lead or ISO (second approver if requester is DevOps Lead)
|
||||||
|
3. Time-limited (maximum 4 hours)
|
||||||
|
4. All actions logged and reviewed within 48 hours post-event
|
||||||
|
5. Credentials rotated if break-glass credentials were used
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Remote Access
|
||||||
|
|
||||||
|
- Production infrastructure accessible only via VPN or bastion host
|
||||||
|
- No direct public exposure of MongoDB, Redis, or MinIO ports
|
||||||
|
- Admin panel and API served over HTTPS only in production
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Violations
|
||||||
|
|
||||||
|
Unauthorized access attempts, credential sharing, or bypassing access controls must be reported immediately per the [Incident Response Plan](./incident-response-plan.md).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial policy |
|
||||||
|
|
||||||
|
**Approval**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
|
| Executive Sponsor | _Pending_ | | |
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
# Backup & Recovery Policy
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | POL-004 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Draft — Pending approval |
|
||||||
|
| **Owner** | DevOps Lead |
|
||||||
|
| **Maintained By** | DevOps Lead |
|
||||||
|
| **Effective Date** | _Pending approval_ |
|
||||||
|
| **Review Cycle** | Annual |
|
||||||
|
| **ISO 27001 Reference** | A.8.13 — Information backup |
|
||||||
|
|
||||||
|
Related: [Information Security Policy](./information-security-policy.md) | [Incident Response Plan](./incident-response-plan.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This policy defines backup requirements, retention periods, and recovery objectives for Tender Management System data to ensure business continuity and data protection.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Scope
|
||||||
|
|
||||||
|
Covers backup and recovery for:
|
||||||
|
|
||||||
|
| Asset | ID | Backup Method |
|
||||||
|
|-------|-----|---------------|
|
||||||
|
| MongoDB (`tm` database) | A-001 | Automated snapshots / mongodump |
|
||||||
|
| MinIO object storage | A-003 | Bucket replication or periodic sync |
|
||||||
|
| Redis | A-002 | RDB snapshots (cache — lower priority) |
|
||||||
|
| Application configuration | A-010 | Version-controlled templates; secrets in vault |
|
||||||
|
| Application logs | A-011 | Log aggregator retention |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Recovery Objectives
|
||||||
|
|
||||||
|
| Metric | Definition | Target |
|
||||||
|
|--------|------------|--------|
|
||||||
|
| **RTO** (Recovery Time Objective) | Maximum acceptable downtime | **4 hours** (production API) |
|
||||||
|
| **RPO** (Recovery Point Objective) | Maximum acceptable data loss | **1 hour** (MongoDB); **24 hours** (MinIO documents) |
|
||||||
|
|
||||||
|
RTO/RPO are reviewed annually and after major architecture changes.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Backup Requirements
|
||||||
|
|
||||||
|
### 4.1 MongoDB
|
||||||
|
|
||||||
|
| Parameter | Requirement |
|
||||||
|
|-----------|-------------|
|
||||||
|
| Frequency | Continuous oplog / hourly snapshots (production) |
|
||||||
|
| Retention | Daily: 30 days; Weekly: 12 weeks; Monthly: 12 months |
|
||||||
|
| Storage | Separate region/account from production |
|
||||||
|
| Encryption | Encrypted at rest (AES-256 or provider equivalent) |
|
||||||
|
| Access | DevOps Lead + break-glass only |
|
||||||
|
|
||||||
|
**Collections in scope:** `companies`, `customers`, `users`, `tenders`, `notices`, `feedback`, `notifications`, and all domain collections.
|
||||||
|
|
||||||
|
### 4.2 MinIO
|
||||||
|
|
||||||
|
| Parameter | Requirement |
|
||||||
|
|-----------|-------------|
|
||||||
|
| Frequency | Daily incremental; weekly full sync |
|
||||||
|
| Retention | 30 days rolling |
|
||||||
|
| Scope | `files` and `documents` buckets (company docs, tender JSON, translations) |
|
||||||
|
| Encryption | Server-side encryption enabled |
|
||||||
|
| Versioning | Bucket versioning enabled where supported |
|
||||||
|
|
||||||
|
### 4.3 Redis
|
||||||
|
|
||||||
|
| Parameter | Requirement |
|
||||||
|
|-----------|-------------|
|
||||||
|
| Frequency | Daily RDB snapshot |
|
||||||
|
| Retention | 7 days |
|
||||||
|
| Note | Redis holds ephemeral cache/session data; rebuild acceptable within RTO |
|
||||||
|
|
||||||
|
### 4.4 Configuration & Secrets
|
||||||
|
|
||||||
|
- Infrastructure-as-code and config templates in Git
|
||||||
|
- Production secrets in secret manager (not backed up in plaintext)
|
||||||
|
- Secret manager backup per provider documentation
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Backup Security
|
||||||
|
|
||||||
|
1. Backups classified **C3 Confidential** — same protection as source data
|
||||||
|
2. Backup storage not publicly accessible
|
||||||
|
3. Backup access logged and restricted to authorized DevOps personnel
|
||||||
|
4. Backup encryption keys managed separately from backup data
|
||||||
|
5. No backup data transferred to unauthorized regions (GDPR data residency compliance)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Recovery Procedures
|
||||||
|
|
||||||
|
### 6.1 MongoDB Full Restore
|
||||||
|
|
||||||
|
1. Identify target recovery point (timestamp before incident)
|
||||||
|
2. Provision clean MongoDB instance or restore to staging first
|
||||||
|
3. Restore from snapshot / mongodump
|
||||||
|
4. Verify document counts and sample integrity checks
|
||||||
|
5. Update application connection string
|
||||||
|
6. Run application smoke tests (auth, tender list, company profile)
|
||||||
|
7. Monitor for 24 hours post-restore
|
||||||
|
|
||||||
|
### 6.2 MinIO Restore
|
||||||
|
|
||||||
|
1. Identify affected buckets/prefixes
|
||||||
|
2. Restore from backup or cross-region replica
|
||||||
|
3. Verify file accessibility via file store service
|
||||||
|
4. Re-run worker jobs for any missing derived data (translations)
|
||||||
|
|
||||||
|
### 6.3 Partial Restore (Single Collection / Document)
|
||||||
|
|
||||||
|
- Use point-in-time recovery or selective mongorestore
|
||||||
|
- Requires IC approval for production partial restores
|
||||||
|
- Audit log entry required
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Backup Testing
|
||||||
|
|
||||||
|
| Test | Frequency | Success Criteria |
|
||||||
|
|------|-----------|------------------|
|
||||||
|
| MongoDB restore to staging | **Monthly** | Data integrity verified; app connects successfully |
|
||||||
|
| MinIO file restore (sample) | **Quarterly** | Random files readable and checksum-valid |
|
||||||
|
| Full DR simulation | **Annual** | RTO/RPO met in tabletop or live test |
|
||||||
|
| Backup job monitoring | **Daily** | All scheduled backups complete without error |
|
||||||
|
|
||||||
|
Test results documented with date, tester, outcome, and remediation items.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Responsibilities
|
||||||
|
|
||||||
|
| Role | Responsibility |
|
||||||
|
|------|----------------|
|
||||||
|
| DevOps Lead | Implement and monitor backups; conduct restore tests |
|
||||||
|
| ISO | Verify policy compliance; include backup status in management review |
|
||||||
|
| Engineering Lead | Validate application behavior post-restore |
|
||||||
|
| IC (during incident) | Authorize production restore; communicate status |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Failure Handling
|
||||||
|
|
||||||
|
If a scheduled backup fails:
|
||||||
|
|
||||||
|
1. Automated alert to DevOps on-call
|
||||||
|
2. Investigate and re-run within **4 hours**
|
||||||
|
3. If unrecoverable, escalate to P2 incident if production data at risk
|
||||||
|
4. Document failure and resolution in backup log
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial policy |
|
||||||
|
|
||||||
|
**Approval**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| DevOps Lead | _Pending_ | | |
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
# Incident Response Plan
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | POL-003 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Draft — Pending approval |
|
||||||
|
| **Owner** | Information Security Officer (ISO) |
|
||||||
|
| **Effective Date** | _Pending approval_ |
|
||||||
|
| **Review Cycle** | Annual (+ after every P1/P2 incident) |
|
||||||
|
| **ISO 27001 Reference** | A.5.24–A.5.28 — Incident management |
|
||||||
|
|
||||||
|
Related: [Information Security Policy](./information-security-policy.md) | [Risk Assessment Matrix](../RISK_ASSESSMENT_MATRIX.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This plan defines how the organization detects, responds to, contains, and recovers from information security incidents affecting the Tender Management System. It ensures consistent handling, timely notification, and post-incident improvement.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Scope
|
||||||
|
|
||||||
|
Covers security incidents involving:
|
||||||
|
|
||||||
|
- Unauthorized access to TM systems or data
|
||||||
|
- Data breaches (PII exposure)
|
||||||
|
- Malware or ransomware
|
||||||
|
- Denial of service affecting production
|
||||||
|
- Secret/credential leakage
|
||||||
|
- Insider threats
|
||||||
|
- Third-party service compromises affecting TM
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Incident Classification
|
||||||
|
|
||||||
|
| Severity | Label | Criteria | Response Time |
|
||||||
|
|----------|-------|----------|---------------|
|
||||||
|
| **P1** | Critical | Active data breach; production down; credential leak in public repo | Acknowledge ≤ 1 h; escalate immediately |
|
||||||
|
| **P2** | High | Suspected breach; partial outage; unpatched critical CVE exploited | Acknowledge ≤ 4 h |
|
||||||
|
| **P3** | Medium | Failed attack attempt; non-production compromise; policy violation | Acknowledge ≤ 1 business day |
|
||||||
|
| **P4** | Low | Near-miss; informational alert; minor misconfiguration | Acknowledge ≤ 3 business days |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Incident Response Team (IRT)
|
||||||
|
|
||||||
|
| Role | Responsibility | Primary | Backup |
|
||||||
|
|------|----------------|---------|--------|
|
||||||
|
| **Incident Commander (IC)** | Overall coordination, decisions, communications | ISO | Engineering Lead |
|
||||||
|
| **Technical Lead** | Investigation, containment, remediation | DevOps Lead | Senior Backend Dev |
|
||||||
|
| **Communications Lead** | Internal/external notifications, status updates | Product Owner | Executive Sponsor |
|
||||||
|
| **Legal / DPO** | Regulatory notification, GDPR assessment | DPO | External counsel |
|
||||||
|
| **Scribe** | Timeline, evidence, post-incident report | Assigned per incident | — |
|
||||||
|
|
||||||
|
Contact list maintained separately (not in repository) with 24/7 reachability for P1/P2.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Response Phases
|
||||||
|
|
||||||
|
```
|
||||||
|
Detect → Triage → Contain → Eradicate → Recover → Learn
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.1 Detect & Report
|
||||||
|
|
||||||
|
**Anyone** who suspects an incident must report immediately via:
|
||||||
|
|
||||||
|
1. Direct message to ISO or DevOps Lead
|
||||||
|
2. Dedicated security channel (e.g. `#security-incidents`)
|
||||||
|
3. Email: `security@<company-domain>` (to be configured)
|
||||||
|
|
||||||
|
Do **not** discuss suspected breaches on public channels until IC approves.
|
||||||
|
|
||||||
|
**Automated detection sources (target state):**
|
||||||
|
|
||||||
|
- SIEM alerts (auth failure spikes, error rate anomalies)
|
||||||
|
- Secret scanning in CI
|
||||||
|
- Dependency vulnerability alerts
|
||||||
|
- Cloud provider security notifications
|
||||||
|
- User/customer reports
|
||||||
|
|
||||||
|
### 5.2 Triage
|
||||||
|
|
||||||
|
IC performs within response time SLA:
|
||||||
|
|
||||||
|
1. Confirm or rule out incident
|
||||||
|
2. Assign severity (P1–P4)
|
||||||
|
3. Activate IRT if P1/P2
|
||||||
|
4. Open incident ticket with unique ID: `INC-YYYY-NNN`
|
||||||
|
5. Begin incident timeline log
|
||||||
|
|
||||||
|
### 5.3 Contain
|
||||||
|
|
||||||
|
Short-term actions to limit damage:
|
||||||
|
|
||||||
|
| Scenario | Containment Actions |
|
||||||
|
|----------|---------------------|
|
||||||
|
| Compromised admin account | Disable account; revoke JWT; force password reset |
|
||||||
|
| Leaked API key / secret | Rotate credential; invalidate old tokens; audit access logs |
|
||||||
|
| Active data exfiltration | Block IP/rate limit; disable affected endpoint; snapshot logs |
|
||||||
|
| Ransomware / malware | Isolate affected hosts; preserve forensic images |
|
||||||
|
| DDoS | Enable WAF/CDN rules; rate limiting; contact provider |
|
||||||
|
|
||||||
|
Preserve evidence: do not delete logs; snapshot affected systems before remediation where feasible.
|
||||||
|
|
||||||
|
### 5.4 Eradicate
|
||||||
|
|
||||||
|
- Remove attacker access (patch vulnerability, close misconfiguration)
|
||||||
|
- Scan for persistence (backdoors, unauthorized accounts)
|
||||||
|
- Verify root cause is addressed
|
||||||
|
|
||||||
|
### 5.5 Recover
|
||||||
|
|
||||||
|
- Restore services from clean backups if needed
|
||||||
|
- Monitor for recurrence (enhanced logging, 72-hour watch period)
|
||||||
|
- Confirm normal operation with smoke tests
|
||||||
|
- Communicate all-clear to stakeholders
|
||||||
|
|
||||||
|
### 5.6 Post-Incident Review (Learn)
|
||||||
|
|
||||||
|
Required for all P1/P2 incidents within **10 business days**:
|
||||||
|
|
||||||
|
- Timeline reconstruction
|
||||||
|
- Root cause analysis
|
||||||
|
- Control gaps identified
|
||||||
|
- Corrective/preventive actions with owners and dates
|
||||||
|
- Update [Risk Assessment Matrix](../RISK_ASSESSMENT_MATRIX.md) if new risks identified
|
||||||
|
- Tabletop or walkthrough if process gaps found
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Communication Plan
|
||||||
|
|
||||||
|
### 6.1 Internal
|
||||||
|
|
||||||
|
| Severity | Notify |
|
||||||
|
|----------|--------|
|
||||||
|
| P1 | IRT, Executive Sponsor, all engineering — immediately |
|
||||||
|
| P2 | IRT, Engineering Lead, DevOps — within 4 h |
|
||||||
|
| P3 | ISO, relevant team lead — within 1 business day |
|
||||||
|
| P4 | ISO — log only |
|
||||||
|
|
||||||
|
### 6.2 External
|
||||||
|
|
||||||
|
| Condition | Action | Timeline |
|
||||||
|
|-----------|--------|----------|
|
||||||
|
| Personal data breach (GDPR) | Notify supervisory authority | ≤ 72 hours of awareness |
|
||||||
|
| Personal data breach (high risk to individuals) | Notify affected data subjects | Without undue delay |
|
||||||
|
| Contractual obligation | Notify affected customers | Per contract terms |
|
||||||
|
| Public disclosure | Press/ status page | IC approval only |
|
||||||
|
|
||||||
|
DPO assesses GDPR notification requirement for any incident involving customer/company PII (assets A-004, A-005).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Evidence Handling
|
||||||
|
|
||||||
|
- Logs exported and stored in tamper-evident location
|
||||||
|
- Chain of custody documented for forensic artifacts
|
||||||
|
- Incident records retained for **3 years** minimum
|
||||||
|
- Access to incident evidence restricted to IRT and Legal/DPO
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Playbooks
|
||||||
|
|
||||||
|
### 8.1 Credential Leak in Git Repository
|
||||||
|
|
||||||
|
1. **Contain:** Revoke/rotate leaked secret immediately
|
||||||
|
2. **Eradicate:** Remove secret from git history (`git filter-repo` or BFG); force push with approval
|
||||||
|
3. **Investigate:** Audit usage logs for unauthorized access during exposure window
|
||||||
|
4. **Recover:** Deploy rotated credentials; verify services operational
|
||||||
|
5. **Learn:** Enable secret scanning in CI; review commit practices
|
||||||
|
|
||||||
|
*Applies to risk R-013; FCM keys and MinIO credentials are high priority.*
|
||||||
|
|
||||||
|
### 8.2 Suspected Unauthorized Admin Access
|
||||||
|
|
||||||
|
1. Disable affected admin account(s)
|
||||||
|
2. Blacklist active JWTs in Redis
|
||||||
|
3. Review `pkg/audit` logs for anomalous actions
|
||||||
|
4. Force password reset and enable MFA
|
||||||
|
5. RBAC audit of actions performed during compromise window
|
||||||
|
|
||||||
|
### 8.3 Production API Outage (Security-Related)
|
||||||
|
|
||||||
|
1. Determine if attack-related (DoS) vs. infrastructure failure
|
||||||
|
2. If DoS: enable rate limiting, WAF rules, scale resources
|
||||||
|
3. If compromise: isolate, preserve logs, follow containment playbook
|
||||||
|
4. Status communication via Communications Lead
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Testing and Training
|
||||||
|
|
||||||
|
| Activity | Frequency |
|
||||||
|
|----------|-----------|
|
||||||
|
| Tabletop exercise (simulated P1) | Annual |
|
||||||
|
| Contact list verification | Quarterly |
|
||||||
|
| Playbook review | Annual or after P1/P2 |
|
||||||
|
| Security awareness (incident reporting) | Onboarding + annual |
|
||||||
|
|
||||||
|
First tabletop exercise target: **Q2 2026** (per [Risk R-023](../RISK_ASSESSMENT_MATRIX.md)).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial plan |
|
||||||
|
|
||||||
|
**Approval**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
|
| Executive Sponsor | _Pending_ | | |
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
# Information Security Policy
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | POL-001 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Draft — Pending approval |
|
||||||
|
| **Owner** | Executive Sponsor |
|
||||||
|
| **Maintained By** | Information Security Officer (ISO) |
|
||||||
|
| **Effective Date** | _Pending approval_ |
|
||||||
|
| **Review Cycle** | Annual |
|
||||||
|
| **ISO 27001 Reference** | A.5.1 — Policies for information security |
|
||||||
|
|
||||||
|
Related: [ISMS Foundation](../ISMS_FOUNDATION.md) | [ISO 27001 Roadmap](../ISO27001_ROADMAP.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This policy establishes the organization's commitment to protecting information assets processed by the **Tender Management (TM) System**. It defines principles, responsibilities, and requirements that all personnel, contractors, and third parties must follow.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Scope
|
||||||
|
|
||||||
|
This policy applies to:
|
||||||
|
|
||||||
|
- All employees, contractors, and third parties with access to TM systems or data
|
||||||
|
- The ISMS scope defined in [ISMS Foundation §3](../ISMS_FOUNDATION.md#3-isms-scope): `cmd/web`, `cmd/worker`, `cmd/scraper`, MongoDB, Redis, MinIO, and integrated services
|
||||||
|
- All information classified C2 (Internal) and above
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Policy Statement
|
||||||
|
|
||||||
|
Management is committed to:
|
||||||
|
|
||||||
|
1. **Protecting** the confidentiality, integrity, and availability of information assets
|
||||||
|
2. **Complying** with applicable legal and regulatory requirements, including GDPR
|
||||||
|
3. **Implementing** an ISMS aligned with ISO/IEC 27001:2022
|
||||||
|
4. **Managing** information security risks through a documented risk assessment process
|
||||||
|
5. **Continuously improving** security controls via the Plan-Do-Check-Act (PDCA) cycle
|
||||||
|
|
||||||
|
Information security is everyone's responsibility. No business objective justifies unacceptable risk to customer data or system integrity without documented risk acceptance by authorized management.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Information Security Objectives
|
||||||
|
|
||||||
|
| # | Objective | Measure | Target |
|
||||||
|
|---|-----------|---------|--------|
|
||||||
|
| O-1 | Protect customer and company PII | Data breach incidents | Zero confirmed breaches per year |
|
||||||
|
| O-2 | Maintain platform availability | Uptime of production API | ≥ 99.5% monthly |
|
||||||
|
| O-3 | Respond to security incidents promptly | P1 incident acknowledgment time | ≤ 1 hour |
|
||||||
|
| O-4 | Remediate critical vulnerabilities | Mean time to patch (Critical CVE) | ≤ 7 days |
|
||||||
|
| O-5 | Maintain security awareness | Training completion rate | 100% of in-scope personnel |
|
||||||
|
| O-6 | Progress toward ISO 27001 certification | Roadmap milestone completion | Per [ISO27001_ROADMAP.md](../ISO27001_ROADMAP.md) |
|
||||||
|
|
||||||
|
Objectives are reviewed quarterly during management review.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Roles and Responsibilities
|
||||||
|
|
||||||
|
| Role | Responsibilities |
|
||||||
|
|------|------------------|
|
||||||
|
| **Executive Sponsor** | Approves this policy; allocates resources; chairs management review |
|
||||||
|
| **Information Security Officer (ISO)** | Operates ISMS; maintains risk register; coordinates audits and incidents |
|
||||||
|
| **Data Protection Officer (DPO)** | Ensures GDPR compliance; oversees DPIAs and data subject requests |
|
||||||
|
| **Engineering Lead** | Secure SDLC; code review; vulnerability remediation |
|
||||||
|
| **DevOps Lead** | Infrastructure security; secrets management; backups and monitoring |
|
||||||
|
| **All personnel** | Comply with policies; report incidents; complete security training |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Core Security Principles
|
||||||
|
|
||||||
|
### 6.1 Confidentiality
|
||||||
|
|
||||||
|
- Access to information is granted on a need-to-know basis
|
||||||
|
- PII and credentials must not be disclosed to unauthorized parties
|
||||||
|
- Secrets must never be committed to source control or included in logs
|
||||||
|
|
||||||
|
### 6.2 Integrity
|
||||||
|
|
||||||
|
- Production changes require peer review and follow change management procedures
|
||||||
|
- Data modifications must be traceable through audit logs where applicable
|
||||||
|
- Input validation is mandatory at all API boundaries
|
||||||
|
|
||||||
|
### 6.3 Availability
|
||||||
|
|
||||||
|
- Critical services (API, database, object storage) must have documented recovery procedures
|
||||||
|
- Capacity and rate limiting must protect against abuse-related outages
|
||||||
|
|
||||||
|
### 6.4 Defense in Depth
|
||||||
|
|
||||||
|
Security controls are layered across application, infrastructure, and process levels. No single control is relied upon exclusively.
|
||||||
|
|
||||||
|
### 6.5 Least Privilege
|
||||||
|
|
||||||
|
Users, services, and processes receive the minimum access required to perform their function.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Supporting Policies and Procedures
|
||||||
|
|
||||||
|
This policy is supported by:
|
||||||
|
|
||||||
|
| Document | ID |
|
||||||
|
|----------|-----|
|
||||||
|
| [Access Control Policy](./access-control-policy.md) | POL-002 |
|
||||||
|
| [Incident Response Plan](./incident-response-plan.md) | POL-003 |
|
||||||
|
| [Backup & Recovery Policy](./backup-and-recovery-policy.md) | POL-004 |
|
||||||
|
| [Acceptable Use Policy](./acceptable-use-policy.md) | POL-005 |
|
||||||
|
| [Risk Assessment Procedure](../procedures/risk-assessment-procedure.md) | PROC-001 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Compliance and Enforcement
|
||||||
|
|
||||||
|
- Violations of this policy may result in disciplinary action, up to and including termination of employment or contract
|
||||||
|
- Regulatory breaches may result in legal liability and notification to supervisory authorities
|
||||||
|
- All personnel must acknowledge this policy upon onboarding and annually thereafter
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Exceptions
|
||||||
|
|
||||||
|
Exceptions to this policy require:
|
||||||
|
|
||||||
|
1. Written request with business justification
|
||||||
|
2. Risk assessment of the exception
|
||||||
|
3. Approval by ISO and Executive Sponsor
|
||||||
|
4. Time-bound validity (maximum 12 months)
|
||||||
|
5. Entry in the risk acceptance log ([Risk Assessment Matrix §5](../RISK_ASSESSMENT_MATRIX.md#5-risk-acceptance-log))
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 10. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial policy |
|
||||||
|
|
||||||
|
**Approval**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| Executive Sponsor | _Pending_ | | |
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 11. Distribution
|
||||||
|
|
||||||
|
This policy is available to all in-scope personnel via the `docs/security/policies/` repository path and must be communicated during onboarding.
|
||||||
@@ -0,0 +1,177 @@
|
|||||||
|
# Risk Assessment Procedure
|
||||||
|
|
||||||
|
| Field | Value |
|
||||||
|
|-------|-------|
|
||||||
|
| **Document ID** | PROC-001 |
|
||||||
|
| **Version** | 1.0 |
|
||||||
|
| **Status** | Draft — Pending approval |
|
||||||
|
| **Owner** | Information Security Officer (ISO) |
|
||||||
|
| **Effective Date** | _Pending approval_ |
|
||||||
|
| **Review Cycle** | Annual |
|
||||||
|
| **ISO 27001 Reference** | Clause 6.1.2 — Information security risk assessment |
|
||||||
|
|
||||||
|
Related: [Risk Assessment Matrix](../RISK_ASSESSMENT_MATRIX.md) | [Information Security Policy](../policies/information-security-policy.md)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Purpose
|
||||||
|
|
||||||
|
This procedure defines how information security risks are identified, analyzed, evaluated, treated, and reviewed for the Tender Management ISMS.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Scope
|
||||||
|
|
||||||
|
Applies to all assets within the ISMS scope ([ISMS Foundation §3](../ISMS_FOUNDATION.md#3-isms-scope)) and all personnel involved in risk management activities.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Roles
|
||||||
|
|
||||||
|
| Role | Responsibility |
|
||||||
|
|------|----------------|
|
||||||
|
| **ISO** | Facilitates assessment; maintains risk register; reports to management |
|
||||||
|
| **Asset Owners** | Provide context on threats and controls for their assets |
|
||||||
|
| **Engineering / DevOps Leads** | Identify technical vulnerabilities and treatment options |
|
||||||
|
| **Executive Sponsor** | Accepts residual risks above threshold |
|
||||||
|
| **DPO** | Assesses privacy impact for PII-related risks |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Risk Assessment Process
|
||||||
|
|
||||||
|
```
|
||||||
|
Context → Identify → Analyze → Evaluate → Treat → Monitor → Review
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.1 Establish Context
|
||||||
|
|
||||||
|
Before each assessment cycle, confirm:
|
||||||
|
|
||||||
|
- Current ISMS scope and asset inventory ([ISMS Foundation §4](../ISMS_FOUNDATION.md#4-information-asset-inventory))
|
||||||
|
- Applicable legal/regulatory requirements (GDPR, contracts)
|
||||||
|
- Risk criteria (scales defined in [Risk Assessment Matrix §2](../RISK_ASSESSMENT_MATRIX.md#2-methodology))
|
||||||
|
|
||||||
|
### 4.2 Risk Identification
|
||||||
|
|
||||||
|
Sources of risk information:
|
||||||
|
|
||||||
|
| Source | Examples |
|
||||||
|
|--------|----------|
|
||||||
|
| Asset-based | Unauthorized access to MongoDB, MinIO data loss |
|
||||||
|
| Threat-based | Credential stuffing, supply chain CVE, insider threat |
|
||||||
|
| Vulnerability scans | govulncheck, container scans, penetration test findings |
|
||||||
|
| Incidents | Post-incident review findings |
|
||||||
|
| Audit findings | Internal/external audit nonconformities |
|
||||||
|
| Change requests | New AI integration, new data store, architecture change |
|
||||||
|
|
||||||
|
Each risk receives a unique ID: `R-NNN` (sequential in risk register).
|
||||||
|
|
||||||
|
### 4.3 Risk Analysis
|
||||||
|
|
||||||
|
For each identified risk, document:
|
||||||
|
|
||||||
|
| Field | Description |
|
||||||
|
|-------|-------------|
|
||||||
|
| Risk ID | Unique identifier |
|
||||||
|
| Threat | What could happen |
|
||||||
|
| Vulnerability | Weakness exploited |
|
||||||
|
| Affected asset(s) | Asset IDs from inventory |
|
||||||
|
| Likelihood (1–5) | Per scale in risk matrix |
|
||||||
|
| Impact (1–5) | Per scale in risk matrix |
|
||||||
|
| Inherent score | L × I before controls |
|
||||||
|
| Existing controls | Current mitigations |
|
||||||
|
| Residual L / I / Score | After existing controls |
|
||||||
|
|
||||||
|
### 4.4 Risk Evaluation
|
||||||
|
|
||||||
|
Compare residual score against acceptance criteria:
|
||||||
|
|
||||||
|
| Score | Level | Decision |
|
||||||
|
|-------|-------|----------|
|
||||||
|
| 1–4 | Low | Accept with monitoring |
|
||||||
|
| 5–9 | Medium | Treat within 6 months |
|
||||||
|
| 10–15 | High | Treat within 3 months |
|
||||||
|
| 16–25 | Critical | Immediate treatment; no acceptance without Executive approval |
|
||||||
|
|
||||||
|
Risks above **Medium** must have a documented treatment plan before acceptance.
|
||||||
|
|
||||||
|
### 4.5 Risk Treatment
|
||||||
|
|
||||||
|
Select one or more options:
|
||||||
|
|
||||||
|
- **Mitigate** — Implement control (preferred)
|
||||||
|
- **Transfer** — Insurance, vendor contract
|
||||||
|
- **Avoid** — Remove activity from scope
|
||||||
|
- **Accept** — Document in risk acceptance log with approver signature
|
||||||
|
|
||||||
|
Treatment plan must include: owner, target date, actions, and expected residual score.
|
||||||
|
|
||||||
|
### 4.6 Monitor and Review
|
||||||
|
|
||||||
|
- Track treatment plan progress in risk register
|
||||||
|
- Re-score risks when controls are implemented
|
||||||
|
- Close risks only when residual score is at or below accepted level
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Assessment Triggers
|
||||||
|
|
||||||
|
| Trigger | Action |
|
||||||
|
|---------|--------|
|
||||||
|
| **Scheduled review** | Full register review quarterly |
|
||||||
|
| **Major change** | Assess new/changed assets before production deployment |
|
||||||
|
| **Incident (P1/P2)** | Ad-hoc assessment within 10 business days |
|
||||||
|
| **New vulnerability (Critical)** | Assess within 48 hours |
|
||||||
|
| **Audit finding** | Add/update risk within 5 business days |
|
||||||
|
| **Annual comprehensive** | Full methodology re-validation |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Major Change Assessment Checklist
|
||||||
|
|
||||||
|
Use before deploying significant changes (new service, data store, third-party integration):
|
||||||
|
|
||||||
|
- [ ] New assets added to inventory?
|
||||||
|
- [ ] Data classification determined?
|
||||||
|
- [ ] Threats and vulnerabilities identified?
|
||||||
|
- [ ] Controls designed before go-live?
|
||||||
|
- [ ] DPO consulted if PII involved?
|
||||||
|
- [ ] Residual risk acceptable or treatment plan in place?
|
||||||
|
- [ ] Risk register updated?
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Reporting
|
||||||
|
|
||||||
|
| Report | Audience | Frequency |
|
||||||
|
|--------|----------|-----------|
|
||||||
|
| Risk register (current) | ISO, Engineering/DevOps leads | Continuous |
|
||||||
|
| Risk summary dashboard | Management review | Quarterly |
|
||||||
|
| Critical/High open risks | Executive Sponsor | Immediate + monthly |
|
||||||
|
| Treatment plan status | ISO | Monthly |
|
||||||
|
|
||||||
|
Report format: [Risk Assessment Matrix](../RISK_ASSESSMENT_MATRIX.md) sections 3–4.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. Records Retention
|
||||||
|
|
||||||
|
- Risk register versions: **3 years**
|
||||||
|
- Risk acceptance decisions: **3 years**
|
||||||
|
- Assessment meeting notes: **3 years**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. Document Control
|
||||||
|
|
||||||
|
| Version | Date | Author | Changes |
|
||||||
|
|---------|------|--------|---------|
|
||||||
|
| 1.0 | 2026-06-11 | Engineering | Initial procedure |
|
||||||
|
|
||||||
|
**Approval**
|
||||||
|
|
||||||
|
| Role | Name | Signature | Date |
|
||||||
|
|------|------|-----------|------|
|
||||||
|
| ISO | _Pending_ | | |
|
||||||
|
| Executive Sponsor | _Pending_ | | |
|
||||||
@@ -5,6 +5,7 @@ go 1.24
|
|||||||
toolchain go1.24.4
|
toolchain go1.24.4
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/elastic/go-elasticsearch/v8 v8.17.1
|
||||||
github.com/golang-jwt/jwt/v5 v5.3.0
|
github.com/golang-jwt/jwt/v5 v5.3.0
|
||||||
github.com/labstack/echo/v4 v4.13.4
|
github.com/labstack/echo/v4 v4.13.4
|
||||||
github.com/microcosm-cc/bluemonday v1.0.27
|
github.com/microcosm-cc/bluemonday v1.0.27
|
||||||
@@ -28,8 +29,11 @@ require (
|
|||||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||||
|
github.com/elastic/elastic-transport-go/v8 v8.6.1 // indirect
|
||||||
github.com/ghodss/yaml v1.0.0 // indirect
|
github.com/ghodss/yaml v1.0.0 // indirect
|
||||||
github.com/go-ini/ini v1.67.0 // indirect
|
github.com/go-ini/ini v1.67.0 // indirect
|
||||||
|
github.com/go-logr/logr v1.4.2 // indirect
|
||||||
|
github.com/go-logr/stdr v1.2.2 // indirect
|
||||||
github.com/go-openapi/jsonpointer v0.21.1 // indirect
|
github.com/go-openapi/jsonpointer v0.21.1 // indirect
|
||||||
github.com/go-openapi/jsonreference v0.21.0 // indirect
|
github.com/go-openapi/jsonreference v0.21.0 // indirect
|
||||||
github.com/go-openapi/spec v0.21.0 // indirect
|
github.com/go-openapi/spec v0.21.0 // indirect
|
||||||
@@ -48,6 +52,9 @@ require (
|
|||||||
github.com/stretchr/objx v0.5.2 // indirect
|
github.com/stretchr/objx v0.5.2 // indirect
|
||||||
github.com/swaggo/files/v2 v2.0.2 // indirect
|
github.com/swaggo/files/v2 v2.0.2 // indirect
|
||||||
github.com/tinylib/msgp v1.3.0 // indirect
|
github.com/tinylib/msgp v1.3.0 // indirect
|
||||||
|
go.opentelemetry.io/otel v1.28.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/metric v1.28.0 // indirect
|
||||||
|
go.opentelemetry.io/otel/trace v1.28.0 // indirect
|
||||||
golang.org/x/mod v0.26.0 // indirect
|
golang.org/x/mod v0.26.0 // indirect
|
||||||
golang.org/x/time v0.11.0 // indirect
|
golang.org/x/time v0.11.0 // indirect
|
||||||
golang.org/x/tools v0.35.0 // indirect
|
golang.org/x/tools v0.35.0 // indirect
|
||||||
|
|||||||
@@ -16,10 +16,19 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
|
|||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||||
|
github.com/elastic/elastic-transport-go/v8 v8.6.1 h1:h2jQRqH6eLGiBSN4eZbQnJLtL4bC5b4lfVFRjw2R4e4=
|
||||||
|
github.com/elastic/elastic-transport-go/v8 v8.6.1/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk=
|
||||||
|
github.com/elastic/go-elasticsearch/v8 v8.17.1 h1:bOXChDoCMB4TIwwGqKd031U8OXssmWLT3UrAr9EGs3Q=
|
||||||
|
github.com/elastic/go-elasticsearch/v8 v8.17.1/go.mod h1:MVJCtL+gJJ7x5jFeUmA20O7rvipX8GcQmo5iBcmaJn4=
|
||||||
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
|
||||||
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
|
||||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||||
|
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||||
|
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||||
|
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||||
|
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||||
|
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||||
github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic=
|
github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic=
|
||||||
github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk=
|
github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk=
|
||||||
github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
|
github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
|
||||||
@@ -108,6 +117,14 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS
|
|||||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||||
go.mongodb.org/mongo-driver/v2 v2.5.1 h1:j2U/Qp+wvueSpqitLCSZPT/+ZpVc1xzuwdHWwl7d8ro=
|
go.mongodb.org/mongo-driver/v2 v2.5.1 h1:j2U/Qp+wvueSpqitLCSZPT/+ZpVc1xzuwdHWwl7d8ro=
|
||||||
go.mongodb.org/mongo-driver/v2 v2.5.1/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0=
|
go.mongodb.org/mongo-driver/v2 v2.5.1/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0=
|
||||||
|
go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
|
||||||
|
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
|
||||||
|
go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
|
||||||
|
go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
|
||||||
|
go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
|
||||||
|
go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
|
||||||
|
go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
|
||||||
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
|
go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=
|
||||||
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
|
go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo=
|
||||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||||
|
|||||||
@@ -0,0 +1,14 @@
|
|||||||
|
package ai_pipeline
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrAINotConfigured is returned when the Opplens AI client is not configured.
|
||||||
|
ErrAINotConfigured = errors.New("ai_pipeline: AI service not configured")
|
||||||
|
|
||||||
|
// ErrEmptyTenderBatch is returned when a batch request has no tenders.
|
||||||
|
ErrEmptyTenderBatch = errors.New("ai_pipeline: at least one tender is required")
|
||||||
|
|
||||||
|
// ErrEmptyLanguages is returned when languages are required but missing.
|
||||||
|
ErrEmptyLanguages = errors.New("ai_pipeline: at least one language is required")
|
||||||
|
)
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package ai_pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TenderRefForm identifies one tender in batch and pipeline requests.
|
||||||
|
type TenderRefForm struct {
|
||||||
|
ContractFolderID string `json:"contract_folder_id" valid:"required"`
|
||||||
|
NoticePublicationID string `json:"notice_publication_id" valid:"required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScrapeDocumentsForm is the request for POST /admin/v1/ai-pipeline/scrape/documents.
|
||||||
|
type ScrapeDocumentsForm struct {
|
||||||
|
ContractFolderID string `json:"contract_folder_id" valid:"required"`
|
||||||
|
NoticePublicationID string `json:"notice_publication_id" valid:"required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TenderBatchForm is the request body for batch scrape and summarize endpoints.
|
||||||
|
type TenderBatchForm struct {
|
||||||
|
Tenders []TenderRefForm `json:"tenders" valid:"required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranslateBatchForm is the request body for POST /admin/v1/ai-pipeline/translate/batch.
|
||||||
|
type TranslateBatchForm struct {
|
||||||
|
Tenders []TenderRefForm `json:"tenders" valid:"required"`
|
||||||
|
Languages []string `json:"languages" valid:"required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PipelineRunForm is the request body for POST /admin/v1/ai-pipeline/run.
|
||||||
|
type PipelineRunForm struct {
|
||||||
|
ContractFolderID string `json:"contract_folder_id" valid:"required"`
|
||||||
|
NoticePublicationID string `json:"notice_publication_id" valid:"required"`
|
||||||
|
Languages []string `json:"languages" valid:"required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PipelineRunBatchForm is the request body for POST /admin/v1/ai-pipeline/run/batch.
|
||||||
|
type PipelineRunBatchForm struct {
|
||||||
|
Tenders []TenderRefForm `json:"tenders" valid:"required"`
|
||||||
|
Languages []string `json:"languages" valid:"required"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PipelineWindowQueryForm is the query form for report and stats endpoints.
|
||||||
|
type PipelineWindowQueryForm struct {
|
||||||
|
Window string `query:"window" valid:"optional,in(last_hour|last_24h|today|yesterday|last_7_days|last_30_days|all|daily)"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func toTenderRefs(forms []TenderRefForm) []ai_summarizer.TenderRef {
|
||||||
|
refs := make([]ai_summarizer.TenderRef, 0, len(forms))
|
||||||
|
for _, form := range forms {
|
||||||
|
refs = append(refs, ai_summarizer.NewTenderRef(form.ContractFolderID, form.NoticePublicationID))
|
||||||
|
}
|
||||||
|
return refs
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeLanguages(languages []string) []string {
|
||||||
|
out := make([]string, 0, len(languages))
|
||||||
|
seen := make(map[string]struct{}, len(languages))
|
||||||
|
for _, language := range languages {
|
||||||
|
lang := strings.ToLower(strings.TrimSpace(language))
|
||||||
|
if lang == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, dup := seen[lang]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[lang] = struct{}{}
|
||||||
|
out = append(out, lang)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,451 @@
|
|||||||
|
package ai_pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
"tm/pkg/response"
|
||||||
|
|
||||||
|
"github.com/labstack/echo/v4"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Handler handles admin HTTP requests for Opplens AI pipeline operations.
|
||||||
|
type Handler struct {
|
||||||
|
service Service
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHandler creates a new AI pipeline handler.
|
||||||
|
func NewHandler(service Service) *Handler {
|
||||||
|
return &Handler{service: service}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScrapeDocuments scrapes documents for one tender via the Opplens AI service.
|
||||||
|
// @Summary Scrape tender documents
|
||||||
|
// @Description Download all documents for one tender through the Opplens AI service (synchronous, cached)
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Param request body ScrapeDocumentsForm true "Tender identifiers"
|
||||||
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.ScrapeDocumentsResponse}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 404 {object} response.APIResponse
|
||||||
|
// @Failure 422 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/scrape/documents [post]
|
||||||
|
func (h *Handler) ScrapeDocuments(c echo.Context) error {
|
||||||
|
form, err := response.Parse[ScrapeDocumentsForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.ScrapeDocuments(c.Request().Context(), *form)
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Scrape documents")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Documents scraped successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScrapeDocumentsBatch enqueues background document scraping for many tenders.
|
||||||
|
// @Summary Batch scrape tender documents
|
||||||
|
// @Description Enqueue background document scraping for multiple tenders via the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Param request body TenderBatchForm true "Tenders to scrape"
|
||||||
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 422 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/scrape/documents/batch [post]
|
||||||
|
func (h *Handler) ScrapeDocumentsBatch(c echo.Context) error {
|
||||||
|
form, err := response.Parse[TenderBatchForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.ScrapeDocumentsBatch(c.Request().Context(), *form)
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Scrape documents batch")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Accepted(c, result, "Scrape batch accepted")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetScrapePortals lists document scraping portals supported by the Opplens AI service.
|
||||||
|
// @Summary List document scraping portals
|
||||||
|
// @Description Retrieve the list of document scraping portals supported by the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Success 200 {object} response.APIResponse{data=[]string} "Scrape portals retrieved successfully"
|
||||||
|
// @Failure 401 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/scrape/portals [get]
|
||||||
|
func (h *Handler) GetScrapePortals(c echo.Context) error {
|
||||||
|
result, err := h.service.GetScrapePortals(c.Request().Context())
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Get scrape portals")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Scrape portals retrieved successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// SummarizeBatch enqueues background summarization for many tenders.
|
||||||
|
// @Summary Batch summarize tenders
|
||||||
|
// @Description Enqueue background AI summarization for multiple tenders via the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Param request body TenderBatchForm true "Tenders to summarize"
|
||||||
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 422 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/summarize/batch [post]
|
||||||
|
func (h *Handler) SummarizeBatch(c echo.Context) error {
|
||||||
|
form, err := response.Parse[TenderBatchForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.SummarizeBatch(c.Request().Context(), *form)
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Summarize batch")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Accepted(c, result, "Summarize batch accepted")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TranslateBatch enqueues background translation for many tenders.
|
||||||
|
// @Summary Batch translate tenders
|
||||||
|
// @Description Enqueue background AI translation for multiple tenders via the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Param request body TranslateBatchForm true "Tenders and target languages"
|
||||||
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 422 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/translate/batch [post]
|
||||||
|
func (h *Handler) TranslateBatch(c echo.Context) error {
|
||||||
|
form, err := response.Parse[TranslateBatchForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.TranslateBatch(c.Request().Context(), *form)
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Translate batch")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Accepted(c, result, "Translate batch accepted")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sync pulls the tender list from this backend into the Opplens AI service.
|
||||||
|
// @Summary Sync tenders to AI pipeline
|
||||||
|
// @Description Pull the tender list from the backend and store it in the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineSyncResponse}
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/sync [post]
|
||||||
|
func (h *Handler) Sync(c echo.Context) error {
|
||||||
|
result, err := h.service.Sync(c.Request().Context())
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Pipeline sync")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Pipeline sync completed successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run runs the full AI pipeline for one tender (scrape → translate → summarize).
|
||||||
|
// @Summary Run full AI pipeline for one tender
|
||||||
|
// @Description Run scrape, translate, and summarize for one tender via the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Param request body PipelineRunForm true "Tender and languages"
|
||||||
|
// @Success 200 {object} response.APIResponse{data=object}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 404 {object} response.APIResponse
|
||||||
|
// @Failure 422 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/run [post]
|
||||||
|
func (h *Handler) Run(c echo.Context) error {
|
||||||
|
form, err := response.Parse[PipelineRunForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.Run(c.Request().Context(), *form)
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Pipeline run")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Pipeline run completed successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunBatch enqueues the full AI pipeline for many tenders.
|
||||||
|
// @Summary Batch run full AI pipeline
|
||||||
|
// @Description Enqueue scrape, translate, and summarize for multiple tenders via the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Param request body PipelineRunBatchForm true "Tenders and languages"
|
||||||
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 422 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/run/batch [post]
|
||||||
|
func (h *Handler) RunBatch(c echo.Context) error {
|
||||||
|
form, err := response.Parse[PipelineRunBatchForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.RunBatch(c.Request().Context(), *form)
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Pipeline run batch")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Accepted(c, result, "Pipeline run batch accepted")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Analyze triggers analysis across stored tenders in the Opplens AI service.
|
||||||
|
// @Summary Trigger AI analysis pipeline
|
||||||
|
// @Description Trigger agentic analysis across stored tenders via the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineActionResponse}
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/analyze [post]
|
||||||
|
func (h *Handler) Analyze(c echo.Context) error {
|
||||||
|
result, err := h.service.Analyze(c.Request().Context())
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Pipeline analyze")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Pipeline analyze triggered successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// DailyRun syncs tenders and runs the full pipeline on newly synced tenders.
|
||||||
|
// @Summary Start daily AI pipeline run
|
||||||
|
// @Description Sync tenders, then run the full pipeline on newly synced tenders (background, single-flight)
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.PipelineStartedResponse}
|
||||||
|
// @Failure 409 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/daily-run [post]
|
||||||
|
func (h *Handler) DailyRun(c echo.Context) error {
|
||||||
|
result, err := h.service.DailyRun(c.Request().Context())
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Pipeline daily-run")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Accepted(c, result, "Pipeline daily-run started")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Auto syncs tenders and completes all missing AI work across stored tenders.
|
||||||
|
// @Summary Start auto AI pipeline run
|
||||||
|
// @Description Sync tenders, then complete all missing scrape/translate/summarize work (background, single-flight, idempotent)
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Success 202 {object} response.APIResponse{data=ai_summarizer.PipelineStartedResponse}
|
||||||
|
// @Failure 409 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/auto [post]
|
||||||
|
func (h *Handler) Auto(c echo.Context) error {
|
||||||
|
result, err := h.service.Auto(c.Request().Context())
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Pipeline auto")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Accepted(c, result, "Pipeline auto started")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetLastRun returns the result of the last daily-run.
|
||||||
|
// @Summary Get last daily-run report
|
||||||
|
// @Description Retrieve the report for the last pipeline daily-run from the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/last-run [get]
|
||||||
|
func (h *Handler) GetLastRun(c echo.Context) error {
|
||||||
|
result, err := h.service.GetLastRun(c.Request().Context())
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Get last pipeline run")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Last pipeline run retrieved successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetLastAutoRun returns the result of the last auto run.
|
||||||
|
// @Summary Get last auto-run report
|
||||||
|
// @Description Retrieve the report for the last pipeline auto run from the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/last-auto-run [get]
|
||||||
|
func (h *Handler) GetLastAutoRun(c echo.Context) error {
|
||||||
|
result, err := h.service.GetLastAutoRun(c.Request().Context())
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Get last pipeline auto run")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Last pipeline auto run retrieved successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetReport returns an event-log summary for a time window.
|
||||||
|
// @Summary Get AI pipeline event report
|
||||||
|
// @Description Retrieve an event-log summary for a time window from the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Param window query string false "Time window" Enums(last_hour,last_24h,today,yesterday,last_7_days,last_30_days,all,daily) default(last_24h)
|
||||||
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 422 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/report [get]
|
||||||
|
func (h *Handler) GetReport(c echo.Context) error {
|
||||||
|
form, err := response.Parse[PipelineWindowQueryForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid query parameters", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.GetReport(c.Request().Context(), form.Window)
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Get pipeline report")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Pipeline report retrieved successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns event-log and storage inventory for a time window.
|
||||||
|
// @Summary Get AI pipeline stats
|
||||||
|
// @Description Retrieve event-log summary and MinIO inventory for a time window from the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Param window query string false "Time window" Enums(last_hour,last_24h,today,yesterday,last_7_days,last_30_days,all,daily) default(last_24h)
|
||||||
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineStatsResponse}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 422 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/stats [get]
|
||||||
|
func (h *Handler) GetStats(c echo.Context) error {
|
||||||
|
form, err := response.Parse[PipelineWindowQueryForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid query parameters", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.GetStats(c.Request().Context(), form.Window)
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Get pipeline stats")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Pipeline stats retrieved successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetMinioStats returns storage inventory from the Opplens AI service.
|
||||||
|
// @Summary Get AI pipeline MinIO stats
|
||||||
|
// @Description Retrieve storage inventory only from the Opplens AI service
|
||||||
|
// @Tags Admin-AI-Pipeline
|
||||||
|
// @Produce json
|
||||||
|
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineMinioStatsResponse}
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/ai-pipeline/minio-stats [get]
|
||||||
|
func (h *Handler) GetMinioStats(c echo.Context) error {
|
||||||
|
result, err := h.service.GetMinioStats(c.Request().Context())
|
||||||
|
if err != nil {
|
||||||
|
return mapPipelineHTTPError(c, err, "Get pipeline minio stats")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "Pipeline MinIO stats retrieved successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
func mapPipelineHTTPError(c echo.Context, err error, action string) error {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, ErrAINotConfigured):
|
||||||
|
return response.ServiceUnavailable(c, "AI service is not configured")
|
||||||
|
case errors.Is(err, ErrEmptyTenderBatch), errors.Is(err, ErrEmptyLanguages):
|
||||||
|
return response.BadRequest(c, "Invalid request", err.Error())
|
||||||
|
case errors.Is(err, ai_summarizer.ErrPipelineJobRunning):
|
||||||
|
return response.Conflict(c, "A pipeline job is already running")
|
||||||
|
case errors.Is(err, ai_summarizer.ErrObjectNotFound):
|
||||||
|
return response.NotFound(c, "Tender not found in AI service")
|
||||||
|
}
|
||||||
|
|
||||||
|
var upstream *ai_summarizer.APIStatusError
|
||||||
|
if errors.As(err, &upstream) {
|
||||||
|
details := upstream.Body
|
||||||
|
if details == "" {
|
||||||
|
details = fmt.Sprintf("upstream status %d", upstream.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
switch upstream.StatusCode {
|
||||||
|
case http.StatusNotFound:
|
||||||
|
return response.NotFound(c, "Tender not found in AI service")
|
||||||
|
case http.StatusUnprocessableEntity:
|
||||||
|
return response.ValidationError(c, "AI service rejected the request", details)
|
||||||
|
case http.StatusBadGateway:
|
||||||
|
return c.JSON(http.StatusBadGateway, response.APIResponse{
|
||||||
|
Success: false,
|
||||||
|
Message: action + " failed",
|
||||||
|
Error: &response.APIError{
|
||||||
|
Code: "UPSTREAM_AI_ERROR",
|
||||||
|
Message: "Opplens AI service returned status 502",
|
||||||
|
Details: details,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
default:
|
||||||
|
return c.JSON(http.StatusBadGateway, response.APIResponse{
|
||||||
|
Success: false,
|
||||||
|
Message: action + " failed",
|
||||||
|
Error: &response.APIError{
|
||||||
|
Code: "UPSTREAM_AI_ERROR",
|
||||||
|
Message: fmt.Sprintf("Opplens AI service returned status %d", upstream.StatusCode),
|
||||||
|
Details: details,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.InternalServerError(c, action+" failed")
|
||||||
|
}
|
||||||
@@ -0,0 +1,471 @@
|
|||||||
|
package ai_pipeline
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
"tm/pkg/logger"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Client defines Opplens AI operations exposed to the admin API.
|
||||||
|
type Client interface {
|
||||||
|
ScrapeDocuments(ctx context.Context, reqBody ai_summarizer.ScrapeDocumentsRequest) (*ai_summarizer.ScrapeDocumentsResponse, error)
|
||||||
|
ScrapeDocumentsBatch(ctx context.Context, tenders []ai_summarizer.TenderRef) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||||
|
GetScrapePortals(ctx context.Context) (*ai_summarizer.ScrapePortalsResponse, error)
|
||||||
|
TriggerSummarizeBatch(ctx context.Context, tenders []ai_summarizer.TenderRef) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||||
|
TriggerTranslateBatch(ctx context.Context, tenders []ai_summarizer.TenderRef, languages []string) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||||
|
PipelineSync(ctx context.Context) (*ai_summarizer.PipelineSyncResponse, error)
|
||||||
|
PipelineRun(ctx context.Context, reqBody ai_summarizer.PipelineRunRequest) (map[string]interface{}, error)
|
||||||
|
PipelineRunBatch(ctx context.Context, tenders []ai_summarizer.TenderRef, languages []string) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||||
|
PipelineAnalyze(ctx context.Context) (*ai_summarizer.PipelineActionResponse, error)
|
||||||
|
PipelineDailyRun(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||||
|
PipelineAuto(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||||
|
GetPipelineLastRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||||
|
GetPipelineLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||||
|
GetPipelineReport(ctx context.Context, window string) (*ai_summarizer.PipelineReportResponse, error)
|
||||||
|
GetPipelineStats(ctx context.Context, window string) (*ai_summarizer.PipelineStatsResponse, error)
|
||||||
|
GetPipelineMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Service proxies admin pipeline operations to the Opplens AI service.
|
||||||
|
type Service interface {
|
||||||
|
ScrapeDocuments(ctx context.Context, form ScrapeDocumentsForm) (*ai_summarizer.ScrapeDocumentsResponse, error)
|
||||||
|
ScrapeDocumentsBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||||
|
GetScrapePortals(ctx context.Context) (*ai_summarizer.ScrapePortalsResponse, error)
|
||||||
|
SummarizeBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||||
|
TranslateBatch(ctx context.Context, form TranslateBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||||
|
Sync(ctx context.Context) (*ai_summarizer.PipelineSyncResponse, error)
|
||||||
|
Run(ctx context.Context, form PipelineRunForm) (map[string]interface{}, error)
|
||||||
|
RunBatch(ctx context.Context, form PipelineRunBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||||
|
Analyze(ctx context.Context) (*ai_summarizer.PipelineActionResponse, error)
|
||||||
|
DailyRun(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||||
|
Auto(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||||
|
GetLastRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||||
|
GetLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||||
|
GetReport(ctx context.Context, window string) (*ai_summarizer.PipelineReportResponse, error)
|
||||||
|
GetStats(ctx context.Context, window string) (*ai_summarizer.PipelineStatsResponse, error)
|
||||||
|
GetMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CachedRecommendationRefresher schedules a background refresh of company recommendation caches.
|
||||||
|
type CachedRecommendationRefresher interface {
|
||||||
|
ScheduleRefreshCachedAIRecommendationsAfterPipeline()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ScrapedDocumentMetadataSyncer persists MinIO scraped-document metadata onto tender records.
|
||||||
|
type ScrapedDocumentMetadataSyncer interface {
|
||||||
|
SyncScrapedDocumentsFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) error
|
||||||
|
}
|
||||||
|
|
||||||
|
type service struct {
|
||||||
|
client Client
|
||||||
|
logger logger.Logger
|
||||||
|
metadataSyncer ScrapedDocumentMetadataSyncer
|
||||||
|
recommendationRefresher CachedRecommendationRefresher
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewService creates an AI pipeline admin service.
|
||||||
|
func NewService(
|
||||||
|
client Client,
|
||||||
|
log logger.Logger,
|
||||||
|
metadataSyncer ScrapedDocumentMetadataSyncer,
|
||||||
|
recommendationRefresher CachedRecommendationRefresher,
|
||||||
|
) Service {
|
||||||
|
return &service{
|
||||||
|
client: client,
|
||||||
|
logger: log,
|
||||||
|
metadataSyncer: metadataSyncer,
|
||||||
|
recommendationRefresher: recommendationRefresher,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) requireClient() error {
|
||||||
|
if s.client == nil {
|
||||||
|
return ErrAINotConfigured
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) ScrapeDocuments(ctx context.Context, form ScrapeDocumentsForm) (*ai_summarizer.ScrapeDocumentsResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin scrape documents requested", map[string]interface{}{
|
||||||
|
"contract_folder_id": form.ContractFolderID,
|
||||||
|
"notice_publication_id": form.NoticePublicationID,
|
||||||
|
})
|
||||||
|
|
||||||
|
resp, err := s.client.ScrapeDocuments(ctx, ai_summarizer.ScrapeDocumentsRequest{
|
||||||
|
ContractFolderID: form.ContractFolderID,
|
||||||
|
NoticePublicationID: form.NoticePublicationID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin scrape documents failed", map[string]interface{}{
|
||||||
|
"contract_folder_id": form.ContractFolderID,
|
||||||
|
"notice_publication_id": form.NoticePublicationID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("scrape documents failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp != nil && resp.Success && resp.FilesDownloaded > 0 {
|
||||||
|
s.syncScrapedDocumentMetadata(ctx, form.ContractFolderID, form.NoticePublicationID)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) ScrapeDocumentsBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
refs, err := validateTenderBatch(form.Tenders)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin scrape documents batch requested", map[string]interface{}{
|
||||||
|
"tender_count": len(refs),
|
||||||
|
})
|
||||||
|
|
||||||
|
resp, err := s.client.ScrapeDocumentsBatch(ctx, refs)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin scrape documents batch failed", map[string]interface{}{
|
||||||
|
"tender_count": len(refs),
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("scrape documents batch failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) GetScrapePortals(ctx context.Context) (*ai_summarizer.ScrapePortalsResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin get scrape portals requested", map[string]interface{}{})
|
||||||
|
|
||||||
|
resp, err := s.client.GetScrapePortals(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin get scrape portals failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("get scrape portals failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) SummarizeBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
refs, err := validateTenderBatch(form.Tenders)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin summarize batch requested", map[string]interface{}{
|
||||||
|
"tender_count": len(refs),
|
||||||
|
})
|
||||||
|
|
||||||
|
resp, err := s.client.TriggerSummarizeBatch(ctx, refs)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin summarize batch failed", map[string]interface{}{
|
||||||
|
"tender_count": len(refs),
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("summarize batch failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) TranslateBatch(ctx context.Context, form TranslateBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
refs, err := validateTenderBatch(form.Tenders)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
languages := normalizeLanguages(form.Languages)
|
||||||
|
if len(languages) == 0 {
|
||||||
|
return nil, ErrEmptyLanguages
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin translate batch requested", map[string]interface{}{
|
||||||
|
"tender_count": len(refs),
|
||||||
|
"languages": languages,
|
||||||
|
})
|
||||||
|
|
||||||
|
resp, err := s.client.TriggerTranslateBatch(ctx, refs, languages)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin translate batch failed", map[string]interface{}{
|
||||||
|
"tender_count": len(refs),
|
||||||
|
"languages": languages,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("translate batch failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) Sync(ctx context.Context) (*ai_summarizer.PipelineSyncResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin pipeline sync requested", map[string]interface{}{})
|
||||||
|
|
||||||
|
resp, err := s.client.PipelineSync(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin pipeline sync failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("pipeline sync failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.scheduleRecommendationRefreshAfterPipeline()
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) Run(ctx context.Context, form PipelineRunForm) (map[string]interface{}, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
languages := normalizeLanguages(form.Languages)
|
||||||
|
if len(languages) == 0 {
|
||||||
|
return nil, ErrEmptyLanguages
|
||||||
|
}
|
||||||
|
|
||||||
|
req := ai_summarizer.PipelineRunRequest{
|
||||||
|
ContractFolderID: form.ContractFolderID,
|
||||||
|
NoticePublicationID: form.NoticePublicationID,
|
||||||
|
Languages: languages,
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin pipeline run requested", map[string]interface{}{
|
||||||
|
"contract_folder_id": req.ContractFolderID,
|
||||||
|
"notice_publication_id": req.NoticePublicationID,
|
||||||
|
"languages": languages,
|
||||||
|
})
|
||||||
|
|
||||||
|
resp, err := s.client.PipelineRun(ctx, req)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin pipeline run failed", map[string]interface{}{
|
||||||
|
"contract_folder_id": req.ContractFolderID,
|
||||||
|
"notice_publication_id": req.NoticePublicationID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("pipeline run failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.syncScrapedDocumentMetadata(ctx, req.ContractFolderID, req.NoticePublicationID)
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) RunBatch(ctx context.Context, form PipelineRunBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
refs, err := validateTenderBatch(form.Tenders)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
languages := normalizeLanguages(form.Languages)
|
||||||
|
if len(languages) == 0 {
|
||||||
|
return nil, ErrEmptyLanguages
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin pipeline run batch requested", map[string]interface{}{
|
||||||
|
"tender_count": len(refs),
|
||||||
|
"languages": languages,
|
||||||
|
})
|
||||||
|
|
||||||
|
resp, err := s.client.PipelineRunBatch(ctx, refs, languages)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin pipeline run batch failed", map[string]interface{}{
|
||||||
|
"tender_count": len(refs),
|
||||||
|
"languages": languages,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("pipeline run batch failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) Analyze(ctx context.Context) (*ai_summarizer.PipelineActionResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin pipeline analyze requested", map[string]interface{}{})
|
||||||
|
|
||||||
|
resp, err := s.client.PipelineAnalyze(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin pipeline analyze failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("pipeline analyze failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) DailyRun(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin pipeline daily-run requested", map[string]interface{}{})
|
||||||
|
|
||||||
|
resp, err := s.client.PipelineDailyRun(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin pipeline daily-run failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("pipeline daily-run failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.scheduleRecommendationRefreshAfterPipeline()
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) Auto(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Admin pipeline auto requested", map[string]interface{}{})
|
||||||
|
|
||||||
|
resp, err := s.client.PipelineAuto(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin pipeline auto failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("pipeline auto failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.scheduleRecommendationRefreshAfterPipeline()
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) GetLastRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := s.client.GetPipelineLastRun(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin get pipeline last-run failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("get pipeline last-run failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) GetLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := s.client.GetPipelineLastAutoRun(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin get pipeline last-auto-run failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("get pipeline last-auto-run failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) GetReport(ctx context.Context, window string) (*ai_summarizer.PipelineReportResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := s.client.GetPipelineReport(ctx, window)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin get pipeline report failed", map[string]interface{}{
|
||||||
|
"window": window,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("get pipeline report failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) GetStats(ctx context.Context, window string) (*ai_summarizer.PipelineStatsResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := s.client.GetPipelineStats(ctx, window)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin get pipeline stats failed", map[string]interface{}{
|
||||||
|
"window": window,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("get pipeline stats failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) GetMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error) {
|
||||||
|
if err := s.requireClient(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := s.client.GetPipelineMinioStats(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Admin get pipeline minio stats failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("get pipeline minio stats failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return resp, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) scheduleRecommendationRefreshAfterPipeline() {
|
||||||
|
if s.recommendationRefresher == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.recommendationRefresher.ScheduleRefreshCachedAIRecommendationsAfterPipeline()
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateTenderBatch(forms []TenderRefForm) ([]ai_summarizer.TenderRef, error) {
|
||||||
|
if len(forms) == 0 {
|
||||||
|
return nil, ErrEmptyTenderBatch
|
||||||
|
}
|
||||||
|
return toTenderRefs(forms), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) syncScrapedDocumentMetadata(ctx context.Context, contractFolderID, noticePublicationID string) {
|
||||||
|
if s.metadataSyncer == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := s.metadataSyncer.SyncScrapedDocumentsFromStorage(ctx, contractFolderID, noticePublicationID); err != nil {
|
||||||
|
s.logger.Warn("Failed to sync scraped document metadata to Mongo", map[string]interface{}{
|
||||||
|
"contract_folder_id": contractFolderID,
|
||||||
|
"notice_publication_id": noticePublicationID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package auditlog
|
||||||
|
|
||||||
|
// Entry represents an audit log record returned by the API.
|
||||||
|
type Entry struct {
|
||||||
|
ActorID string `json:"actor_id"`
|
||||||
|
ActorType string `json:"actor_type"`
|
||||||
|
Action string `json:"action"`
|
||||||
|
TargetType string `json:"target_type"`
|
||||||
|
TargetID string `json:"target_id"`
|
||||||
|
At int64 `json:"at"`
|
||||||
|
IP string `json:"ip,omitempty"`
|
||||||
|
RequestID string `json:"request_id,omitempty"`
|
||||||
|
UserAgent string `json:"user_agent,omitempty"`
|
||||||
|
Success bool `json:"success"`
|
||||||
|
Metadata map[string]string `json:"metadata,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListResponse is the paginated audit log list response.
|
||||||
|
type ListResponse struct {
|
||||||
|
Logs []*Entry `json:"logs"`
|
||||||
|
Meta *ListMeta `json:"meta,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListMeta contains pagination metadata.
|
||||||
|
type ListMeta struct {
|
||||||
|
Total int64 `json:"total"`
|
||||||
|
Limit int `json:"limit"`
|
||||||
|
Offset int `json:"offset"`
|
||||||
|
Page int `json:"page"`
|
||||||
|
Pages int `json:"pages"`
|
||||||
|
}
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
package auditlog
|
||||||
|
|
||||||
|
// SearchForm defines query parameters for audit log search.
|
||||||
|
type SearchForm struct {
|
||||||
|
ActorID string `query:"actor_id" valid:"optional"`
|
||||||
|
ActorType string `query:"actor_type" valid:"optional,in(admin|customer|system)"`
|
||||||
|
Action string `query:"action" valid:"optional"`
|
||||||
|
TargetType string `query:"target_type" valid:"optional"`
|
||||||
|
TargetID string `query:"target_id" valid:"optional"`
|
||||||
|
From int64 `query:"from" valid:"optional"`
|
||||||
|
To int64 `query:"to" valid:"optional"`
|
||||||
|
}
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
package auditlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"tm/pkg/response"
|
||||||
|
|
||||||
|
"github.com/asaskevich/govalidator"
|
||||||
|
"github.com/labstack/echo/v4"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Handler handles HTTP requests for audit log operations.
|
||||||
|
type Handler struct {
|
||||||
|
service Service
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHandler creates an audit log handler.
|
||||||
|
func NewHandler(service Service) *Handler {
|
||||||
|
return &Handler{service: service}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Search returns paginated audit logs with optional filters.
|
||||||
|
// @Summary Search audit logs
|
||||||
|
// @Description Retrieve paginated user action audit logs with optional filtering by actor, action, target, and time range
|
||||||
|
// @Tags Admin-AuditLogs
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Param actor_id query string false "Filter by actor ID"
|
||||||
|
// @Param actor_type query string false "Filter by actor type" Enums(admin,customer,system)
|
||||||
|
// @Param action query string false "Filter by action (e.g. auth.login.success)"
|
||||||
|
// @Param target_type query string false "Filter by target type"
|
||||||
|
// @Param target_id query string false "Filter by target ID"
|
||||||
|
// @Param from query int64 false "Filter by timestamp from (Unix seconds)"
|
||||||
|
// @Param to query int64 false "Filter by timestamp to (Unix seconds)"
|
||||||
|
// @Param limit query int false "Number of items per page (default: 10, max: 100)"
|
||||||
|
// @Param offset query int false "Number of items to skip (default: 0)"
|
||||||
|
// @Success 200 {object} response.APIResponse{data=ListResponse} "Audit logs retrieved successfully"
|
||||||
|
// @Failure 400 {object} response.APIResponse "Bad request"
|
||||||
|
// @Failure 401 {object} response.APIResponse "Unauthorized"
|
||||||
|
// @Failure 403 {object} response.APIResponse "Forbidden"
|
||||||
|
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||||
|
// @Router /admin/v1/audit-logs [get]
|
||||||
|
func (h *Handler) Search(c echo.Context) error {
|
||||||
|
form, err := response.Parse[SearchForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := govalidator.ValidateStruct(form); err != nil {
|
||||||
|
return response.ValidationError(c, "Validation failed", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
pagination, err := response.NewPagination(c)
|
||||||
|
if err != nil {
|
||||||
|
return response.PaginationBadRequest(c, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.Search(c.Request().Context(), *form, pagination)
|
||||||
|
if err != nil {
|
||||||
|
return response.InternalServerError(c, "Failed to retrieve audit logs")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.SuccessWithMeta(c, result, &response.Meta{
|
||||||
|
Total: int(result.Meta.Total),
|
||||||
|
Limit: result.Meta.Limit,
|
||||||
|
Offset: result.Meta.Offset,
|
||||||
|
Page: result.Meta.Page,
|
||||||
|
Pages: result.Meta.Pages,
|
||||||
|
}, "audit logs retrieved successfully")
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
package auditlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"tm/pkg/audit"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Repository defines audit log data access.
|
||||||
|
type Repository interface {
|
||||||
|
Search(ctx context.Context, filter audit.SearchFilter) (*audit.SearchResult, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type repository struct {
|
||||||
|
store audit.Store
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRepository creates an audit log repository.
|
||||||
|
func NewRepository(store audit.Store) Repository {
|
||||||
|
return &repository{store: store}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *repository) Search(ctx context.Context, filter audit.SearchFilter) (*audit.SearchResult, error) {
|
||||||
|
return r.store.Search(ctx, filter)
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package auditlog
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"math"
|
||||||
|
|
||||||
|
"tm/pkg/audit"
|
||||||
|
"tm/pkg/logger"
|
||||||
|
"tm/pkg/response"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Service defines audit log query operations.
|
||||||
|
type Service interface {
|
||||||
|
Search(ctx context.Context, form SearchForm, pagination *response.Pagination) (*ListResponse, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type service struct {
|
||||||
|
repository Repository
|
||||||
|
logger logger.Logger
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewService creates an audit log service.
|
||||||
|
func NewService(repository Repository, log logger.Logger) Service {
|
||||||
|
return &service{
|
||||||
|
repository: repository,
|
||||||
|
logger: log,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) Search(ctx context.Context, form SearchForm, pagination *response.Pagination) (*ListResponse, error) {
|
||||||
|
filter := audit.SearchFilter{
|
||||||
|
ActorID: form.ActorID,
|
||||||
|
ActorType: form.ActorType,
|
||||||
|
Action: form.Action,
|
||||||
|
TargetType: form.TargetType,
|
||||||
|
TargetID: form.TargetID,
|
||||||
|
From: form.From,
|
||||||
|
To: form.To,
|
||||||
|
Limit: pagination.Limit,
|
||||||
|
Offset: pagination.Offset,
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := s.repository.Search(ctx, filter)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to search audit logs", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
entries := make([]*Entry, 0, len(result.Items))
|
||||||
|
for _, item := range result.Items {
|
||||||
|
entries = append(entries, &Entry{
|
||||||
|
ActorID: item.ActorID,
|
||||||
|
ActorType: item.ActorType,
|
||||||
|
Action: item.Action,
|
||||||
|
TargetType: item.TargetType,
|
||||||
|
TargetID: item.TargetID,
|
||||||
|
At: item.At,
|
||||||
|
IP: item.IP,
|
||||||
|
RequestID: item.RequestID,
|
||||||
|
UserAgent: item.UserAgent,
|
||||||
|
Success: item.Success,
|
||||||
|
Metadata: item.Metadata,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
total := result.TotalCount
|
||||||
|
pages := 0
|
||||||
|
if pagination.Limit > 0 {
|
||||||
|
pages = int(math.Ceil(float64(total) / float64(pagination.Limit)))
|
||||||
|
}
|
||||||
|
page := 1
|
||||||
|
if pagination.Limit > 0 {
|
||||||
|
page = (pagination.Offset / pagination.Limit) + 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return &ListResponse{
|
||||||
|
Logs: entries,
|
||||||
|
Meta: &ListMeta{
|
||||||
|
Total: total,
|
||||||
|
Limit: pagination.Limit,
|
||||||
|
Offset: pagination.Offset,
|
||||||
|
Page: page,
|
||||||
|
Pages: pages,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
@@ -75,12 +75,13 @@ func (s *companyService) UploadDocuments(ctx context.Context, companyID, uploade
|
|||||||
"file_count": len(files),
|
"file_count": len(files),
|
||||||
})
|
})
|
||||||
return &UploadDocumentsResponse{
|
return &UploadDocumentsResponse{
|
||||||
DocumentFileIDs: company.DocumentFileIDs,
|
DocumentFileIDs: s.sanitizeDocumentFileIDs(company.DocumentFileIDs),
|
||||||
Uploaded: uploaded,
|
Uploaded: uploaded,
|
||||||
Failed: failed,
|
Failed: failed,
|
||||||
}, errors.New("all file uploads failed")
|
}, errors.New("all file uploads failed")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
company.DocumentFileIDs = s.sanitizeDocumentFileIDs(company.DocumentFileIDs)
|
||||||
company.DocumentFileIDs = mergeDocumentFileIDs(company.DocumentFileIDs, newIDs)
|
company.DocumentFileIDs = mergeDocumentFileIDs(company.DocumentFileIDs, newIDs)
|
||||||
if err := s.repository.Update(ctx, company); err != nil {
|
if err := s.repository.Update(ctx, company); err != nil {
|
||||||
s.logger.Error("Failed to save company documents", map[string]interface{}{
|
s.logger.Error("Failed to save company documents", map[string]interface{}{
|
||||||
@@ -97,6 +98,8 @@ func (s *companyService) UploadDocuments(ctx context.Context, companyID, uploade
|
|||||||
"documents_total": len(company.DocumentFileIDs),
|
"documents_total": len(company.DocumentFileIDs),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.triggerAIOnboardingAsync(companyID)
|
||||||
|
|
||||||
return &UploadDocumentsResponse{
|
return &UploadDocumentsResponse{
|
||||||
DocumentFileIDs: company.DocumentFileIDs,
|
DocumentFileIDs: company.DocumentFileIDs,
|
||||||
Uploaded: uploaded,
|
Uploaded: uploaded,
|
||||||
@@ -125,3 +128,40 @@ func mergeDocumentFileIDs(existing, additional []string) []string {
|
|||||||
appendUnique(additional)
|
appendUnique(additional)
|
||||||
return merged
|
return merged
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DetachDocumentFileID removes a file ID from every company that still references it.
|
||||||
|
func (s *companyService) DetachDocumentFileID(ctx context.Context, fileID string) error {
|
||||||
|
if err := s.repository.RemoveDocumentFileID(ctx, fileID); err != nil {
|
||||||
|
s.logger.Error("Failed to detach document file ID from companies", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"file_id": fileID,
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) sanitizeDocumentFileIDs(ids []string) []string {
|
||||||
|
if len(ids) == 0 || s.fileStore == nil {
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
sanitized := make([]string, 0, len(ids))
|
||||||
|
for _, id := range ids {
|
||||||
|
if id == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
exists, err := s.fileStore.Exists(id)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("Failed to verify company document file", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"file_id": id,
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
sanitized = append(sanitized, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return sanitized
|
||||||
|
}
|
||||||
|
|||||||
@@ -51,7 +51,10 @@ type Company struct {
|
|||||||
Email *string `bson:"email,omitempty" json:"email,omitempty"`
|
Email *string `bson:"email,omitempty" json:"email,omitempty"`
|
||||||
|
|
||||||
// File references stored in GridFS
|
// File references stored in GridFS
|
||||||
DocumentFileIDs []string `bson:"document_file_ids,omitempty" json:"document_file_ids,omitempty"`
|
DocumentFileIDs []string `bson:"document_file_ids" json:"document_file_ids,omitempty"`
|
||||||
|
|
||||||
|
// External resource links (e.g. company profile pages, portfolios)
|
||||||
|
Links []CompanyLink `bson:"links,omitempty" json:"links,omitempty"`
|
||||||
|
|
||||||
// Tags for categorization and search
|
// Tags for categorization and search
|
||||||
Tags *CompanyTags `bson:"tags,omitempty" json:"tags,omitempty"`
|
Tags *CompanyTags `bson:"tags,omitempty" json:"tags,omitempty"`
|
||||||
@@ -97,6 +100,12 @@ func (c *Company) GetUpdatedAt() int64 {
|
|||||||
return c.UpdatedAt
|
return c.UpdatedAt
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CompanyLink represents an external URL associated with a company.
|
||||||
|
type CompanyLink struct {
|
||||||
|
URL string `bson:"url" json:"url"`
|
||||||
|
Title *string `bson:"title,omitempty" json:"title,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// Address represents a company's address
|
// Address represents a company's address
|
||||||
type Address struct {
|
type Address struct {
|
||||||
Street string `bson:"street" json:"street"`
|
Street string `bson:"street" json:"street"`
|
||||||
|
|||||||
+42
-10
@@ -7,9 +7,9 @@ type (
|
|||||||
CompanyForm struct {
|
CompanyForm struct {
|
||||||
Name string `json:"name" valid:"required,length(2|200)" example:"Opplens"`
|
Name string `json:"name" valid:"required,length(2|200)" example:"Opplens"`
|
||||||
Type string `json:"type" valid:"required,in(private|public|government|ngo|startup)" example:"private"`
|
Type string `json:"type" valid:"required,in(private|public|government|ngo|startup)" example:"private"`
|
||||||
RegistrationNumber string `json:"registration_number" valid:"required,length(5|50)" example:"1234567890"`
|
RegistrationNumber string `json:"registration_number" valid:"optional,length(5|50)" example:"1234567890"`
|
||||||
TaxID string `json:"tax_id" valid:"required,length(5|50)" example:"1234567890"`
|
TaxID string `json:"tax_id" valid:"optional,length(5|50)" example:"1234567890"`
|
||||||
Industry string `json:"industry" valid:"required,length(2|100)" example:"Technology"`
|
Industry string `json:"industry" valid:"optional,length(2|100)" example:"Technology"`
|
||||||
Description *string `json:"description,omitempty" valid:"optional,length(10|1000)" example:"Opplens is a technology company"`
|
Description *string `json:"description,omitempty" valid:"optional,length(10|1000)" example:"Opplens is a technology company"`
|
||||||
Website *string `json:"website,omitempty" valid:"optional,url" example:"https://opplens.com"`
|
Website *string `json:"website,omitempty" valid:"optional,url" example:"https://opplens.com"`
|
||||||
|
|
||||||
@@ -25,6 +25,7 @@ type (
|
|||||||
Phone *string `json:"phone,omitempty" valid:"optional,length(10|20)" example:"+1234567890"`
|
Phone *string `json:"phone,omitempty" valid:"optional,length(10|20)" example:"+1234567890"`
|
||||||
Email *string `json:"email,omitempty" valid:"optional,email" example:"info@opplens.com"`
|
Email *string `json:"email,omitempty" valid:"optional,email" example:"info@opplens.com"`
|
||||||
DocumentFileIDs []string `json:"document_file_ids,omitempty" valid:"optional"`
|
DocumentFileIDs []string `json:"document_file_ids,omitempty" valid:"optional"`
|
||||||
|
Links []CompanyLinkForm `json:"links,omitempty" valid:"optional"`
|
||||||
|
|
||||||
// Tags for categorization and search
|
// Tags for categorization and search
|
||||||
Tags *CompanyTagsForm `json:"tags,omitempty"`
|
Tags *CompanyTagsForm `json:"tags,omitempty"`
|
||||||
@@ -35,13 +36,19 @@ type (
|
|||||||
Timezone *string `json:"timezone,omitempty" valid:"optional,length(3|50)" example:"UTC"`
|
Timezone *string `json:"timezone,omitempty" valid:"optional,length(3|50)" example:"UTC"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CompanyLinkForm represents the form for a company link.
|
||||||
|
CompanyLinkForm struct {
|
||||||
|
URL string `json:"url" valid:"required,url" example:"https://example.com/profile"`
|
||||||
|
Title *string `json:"title,omitempty" valid:"optional,length(1|200)" example:"Company profile"`
|
||||||
|
}
|
||||||
|
|
||||||
// AddressForm represents the form for company address
|
// AddressForm represents the form for company address
|
||||||
AddressForm struct {
|
AddressForm struct {
|
||||||
Street string `json:"street" valid:"required,length(5|200)" example:"123 Main St"`
|
Street string `json:"street" valid:"optional,length(5|200)" example:"123 Main St"`
|
||||||
City string `json:"city" valid:"required,length(2|100)" example:"New York"`
|
City string `json:"city" valid:"optional,length(2|100)" example:"New York"`
|
||||||
State string `json:"state" valid:"required,length(2|100)" example:"NY"`
|
State string `json:"state" valid:"optional,length(2|100)" example:"NY"`
|
||||||
PostalCode string `json:"postal_code" valid:"required,length(3|20)" example:"10001"`
|
PostalCode string `json:"postal_code" valid:"optional,length(3|20)" example:"10001"`
|
||||||
Country string `json:"country" valid:"required,length(2|100)" example:"US"`
|
Country string `json:"country" valid:"optional,length(2|100)" example:"US"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// CompanyTagsForm represents the form for company tags
|
// CompanyTagsForm represents the form for company tags
|
||||||
@@ -57,6 +64,10 @@ type (
|
|||||||
// SearchForm represents the form for searching companies with filters
|
// SearchForm represents the form for searching companies with filters
|
||||||
type SearchForm struct {
|
type SearchForm struct {
|
||||||
Search *string `query:"q" valid:"optional"`
|
Search *string `query:"q" valid:"optional"`
|
||||||
|
Name *string `query:"name" valid:"optional,length(1|200)"`
|
||||||
|
Email *string `query:"email" valid:"optional,length(1|254)"`
|
||||||
|
Phone *string `query:"phone" valid:"optional,length(1|20)"`
|
||||||
|
EmployeeCount *int `query:"employee_count" valid:"optional,range(1|100000)"`
|
||||||
Type *string `query:"type" valid:"optional,in(private|public|government|ngo|startup)"`
|
Type *string `query:"type" valid:"optional,in(private|public|government|ngo|startup)"`
|
||||||
Status *string `query:"status" valid:"optional,in(active|inactive|suspended|pending)"`
|
Status *string `query:"status" valid:"optional,in(active|inactive|suspended|pending)"`
|
||||||
Industry *string `query:"industry" valid:"optional"`
|
Industry *string `query:"industry" valid:"optional"`
|
||||||
@@ -68,8 +79,8 @@ type SearchForm struct {
|
|||||||
Categories []string `query:"categories" valid:"optional"`
|
Categories []string `query:"categories" valid:"optional"`
|
||||||
Keywords []string `query:"keywords" valid:"optional"`
|
Keywords []string `query:"keywords" valid:"optional"`
|
||||||
Specializations []string `query:"specializations" valid:"optional"`
|
Specializations []string `query:"specializations" valid:"optional"`
|
||||||
EmployeeCountMin *int `query:"employee_count_min" valid:"optional,min(1)"`
|
EmployeeCountMin *int `query:"employee_count_min" valid:"optional,range(1|100000)"`
|
||||||
EmployeeCountMax *int `query:"employee_count_max" valid:"optional,min(1)"`
|
EmployeeCountMax *int `query:"employee_count_max" valid:"optional,range(1|100000)"`
|
||||||
AnnualRevenueMin *float64 `query:"annual_revenue_min" valid:"optional,min(0)"`
|
AnnualRevenueMin *float64 `query:"annual_revenue_min" valid:"optional,min(0)"`
|
||||||
AnnualRevenueMax *float64 `query:"annual_revenue_max" valid:"optional,min(0)"`
|
AnnualRevenueMax *float64 `query:"annual_revenue_max" valid:"optional,min(0)"`
|
||||||
FoundedYearMin *int `query:"founded_year_min" valid:"optional,range(1800|2100)"`
|
FoundedYearMin *int `query:"founded_year_min" valid:"optional,range(1800|2100)"`
|
||||||
@@ -86,6 +97,11 @@ type StatusForm struct {
|
|||||||
Reason *string `json:"reason" valid:"optional,length(10|500)"`
|
Reason *string `json:"reason" valid:"optional,length(10|500)"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AddLinksForm is the request body for appending links to a company.
|
||||||
|
type AddLinksForm struct {
|
||||||
|
Links []CompanyLinkForm `json:"links" valid:"required"`
|
||||||
|
}
|
||||||
|
|
||||||
// VerificationForm represents the form for updating company verification status
|
// VerificationForm represents the form for updating company verification status
|
||||||
type VerificationForm struct {
|
type VerificationForm struct {
|
||||||
IsVerified bool `json:"is_verified"`
|
IsVerified bool `json:"is_verified"`
|
||||||
@@ -124,6 +140,7 @@ type (
|
|||||||
Phone *string `json:"phone,omitempty"`
|
Phone *string `json:"phone,omitempty"`
|
||||||
Email *string `json:"email,omitempty"`
|
Email *string `json:"email,omitempty"`
|
||||||
DocumentFileIDs []string `json:"document_file_ids,omitempty"`
|
DocumentFileIDs []string `json:"document_file_ids,omitempty"`
|
||||||
|
Links []CompanyLink `json:"links,omitempty"`
|
||||||
Tags *CompanyTagsResponse `json:"tags,omitempty"`
|
Tags *CompanyTagsResponse `json:"tags,omitempty"`
|
||||||
IsVerified bool `json:"is_verified"`
|
IsVerified bool `json:"is_verified"`
|
||||||
IsCompliant bool `json:"is_compliant"`
|
IsCompliant bool `json:"is_compliant"`
|
||||||
@@ -163,6 +180,7 @@ func (c *Company) ToResponse(categories []*CategoryResponse) *CompanyResponse {
|
|||||||
Phone: c.Phone,
|
Phone: c.Phone,
|
||||||
Email: c.Email,
|
Email: c.Email,
|
||||||
DocumentFileIDs: c.DocumentFileIDs,
|
DocumentFileIDs: c.DocumentFileIDs,
|
||||||
|
Links: c.Links,
|
||||||
Tags: c.Tags.ToResponse(categories),
|
Tags: c.Tags.ToResponse(categories),
|
||||||
IsVerified: c.IsVerified,
|
IsVerified: c.IsVerified,
|
||||||
IsCompliant: c.IsCompliant,
|
IsCompliant: c.IsCompliant,
|
||||||
@@ -185,6 +203,18 @@ func (c *CompanyTags) ToResponse(categories []*CategoryResponse) *CompanyTagsRes
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// OnboardingResponse is returned after starting AI company onboarding.
|
||||||
|
type OnboardingResponse struct {
|
||||||
|
Status string `json:"status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecommendedTenderResponse is one ranked tender recommendation from the AI service.
|
||||||
|
type RecommendedTenderResponse struct {
|
||||||
|
Rank int `json:"rank"`
|
||||||
|
TenderID string `json:"tender_id"`
|
||||||
|
Analysis string `json:"analysis"`
|
||||||
|
}
|
||||||
|
|
||||||
// CompanyProfileResponse represents the company profile data sent in API responses
|
// CompanyProfileResponse represents the company profile data sent in API responses
|
||||||
type CompanyProfileResponse struct {
|
type CompanyProfileResponse struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
@@ -193,6 +223,7 @@ type CompanyProfileResponse struct {
|
|||||||
Industry string `json:"industry"`
|
Industry string `json:"industry"`
|
||||||
FoundedYear *int `json:"founded_year,omitempty"`
|
FoundedYear *int `json:"founded_year,omitempty"`
|
||||||
DocumentFileIDs []string `json:"document_file_ids,omitempty"`
|
DocumentFileIDs []string `json:"document_file_ids,omitempty"`
|
||||||
|
Links []CompanyLink `json:"links,omitempty"`
|
||||||
CreatedAt int64 `json:"created_at"`
|
CreatedAt int64 `json:"created_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,6 +236,7 @@ func (c *Company) ToProfileResponse() *CompanyProfileResponse {
|
|||||||
Industry: c.Industry,
|
Industry: c.Industry,
|
||||||
FoundedYear: c.FoundedYear,
|
FoundedYear: c.FoundedYear,
|
||||||
DocumentFileIDs: c.DocumentFileIDs,
|
DocumentFileIDs: c.DocumentFileIDs,
|
||||||
|
Links: c.Links,
|
||||||
CreatedAt: c.CreatedAt,
|
CreatedAt: c.CreatedAt,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package company
|
package company
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
|
|
||||||
"tm/internal/user"
|
"tm/internal/user"
|
||||||
@@ -53,6 +54,87 @@ func (h *Handler) MyCompany(c echo.Context) error {
|
|||||||
return response.Success(c, company, "Company retrieved successfully")
|
return response.Success(c, company, "Company retrieved successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StartOnboarding starts AI onboarding for the authenticated customer's company (Mobile App)
|
||||||
|
// @Summary Start AI company onboarding
|
||||||
|
// @Description Send company profile, uploaded documents, and website URL to the AI team to start tender recommendation onboarding
|
||||||
|
// @Tags Company
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Success 200 {object} response.APIResponse{data=OnboardingResponse}
|
||||||
|
// @Failure 401 {object} response.APIResponse "Unauthorized - User not authenticated"
|
||||||
|
// @Failure 404 {object} response.APIResponse "Company not found"
|
||||||
|
// @Failure 503 {object} response.APIResponse "AI recommendation service unavailable"
|
||||||
|
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /api/v1/onboarding [post]
|
||||||
|
func (h *Handler) StartOnboarding(c echo.Context) error {
|
||||||
|
companyID, err := user.GetCompanyIDFromContext(c)
|
||||||
|
if err != nil {
|
||||||
|
return response.Unauthorized(c, "User not authenticated")
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.StartAIOnboarding(c.Request().Context(), companyID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrAIRecommendationNotConfigured) {
|
||||||
|
return response.ServiceUnavailable(c, "AI recommendation service is not configured")
|
||||||
|
}
|
||||||
|
if err.Error() == "company not found" {
|
||||||
|
return response.NotFound(c, "Company not found")
|
||||||
|
}
|
||||||
|
return response.InternalServerError(c, "Failed to start AI onboarding")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "AI onboarding started successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// RecommendTenders returns AI-ranked tender recommendations for the authenticated customer's company (Mobile App)
|
||||||
|
// @Summary Get AI tender recommendations
|
||||||
|
// @Description Retrieve ranked tender recommendations from the AI team for the authenticated customer's company
|
||||||
|
// @Tags Company
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Success 200 {object} response.APIResponse{data=[]RecommendedTenderResponse}
|
||||||
|
// @Failure 401 {object} response.APIResponse "Unauthorized - User not authenticated"
|
||||||
|
// @Failure 404 {object} response.APIResponse "Company not found"
|
||||||
|
// @Failure 503 {object} response.APIResponse "AI recommendation service unavailable"
|
||||||
|
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /api/v1/recommend [post]
|
||||||
|
func (h *Handler) RecommendTenders(c echo.Context) error {
|
||||||
|
if companyIDs, ok := c.Get("company_ids").([]string); ok && len(companyIDs) > 0 {
|
||||||
|
result, err := h.service.GetMergedAIRecommendations(c.Request().Context(), companyIDs)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrAIRecommendationNotConfigured) {
|
||||||
|
return response.ServiceUnavailable(c, "AI recommendation service is not configured")
|
||||||
|
}
|
||||||
|
if err.Error() == "company not found" {
|
||||||
|
return response.NotFound(c, "Company not found")
|
||||||
|
}
|
||||||
|
return response.InternalServerError(c, "Failed to retrieve AI recommendations")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "AI recommendations retrieved successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
companyID, err := user.GetCompanyIDFromContext(c)
|
||||||
|
if err != nil {
|
||||||
|
return response.Unauthorized(c, "User not authenticated")
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.GetAIRecommendations(c.Request().Context(), companyID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrAIRecommendationNotConfigured) {
|
||||||
|
return response.ServiceUnavailable(c, "AI recommendation service is not configured")
|
||||||
|
}
|
||||||
|
if err.Error() == "company not found" {
|
||||||
|
return response.NotFound(c, "Company not found")
|
||||||
|
}
|
||||||
|
return response.InternalServerError(c, "Failed to retrieve AI recommendations")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, "AI recommendations retrieved successfully")
|
||||||
|
}
|
||||||
|
|
||||||
// **************************** ADMIN ENDPOINTS ****************************
|
// **************************** ADMIN ENDPOINTS ****************************
|
||||||
|
|
||||||
// Create creates a new company (Web Panel)
|
// Create creates a new company (Web Panel)
|
||||||
@@ -86,6 +168,9 @@ func (h *Handler) Create(c echo.Context) error {
|
|||||||
err.Error() == "failed to validate categories" {
|
err.Error() == "failed to validate categories" {
|
||||||
return response.BadRequest(c, err.Error(), "Invalid category data")
|
return response.BadRequest(c, err.Error(), "Invalid category data")
|
||||||
}
|
}
|
||||||
|
if err.Error() == "invalid link URL" || err.Error() == "too many links provided" {
|
||||||
|
return response.BadRequest(c, err.Error(), "")
|
||||||
|
}
|
||||||
return response.InternalServerError(c, "Failed to create company")
|
return response.InternalServerError(c, "Failed to create company")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -156,6 +241,9 @@ func (h *Handler) Update(c echo.Context) error {
|
|||||||
err.Error() == "failed to validate categories" {
|
err.Error() == "failed to validate categories" {
|
||||||
return response.BadRequest(c, err.Error(), "Invalid category data")
|
return response.BadRequest(c, err.Error(), "Invalid category data")
|
||||||
}
|
}
|
||||||
|
if err.Error() == "invalid link URL" || err.Error() == "too many links provided" {
|
||||||
|
return response.BadRequest(c, err.Error(), "")
|
||||||
|
}
|
||||||
return response.InternalServerError(c, "Failed to update company")
|
return response.InternalServerError(c, "Failed to update company")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -198,6 +286,10 @@ func (h *Handler) Delete(c echo.Context) error {
|
|||||||
// @Accept json
|
// @Accept json
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param q query string false "Search query"
|
// @Param q query string false "Search query"
|
||||||
|
// @Param name query string false "Filter by company name"
|
||||||
|
// @Param email query string false "Filter by email"
|
||||||
|
// @Param phone query string false "Filter by phone"
|
||||||
|
// @Param employee_count query integer false "Filter by exact employee count"
|
||||||
// @Param cpv_codes query array false "CPV codes filter"
|
// @Param cpv_codes query array false "CPV codes filter"
|
||||||
// @Param categories query array false "Categories filter"
|
// @Param categories query array false "Categories filter"
|
||||||
// @Param keywords query array false "Keywords filter"
|
// @Param keywords query array false "Keywords filter"
|
||||||
@@ -369,6 +461,54 @@ func (h *Handler) UploadDocuments(c echo.Context) error {
|
|||||||
return response.Success(c, result, message)
|
return response.Success(c, result, message)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AddLinks appends external links to a company (Web Panel)
|
||||||
|
// @Summary Add company links
|
||||||
|
// @Description Append one or more external URLs to the company (max 20 links per request)
|
||||||
|
// @Tags Admin-Companies
|
||||||
|
// @Accept json
|
||||||
|
// @Produce json
|
||||||
|
// @Param id path string true "Company ID"
|
||||||
|
// @Param links body AddLinksForm true "Links to append"
|
||||||
|
// @Success 200 {object} response.APIResponse{data=AddLinksResponse} "Links added successfully"
|
||||||
|
// @Failure 400 {object} response.APIResponse "Bad request - No links or invalid URLs"
|
||||||
|
// @Failure 404 {object} response.APIResponse "Not found - Company not found"
|
||||||
|
// @Failure 422 {object} response.APIResponse "Validation error - Invalid request data"
|
||||||
|
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/companies/{id}/links [post]
|
||||||
|
func (h *Handler) AddLinks(c echo.Context) error {
|
||||||
|
id := c.Param("id")
|
||||||
|
|
||||||
|
form, err := response.Parse[AddLinksForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||||
|
}
|
||||||
|
if len(form.Links) > maxCompanyLinks {
|
||||||
|
return response.ValidationError(c, "Too many links", "Maximum 20 links per request")
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := h.service.AddLinks(c.Request().Context(), id, form)
|
||||||
|
if err != nil {
|
||||||
|
switch err.Error() {
|
||||||
|
case "company not found":
|
||||||
|
return response.NotFound(c, "Company not found")
|
||||||
|
case "no links provided", "no valid links provided", "invalid link URL":
|
||||||
|
return response.BadRequest(c, err.Error(), "")
|
||||||
|
case "too many links in request", "company link limit exceeded":
|
||||||
|
return response.ValidationError(c, err.Error(), "")
|
||||||
|
default:
|
||||||
|
return response.InternalServerError(c, "Failed to add company links")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
message := "Company links added successfully"
|
||||||
|
if len(result.Added) == 0 {
|
||||||
|
message = "No new company links were added"
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, result, message)
|
||||||
|
}
|
||||||
|
|
||||||
func collectCompanyUploadFiles(form *multipart.Form) []*multipart.FileHeader {
|
func collectCompanyUploadFiles(form *multipart.Form) []*multipart.FileHeader {
|
||||||
if form == nil {
|
if form == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -0,0 +1,164 @@
|
|||||||
|
package company
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/asaskevich/govalidator"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxCompanyLinks = 20
|
||||||
|
|
||||||
|
// AddLinksResponse is returned after appending links to a company.
|
||||||
|
type AddLinksResponse struct {
|
||||||
|
Links []CompanyLink `json:"links"`
|
||||||
|
Added []CompanyLink `json:"added"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddLinks appends validated links to a company.
|
||||||
|
func (s *companyService) AddLinks(ctx context.Context, companyID string, form *AddLinksForm) (*AddLinksResponse, error) {
|
||||||
|
if form == nil || len(form.Links) == 0 {
|
||||||
|
return nil, errors.New("no links provided")
|
||||||
|
}
|
||||||
|
if len(form.Links) > maxCompanyLinks {
|
||||||
|
return nil, errors.New("too many links in request")
|
||||||
|
}
|
||||||
|
|
||||||
|
company, err := s.repository.GetByID(ctx, companyID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("company not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
newLinks, err := sanitizeCompanyLinks(convertCompanyLinkForms(form.Links))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(newLinks) == 0 {
|
||||||
|
return nil, errors.New("no valid links provided")
|
||||||
|
}
|
||||||
|
|
||||||
|
existing := sanitizeStoredCompanyLinks(company.Links)
|
||||||
|
merged, added := mergeCompanyLinks(existing, newLinks)
|
||||||
|
if len(merged) > maxCompanyLinks {
|
||||||
|
return nil, errors.New("company link limit exceeded")
|
||||||
|
}
|
||||||
|
|
||||||
|
company.Links = merged
|
||||||
|
if err := s.repository.Update(ctx, company); err != nil {
|
||||||
|
s.logger.Error("Failed to save company links", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"company_id": companyID,
|
||||||
|
})
|
||||||
|
return nil, errors.New("failed to update company links")
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Company links added", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"added_count": len(added),
|
||||||
|
"links_total": len(company.Links),
|
||||||
|
})
|
||||||
|
|
||||||
|
s.triggerAIOnboardingAsync(companyID)
|
||||||
|
|
||||||
|
return &AddLinksResponse{
|
||||||
|
Links: company.Links,
|
||||||
|
Added: added,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertCompanyLinkForms(forms []CompanyLinkForm) []CompanyLink {
|
||||||
|
if len(forms) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
links := make([]CompanyLink, 0, len(forms))
|
||||||
|
for _, form := range forms {
|
||||||
|
links = append(links, CompanyLink{
|
||||||
|
URL: form.URL,
|
||||||
|
Title: form.Title,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return links
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanitizeCompanyLinks(links []CompanyLink) ([]CompanyLink, error) {
|
||||||
|
if len(links) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sanitized := make([]CompanyLink, 0, len(links))
|
||||||
|
for _, link := range links {
|
||||||
|
url := strings.TrimSpace(link.URL)
|
||||||
|
if url == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !govalidator.IsURL(url) {
|
||||||
|
return nil, errors.New("invalid link URL")
|
||||||
|
}
|
||||||
|
|
||||||
|
var title *string
|
||||||
|
if link.Title != nil {
|
||||||
|
trimmed := strings.TrimSpace(*link.Title)
|
||||||
|
if trimmed != "" {
|
||||||
|
title = &trimmed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sanitized = append(sanitized, CompanyLink{
|
||||||
|
URL: url,
|
||||||
|
Title: title,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return sanitized, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func sanitizeStoredCompanyLinks(links []CompanyLink) []CompanyLink {
|
||||||
|
sanitized, err := sanitizeCompanyLinks(links)
|
||||||
|
if err != nil || len(sanitized) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return sanitized
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeCompanyLinks(existing, additional []CompanyLink) ([]CompanyLink, []CompanyLink) {
|
||||||
|
seen := make(map[string]struct{}, len(existing)+len(additional))
|
||||||
|
merged := make([]CompanyLink, 0, len(existing)+len(additional))
|
||||||
|
added := make([]CompanyLink, 0, len(additional))
|
||||||
|
|
||||||
|
appendUnique := func(items []CompanyLink, trackAdded bool) {
|
||||||
|
for _, item := range items {
|
||||||
|
key := strings.ToLower(strings.TrimSpace(item.URL))
|
||||||
|
if key == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[key]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
merged = append(merged, item)
|
||||||
|
if trackAdded {
|
||||||
|
added = append(added, item)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
appendUnique(existing, false)
|
||||||
|
appendUnique(additional, true)
|
||||||
|
return merged, added
|
||||||
|
}
|
||||||
|
|
||||||
|
func companyLinkURLs(links []CompanyLink) []string {
|
||||||
|
if len(links) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
urls := make([]string, 0, len(links))
|
||||||
|
for _, link := range links {
|
||||||
|
url := strings.TrimSpace(link.URL)
|
||||||
|
if url != "" {
|
||||||
|
urls = append(urls, url)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return urls
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package company
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestMergeCompanyLinks(t *testing.T) {
|
||||||
|
existing := []CompanyLink{
|
||||||
|
{URL: "https://example.com/a"},
|
||||||
|
{URL: "https://example.com/b"},
|
||||||
|
}
|
||||||
|
additional := []CompanyLink{
|
||||||
|
{URL: " https://example.com/b "},
|
||||||
|
{URL: "https://Example.com/C"},
|
||||||
|
{URL: " "},
|
||||||
|
}
|
||||||
|
|
||||||
|
merged, added := mergeCompanyLinks(existing, additional)
|
||||||
|
if len(merged) != 3 {
|
||||||
|
t.Fatalf("mergeCompanyLinks() merged len = %d, want 3", len(merged))
|
||||||
|
}
|
||||||
|
if len(added) != 1 {
|
||||||
|
t.Fatalf("mergeCompanyLinks() added len = %d, want 1", len(added))
|
||||||
|
}
|
||||||
|
if added[0].URL != "https://Example.com/C" {
|
||||||
|
t.Fatalf("added link = %+v, want https://Example.com/C", added[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSanitizeCompanyLinks(t *testing.T) {
|
||||||
|
title := " Profile "
|
||||||
|
got, err := sanitizeCompanyLinks([]CompanyLink{
|
||||||
|
{URL: " https://example.com ", Title: &title},
|
||||||
|
{URL: "not-a-url"},
|
||||||
|
})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("sanitizeCompanyLinks() expected invalid URL error")
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err = sanitizeCompanyLinks([]CompanyLink{
|
||||||
|
{URL: "https://example.com", Title: &title},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("sanitizeCompanyLinks() unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("sanitizeCompanyLinks() len = %d, want 1", len(got))
|
||||||
|
}
|
||||||
|
if got[0].URL != "https://example.com" {
|
||||||
|
t.Fatalf("sanitized URL = %q, want https://example.com", got[0].URL)
|
||||||
|
}
|
||||||
|
if got[0].Title == nil || *got[0].Title != "Profile" {
|
||||||
|
t.Fatalf("sanitized title = %+v, want Profile", got[0].Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCompanyLinkURLs(t *testing.T) {
|
||||||
|
got := companyLinkURLs([]CompanyLink{
|
||||||
|
{URL: " https://a.test "},
|
||||||
|
{URL: " "},
|
||||||
|
{URL: "https://b.test"},
|
||||||
|
})
|
||||||
|
want := []string{"https://a.test", "https://b.test"}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("companyLinkURLs() len = %d, want %d", len(got), len(want))
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
t.Fatalf("companyLinkURLs()[%d] = %q, want %q", i, got[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
package company
|
||||||
|
|
||||||
|
// RecommendedTendersPageCacheRefresher pre-builds resolved recommendation pages in Redis.
|
||||||
|
// Implemented by the tender service on the web API; optional on the worker.
|
||||||
|
type RecommendedTendersPageCacheRefresher interface {
|
||||||
|
ScheduleRefreshRecommendedTendersPageCache(companyID string)
|
||||||
|
InvalidateRecommendedTendersPageCache(companyID string)
|
||||||
|
}
|
||||||
@@ -0,0 +1,554 @@
|
|||||||
|
package company
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
)
|
||||||
|
|
||||||
|
var ErrAIRecommendationNotConfigured = errors.New("AI recommendation service is not configured")
|
||||||
|
|
||||||
|
const aiRecommendationCacheKeyPrefix = "ai:recommendations:"
|
||||||
|
|
||||||
|
const (
|
||||||
|
recommendationFetchMaxAttempts = 6
|
||||||
|
recommendationFetchInitialDelay = 10 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
func normalizeRecommendationCompanyIDs(companyIDs []string) []string {
|
||||||
|
normalized := make([]string, 0, len(companyIDs))
|
||||||
|
seen := make(map[string]struct{}, len(companyIDs))
|
||||||
|
for _, companyID := range companyIDs {
|
||||||
|
clean := strings.TrimSpace(companyID)
|
||||||
|
if clean == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[clean]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[clean] = struct{}{}
|
||||||
|
normalized = append(normalized, clean)
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeRecommendedTenders(recommendationLists ...[]RecommendedTenderResponse) []RecommendedTenderResponse {
|
||||||
|
merged := make([]RecommendedTenderResponse, 0)
|
||||||
|
for _, recommendations := range recommendationLists {
|
||||||
|
merged = append(merged, recommendations...)
|
||||||
|
}
|
||||||
|
return dedupeRecommendationResponsesByTenderID(merged)
|
||||||
|
}
|
||||||
|
|
||||||
|
func aiRecommendationCacheKey(companyID string) string {
|
||||||
|
return aiRecommendationCacheKeyPrefix + companyID
|
||||||
|
}
|
||||||
|
|
||||||
|
// AIRecommendationClient defines the interface for company tender recommendation AI operations.
|
||||||
|
type AIRecommendationClient interface {
|
||||||
|
StartOnboarding(ctx context.Context, reqBody ai_summarizer.OnboardingRequest) (*ai_summarizer.OnboardingResponse, error)
|
||||||
|
FetchRecommendations(ctx context.Context, reqBody ai_summarizer.RecommendRequest) ([]ai_summarizer.RecommendedTender, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type onboardingDocumentPayload struct {
|
||||||
|
Filename string `json:"filename"`
|
||||||
|
ContentType string `json:"content_type"`
|
||||||
|
Content string `json:"content"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// StartAIOnboarding sends company profile data to the AI team to start recommendation onboarding.
|
||||||
|
func (s *companyService) StartAIOnboarding(ctx context.Context, companyID string) (*OnboardingResponse, error) {
|
||||||
|
if s.aiRecommendationClient == nil {
|
||||||
|
return nil, ErrAIRecommendationNotConfigured
|
||||||
|
}
|
||||||
|
|
||||||
|
companyID = strings.TrimSpace(companyID)
|
||||||
|
if companyID == "" {
|
||||||
|
return nil, errors.New("company ID is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
s.invalidateAIRecommendationCache(ctx, companyID)
|
||||||
|
|
||||||
|
result, err := s.startAIOnboardingInternal(ctx, companyID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.scheduleRecommendationRefreshAfterOnboarding(companyID)
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// triggerAIOnboardingAsync re-indexes the company on the AI service and refreshes the recommendation cache.
|
||||||
|
// This is the only path (besides StartAIOnboarding) that calls the AI recommend endpoint.
|
||||||
|
func (s *companyService) triggerAIOnboardingAsync(companyID string) {
|
||||||
|
if s.aiRecommendationClient == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
companyID = strings.TrimSpace(companyID)
|
||||||
|
if companyID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
ctx := context.Background()
|
||||||
|
s.logger.Info("Starting async AI company onboarding and recommendation refresh", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
})
|
||||||
|
|
||||||
|
s.invalidateAIRecommendationCache(ctx, companyID)
|
||||||
|
|
||||||
|
if _, err := s.startAIOnboardingInternal(ctx, companyID); err != nil {
|
||||||
|
s.logger.Error("Async AI company onboarding failed", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.fetchAndCacheAIRecommendationsWithRetry(ctx, companyID); err != nil {
|
||||||
|
s.logger.Error("Failed to refresh AI recommendations after onboarding", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Async AI company onboarding and recommendation refresh completed", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) startAIOnboardingInternal(ctx context.Context, companyID string) (*OnboardingResponse, error) {
|
||||||
|
company, err := s.repository.GetByID(ctx, companyID)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to load company for AI onboarding", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, errors.New("company not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
documents, err := s.buildOnboardingDocuments(company.DocumentFileIDs)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to prepare company documents for AI onboarding", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to prepare company documents: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
websiteURL := ""
|
||||||
|
if company.Website != nil {
|
||||||
|
websiteURL = strings.TrimSpace(*company.Website)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Starting AI company onboarding", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"documents_count": len(company.DocumentFileIDs),
|
||||||
|
"links_count": len(company.Links),
|
||||||
|
"has_website_url": websiteURL != "",
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := s.aiRecommendationClient.StartOnboarding(ctx, ai_summarizer.OnboardingRequest{
|
||||||
|
CompanyID: companyID,
|
||||||
|
Documents: documents,
|
||||||
|
WebsiteURL: websiteURL,
|
||||||
|
Links: companyLinkURLs(company.Links),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("AI onboarding request failed", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to start AI onboarding: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &OnboardingResponse{Status: result.Status}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetAIRecommendations returns cached recommendations only. The AI service is not called on read;
|
||||||
|
// cache is populated when a company is created, updated, or onboarded.
|
||||||
|
func (s *companyService) GetAIRecommendations(ctx context.Context, companyID string) ([]RecommendedTenderResponse, error) {
|
||||||
|
if s.aiRecommendationClient == nil {
|
||||||
|
return nil, ErrAIRecommendationNotConfigured
|
||||||
|
}
|
||||||
|
|
||||||
|
companyID = strings.TrimSpace(companyID)
|
||||||
|
if companyID == "" {
|
||||||
|
return nil, errors.New("company ID is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
if cached, ok := s.getCachedAIRecommendations(ctx, companyID); ok {
|
||||||
|
return cached, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := s.repository.GetByID(ctx, companyID); err != nil {
|
||||||
|
s.logger.Error("Failed to load company for AI recommendations", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, errors.New("company not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Debug("AI recommendations cache miss; returning empty list", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
})
|
||||||
|
|
||||||
|
return []RecommendedTenderResponse{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) validateRecommendationCompanyIDs(ctx context.Context, companyIDs []string) error {
|
||||||
|
companies, err := s.repository.GetByIDs(ctx, companyIDs)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to load companies for merged AI recommendations", map[string]interface{}{
|
||||||
|
"company_ids": companyIDs,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return errors.New("company not found")
|
||||||
|
}
|
||||||
|
if len(companies) != len(companyIDs) {
|
||||||
|
s.logger.Error("Some companies were not found for merged AI recommendations", map[string]interface{}{
|
||||||
|
"requested_count": len(companyIDs),
|
||||||
|
"loaded_count": len(companies),
|
||||||
|
})
|
||||||
|
return errors.New("company not found")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) getCachedAIRecommendationsOrEmpty(ctx context.Context, companyID string) []RecommendedTenderResponse {
|
||||||
|
if cached, ok := s.getCachedAIRecommendations(ctx, companyID); ok {
|
||||||
|
return cached
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Debug("AI recommendations cache miss; returning empty list", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
})
|
||||||
|
return []RecommendedTenderResponse{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetMergedAIRecommendations unions cached AI recommendations across multiple companies.
|
||||||
|
func (s *companyService) GetMergedAIRecommendations(ctx context.Context, companyIDs []string) ([]RecommendedTenderResponse, error) {
|
||||||
|
if s.aiRecommendationClient == nil {
|
||||||
|
return nil, ErrAIRecommendationNotConfigured
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized := normalizeRecommendationCompanyIDs(companyIDs)
|
||||||
|
if len(normalized) == 0 {
|
||||||
|
return []RecommendedTenderResponse{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
recommendationLists := make([][]RecommendedTenderResponse, len(normalized))
|
||||||
|
group, groupCtx := errgroup.WithContext(ctx)
|
||||||
|
group.Go(func() error {
|
||||||
|
return s.validateRecommendationCompanyIDs(groupCtx, normalized)
|
||||||
|
})
|
||||||
|
for i, companyID := range normalized {
|
||||||
|
i := i
|
||||||
|
companyID := companyID
|
||||||
|
group.Go(func() error {
|
||||||
|
recommendationLists[i] = s.getCachedAIRecommendationsOrEmpty(groupCtx, companyID)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err := group.Wait(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return mergeRecommendedTenders(recommendationLists...), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) scheduleRecommendationRefreshAfterOnboarding(companyID string) {
|
||||||
|
if s.aiRecommendationClient == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
companyID = strings.TrimSpace(companyID)
|
||||||
|
if companyID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
ctx := context.Background()
|
||||||
|
if err := s.fetchAndCacheAIRecommendationsWithRetry(ctx, companyID); err != nil {
|
||||||
|
s.logger.Error("Failed to refresh AI recommendation cache after onboarding", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchAndCacheAIRecommendationsWithRetry waits for the AI service to finish indexing after onboarding.
|
||||||
|
// Empty results are not cached so reads keep returning empty until real recommendations exist.
|
||||||
|
func (s *companyService) fetchAndCacheAIRecommendationsWithRetry(ctx context.Context, companyID string) error {
|
||||||
|
delay := recommendationFetchInitialDelay
|
||||||
|
|
||||||
|
for attempt := 1; attempt <= recommendationFetchMaxAttempts; attempt++ {
|
||||||
|
if attempt > 1 {
|
||||||
|
s.logger.Info("Retrying AI recommendation fetch after onboarding", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"attempt": attempt,
|
||||||
|
"delay_sec": int(delay.Seconds()),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
s.logger.Info("Waiting before first AI recommendation fetch after onboarding", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"delay_sec": int(delay.Seconds()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(delay):
|
||||||
|
}
|
||||||
|
|
||||||
|
if attempt > 1 {
|
||||||
|
delay *= 2
|
||||||
|
}
|
||||||
|
|
||||||
|
responses, err := s.fetchAIRecommendations(ctx, companyID)
|
||||||
|
if err != nil {
|
||||||
|
if attempt == recommendationFetchMaxAttempts {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(responses) == 0 {
|
||||||
|
if attempt == recommendationFetchMaxAttempts {
|
||||||
|
s.logger.Warn("AI returned no recommendations after onboarding retries; cache not updated", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"attempts": attempt,
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
s.cacheAIRecommendations(ctx, companyID, responses)
|
||||||
|
s.logger.Info("AI recommendation cache populated after onboarding", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"count": len(responses),
|
||||||
|
"attempt": attempt,
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) fetchAIRecommendations(ctx context.Context, companyID string) ([]RecommendedTenderResponse, error) {
|
||||||
|
s.logger.Info("Fetching AI tender recommendations from AI service", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
})
|
||||||
|
|
||||||
|
items, err := s.aiRecommendationClient.FetchRecommendations(ctx, ai_summarizer.RecommendRequest{
|
||||||
|
CompanyID: companyID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("AI recommend request failed", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to fetch AI recommendations: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return dedupeRecommendationResponsesByTenderID(itemsToRecommendationResponses(items)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func itemsToRecommendationResponses(items []ai_summarizer.RecommendedTender) []RecommendedTenderResponse {
|
||||||
|
responses := make([]RecommendedTenderResponse, 0, len(items))
|
||||||
|
for _, item := range items {
|
||||||
|
responses = append(responses, RecommendedTenderResponse{
|
||||||
|
Rank: item.Rank,
|
||||||
|
TenderID: item.TenderID,
|
||||||
|
Analysis: item.Analysis,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return responses
|
||||||
|
}
|
||||||
|
|
||||||
|
func dedupeRecommendationResponsesByTenderID(responses []RecommendedTenderResponse) []RecommendedTenderResponse {
|
||||||
|
seen := make(map[string]struct{}, len(responses))
|
||||||
|
out := make([]RecommendedTenderResponse, 0, len(responses))
|
||||||
|
for _, recommendation := range responses {
|
||||||
|
tenderID := strings.TrimSpace(recommendation.TenderID)
|
||||||
|
if tenderID == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[tenderID]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[tenderID] = struct{}{}
|
||||||
|
recommendation.TenderID = tenderID
|
||||||
|
recommendation.Analysis = strings.TrimSpace(recommendation.Analysis)
|
||||||
|
out = append(out, recommendation)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) getCachedAIRecommendations(ctx context.Context, companyID string) ([]RecommendedTenderResponse, bool) {
|
||||||
|
if s.redisClient == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
readCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
raw, err := s.redisClient.Get(readCtx, aiRecommendationCacheKey(companyID))
|
||||||
|
if err != nil {
|
||||||
|
if err != redis.Nil {
|
||||||
|
s.logger.Warn("Failed to read AI recommendation cache", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
var responses []RecommendedTenderResponse
|
||||||
|
if err := json.Unmarshal([]byte(raw), &responses); err != nil {
|
||||||
|
s.logger.Warn("Failed to decode AI recommendation cache", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
_ = s.redisClient.Del(ctx, aiRecommendationCacheKey(companyID))
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(responses) == 0 {
|
||||||
|
_ = s.redisClient.Del(ctx, aiRecommendationCacheKey(companyID))
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Debug("AI tender recommendations served from cache", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"count": len(responses),
|
||||||
|
})
|
||||||
|
|
||||||
|
return responses, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) cacheAIRecommendations(ctx context.Context, companyID string, responses []RecommendedTenderResponse) {
|
||||||
|
if s.redisClient == nil || len(responses) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
encoded, err := json.Marshal(responses)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("Failed to encode AI recommendations for cache", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
expiration := s.recommendationCacheExpiration()
|
||||||
|
if err := s.redisClient.Set(ctx, aiRecommendationCacheKey(companyID), string(encoded), expiration); err != nil {
|
||||||
|
s.logger.Warn("Failed to store AI recommendation cache", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.invalidateRecommendedTendersPageCache(companyID)
|
||||||
|
s.scheduleRecommendedTendersPageCacheRefresh(companyID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) recommendationCacheExpiration() time.Duration {
|
||||||
|
if s.recommendationCacheTTL > 0 {
|
||||||
|
return s.recommendationCacheTTL
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) invalidateAIRecommendationCache(ctx context.Context, companyID string) {
|
||||||
|
if s.redisClient == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.redisClient.Del(ctx, aiRecommendationCacheKey(companyID)); err != nil {
|
||||||
|
s.logger.Warn("Failed to invalidate AI recommendation cache", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
s.invalidateRecommendedTendersPageCache(companyID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) buildOnboardingDocuments(fileIDs []string) (string, error) {
|
||||||
|
if len(fileIDs) == 0 {
|
||||||
|
return "[]", nil
|
||||||
|
}
|
||||||
|
if s.fileStore == nil {
|
||||||
|
return "", errors.New("file storage is not configured")
|
||||||
|
}
|
||||||
|
|
||||||
|
payloads := make([]onboardingDocumentPayload, 0, len(fileIDs))
|
||||||
|
for _, fileID := range fileIDs {
|
||||||
|
if strings.TrimSpace(fileID) == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, info, err := s.fileStore.Download(fileID)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("Skipping company document for AI onboarding", map[string]interface{}{
|
||||||
|
"file_id": fileID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
content, readErr := io.ReadAll(reader)
|
||||||
|
closeErr := reader.Close()
|
||||||
|
if readErr != nil {
|
||||||
|
return "", fmt.Errorf("failed to read document %s: %w", fileID, readErr)
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
s.logger.Warn("Failed to close company document reader", map[string]interface{}{
|
||||||
|
"file_id": fileID,
|
||||||
|
"error": closeErr.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := fileID
|
||||||
|
contentType := "application/octet-stream"
|
||||||
|
if info != nil {
|
||||||
|
if info.Filename != "" {
|
||||||
|
filename = info.Filename
|
||||||
|
}
|
||||||
|
if info.ContentType != "" {
|
||||||
|
contentType = info.ContentType
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
payloads = append(payloads, onboardingDocumentPayload{
|
||||||
|
Filename: filename,
|
||||||
|
ContentType: contentType,
|
||||||
|
Content: base64.StdEncoding.EncodeToString(content),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
encoded, err := json.Marshal(payloads)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("failed to encode company documents: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return string(encoded), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
package company
|
||||||
|
|
||||||
|
func (s *companyService) invalidateRecommendedTendersPageCache(companyID string) {
|
||||||
|
if s.pageCacheRefresher == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.pageCacheRefresher.InvalidateRecommendedTendersPageCache(companyID)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) scheduleRecommendedTendersPageCacheRefresh(companyID string) {
|
||||||
|
if s.pageCacheRefresher == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.pageCacheRefresher.ScheduleRefreshRecommendedTendersPageCache(companyID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetRecommendedTendersPageCacheRefresher wires async page-cache rebuilds from the tender service.
|
||||||
|
func (s *companyService) SetRecommendedTendersPageCacheRefresher(refresher RecommendedTendersPageCacheRefresher) {
|
||||||
|
s.pageCacheRefresher = refresher
|
||||||
|
}
|
||||||
@@ -0,0 +1,221 @@
|
|||||||
|
package company
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AIPipelineStatusClient reports pipeline job status from the AI service.
|
||||||
|
type AIPipelineStatusClient interface {
|
||||||
|
GetPipelineLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
recommendationPipelineWaitInitialDelay = 5 * time.Minute
|
||||||
|
recommendationPipelineWaitMaxAttempts = 24
|
||||||
|
recommendationRefreshConcurrency = 3
|
||||||
|
)
|
||||||
|
|
||||||
|
// ScheduleRefreshCachedAIRecommendationsAfterPipeline waits for the AI pipeline auto run to finish,
|
||||||
|
// then re-fetches ranked tenders for every company that already has a recommendation cache.
|
||||||
|
func (s *companyService) ScheduleRefreshCachedAIRecommendationsAfterPipeline() {
|
||||||
|
if s.aiRecommendationClient == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
ctx := context.Background()
|
||||||
|
s.logger.Info("Scheduling AI recommendation cache refresh after pipeline sync", map[string]interface{}{})
|
||||||
|
|
||||||
|
if err := s.refreshCachedAIRecommendationsAfterPipeline(ctx); err != nil {
|
||||||
|
s.logger.Error("AI recommendation cache refresh after pipeline failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("AI recommendation cache refresh after pipeline completed", map[string]interface{}{})
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) refreshCachedAIRecommendationsAfterPipeline(ctx context.Context) error {
|
||||||
|
if err := s.waitForPipelineAutoCompletion(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
companyIDs, err := s.listCompanyIDsWithRecommendationCache(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(companyIDs) == 0 {
|
||||||
|
s.logger.Info("No companies with recommendation cache to refresh after pipeline", map[string]interface{}{})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Refreshing AI recommendation caches after pipeline", map[string]interface{}{
|
||||||
|
"company_count": len(companyIDs),
|
||||||
|
})
|
||||||
|
|
||||||
|
group, groupCtx := errgroup.WithContext(ctx)
|
||||||
|
group.SetLimit(recommendationRefreshConcurrency)
|
||||||
|
|
||||||
|
var refreshed atomic.Int32
|
||||||
|
var failed atomic.Int32
|
||||||
|
for _, companyID := range companyIDs {
|
||||||
|
companyID := companyID
|
||||||
|
group.Go(func() error {
|
||||||
|
if err := s.refreshAIRecommendationsCache(groupCtx, companyID); err != nil {
|
||||||
|
s.logger.Error("Failed to refresh AI recommendation cache for company", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
failed.Add(1)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
refreshed.Add(1)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := group.Wait(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("AI recommendation cache refresh summary", map[string]interface{}{
|
||||||
|
"company_count": len(companyIDs),
|
||||||
|
"refreshed": refreshed.Load(),
|
||||||
|
"failed": failed.Load(),
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) waitForPipelineAutoCompletion(ctx context.Context) error {
|
||||||
|
if s.aiPipelineStatusClient == nil {
|
||||||
|
s.logger.Info("AI pipeline status client not configured; waiting before recommendation refresh", map[string]interface{}{
|
||||||
|
"delay_sec": int(recommendationPipelineWaitInitialDelay.Seconds()),
|
||||||
|
})
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(recommendationPipelineWaitInitialDelay):
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
delay := recommendationPipelineWaitInitialDelay
|
||||||
|
for attempt := 1; attempt <= recommendationPipelineWaitMaxAttempts; attempt++ {
|
||||||
|
if attempt == 1 {
|
||||||
|
s.logger.Info("Waiting before first AI pipeline status check", map[string]interface{}{
|
||||||
|
"delay_sec": int(delay.Seconds()),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
s.logger.Info("Retrying AI pipeline status check", map[string]interface{}{
|
||||||
|
"attempt": attempt,
|
||||||
|
"delay_sec": int(delay.Seconds()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
case <-time.After(delay):
|
||||||
|
}
|
||||||
|
|
||||||
|
if attempt > 1 {
|
||||||
|
delay *= 2
|
||||||
|
if delay > 30*time.Minute {
|
||||||
|
delay = 30 * time.Minute
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
report, err := s.aiPipelineStatusClient.GetPipelineLastAutoRun(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("Failed to read AI pipeline last-auto-run status", map[string]interface{}{
|
||||||
|
"attempt": attempt,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
if attempt == recommendationPipelineWaitMaxAttempts {
|
||||||
|
s.logger.Warn("Proceeding with recommendation refresh without pipeline status", map[string]interface{}{})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !isPipelineRunInProgress(report) {
|
||||||
|
s.logger.Info("AI pipeline auto run finished; refreshing recommendation caches", map[string]interface{}{
|
||||||
|
"attempt": attempt,
|
||||||
|
"status": strings.TrimSpace(report.Status),
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if attempt == recommendationPipelineWaitMaxAttempts {
|
||||||
|
s.logger.Warn("AI pipeline still in progress after max wait; refreshing recommendation caches anyway", map[string]interface{}{
|
||||||
|
"attempts": attempt,
|
||||||
|
"status": strings.TrimSpace(report.Status),
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isPipelineRunInProgress(report *ai_summarizer.PipelineReportResponse) bool {
|
||||||
|
if report == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
status := strings.ToLower(strings.TrimSpace(report.Status))
|
||||||
|
switch status {
|
||||||
|
case "", "running", "in_progress", "started", "processing", "pending", "queued":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *companyService) listCompanyIDsWithRecommendationCache(ctx context.Context) ([]string, error) {
|
||||||
|
allIDs, err := s.repository.ListIDs(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to list companies for recommendation refresh", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
cached := make([]string, 0, len(allIDs))
|
||||||
|
for _, companyID := range allIDs {
|
||||||
|
if _, ok := s.getCachedAIRecommendations(ctx, companyID); ok {
|
||||||
|
cached = append(cached, companyID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return cached, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// refreshAIRecommendationsCache updates Redis when the AI service returns a non-empty ranked list.
|
||||||
|
// Empty or failed fetches leave the existing cache untouched.
|
||||||
|
func (s *companyService) refreshAIRecommendationsCache(ctx context.Context, companyID string) error {
|
||||||
|
responses, err := s.fetchAIRecommendations(ctx, companyID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if len(responses) == 0 {
|
||||||
|
s.logger.Info("Skipping recommendation cache update after pipeline: AI returned empty list", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s.cacheAIRecommendations(ctx, companyID, responses)
|
||||||
|
s.logger.Info("AI recommendation cache refreshed after pipeline", map[string]interface{}{
|
||||||
|
"company_id": companyID,
|
||||||
|
"count": len(responses),
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package company
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestIsPipelineRunInProgress(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
report *ai_summarizer.PipelineReportResponse
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{name: "nil report", report: nil, want: true},
|
||||||
|
{name: "empty status", report: &ai_summarizer.PipelineReportResponse{}, want: true},
|
||||||
|
{name: "running", report: &ai_summarizer.PipelineReportResponse{Status: "running"}, want: true},
|
||||||
|
{name: "in progress", report: &ai_summarizer.PipelineReportResponse{Status: "in_progress"}, want: true},
|
||||||
|
{name: "completed", report: &ai_summarizer.PipelineReportResponse{Status: "completed"}, want: false},
|
||||||
|
{name: "failed", report: &ai_summarizer.PipelineReportResponse{Status: "failed"}, want: false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := isPipelineRunInProgress(tt.report); got != tt.want {
|
||||||
|
t.Fatalf("isPipelineRunInProgress() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,64 @@
|
|||||||
|
package company
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestDedupeRecommendationResponsesByTenderID(t *testing.T) {
|
||||||
|
input := []RecommendedTenderResponse{
|
||||||
|
{Rank: 1, TenderID: " PROC-1 ", Analysis: " first "},
|
||||||
|
{Rank: 2, TenderID: "PROC-2", Analysis: "second"},
|
||||||
|
{Rank: 1, TenderID: "PROC-2", Analysis: "duplicate"},
|
||||||
|
{Rank: 4, TenderID: " ", Analysis: "ignored"},
|
||||||
|
}
|
||||||
|
|
||||||
|
got := dedupeRecommendationResponsesByTenderID(input)
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("dedupeRecommendationResponsesByTenderID() len = %d, want 2", len(got))
|
||||||
|
}
|
||||||
|
if got[0].TenderID != "PROC-1" || got[0].Analysis != "first" {
|
||||||
|
t.Fatalf("first recommendation = %+v, want trimmed PROC-1/first", got[0])
|
||||||
|
}
|
||||||
|
if got[1].TenderID != "PROC-2" || got[1].Rank != 2 {
|
||||||
|
t.Fatalf("second recommendation = %+v, want original PROC-2 rank 2", got[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeRecommendedTenders(t *testing.T) {
|
||||||
|
first := []RecommendedTenderResponse{
|
||||||
|
{Rank: 1, TenderID: " PROC-1 ", Analysis: " first "},
|
||||||
|
{Rank: 2, TenderID: "PROC-2", Analysis: "second"},
|
||||||
|
}
|
||||||
|
second := []RecommendedTenderResponse{
|
||||||
|
{Rank: 1, TenderID: "PROC-2", Analysis: "duplicate"},
|
||||||
|
{Rank: 3, TenderID: "PROC-3", Analysis: "third"},
|
||||||
|
{Rank: 4, TenderID: " ", Analysis: "ignored"},
|
||||||
|
}
|
||||||
|
|
||||||
|
got := mergeRecommendedTenders(first, second)
|
||||||
|
if len(got) != 3 {
|
||||||
|
t.Fatalf("mergeRecommendedTenders() len = %d, want 3", len(got))
|
||||||
|
}
|
||||||
|
|
||||||
|
if got[0].TenderID != "PROC-1" || got[0].Analysis != "first" {
|
||||||
|
t.Fatalf("first recommendation = %+v, want trimmed PROC-1/first", got[0])
|
||||||
|
}
|
||||||
|
if got[1].TenderID != "PROC-2" || got[1].Rank != 2 {
|
||||||
|
t.Fatalf("second recommendation = %+v, want original PROC-2 rank 2", got[1])
|
||||||
|
}
|
||||||
|
if got[2].TenderID != "PROC-3" {
|
||||||
|
t.Fatalf("third recommendation = %+v, want PROC-3", got[2])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNormalizeRecommendationCompanyIDs(t *testing.T) {
|
||||||
|
got := normalizeRecommendationCompanyIDs([]string{" company-a ", "", "company-b", "company-a"})
|
||||||
|
want := []string{"company-a", "company-b"}
|
||||||
|
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("normalizeRecommendationCompanyIDs() len = %d, want %d", len(got), len(want))
|
||||||
|
}
|
||||||
|
for i := range want {
|
||||||
|
if got[i] != want[i] {
|
||||||
|
t.Fatalf("normalizeRecommendationCompanyIDs()[%d] = %q, want %q", i, got[i], want[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,12 +3,15 @@ package company
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
orm "tm/pkg/mongo"
|
orm "tm/pkg/mongo"
|
||||||
"tm/pkg/response"
|
"tm/pkg/response"
|
||||||
|
|
||||||
"go.mongodb.org/mongo-driver/v2/bson"
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Repository defines the interface for company data operations
|
// Repository defines the interface for company data operations
|
||||||
@@ -21,14 +24,17 @@ type Repository interface {
|
|||||||
GetByRegistrationNumber(ctx context.Context, registrationNumber string) (*Company, error)
|
GetByRegistrationNumber(ctx context.Context, registrationNumber string) (*Company, error)
|
||||||
GetByTaxID(ctx context.Context, taxID string) (*Company, error)
|
GetByTaxID(ctx context.Context, taxID string) (*Company, error)
|
||||||
Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*orm.PaginatedResult[Company], error)
|
Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*orm.PaginatedResult[Company], error)
|
||||||
|
ListIDs(ctx context.Context) ([]string, error)
|
||||||
Delete(ctx context.Context, id string) error
|
Delete(ctx context.Context, id string) error
|
||||||
UpdateStatus(ctx context.Context, id string, status CompanyStatus) error
|
UpdateStatus(ctx context.Context, id string, status CompanyStatus) error
|
||||||
UpdateVerification(ctx context.Context, id string, isVerified, isCompliant bool, complianceNotes *string) error
|
UpdateVerification(ctx context.Context, id string, isVerified, isCompliant bool, complianceNotes *string) error
|
||||||
|
RemoveDocumentFileID(ctx context.Context, fileID string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// companyRepository implements the Repository interface using the MongoDB ORM
|
// companyRepository implements the Repository interface using the MongoDB ORM
|
||||||
type companyRepository struct {
|
type companyRepository struct {
|
||||||
ormRepo orm.Repository[Company]
|
ormRepo orm.Repository[Company]
|
||||||
|
collection *mongo.Collection
|
||||||
logger logger.Logger
|
logger logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -47,11 +53,14 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
collection := mongoManager.GetCollection("companies")
|
||||||
|
|
||||||
// Create ORM repository
|
// Create ORM repository
|
||||||
ormRepo := orm.NewRepository[Company](mongoManager.GetCollection("companies"), logger)
|
ormRepo := orm.NewRepository[Company](collection, logger)
|
||||||
|
|
||||||
return &companyRepository{
|
return &companyRepository{
|
||||||
ormRepo: ormRepo,
|
ormRepo: ormRepo,
|
||||||
|
collection: collection,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -161,6 +170,9 @@ func (r *companyRepository) Update(ctx context.Context, company *Company) error
|
|||||||
// Set updated timestamp using Unix timestamp
|
// Set updated timestamp using Unix timestamp
|
||||||
company.SetUpdatedAt(time.Now().Unix())
|
company.SetUpdatedAt(time.Now().Unix())
|
||||||
|
|
||||||
|
clearDocuments := company.DocumentFileIDs != nil && len(company.DocumentFileIDs) == 0
|
||||||
|
clearLinks := company.Links != nil && len(company.Links) == 0
|
||||||
|
|
||||||
// Use ORM to update company
|
// Use ORM to update company
|
||||||
err := r.ormRepo.Update(ctx, company)
|
err := r.ormRepo.Update(ctx, company)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -171,6 +183,30 @@ func (r *companyRepository) Update(ctx context.Context, company *Company) error
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BSON omitempty can skip empty slices in $set, so persist explicit empty arrays.
|
||||||
|
if clearDocuments || clearLinks {
|
||||||
|
setFields := bson.M{
|
||||||
|
"updated_at": company.UpdatedAt,
|
||||||
|
}
|
||||||
|
if clearDocuments {
|
||||||
|
setFields["document_file_ids"] = []string{}
|
||||||
|
}
|
||||||
|
if clearLinks {
|
||||||
|
setFields["links"] = []CompanyLink{}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = r.collection.UpdateOne(ctx, bson.M{"_id": company.ID}, bson.M{
|
||||||
|
"$set": setFields,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
r.logger.Error("Failed to clear company collection fields", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"company_id": company.ID,
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
r.logger.Info("Company updated successfully", map[string]interface{}{
|
r.logger.Info("Company updated successfully", map[string]interface{}{
|
||||||
"company_id": company.ID,
|
"company_id": company.ID,
|
||||||
"name": company.Name,
|
"name": company.Name,
|
||||||
@@ -213,6 +249,36 @@ func (r *companyRepository) Search(ctx context.Context, form *SearchForm, pagina
|
|||||||
filter["$text"] = bson.M{"$search": *form.Search}
|
filter["$text"] = bson.M{"$search": *form.Search}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if form.Name != nil {
|
||||||
|
name := strings.TrimSpace(*form.Name)
|
||||||
|
if name != "" {
|
||||||
|
filter["name"] = bson.M{
|
||||||
|
"$regex": regexp.QuoteMeta(name),
|
||||||
|
"$options": "i",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if form.Email != nil {
|
||||||
|
email := strings.TrimSpace(*form.Email)
|
||||||
|
if email != "" {
|
||||||
|
filter["email"] = bson.M{
|
||||||
|
"$regex": regexp.QuoteMeta(email),
|
||||||
|
"$options": "i",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if form.Phone != nil {
|
||||||
|
phone := strings.TrimSpace(*form.Phone)
|
||||||
|
if phone != "" {
|
||||||
|
filter["phone"] = bson.M{
|
||||||
|
"$regex": regexp.QuoteMeta(phone),
|
||||||
|
"$options": "i",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if form.Type != nil {
|
if form.Type != nil {
|
||||||
filter["type"] = *form.Type
|
filter["type"] = *form.Type
|
||||||
}
|
}
|
||||||
@@ -259,7 +325,9 @@ func (r *companyRepository) Search(ctx context.Context, form *SearchForm, pagina
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Range filters
|
// Range filters
|
||||||
if form.EmployeeCountMin != nil || form.EmployeeCountMax != nil {
|
if form.EmployeeCount != nil {
|
||||||
|
filter["employee_count"] = *form.EmployeeCount
|
||||||
|
} else if form.EmployeeCountMin != nil || form.EmployeeCountMax != nil {
|
||||||
rangeFilter := bson.M{}
|
rangeFilter := bson.M{}
|
||||||
if form.EmployeeCountMin != nil {
|
if form.EmployeeCountMin != nil {
|
||||||
rangeFilter["$gte"] = *form.EmployeeCountMin
|
rangeFilter["$gte"] = *form.EmployeeCountMin
|
||||||
@@ -316,6 +384,49 @@ func (r *companyRepository) Search(ctx context.Context, form *SearchForm, pagina
|
|||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListIDs returns every company document ID.
|
||||||
|
func (r *companyRepository) ListIDs(ctx context.Context) ([]string, error) {
|
||||||
|
const pageSize = 500
|
||||||
|
ids := make([]string, 0)
|
||||||
|
skip := 0
|
||||||
|
|
||||||
|
for {
|
||||||
|
result, err := r.ormRepo.FindAll(ctx, bson.M{}, orm.Pagination{
|
||||||
|
Limit: pageSize,
|
||||||
|
Skip: skip,
|
||||||
|
SortField: "_id",
|
||||||
|
SortOrder: 1,
|
||||||
|
SkipCount: true,
|
||||||
|
Projection: bson.M{"_id": 1},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
r.logger.Error("Failed to list company IDs", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"skip": skip,
|
||||||
|
})
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Items) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := range result.Items {
|
||||||
|
id := strings.TrimSpace(result.Items[i].GetID())
|
||||||
|
if id != "" {
|
||||||
|
ids = append(ids, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(result.Items) < pageSize {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
skip += pageSize
|
||||||
|
}
|
||||||
|
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
// UpdateStatus updates company status
|
// UpdateStatus updates company status
|
||||||
func (r *companyRepository) UpdateStatus(ctx context.Context, id string, status CompanyStatus) error {
|
func (r *companyRepository) UpdateStatus(ctx context.Context, id string, status CompanyStatus) error {
|
||||||
// Get company first
|
// Get company first
|
||||||
@@ -379,3 +490,34 @@ func (r *companyRepository) UpdateVerification(ctx context.Context, id string, i
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RemoveDocumentFileID removes a file ID from all companies that reference it.
|
||||||
|
func (r *companyRepository) RemoveDocumentFileID(ctx context.Context, fileID string) error {
|
||||||
|
if fileID == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
filter := bson.M{"document_file_ids": fileID}
|
||||||
|
update := bson.M{
|
||||||
|
"$pull": bson.M{"document_file_ids": fileID},
|
||||||
|
"$set": bson.M{"updated_at": time.Now().Unix()},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := r.collection.UpdateMany(ctx, filter, update)
|
||||||
|
if err != nil {
|
||||||
|
r.logger.Error("Failed to remove document file ID from companies", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"file_id": fileID,
|
||||||
|
})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.ModifiedCount > 0 {
|
||||||
|
r.logger.Info("Removed document file ID from companies", map[string]interface{}{
|
||||||
|
"file_id": fileID,
|
||||||
|
"modified_count": result.ModifiedCount,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,12 +3,13 @@ package company
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
|
"time"
|
||||||
|
|
||||||
"tm/internal/company_category"
|
"tm/internal/company_category"
|
||||||
"tm/pkg/filestore"
|
"tm/pkg/filestore"
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
|
"tm/pkg/redis"
|
||||||
"tm/pkg/response"
|
"tm/pkg/response"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,6 +35,22 @@ type Service interface {
|
|||||||
|
|
||||||
// UploadDocuments uploads files and attaches them to a company
|
// UploadDocuments uploads files and attaches them to a company
|
||||||
UploadDocuments(ctx context.Context, companyID, uploadedBy string, files []*multipart.FileHeader) (*UploadDocumentsResponse, error)
|
UploadDocuments(ctx context.Context, companyID, uploadedBy string, files []*multipart.FileHeader) (*UploadDocumentsResponse, error)
|
||||||
|
|
||||||
|
// AddLinks appends external URLs to a company
|
||||||
|
AddLinks(ctx context.Context, companyID string, form *AddLinksForm) (*AddLinksResponse, error)
|
||||||
|
|
||||||
|
// DetachDocumentFileID removes a file ID from companies when the file is deleted
|
||||||
|
DetachDocumentFileID(ctx context.Context, fileID string) error
|
||||||
|
|
||||||
|
// StartAIOnboarding sends company data to the AI team for recommendation onboarding
|
||||||
|
StartAIOnboarding(ctx context.Context, companyID string) (*OnboardingResponse, error)
|
||||||
|
|
||||||
|
// GetAIRecommendations returns ranked tender recommendations from the AI team
|
||||||
|
GetAIRecommendations(ctx context.Context, companyID string) ([]RecommendedTenderResponse, error)
|
||||||
|
GetMergedAIRecommendations(ctx context.Context, companyIDs []string) ([]RecommendedTenderResponse, error)
|
||||||
|
|
||||||
|
// ScheduleRefreshCachedAIRecommendationsAfterPipeline re-fetches cached company recommendations after tender sync.
|
||||||
|
ScheduleRefreshCachedAIRecommendationsAfterPipeline()
|
||||||
}
|
}
|
||||||
|
|
||||||
// companyService implements the Service interface
|
// companyService implements the Service interface
|
||||||
@@ -41,15 +58,33 @@ type companyService struct {
|
|||||||
repository Repository
|
repository Repository
|
||||||
categoryService company_category.Service
|
categoryService company_category.Service
|
||||||
fileStore filestore.FileStoreService
|
fileStore filestore.FileStoreService
|
||||||
|
aiRecommendationClient AIRecommendationClient
|
||||||
|
aiPipelineStatusClient AIPipelineStatusClient
|
||||||
|
pageCacheRefresher RecommendedTendersPageCacheRefresher
|
||||||
|
redisClient redis.Client
|
||||||
|
recommendationCacheTTL time.Duration
|
||||||
logger logger.Logger
|
logger logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewService creates a new company service
|
// NewService creates a new company service
|
||||||
func NewService(repository Repository, categoryService company_category.Service, fileStore filestore.FileStoreService, logger logger.Logger) Service {
|
func NewService(
|
||||||
|
repository Repository,
|
||||||
|
categoryService company_category.Service,
|
||||||
|
fileStore filestore.FileStoreService,
|
||||||
|
aiRecommendationClient AIRecommendationClient,
|
||||||
|
aiPipelineStatusClient AIPipelineStatusClient,
|
||||||
|
redisClient redis.Client,
|
||||||
|
recommendationCacheTTL time.Duration,
|
||||||
|
logger logger.Logger,
|
||||||
|
) Service {
|
||||||
return &companyService{
|
return &companyService{
|
||||||
repository: repository,
|
repository: repository,
|
||||||
categoryService: categoryService,
|
categoryService: categoryService,
|
||||||
fileStore: fileStore,
|
fileStore: fileStore,
|
||||||
|
aiRecommendationClient: aiRecommendationClient,
|
||||||
|
aiPipelineStatusClient: aiPipelineStatusClient,
|
||||||
|
redisClient: redisClient,
|
||||||
|
recommendationCacheTTL: recommendationCacheTTL,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -79,17 +114,27 @@ func (s *companyService) Create(ctx context.Context, form *CompanyForm) (*Compan
|
|||||||
return nil, errors.New("company with this name already exists")
|
return nil, errors.New("company with this name already exists")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if registration number already exists
|
if form.RegistrationNumber != "" {
|
||||||
existingCompany, _ = s.repository.GetByRegistrationNumber(ctx, form.RegistrationNumber)
|
existingCompany, _ = s.repository.GetByRegistrationNumber(ctx, form.RegistrationNumber)
|
||||||
if existingCompany != nil {
|
if existingCompany != nil {
|
||||||
return nil, errors.New("company with this registration number already exists")
|
return nil, errors.New("company with this registration number already exists")
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Check if tax ID already exists
|
if form.TaxID != "" {
|
||||||
existingCompany, _ = s.repository.GetByTaxID(ctx, form.TaxID)
|
existingCompany, _ = s.repository.GetByTaxID(ctx, form.TaxID)
|
||||||
if existingCompany != nil {
|
if existingCompany != nil {
|
||||||
return nil, errors.New("company with this tax ID already exists")
|
return nil, errors.New("company with this tax ID already exists")
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
links, err := sanitizeCompanyLinks(convertCompanyLinkForms(form.Links))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(links) > maxCompanyLinks {
|
||||||
|
return nil, errors.New("too many links provided")
|
||||||
|
}
|
||||||
|
|
||||||
// Create company
|
// Create company
|
||||||
company := &Company{
|
company := &Company{
|
||||||
@@ -107,7 +152,8 @@ func (s *companyService) Create(ctx context.Context, form *CompanyForm) (*Compan
|
|||||||
Address: s.convertAddressForm(form.Address),
|
Address: s.convertAddressForm(form.Address),
|
||||||
Phone: form.Phone,
|
Phone: form.Phone,
|
||||||
Email: form.Email,
|
Email: form.Email,
|
||||||
DocumentFileIDs: form.DocumentFileIDs,
|
DocumentFileIDs: s.sanitizeDocumentFileIDs(form.DocumentFileIDs),
|
||||||
|
Links: links,
|
||||||
Tags: s.convertCompanyTagsForm(form.Tags),
|
Tags: s.convertCompanyTagsForm(form.Tags),
|
||||||
IsVerified: false,
|
IsVerified: false,
|
||||||
IsCompliant: false,
|
IsCompliant: false,
|
||||||
@@ -117,7 +163,7 @@ func (s *companyService) Create(ctx context.Context, form *CompanyForm) (*Compan
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Save to database
|
// Save to database
|
||||||
err := s.repository.Create(ctx, company)
|
err = s.repository.Create(ctx, company)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to create company", map[string]interface{}{
|
s.logger.Error("Failed to create company", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -133,6 +179,8 @@ func (s *companyService) Create(ctx context.Context, form *CompanyForm) (*Compan
|
|||||||
"type": company.Type,
|
"type": company.Type,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.triggerAIOnboardingAsync(company.GetID())
|
||||||
|
|
||||||
return company.ToResponse(nil), nil
|
return company.ToResponse(nil), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -169,6 +217,8 @@ func (s *companyService) GetByID(ctx context.Context, id string) (*CompanyRespon
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
company.DocumentFileIDs = s.sanitizeDocumentFileIDs(company.DocumentFileIDs)
|
||||||
|
|
||||||
return company.ToResponse(convertedCategories), nil
|
return company.ToResponse(convertedCategories), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -270,7 +320,18 @@ func (s *companyService) Update(ctx context.Context, id string, form *CompanyFor
|
|||||||
}
|
}
|
||||||
|
|
||||||
if form.DocumentFileIDs != nil {
|
if form.DocumentFileIDs != nil {
|
||||||
company.DocumentFileIDs = form.DocumentFileIDs
|
company.DocumentFileIDs = s.sanitizeDocumentFileIDs(form.DocumentFileIDs)
|
||||||
|
}
|
||||||
|
|
||||||
|
if form.Links != nil {
|
||||||
|
links, linkErr := sanitizeCompanyLinks(convertCompanyLinkForms(form.Links))
|
||||||
|
if linkErr != nil {
|
||||||
|
return nil, linkErr
|
||||||
|
}
|
||||||
|
if len(links) > maxCompanyLinks {
|
||||||
|
return nil, errors.New("too many links provided")
|
||||||
|
}
|
||||||
|
company.Links = links
|
||||||
}
|
}
|
||||||
|
|
||||||
if form.Tags != nil {
|
if form.Tags != nil {
|
||||||
@@ -304,6 +365,8 @@ func (s *companyService) Update(ctx context.Context, id string, form *CompanyFor
|
|||||||
"name": company.Name,
|
"name": company.Name,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.triggerAIOnboardingAsync(company.GetID())
|
||||||
|
|
||||||
return company.ToResponse(nil), nil
|
return company.ToResponse(nil), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -483,6 +546,8 @@ func (s *companyService) GetMyCompany(ctx context.Context, id string) (*CompanyP
|
|||||||
return nil, errors.New("company not found")
|
return nil, errors.New("company not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
company.DocumentFileIDs = s.sanitizeDocumentFileIDs(company.DocumentFileIDs)
|
||||||
|
|
||||||
return company.ToProfileResponse(), nil
|
return company.ToProfileResponse(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -88,9 +88,11 @@ func (s *customerService) AdminResetPassword(ctx context.Context, actorID, targe
|
|||||||
|
|
||||||
s.auditLogger.Log(ctx, audit.Entry{
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
ActorID: actorID,
|
ActorID: actorID,
|
||||||
|
ActorType: audit.ActorTypeAdmin,
|
||||||
TargetType: audit.TargetTypeCustomer,
|
TargetType: audit.TargetTypeCustomer,
|
||||||
TargetID: targetID,
|
TargetID: targetID,
|
||||||
Action: audit.ActionPasswordReset,
|
Action: audit.ActionPasswordReset,
|
||||||
|
Success: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
s.logger.Info("Customer password reset completed", map[string]interface{}{
|
s.logger.Info("Customer password reset completed", map[string]interface{}{
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package customer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrCompanyNotAssigned is returned when the requested company is not assigned to the customer.
|
||||||
|
var ErrCompanyNotAssigned = errors.New("company is not assigned to customer")
|
||||||
|
|
||||||
|
var errCompanyNotAssigned = ErrCompanyNotAssigned
|
||||||
|
|
||||||
|
func normalizeAssignedCompanyIDs(assigned []string) []string {
|
||||||
|
normalized := make([]string, 0, len(assigned))
|
||||||
|
seen := make(map[string]struct{}, len(assigned))
|
||||||
|
for _, id := range assigned {
|
||||||
|
clean := strings.TrimSpace(id)
|
||||||
|
if clean == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[clean]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[clean] = struct{}{}
|
||||||
|
normalized = append(normalized, clean)
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
// pickActiveCompanyID chooses the company context for a customer request.
|
||||||
|
// When requestedCompanyID is set it must be in assigned; otherwise a still-valid
|
||||||
|
// token company is kept; if the token company was removed, the first assignment is used.
|
||||||
|
func pickActiveCompanyID(assigned []string, tokenCompanyID, requestedCompanyID string) (string, error) {
|
||||||
|
assigned = normalizeAssignedCompanyIDs(assigned)
|
||||||
|
if len(assigned) == 0 {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
requested := strings.TrimSpace(requestedCompanyID)
|
||||||
|
if requested != "" {
|
||||||
|
for _, id := range assigned {
|
||||||
|
if id == requested {
|
||||||
|
return requested, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", errCompanyNotAssigned
|
||||||
|
}
|
||||||
|
|
||||||
|
token := strings.TrimSpace(tokenCompanyID)
|
||||||
|
if token != "" {
|
||||||
|
for _, id := range assigned {
|
||||||
|
if id == token {
|
||||||
|
return token, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return assigned[0], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// PickAssignedCompanyID chooses the company for a company-scoped write (e.g. tender approval).
|
||||||
|
// When requestedCompanyID is set it must be in assigned; otherwise activeCompanyID is used.
|
||||||
|
func PickAssignedCompanyID(activeCompanyID string, assigned []string, requestedCompanyID string) (string, error) {
|
||||||
|
activeCompanyID = strings.TrimSpace(activeCompanyID)
|
||||||
|
requestedCompanyID = strings.TrimSpace(requestedCompanyID)
|
||||||
|
|
||||||
|
if requestedCompanyID != "" {
|
||||||
|
for _, id := range normalizeAssignedCompanyIDs(assigned) {
|
||||||
|
if id == requestedCompanyID {
|
||||||
|
return requestedCompanyID, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", ErrCompanyNotAssigned
|
||||||
|
}
|
||||||
|
|
||||||
|
if activeCompanyID == "" {
|
||||||
|
return "", errors.New("company ID is required")
|
||||||
|
}
|
||||||
|
return activeCompanyID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveCompanyContext resolves the active company and assigned companies for an authenticated customer.
|
||||||
|
func (s *customerService) ResolveCompanyContext(ctx context.Context, customerID, tokenCompanyID, requestedCompanyID string) (string, []string, error) {
|
||||||
|
customer, err := s.repository.GetByID(ctx, customerID)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, errors.New("customer not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
assigned := normalizeAssignedCompanyIDs(customer.Companies)
|
||||||
|
companyID, err := pickActiveCompanyID(assigned, tokenCompanyID, requestedCompanyID)
|
||||||
|
if err != nil {
|
||||||
|
return "", nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return companyID, assigned, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveActiveCompanyID resolves the active company for an authenticated customer.
|
||||||
|
func (s *customerService) ResolveActiveCompanyID(ctx context.Context, customerID, tokenCompanyID, requestedCompanyID string) (string, error) {
|
||||||
|
companyID, _, err := s.ResolveCompanyContext(ctx, customerID, tokenCompanyID, requestedCompanyID)
|
||||||
|
return companyID, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package customer
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestPickActiveCompanyID(t *testing.T) {
|
||||||
|
assigned := []string{"company-a", "company-b"}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
assigned []string
|
||||||
|
token string
|
||||||
|
requested string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
wantErrType error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "uses requested company when assigned",
|
||||||
|
assigned: assigned,
|
||||||
|
token: "company-a",
|
||||||
|
requested: "company-b",
|
||||||
|
want: "company-b",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "rejects requested company that is not assigned",
|
||||||
|
assigned: assigned,
|
||||||
|
token: "company-a",
|
||||||
|
requested: "company-c",
|
||||||
|
wantErr: true,
|
||||||
|
wantErrType: errCompanyNotAssigned,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "keeps valid token company",
|
||||||
|
assigned: assigned,
|
||||||
|
token: "company-b",
|
||||||
|
want: "company-b",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "falls back to first assignment when token company was removed",
|
||||||
|
assigned: []string{"company-b"},
|
||||||
|
token: "company-a",
|
||||||
|
want: "company-b",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "returns empty when customer has no companies",
|
||||||
|
assigned: nil,
|
||||||
|
token: "company-a",
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "uses first assignment when token is empty",
|
||||||
|
assigned: assigned,
|
||||||
|
want: "company-a",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := pickActiveCompanyID(tt.assigned, tt.token, tt.requested)
|
||||||
|
if tt.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("expected error, got nil")
|
||||||
|
}
|
||||||
|
if tt.wantErrType != nil && err != tt.wantErrType {
|
||||||
|
t.Fatalf("expected error %v, got %v", tt.wantErrType, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.want {
|
||||||
|
t.Fatalf("pickActiveCompanyID() = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,152 @@
|
|||||||
|
package customer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type companySelectionObject struct {
|
||||||
|
ID string `json:"id"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodeCompanySelection parses company IDs sent as strings, legacy company_ids,
|
||||||
|
// or company summary objects returned by GET customer.
|
||||||
|
func decodeCompanySelection(raw json.RawMessage) ([]string, error) {
|
||||||
|
if len(raw) == 0 {
|
||||||
|
return nil, fmt.Errorf("empty company selection")
|
||||||
|
}
|
||||||
|
if string(raw) == "null" {
|
||||||
|
return []string{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var ids []string
|
||||||
|
if err := json.Unmarshal(raw, &ids); err == nil {
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var objects []companySelectionObject
|
||||||
|
if err := json.Unmarshal(raw, &objects); err == nil {
|
||||||
|
result := make([]string, 0, len(objects))
|
||||||
|
for _, object := range objects {
|
||||||
|
if id := strings.TrimSpace(object.ID); id != "" {
|
||||||
|
result = append(result, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("companies must be an array of IDs or objects with id")
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeCompanySelectionFields(raw map[string]json.RawMessage) (json.RawMessage, bool, error) {
|
||||||
|
if value, ok := raw["companies"]; ok {
|
||||||
|
return value, true, nil
|
||||||
|
}
|
||||||
|
if value, ok := raw["company_ids"]; ok {
|
||||||
|
return value, true, nil
|
||||||
|
}
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func unmarshalFormWithoutCompanyFields(data []byte, raw map[string]json.RawMessage, target any) error {
|
||||||
|
delete(raw, "companies")
|
||||||
|
delete(raw, "company_ids")
|
||||||
|
|
||||||
|
trimmed, err := json.Marshal(raw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.Unmarshal(trimmed, target)
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalJSON accepts companies, legacy company_ids, and company summary objects.
|
||||||
|
func (f *CreateCustomerForm) UnmarshalJSON(data []byte) error {
|
||||||
|
var raw map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(data, &raw); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
selectionRaw, present, err := mergeCompanySelectionFields(raw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
type alias CreateCustomerForm
|
||||||
|
if err := unmarshalFormWithoutCompanyFields(data, raw, (*alias)(f)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !present {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
companies, err := decodeCompanySelection(selectionRaw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.Companies = companies
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalJSON accepts companies, legacy company_ids, and company summary objects.
|
||||||
|
func (f *UpdateCustomerForm) UnmarshalJSON(data []byte) error {
|
||||||
|
var raw map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(data, &raw); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
selectionRaw, present, err := mergeCompanySelectionFields(raw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
type alias UpdateCustomerForm
|
||||||
|
if err := unmarshalFormWithoutCompanyFields(data, raw, (*alias)(f)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !present {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
companies, err := decodeCompanySelection(selectionRaw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.Companies = &companies
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalJSON accepts companies, legacy company_ids, and company summary objects.
|
||||||
|
func (f *AssignCompaniesForm) UnmarshalJSON(data []byte) error {
|
||||||
|
var raw map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(data, &raw); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
selectionRaw, present, err := mergeCompanySelectionFields(raw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
type alias AssignCompaniesForm
|
||||||
|
if err := unmarshalFormWithoutCompanyFields(data, raw, (*alias)(f)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !present {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
companies, err := decodeCompanySelection(selectionRaw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.Companies = companies
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
package customer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCreateCustomerForm_UnmarshalJSON_CompanyIDs(t *testing.T) {
|
||||||
|
payload := `{
|
||||||
|
"type": "individual",
|
||||||
|
"role": "analyst",
|
||||||
|
"username": "testuser",
|
||||||
|
"email": "test@example.com",
|
||||||
|
"password": "Test!1234",
|
||||||
|
"company_ids": ["507f1f77bcf86cd799439011", "507f1f77bcf86cd799439012"]
|
||||||
|
}`
|
||||||
|
|
||||||
|
var form CreateCustomerForm
|
||||||
|
if err := json.Unmarshal([]byte(payload), &form); err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{"507f1f77bcf86cd799439011", "507f1f77bcf86cd799439012"}
|
||||||
|
if len(form.Companies) != len(want) {
|
||||||
|
t.Fatalf("Companies len = %d, want %d", len(form.Companies), len(want))
|
||||||
|
}
|
||||||
|
for i, id := range want {
|
||||||
|
if form.Companies[i] != id {
|
||||||
|
t.Fatalf("Companies[%d] = %q, want %q", i, form.Companies[i], id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateCustomerForm_UnmarshalJSON_CompanySummaries(t *testing.T) {
|
||||||
|
payload := `{
|
||||||
|
"type": "individual",
|
||||||
|
"role": "analyst",
|
||||||
|
"username": "testuser",
|
||||||
|
"email": "test@example.com",
|
||||||
|
"companies": [
|
||||||
|
{"id": "507f1f77bcf86cd799439011", "name": "Acme"},
|
||||||
|
{"id": "507f1f77bcf86cd799439012", "name": "Globex"}
|
||||||
|
]
|
||||||
|
}`
|
||||||
|
|
||||||
|
var form UpdateCustomerForm
|
||||||
|
if err := json.Unmarshal([]byte(payload), &form); err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if form.Companies == nil {
|
||||||
|
t.Fatal("Companies should be set")
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{"507f1f77bcf86cd799439011", "507f1f77bcf86cd799439012"}
|
||||||
|
if len(*form.Companies) != len(want) {
|
||||||
|
t.Fatalf("Companies len = %d, want %d", len(*form.Companies), len(want))
|
||||||
|
}
|
||||||
|
for i, id := range want {
|
||||||
|
if (*form.Companies)[i] != id {
|
||||||
|
t.Fatalf("Companies[%d] = %q, want %q", i, (*form.Companies)[i], id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUpdateCustomerForm_UnmarshalJSON_OmittedCompanies(t *testing.T) {
|
||||||
|
payload := `{
|
||||||
|
"type": "individual",
|
||||||
|
"role": "analyst",
|
||||||
|
"username": "testuser",
|
||||||
|
"email": "test@example.com"
|
||||||
|
}`
|
||||||
|
|
||||||
|
var form UpdateCustomerForm
|
||||||
|
if err := json.Unmarshal([]byte(payload), &form); err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if form.Companies != nil {
|
||||||
|
t.Fatalf("Companies = %v, want nil when omitted", form.Companies)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAssignCompaniesForm_UnmarshalJSON_LegacyField(t *testing.T) {
|
||||||
|
payload := `{"company_ids": ["507f1f77bcf86cd799439011"]}`
|
||||||
|
|
||||||
|
var form AssignCompaniesForm
|
||||||
|
if err := json.Unmarshal([]byte(payload), &form); err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if len(form.Companies) != 1 || form.Companies[0] != "507f1f77bcf86cd799439011" {
|
||||||
|
t.Fatalf("Companies = %v", form.Companies)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,10 @@
|
|||||||
package customer
|
package customer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"tm/pkg/audit"
|
||||||
"tm/pkg/response"
|
"tm/pkg/response"
|
||||||
|
|
||||||
"github.com/labstack/echo/v4"
|
"github.com/labstack/echo/v4"
|
||||||
@@ -52,12 +54,48 @@ func (h *Handler) AuthMiddleware() echo.MiddlewareFunc {
|
|||||||
c.Set("company_id", validationResult.Claims.CompanyID)
|
c.Set("company_id", validationResult.Claims.CompanyID)
|
||||||
c.Set("access_token", tokenString)
|
c.Set("access_token", tokenString)
|
||||||
|
|
||||||
|
audit.EnrichEchoContext(c, audit.ActorTypeCustomer)
|
||||||
|
|
||||||
// Continue to next handler
|
// Continue to next handler
|
||||||
return next(c)
|
return next(c)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CompanyContextMiddleware resolves company_id from current customer assignments.
|
||||||
|
// JWT company_id can be stale after admin reassigns companies; this keeps tender
|
||||||
|
// recommendations and other company-scoped APIs in sync with the database.
|
||||||
|
func (h *Handler) CompanyContextMiddleware() echo.MiddlewareFunc {
|
||||||
|
return func(next echo.HandlerFunc) echo.HandlerFunc {
|
||||||
|
return func(c echo.Context) error {
|
||||||
|
customerID, err := GetCustomerIDFromContext(c)
|
||||||
|
if err != nil {
|
||||||
|
return response.Unauthorized(c, "User not authenticated")
|
||||||
|
}
|
||||||
|
|
||||||
|
tokenCompanyID, _ := c.Get("company_id").(string)
|
||||||
|
requestedCompanyID := strings.TrimSpace(c.QueryParam("company_id"))
|
||||||
|
|
||||||
|
companyID, companyIDs, err := h.service.ResolveCompanyContext(
|
||||||
|
c.Request().Context(),
|
||||||
|
customerID,
|
||||||
|
tokenCompanyID,
|
||||||
|
requestedCompanyID,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, errCompanyNotAssigned) {
|
||||||
|
return response.Forbidden(c, "Company is not assigned to customer")
|
||||||
|
}
|
||||||
|
return response.Unauthorized(c, "User not authenticated")
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Set("company_id", companyID)
|
||||||
|
c.Set("company_ids", companyIDs)
|
||||||
|
return next(c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// GetCustomerIDFromContext extracts customer ID from Echo context
|
// GetCustomerIDFromContext extracts customer ID from Echo context
|
||||||
func GetCustomerIDFromContext(c echo.Context) (string, error) {
|
func GetCustomerIDFromContext(c echo.Context) (string, error) {
|
||||||
customerID, ok := c.Get("customer_id").(string)
|
customerID, ok := c.Get("customer_id").(string)
|
||||||
|
|||||||
@@ -249,16 +249,11 @@ func (r *customerRepository) GetByRole(ctx context.Context, role CustomerRole) (
|
|||||||
// GetByCompanies retrieves customers by companies
|
// GetByCompanies retrieves customers by companies
|
||||||
|
|
||||||
func (r *customerRepository) GetByCompanies(ctx context.Context, companies []string) ([]Customer, error) {
|
func (r *customerRepository) GetByCompanies(ctx context.Context, companies []string) ([]Customer, error) {
|
||||||
companyIDs := make([]bson.ObjectID, 0)
|
if len(companies) == 0 {
|
||||||
for _, company := range companies {
|
return []Customer{}, nil
|
||||||
id, err := bson.ObjectIDFromHex(company)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
companyIDs = append(companyIDs, id)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
customers, err := r.ormRepo.FindAll(ctx, bson.M{"companies": bson.M{"$in": companyIDs}}, orm.NewPaginationBuilder().Build())
|
customers, err := r.ormRepo.FindAll(ctx, bson.M{"companies": bson.M{"$in": companies}}, orm.NewPaginationBuilder().Build())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -274,9 +269,9 @@ func (r *customerRepository) GetByCompanyID(ctx context.Context, companyID strin
|
|||||||
SortBy(pagination.SortBy, pagination.SortOrder).
|
SortBy(pagination.SortBy, pagination.SortOrder).
|
||||||
Build()
|
Build()
|
||||||
|
|
||||||
// Filter by company ID and active status
|
// Filter by assigned company ID and active status
|
||||||
filter := bson.M{
|
filter := bson.M{
|
||||||
"company_id": companyID,
|
"companies": companyID,
|
||||||
"status": bson.M{"$ne": CustomerStatusInactive},
|
"status": bson.M{"$ne": CustomerStatusInactive},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+148
-24
@@ -53,6 +53,13 @@ type Service interface {
|
|||||||
// Role assignment operations
|
// Role assignment operations
|
||||||
AssignRole(ctx context.Context, customerID string, form *AssignRoleForm) (*AssignRoleResponse, error)
|
AssignRole(ctx context.Context, customerID string, form *AssignRoleForm) (*AssignRoleResponse, error)
|
||||||
AssignCompanies(ctx context.Context, customerID string, Companies []string) error
|
AssignCompanies(ctx context.Context, customerID string, Companies []string) error
|
||||||
|
|
||||||
|
// ResolveCompanyContext returns the active company and all assigned companies
|
||||||
|
// for an authenticated customer request.
|
||||||
|
ResolveCompanyContext(ctx context.Context, customerID, tokenCompanyID, requestedCompanyID string) (string, []string, error)
|
||||||
|
|
||||||
|
// ResolveActiveCompanyID returns the company context for an authenticated customer.
|
||||||
|
ResolveActiveCompanyID(ctx context.Context, customerID, tokenCompanyID, requestedCompanyID string) (string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// customerService implements the Service interface
|
// customerService implements the Service interface
|
||||||
@@ -181,6 +188,14 @@ func (s *customerService) Register(ctx context.Context, form *CreateCustomerForm
|
|||||||
"company_ids": Companies,
|
"company_ids": Companies,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
Action: audit.ActionCustomerCreate,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: customer.ID.Hex(),
|
||||||
|
Success: true,
|
||||||
|
Metadata: map[string]string{"role": string(customer.Role)},
|
||||||
|
})
|
||||||
|
|
||||||
go s.notification.SendEmail(context.Background(), notification.Model{
|
go s.notification.SendEmail(context.Background(), notification.Model{
|
||||||
Recipient: customer.Email,
|
Recipient: customer.Email,
|
||||||
Title: "Welcome to Opplens",
|
Title: "Welcome to Opplens",
|
||||||
@@ -198,6 +213,13 @@ func (s *customerService) GetByID(ctx context.Context, id string) (*CustomerResp
|
|||||||
customer, err := s.repository.GetByID(ctx, id)
|
customer, err := s.repository.GetByID(ctx, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if err.Error() == "customer not found" {
|
if err.Error() == "customer not found" {
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
Action: audit.ActionCustomerRead,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: id,
|
||||||
|
Success: false,
|
||||||
|
Metadata: map[string]string{"reason": "not_found"},
|
||||||
|
})
|
||||||
return nil, errors.New("customer not found")
|
return nil, errors.New("customer not found")
|
||||||
}
|
}
|
||||||
s.logger.Error("Failed to get customer by ID", map[string]interface{}{
|
s.logger.Error("Failed to get customer by ID", map[string]interface{}{
|
||||||
@@ -223,6 +245,13 @@ func (s *customerService) GetByID(ctx context.Context, id string) (*CustomerResp
|
|||||||
"email": customer.Email,
|
"email": customer.Email,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
Action: audit.ActionCustomerRead,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: id,
|
||||||
|
Success: true,
|
||||||
|
})
|
||||||
|
|
||||||
return customer.ToResponse(companies), nil
|
return customer.ToResponse(companies), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -249,12 +278,37 @@ func (s *customerService) Search(ctx context.Context, form *SearchCustomersForm,
|
|||||||
customerResponses = append(customerResponses, customer.ToResponse(companies))
|
customerResponses = append(customerResponses, customer.ToResponse(companies))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
Action: audit.ActionCustomerList,
|
||||||
|
Success: true,
|
||||||
|
Metadata: audit.MergeMetadata(
|
||||||
|
customerSearchFilterMetadata(form),
|
||||||
|
audit.PaginationMetadata(pagination.Limit, pagination.Offset, page.TotalCount),
|
||||||
|
),
|
||||||
|
})
|
||||||
|
|
||||||
return &CustomerListResponse{
|
return &CustomerListResponse{
|
||||||
Customers: customerResponses,
|
Customers: customerResponses,
|
||||||
Meta: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore, page.PageOffset),
|
Meta: pagination.ListMeta(page.TotalCount, page.NextCursor, page.HasMore, page.PageOffset),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func customerSearchFilterMetadata(form *SearchCustomersForm) map[string]string {
|
||||||
|
if form == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return audit.MergeMetadata(
|
||||||
|
audit.FlagMetadata("has_search", form.Search != nil && *form.Search != ""),
|
||||||
|
audit.FlagMetadata("has_full_name", form.FullName != nil && *form.FullName != ""),
|
||||||
|
audit.FlagMetadata("has_email_filter", form.Email != nil && *form.Email != ""),
|
||||||
|
audit.OptionalStringMetadata("status", form.Status),
|
||||||
|
audit.OptionalStringMetadata("role", form.Role),
|
||||||
|
audit.OptionalStringMetadata("type", form.Type),
|
||||||
|
audit.OptionalStringMetadata("company_id", form.CompanyID),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
func (s *customerService) SearchNotification(ctx context.Context, form *SearchCustomersForm, pagination *response.Pagination) ([]*CustomerNotificationResponse, error) {
|
func (s *customerService) SearchNotification(ctx context.Context, form *SearchCustomersForm, pagination *response.Pagination) ([]*CustomerNotificationResponse, error) {
|
||||||
page, err := s.repository.Search(ctx, form, pagination)
|
page, err := s.repository.Search(ctx, form, pagination)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -452,6 +506,13 @@ func (s *customerService) Update(ctx context.Context, id string, form *UpdateCus
|
|||||||
"email": customer.Email,
|
"email": customer.Email,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
Action: audit.ActionCustomerUpdate,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: id,
|
||||||
|
Success: true,
|
||||||
|
})
|
||||||
|
|
||||||
responseCompanies, err := s.loadCompaniesForCustomer(ctx, customer)
|
responseCompanies, err := s.loadCompaniesForCustomer(ctx, customer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to load companies for customer response", map[string]interface{}{
|
s.logger.Error("Failed to load companies for customer response", map[string]interface{}{
|
||||||
@@ -490,6 +551,13 @@ func (s *customerService) Delete(ctx context.Context, id string) error {
|
|||||||
"customer_id": id,
|
"customer_id": id,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
Action: audit.ActionCustomerDelete,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: id,
|
||||||
|
Success: true,
|
||||||
|
})
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -518,6 +586,14 @@ func (s *customerService) UpdateStatus(ctx context.Context, id string, form *Upd
|
|||||||
"status": form.Status,
|
"status": form.Status,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
Action: audit.ActionCustomerStatusChange,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: id,
|
||||||
|
Success: true,
|
||||||
|
Metadata: map[string]string{"status": form.Status},
|
||||||
|
})
|
||||||
|
|
||||||
go s.notification.SendEmail(context.Background(), notification.Model{
|
go s.notification.SendEmail(context.Background(), notification.Model{
|
||||||
Recipient: customer.Email,
|
Recipient: customer.Email,
|
||||||
Title: "Account Status Updated",
|
Title: "Account Status Updated",
|
||||||
@@ -682,6 +758,12 @@ func (s *customerService) Login(ctx context.Context, form *LoginForm) (*AuthResp
|
|||||||
s.logger.Warn("Customer login failed - username not found", map[string]interface{}{
|
s.logger.Warn("Customer login failed - username not found", map[string]interface{}{
|
||||||
"username": strings.ToLower(form.Username),
|
"username": strings.ToLower(form.Username),
|
||||||
})
|
})
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
Action: audit.ActionAuthLoginFailure,
|
||||||
|
ActorType: audit.ActorTypeCustomer,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
Success: false,
|
||||||
|
})
|
||||||
return nil, errors.New("invalid credentials")
|
return nil, errors.New("invalid credentials")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -692,6 +774,15 @@ func (s *customerService) Login(ctx context.Context, form *LoginForm) (*AuthResp
|
|||||||
"customer_id": customer.ID,
|
"customer_id": customer.ID,
|
||||||
"status": string(customer.Status),
|
"status": string(customer.Status),
|
||||||
})
|
})
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
ActorID: customer.ID.Hex(),
|
||||||
|
ActorType: audit.ActorTypeCustomer,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: customer.ID.Hex(),
|
||||||
|
Action: audit.ActionAuthLoginFailure,
|
||||||
|
Success: false,
|
||||||
|
Metadata: map[string]string{"reason": "inactive"},
|
||||||
|
})
|
||||||
return nil, errors.New("account is not active")
|
return nil, errors.New("account is not active")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -702,13 +793,20 @@ func (s *customerService) Login(ctx context.Context, form *LoginForm) (*AuthResp
|
|||||||
"customer_id": customer.ID,
|
"customer_id": customer.ID,
|
||||||
"email": customer.Email,
|
"email": customer.Email,
|
||||||
})
|
})
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
ActorID: customer.ID.Hex(),
|
||||||
|
ActorType: audit.ActorTypeCustomer,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: customer.ID.Hex(),
|
||||||
|
Action: audit.ActionAuthLoginFailure,
|
||||||
|
Success: false,
|
||||||
|
})
|
||||||
return nil, errors.New("invalid credentials")
|
return nil, errors.New("invalid credentials")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Generate JWT tokens using authorization service
|
companyID, err := pickActiveCompanyID(customer.Companies, "", "")
|
||||||
companyID := ""
|
if err != nil {
|
||||||
if len(customer.Companies) > 0 {
|
return nil, errors.New("failed to resolve customer company")
|
||||||
companyID = customer.Companies[0] // Use first company ID for JWT token
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tokenPair, err := s.authService.GenerateTokenPair(
|
tokenPair, err := s.authService.GenerateTokenPair(
|
||||||
@@ -747,6 +845,16 @@ func (s *customerService) Login(ctx context.Context, form *LoginForm) (*AuthResp
|
|||||||
"customer_type": string(customer.Type),
|
"customer_type": string(customer.Type),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
ActorID: customer.ID.Hex(),
|
||||||
|
ActorType: audit.ActorTypeCustomer,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: customer.ID.Hex(),
|
||||||
|
Action: audit.ActionAuthLoginSuccess,
|
||||||
|
Success: true,
|
||||||
|
Metadata: map[string]string{"role": string(customer.Role)},
|
||||||
|
})
|
||||||
|
|
||||||
return &AuthResponse{
|
return &AuthResponse{
|
||||||
Customer: customer.ToResponse(companies),
|
Customer: customer.ToResponse(companies),
|
||||||
AccessToken: tokenPair.AccessToken,
|
AccessToken: tokenPair.AccessToken,
|
||||||
@@ -761,32 +869,20 @@ func (s *customerService) RefreshToken(ctx context.Context, refreshToken string)
|
|||||||
"refresh_token": refreshToken[:10] + "...", // Log only first 10 chars for security
|
"refresh_token": refreshToken[:10] + "...", // Log only first 10 chars for security
|
||||||
})
|
})
|
||||||
|
|
||||||
// Validate refresh token and generate new access token
|
validationResult, err := s.authService.ValidateRefreshToken(refreshToken)
|
||||||
newAccessToken, err := s.authService.RefreshAccessToken(refreshToken)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Warn("Failed to refresh access token", map[string]interface{}{
|
s.logger.Warn("Failed to validate refresh token", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
return nil, errors.New("invalid or expired refresh token")
|
return nil, errors.New("invalid or expired refresh token")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse the new access token to get customer information
|
|
||||||
validationResult, err := s.authService.ValidateAccessToken(newAccessToken)
|
|
||||||
if err != nil {
|
|
||||||
s.logger.Error("Failed to validate new access token", map[string]interface{}{
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
return nil, errors.New("failed to generate valid access token")
|
|
||||||
}
|
|
||||||
|
|
||||||
if !validationResult.Valid {
|
if !validationResult.Valid {
|
||||||
s.logger.Error("New access token validation failed", map[string]interface{}{
|
s.logger.Warn("Invalid refresh token provided", map[string]interface{}{
|
||||||
"error": validationResult.Error,
|
"error": validationResult.Error,
|
||||||
})
|
})
|
||||||
return nil, errors.New("failed to generate valid access token")
|
return nil, errors.New("invalid or expired refresh token")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get customer information
|
|
||||||
customerID, err := bson.ObjectIDFromHex(validationResult.Claims.UserID)
|
customerID, err := bson.ObjectIDFromHex(validationResult.Claims.UserID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to parse customer ID from token", map[string]interface{}{
|
s.logger.Error("Failed to parse customer ID from token", map[string]interface{}{
|
||||||
@@ -804,11 +900,29 @@ func (s *customerService) RefreshToken(ctx context.Context, refreshToken string)
|
|||||||
return nil, errors.New("customer not found")
|
return nil, errors.New("customer not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if customer is still active
|
|
||||||
if customer.Status != CustomerStatusActive {
|
if customer.Status != CustomerStatusActive {
|
||||||
return nil, errors.New("customer account is inactive")
|
return nil, errors.New("customer account is inactive")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
companyID, err := pickActiveCompanyID(customer.Companies, validationResult.Claims.CompanyID, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New("failed to resolve customer company")
|
||||||
|
}
|
||||||
|
|
||||||
|
tokenPair, err := s.authService.GenerateTokenPair(
|
||||||
|
customer.ID.Hex(),
|
||||||
|
customer.Email,
|
||||||
|
string(customer.Role),
|
||||||
|
companyID,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to generate JWT tokens during refresh", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"customer_id": customer.ID,
|
||||||
|
})
|
||||||
|
return nil, errors.New("failed to generate authentication tokens")
|
||||||
|
}
|
||||||
|
|
||||||
companies, err := s.loadCompaniesForCustomer(ctx, customer)
|
companies, err := s.loadCompaniesForCustomer(ctx, customer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to load companies for customer", map[string]interface{}{
|
s.logger.Error("Failed to load companies for customer", map[string]interface{}{
|
||||||
@@ -820,13 +934,14 @@ func (s *customerService) RefreshToken(ctx context.Context, refreshToken string)
|
|||||||
s.logger.Info("Access token refreshed successfully", map[string]interface{}{
|
s.logger.Info("Access token refreshed successfully", map[string]interface{}{
|
||||||
"customer_id": customer.ID,
|
"customer_id": customer.ID,
|
||||||
"customer_type": string(customer.Type),
|
"customer_type": string(customer.Type),
|
||||||
|
"company_id": companyID,
|
||||||
})
|
})
|
||||||
|
|
||||||
return &AuthResponse{
|
return &AuthResponse{
|
||||||
Customer: customer.ToResponse(companies),
|
Customer: customer.ToResponse(companies),
|
||||||
AccessToken: newAccessToken,
|
AccessToken: tokenPair.AccessToken,
|
||||||
RefreshToken: refreshToken, // Return the same refresh token
|
RefreshToken: tokenPair.RefreshToken,
|
||||||
ExpiresAt: time.Now().Add(15 * time.Minute).Unix(), // 15 minutes from now
|
ExpiresAt: time.Now().Add(15 * time.Minute).Unix(),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -942,6 +1057,15 @@ func (s *customerService) Logout(ctx context.Context, customerID string, accessT
|
|||||||
"customer_id": customerID,
|
"customer_id": customerID,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
s.auditLogger.Log(ctx, audit.Entry{
|
||||||
|
ActorID: customerID,
|
||||||
|
ActorType: audit.ActorTypeCustomer,
|
||||||
|
TargetType: audit.TargetTypeCustomer,
|
||||||
|
TargetID: customerID,
|
||||||
|
Action: audit.ActionAuthLogout,
|
||||||
|
Success: true,
|
||||||
|
})
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ type SummaryResponse struct {
|
|||||||
ClosingSoon int64 `json:"closing_soon"`
|
ClosingSoon int64 `json:"closing_soon"`
|
||||||
TotalEstimatedValue float64 `json:"total_estimated_value"`
|
TotalEstimatedValue float64 `json:"total_estimated_value"`
|
||||||
ValueCurrency string `json:"value_currency"`
|
ValueCurrency string `json:"value_currency"`
|
||||||
|
Days int `json:"days"`
|
||||||
|
ScrapedTED []TrendPoint `json:"scraped_ted"`
|
||||||
GeneratedAt int64 `json:"generated_at"`
|
GeneratedAt int64 `json:"generated_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -96,7 +98,6 @@ type StatisticsReportResponse struct {
|
|||||||
|
|
||||||
// StatisticsDailySeries contains per-day chart series.
|
// StatisticsDailySeries contains per-day chart series.
|
||||||
type StatisticsDailySeries struct {
|
type StatisticsDailySeries struct {
|
||||||
ScrapedTED []TrendPoint `json:"scraped_ted"`
|
|
||||||
ScrapedDocuments []TrendPoint `json:"scraped_documents"`
|
ScrapedDocuments []TrendPoint `json:"scraped_documents"`
|
||||||
TranslatedNotices []TrendPoint `json:"translated_notices"`
|
TranslatedNotices []TrendPoint `json:"translated_notices"`
|
||||||
}
|
}
|
||||||
@@ -105,4 +106,5 @@ type StatisticsDailySeries struct {
|
|||||||
type StatisticsLifetimeTotals struct {
|
type StatisticsLifetimeTotals struct {
|
||||||
ScrapedDocuments int64 `json:"scraped_documents"`
|
ScrapedDocuments int64 `json:"scraped_documents"`
|
||||||
TranslatedTenders int64 `json:"translated_tenders"`
|
TranslatedTenders int64 `json:"translated_tenders"`
|
||||||
|
ScrapedTEDNotices int64 `json:"scraped_ted_notices"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package dashboard
|
|||||||
// SummaryQuery binds query params for GET /dashboard/summary.
|
// SummaryQuery binds query params for GET /dashboard/summary.
|
||||||
type SummaryQuery struct {
|
type SummaryQuery struct {
|
||||||
ClosingWindow int `query:"closing_window"`
|
ClosingWindow int `query:"closing_window"`
|
||||||
|
Days int `query:"days"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TrendQuery binds query params for GET /dashboard/trend.
|
// TrendQuery binds query params for GET /dashboard/trend.
|
||||||
|
|||||||
@@ -19,10 +19,11 @@ func NewHandler(service Service) *Handler {
|
|||||||
|
|
||||||
// Summary returns top-level dashboard counters.
|
// Summary returns top-level dashboard counters.
|
||||||
// @Summary Dashboard summary
|
// @Summary Dashboard summary
|
||||||
// @Description Top-level counters for hero pills and stat cards
|
// @Description Top-level counters for hero pills and stat cards, including daily TED scrape counts
|
||||||
// @Tags Admin-Dashboard
|
// @Tags Admin-Dashboard
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param closing_window query int false "Hours considered closing soon (default 168)"
|
// @Param closing_window query int false "Hours considered closing soon (default 168)"
|
||||||
|
// @Param days query int false "Days back for scraped TED daily series (default 14, max 90)"
|
||||||
// @Success 200 {object} response.APIResponse{data=SummaryResponse}
|
// @Success 200 {object} response.APIResponse{data=SummaryResponse}
|
||||||
// @Failure 500 {object} response.APIResponse
|
// @Failure 500 {object} response.APIResponse
|
||||||
// @Router /admin/v1/dashboard/summary [get]
|
// @Router /admin/v1/dashboard/summary [get]
|
||||||
@@ -30,6 +31,7 @@ func NewHandler(service Service) *Handler {
|
|||||||
func (h *Handler) Summary(c echo.Context) error {
|
func (h *Handler) Summary(c echo.Context) error {
|
||||||
query := SummaryQuery{
|
query := SummaryQuery{
|
||||||
ClosingWindow: parseIntQuery(c, "closing_window", 0),
|
ClosingWindow: parseIntQuery(c, "closing_window", 0),
|
||||||
|
Days: parseIntQuery(c, "days", 0),
|
||||||
}
|
}
|
||||||
|
|
||||||
out, err := h.service.Summary(c.Request().Context(), query)
|
out, err := h.service.Summary(c.Request().Context(), query)
|
||||||
@@ -37,7 +39,7 @@ func (h *Handler) Summary(c echo.Context) error {
|
|||||||
return response.InternalServerError(c, "Failed to load dashboard summary")
|
return response.InternalServerError(c, "Failed to load dashboard summary")
|
||||||
}
|
}
|
||||||
|
|
||||||
setPrivateCache(c, 60)
|
setPrivateCache(c, 300)
|
||||||
return response.Success(c, out, "Dashboard summary retrieved successfully")
|
return response.Success(c, out, "Dashboard summary retrieved successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -163,7 +165,7 @@ func (h *Handler) Recent(c echo.Context) error {
|
|||||||
|
|
||||||
// Statistics returns scraping and translation statistics for charts.
|
// Statistics returns scraping and translation statistics for charts.
|
||||||
// @Summary Dashboard statistics report
|
// @Summary Dashboard statistics report
|
||||||
// @Description Daily and lifetime statistics for TED scraping, document scraping, and AI translation
|
// @Description Daily and lifetime statistics for document scraping and AI translation
|
||||||
// @Tags Admin-Dashboard
|
// @Tags Admin-Dashboard
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param days query int false "Days back for daily series (default 14, max 90)"
|
// @Param days query int false "Days back for daily series (default 14, max 90)"
|
||||||
@@ -181,7 +183,7 @@ func (h *Handler) Statistics(c echo.Context) error {
|
|||||||
return response.InternalServerError(c, "Failed to load dashboard statistics")
|
return response.InternalServerError(c, "Failed to load dashboard statistics")
|
||||||
}
|
}
|
||||||
|
|
||||||
setPrivateCache(c, 60)
|
setPrivateCache(c, 300)
|
||||||
return response.Success(c, out, "Dashboard statistics retrieved successfully")
|
return response.Success(c, out, "Dashboard statistics retrieved successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,20 +4,39 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"tm/internal/tender"
|
"tm/internal/tender"
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
orm "tm/pkg/mongo"
|
orm "tm/pkg/mongo"
|
||||||
|
|
||||||
"go.mongodb.org/mongo-driver/v2/bson"
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
"go.mongodb.org/mongo-driver/v2/mongo"
|
"go.mongodb.org/mongo-driver/v2/mongo"
|
||||||
|
"golang.org/x/sync/singleflight"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ProcedureDocumentsLister lists contract folders that have scraped documents in MinIO.
|
||||||
|
// This is the source of truth for document-scrape statistics, matching tender panel search.
|
||||||
|
type ProcedureDocumentsLister interface {
|
||||||
|
ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// scrapedDocumentsScanner optionally scans MinIO for per-day document counts in the same pass.
|
||||||
|
type scrapedDocumentsScanner interface {
|
||||||
|
ScanScrapedDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, map[string]int64, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// translatedNoticesScanner optionally scans MinIO for per-day translated notice counts.
|
||||||
|
type translatedNoticesScanner interface {
|
||||||
|
ScanTranslatedNotices(ctx context.Context) (map[string]int64, int64, error)
|
||||||
|
}
|
||||||
|
|
||||||
const defaultValueCurrency = "EUR"
|
const defaultValueCurrency = "EUR"
|
||||||
|
|
||||||
// Repository defines dashboard data access.
|
// Repository defines dashboard data access.
|
||||||
type Repository interface {
|
type Repository interface {
|
||||||
Summary(ctx context.Context, closingWindowSec int64) (*SummaryResponse, error)
|
Summary(ctx context.Context, closingWindowSec int64, days int) (*SummaryResponse, error)
|
||||||
Trend(ctx context.Context, days int, metric string, startUnix int64) (map[string]int64, error)
|
Trend(ctx context.Context, days int, metric string, startUnix int64) (map[string]int64, error)
|
||||||
Countries(ctx context.Context, limit int) (*CountriesResponse, error)
|
Countries(ctx context.Context, limit int) (*CountriesResponse, error)
|
||||||
NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
|
NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
|
||||||
@@ -30,15 +49,27 @@ type repository struct {
|
|||||||
ormRepo orm.Repository[tender.Tender]
|
ormRepo orm.Repository[tender.Tender]
|
||||||
mongoManager *orm.ConnectionManager
|
mongoManager *orm.ConnectionManager
|
||||||
metricsCounter *orm.Counter
|
metricsCounter *orm.Counter
|
||||||
|
procedureLister ProcedureDocumentsLister
|
||||||
logger logger.Logger
|
logger logger.Logger
|
||||||
|
scrapedScopeMu sync.Mutex
|
||||||
|
scrapedScope scrapedTendersScope
|
||||||
|
scrapedScopeExpiry time.Time
|
||||||
|
scrapedScopeCached bool
|
||||||
|
scrapedScopeGroup singleflight.Group
|
||||||
|
translatedScopeMu sync.Mutex
|
||||||
|
translatedScope translatedNoticesScope
|
||||||
|
translatedScopeExpiry time.Time
|
||||||
|
translatedScopeCached bool
|
||||||
|
translatedScopeGroup singleflight.Group
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewRepository creates a dashboard repository backed by the tenders collection.
|
// NewRepository creates a dashboard repository backed by the tenders collection.
|
||||||
func NewRepository(mongoManager *orm.ConnectionManager, log logger.Logger) Repository {
|
func NewRepository(mongoManager *orm.ConnectionManager, log logger.Logger, procedureLister ProcedureDocumentsLister) Repository {
|
||||||
return &repository{
|
return &repository{
|
||||||
ormRepo: orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log),
|
ormRepo: orm.NewRepository[tender.Tender](mongoManager.GetCollection(tenderCollectionName()), log),
|
||||||
mongoManager: mongoManager,
|
mongoManager: mongoManager,
|
||||||
metricsCounter: orm.NewCounter(mongoManager),
|
metricsCounter: orm.NewCounter(mongoManager),
|
||||||
|
procedureLister: procedureLister,
|
||||||
logger: log,
|
logger: log,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -47,110 +78,162 @@ func tenderCollectionName() string {
|
|||||||
return "tenders"
|
return "tenders"
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *repository) Summary(ctx context.Context, closingWindowSec int64) (*SummaryResponse, error) {
|
func (r *repository) Summary(ctx context.Context, closingWindowSec int64, days int) (*SummaryResponse, error) {
|
||||||
now := time.Now().Unix()
|
now := time.Now().Unix()
|
||||||
windowEnd := now + closingWindowSec
|
windowEnd := now + closingWindowSec
|
||||||
|
|
||||||
total, err := r.ormRepo.Count(ctx, bson.M{})
|
nowUTC := time.Now().UTC()
|
||||||
if err != nil {
|
endDay := time.Date(nowUTC.Year(), nowUTC.Month(), nowUTC.Day(), 0, 0, 0, 0, time.UTC)
|
||||||
return nil, fmt.Errorf("count total tenders: %w", err)
|
startDay := endDay.AddDate(0, 0, -(days - 1))
|
||||||
|
|
||||||
|
var (
|
||||||
|
statusCounts summaryStatusCounts
|
||||||
|
valueCurrency string
|
||||||
|
totalValue float64
|
||||||
|
scrapedTED map[string]int64
|
||||||
|
wg sync.WaitGroup
|
||||||
|
mu sync.Mutex
|
||||||
|
firstErr error
|
||||||
|
)
|
||||||
|
|
||||||
|
recordErr := func(section string, err error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if firstErr == nil {
|
||||||
|
firstErr = fmt.Errorf("%s: %w", section, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
active, err := r.ormRepo.Count(ctx, bson.M{"status": tender.TenderStatusActive})
|
wg.Add(3)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
counts, err := r.summaryStatusCounts(ctx, now, windowEnd)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("count active tenders: %w", err)
|
recordErr("status_counts", err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
statusCounts = counts
|
||||||
|
}()
|
||||||
|
|
||||||
awarded, err := r.ormRepo.Count(ctx, bson.M{"status": tender.TenderStatusAwarded})
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
currency, value, err := r.summaryCurrencyValue(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("count awarded tenders: %w", err)
|
recordErr("currency_value", err)
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
valueCurrency, totalValue = currency, value
|
||||||
|
}()
|
||||||
|
|
||||||
expired, err := r.countExpired(ctx, now)
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
qctx, cancel := context.WithTimeout(ctx, statisticsMongoQueryTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
counts, err := r.metricsCounter.GetTEDNoticeScrapedDailyCounts(qctx, startDay, endDay)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
if r.logger != nil {
|
||||||
|
r.logger.Warn("Dashboard summary scraped TED daily counts failed", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
scrapedTED = map[string]int64{}
|
||||||
cancelled, err := r.ormRepo.Count(ctx, bson.M{"status": tender.TenderStatusCancelled})
|
return
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("count cancelled tenders: %w", err)
|
|
||||||
}
|
}
|
||||||
|
scrapedTED = counts
|
||||||
|
}()
|
||||||
|
|
||||||
closingSoon, err := r.countClosingSoon(ctx, now, windowEnd)
|
wg.Wait()
|
||||||
if err != nil {
|
if firstErr != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf("dashboard summary aggregation: %w", firstErr)
|
||||||
}
|
|
||||||
|
|
||||||
valueCurrency, totalValue, err := r.dominantCurrencyValue(ctx)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return &SummaryResponse{
|
return &SummaryResponse{
|
||||||
TotalTenders: total,
|
TotalTenders: statusCounts.total,
|
||||||
ActiveTenders: active,
|
ActiveTenders: statusCounts.active,
|
||||||
AwardedTenders: awarded,
|
AwardedTenders: statusCounts.awarded,
|
||||||
ExpiredTenders: expired,
|
ExpiredTenders: statusCounts.expired,
|
||||||
CancelledTenders: cancelled,
|
CancelledTenders: statusCounts.cancelled,
|
||||||
ClosingSoon: closingSoon,
|
ClosingSoon: statusCounts.closingSoon,
|
||||||
TotalEstimatedValue: totalValue,
|
TotalEstimatedValue: totalValue,
|
||||||
ValueCurrency: valueCurrency,
|
ValueCurrency: valueCurrency,
|
||||||
|
Days: days,
|
||||||
|
ScrapedTED: fillTrendSeries(days, scrapedTED),
|
||||||
GeneratedAt: now,
|
GeneratedAt: now,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *repository) countExpired(ctx context.Context, now int64) (int64, error) {
|
type summaryStatusCounts struct {
|
||||||
|
total int64
|
||||||
|
active int64
|
||||||
|
awarded int64
|
||||||
|
expired int64
|
||||||
|
cancelled int64
|
||||||
|
closingSoon int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *repository) summaryStatusCounts(ctx context.Context, now, windowEnd int64) (summaryStatusCounts, error) {
|
||||||
pipeline := mongo.Pipeline{
|
pipeline := mongo.Pipeline{
|
||||||
{{Key: "$addFields", Value: bson.M{"expiry_deadline": expiryDeadlineExpr()}}},
|
{{Key: "$addFields", Value: bson.M{
|
||||||
{{Key: "$match", Value: bson.M{
|
"expiry_deadline": expiryDeadlineExpr(),
|
||||||
"status": bson.M{"$nin": []tender.TenderStatus{
|
"effective_deadline": effectiveDeadlineExpr(),
|
||||||
tender.TenderStatusAwarded,
|
}}},
|
||||||
tender.TenderStatusCancelled,
|
{{Key: "$group", Value: bson.M{
|
||||||
|
"_id": nil,
|
||||||
|
"total": bson.M{"$sum": 1},
|
||||||
|
"active": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||||
|
bson.M{"$eq": bson.A{"$status", tender.TenderStatusActive}}, 1, 0,
|
||||||
|
}}},
|
||||||
|
"awarded": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||||
|
bson.M{"$eq": bson.A{"$status", tender.TenderStatusAwarded}}, 1, 0,
|
||||||
|
}}},
|
||||||
|
"cancelled": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||||
|
bson.M{"$eq": bson.A{"$status", tender.TenderStatusCancelled}}, 1, 0,
|
||||||
|
}}},
|
||||||
|
"expired": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||||
|
bson.M{"$and": bson.A{
|
||||||
|
bson.M{"$not": bson.M{"$in": bson.A{
|
||||||
|
"$status",
|
||||||
|
bson.A{tender.TenderStatusAwarded, tender.TenderStatusCancelled},
|
||||||
|
}}},
|
||||||
|
bson.M{"$or": bson.A{
|
||||||
|
bson.M{"$eq": bson.A{"$status", tender.TenderStatusExpired}},
|
||||||
|
bson.M{"$and": bson.A{
|
||||||
|
bson.M{"$gt": bson.A{"$expiry_deadline", 0}},
|
||||||
|
bson.M{"$lte": bson.A{"$expiry_deadline", now}},
|
||||||
}},
|
}},
|
||||||
"$or": []bson.M{
|
}},
|
||||||
{"status": tender.TenderStatusExpired},
|
}}, 1, 0,
|
||||||
{
|
}}},
|
||||||
"expiry_deadline": bson.M{"$gt": 0, "$lte": now},
|
"closing_soon": bson.M{"$sum": bson.M{"$cond": bson.A{
|
||||||
},
|
bson.M{"$and": bson.A{
|
||||||
},
|
bson.M{"$gt": bson.A{"$effective_deadline", now}},
|
||||||
|
bson.M{"$lte": bson.A{"$effective_deadline", windowEnd}},
|
||||||
|
}}, 1, 0,
|
||||||
|
}}},
|
||||||
}}},
|
}}},
|
||||||
{{Key: "$count", Value: "count"}},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, fmt.Errorf("count expired tenders: %w", err)
|
return summaryStatusCounts{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(results) == 0 {
|
if len(results) == 0 {
|
||||||
return 0, nil
|
return summaryStatusCounts{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return aggregateCount(results[0], "count"), nil
|
row := results[0]
|
||||||
|
return summaryStatusCounts{
|
||||||
|
total: aggregateCount(row, "total"),
|
||||||
|
active: aggregateCount(row, "active"),
|
||||||
|
awarded: aggregateCount(row, "awarded"),
|
||||||
|
expired: aggregateCount(row, "expired"),
|
||||||
|
cancelled: aggregateCount(row, "cancelled"),
|
||||||
|
closingSoon: aggregateCount(row, "closing_soon"),
|
||||||
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *repository) countClosingSoon(ctx context.Context, now, windowEnd int64) (int64, error) {
|
func (r *repository) summaryCurrencyValue(ctx context.Context) (string, float64, error) {
|
||||||
pipeline := mongo.Pipeline{
|
|
||||||
{{Key: "$addFields", Value: bson.M{"effective_deadline": effectiveDeadlineExpr()}}},
|
|
||||||
{{Key: "$match", Value: bson.M{
|
|
||||||
"effective_deadline": bson.M{"$gt": now, "$lte": windowEnd},
|
|
||||||
}}},
|
|
||||||
{{Key: "$count", Value: "count"}},
|
|
||||||
}
|
|
||||||
|
|
||||||
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
|
||||||
if err != nil {
|
|
||||||
return 0, fmt.Errorf("count closing soon: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(results) == 0 {
|
|
||||||
return 0, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return aggregateCount(results[0], "count"), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *repository) dominantCurrencyValue(ctx context.Context) (string, float64, error) {
|
|
||||||
pipeline := mongo.Pipeline{
|
pipeline := mongo.Pipeline{
|
||||||
{{Key: "$match", Value: bson.M{
|
{{Key: "$match", Value: bson.M{
|
||||||
"estimated_value": bson.M{"$gt": 0},
|
"estimated_value": bson.M{"$gt": 0},
|
||||||
@@ -167,19 +250,71 @@ func (r *repository) dominantCurrencyValue(ctx context.Context) (string, float64
|
|||||||
|
|
||||||
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
results, err := r.ormRepo.Aggregate(ctx, pipeline)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return defaultValueCurrency, 0, fmt.Errorf("dominant currency value: %w", err)
|
return defaultValueCurrency, 0, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(results) == 0 {
|
if len(results) == 0 {
|
||||||
return defaultValueCurrency, 0, nil
|
return defaultValueCurrency, 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
currency := defaultValueCurrency
|
currency, total := facetCurrencyValue(bson.M{"currency_value": bson.A{results[0]}})
|
||||||
if c, ok := results[0]["_id"].(string); ok && c != "" {
|
return currency, total, nil
|
||||||
currency = strings.ToUpper(c)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return currency, toFloat64(results[0]["total"]), nil
|
func facetCount(facet bson.M, key string) int64 {
|
||||||
|
rows := facetRows(facet, key)
|
||||||
|
if len(rows) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return aggregateCount(asBSONMap(rows[0]), "count")
|
||||||
|
}
|
||||||
|
|
||||||
|
func facetCurrencyValue(facet bson.M) (string, float64) {
|
||||||
|
rows := facetRows(facet, "currency_value")
|
||||||
|
if len(rows) == 0 {
|
||||||
|
return defaultValueCurrency, 0
|
||||||
|
}
|
||||||
|
row := asBSONMap(rows[0])
|
||||||
|
if row == nil {
|
||||||
|
return defaultValueCurrency, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
currency := defaultValueCurrency
|
||||||
|
if c, ok := row["_id"].(string); ok && c != "" {
|
||||||
|
currency = strings.ToUpper(c)
|
||||||
|
}
|
||||||
|
return currency, toFloat64(row["total"])
|
||||||
|
}
|
||||||
|
|
||||||
|
func facetRows(facet bson.M, key string) []interface{} {
|
||||||
|
raw, ok := facet[key]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch rows := raw.(type) {
|
||||||
|
case bson.A:
|
||||||
|
return []interface{}(rows)
|
||||||
|
case []interface{}:
|
||||||
|
return rows
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func asBSONMap(v interface{}) bson.M {
|
||||||
|
switch doc := v.(type) {
|
||||||
|
case bson.M:
|
||||||
|
return doc
|
||||||
|
case bson.D:
|
||||||
|
out := make(bson.M, len(doc))
|
||||||
|
for _, elem := range doc {
|
||||||
|
out[elem.Key] = elem.Value
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
case map[string]interface{}:
|
||||||
|
return doc
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *repository) Trend(ctx context.Context, days int, metric string, startUnix int64) (map[string]int64, error) {
|
func (r *repository) Trend(ctx context.Context, days int, metric string, startUnix int64) (map[string]int64, error) {
|
||||||
@@ -306,6 +441,22 @@ func (r *repository) ClosingSoon(ctx context.Context, limit int, windowSec int64
|
|||||||
windowEnd := now + windowSec
|
windowEnd := now + windowSec
|
||||||
|
|
||||||
pipeline := mongo.Pipeline{
|
pipeline := mongo.Pipeline{
|
||||||
|
{{Key: "$match", Value: bson.M{
|
||||||
|
"$or": bson.A{
|
||||||
|
bson.M{
|
||||||
|
"$and": bson.A{
|
||||||
|
bson.M{"submission_deadline": bson.M{"$gt": 0}},
|
||||||
|
deadlineWithinWindowMatch("submission_deadline", now, windowEnd),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
bson.M{
|
||||||
|
"$and": bson.A{
|
||||||
|
noSubmissionDeadlineClause(),
|
||||||
|
deadlineWithinWindowMatch("tender_deadline", now, windowEnd),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}}},
|
||||||
{{Key: "$addFields", Value: bson.M{"effective_deadline": effectiveDeadlineExpr()}}},
|
{{Key: "$addFields", Value: bson.M{"effective_deadline": effectiveDeadlineExpr()}}},
|
||||||
{{Key: "$match", Value: bson.M{
|
{{Key: "$match", Value: bson.M{
|
||||||
"effective_deadline": bson.M{"$gt": now, "$lte": windowEnd},
|
"effective_deadline": bson.M{"$gt": now, "$lte": windowEnd},
|
||||||
@@ -445,6 +596,26 @@ func trendTimestampField(metric string) string {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// deadlineWithinWindowMatch matches raw deadline fields stored as Unix seconds or
|
||||||
|
// milliseconds. normalizeTimestampExpr treats values > 1e12 as ms; indexed pre-filters
|
||||||
|
// must accept both encodings to stay equivalent to filtering on normalized deadlines.
|
||||||
|
func deadlineWithinWindowMatch(field string, now, windowEnd int64) bson.M {
|
||||||
|
return bson.M{
|
||||||
|
"$or": bson.A{
|
||||||
|
bson.M{field: bson.M{"$gt": now, "$lte": windowEnd}},
|
||||||
|
bson.M{field: bson.M{"$gt": now * 1000, "$lte": windowEnd * 1000}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func noSubmissionDeadlineClause() bson.M {
|
||||||
|
return bson.M{"$or": bson.A{
|
||||||
|
bson.M{"submission_deadline": bson.M{"$exists": false}},
|
||||||
|
bson.M{"submission_deadline": nil},
|
||||||
|
bson.M{"submission_deadline": bson.M{"$lte": 0}},
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
// effectiveDeadlineExpr is used for closing-soon (submission first, per dashboard-api.md).
|
// effectiveDeadlineExpr is used for closing-soon (submission first, per dashboard-api.md).
|
||||||
func effectiveDeadlineExpr() bson.M {
|
func effectiveDeadlineExpr() bson.M {
|
||||||
return normalizeDeadlineFieldExpr(
|
return normalizeDeadlineFieldExpr(
|
||||||
|
|||||||
@@ -0,0 +1,71 @@
|
|||||||
|
package dashboard
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFacetCountDecodesBSOND(t *testing.T) {
|
||||||
|
facet := bson.M{
|
||||||
|
"total": bson.A{
|
||||||
|
bson.D{{Key: "count", Value: int32(42)}},
|
||||||
|
},
|
||||||
|
"active": bson.A{
|
||||||
|
bson.D{{Key: "count", Value: int64(7)}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
if got := facetCount(facet, "total"); got != 42 {
|
||||||
|
t.Fatalf("expected total 42, got %d", got)
|
||||||
|
}
|
||||||
|
if got := facetCount(facet, "active"); got != 7 {
|
||||||
|
t.Fatalf("expected active 7, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFacetCurrencyValueDecodesBSOND(t *testing.T) {
|
||||||
|
facet := bson.M{
|
||||||
|
"currency_value": bson.A{
|
||||||
|
bson.D{
|
||||||
|
{Key: "_id", Value: "eur"},
|
||||||
|
{Key: "total", Value: float64(12345.67)},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
currency, total := facetCurrencyValue(facet)
|
||||||
|
if currency != "EUR" {
|
||||||
|
t.Fatalf("expected EUR, got %q", currency)
|
||||||
|
}
|
||||||
|
if total != 12345.67 {
|
||||||
|
t.Fatalf("expected 12345.67, got %f", total)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeadlineWithinWindowMatchCoversSecondsAndMilliseconds(t *testing.T) {
|
||||||
|
now := int64(1_700_000_000)
|
||||||
|
windowEnd := now + 86_400
|
||||||
|
|
||||||
|
match := deadlineWithinWindowMatch("submission_deadline", now, windowEnd)
|
||||||
|
orClause, ok := match["$or"].(bson.A)
|
||||||
|
if !ok || len(orClause) != 2 {
|
||||||
|
t.Fatalf("expected two range branches, got %#v", match)
|
||||||
|
}
|
||||||
|
|
||||||
|
secRange, ok := orClause[0].(bson.M)["submission_deadline"].(bson.M)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected seconds range on submission_deadline, got %#v", orClause[0])
|
||||||
|
}
|
||||||
|
if secRange["$gt"] != now || secRange["$lte"] != windowEnd {
|
||||||
|
t.Fatalf("unexpected seconds range: %#v", secRange)
|
||||||
|
}
|
||||||
|
|
||||||
|
msRange, ok := orClause[1].(bson.M)["submission_deadline"].(bson.M)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected milliseconds range on submission_deadline, got %#v", orClause[1])
|
||||||
|
}
|
||||||
|
if msRange["$gt"] != now*1000 || msRange["$lte"] != windowEnd*1000 {
|
||||||
|
t.Fatalf("unexpected milliseconds range: %#v", msRange)
|
||||||
|
}
|
||||||
|
}
|
||||||
+554
-15
@@ -2,10 +2,17 @@ package dashboard
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
|
"tm/pkg/redis"
|
||||||
|
|
||||||
|
goredis "github.com/redis/go-redis/v9"
|
||||||
|
"golang.org/x/sync/singleflight"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -15,6 +22,15 @@ const (
|
|||||||
defaultCountriesLimit = 6
|
defaultCountriesLimit = 6
|
||||||
defaultListLimit = 5
|
defaultListLimit = 5
|
||||||
maxListLimit = 20
|
maxListLimit = 20
|
||||||
|
summaryCacheTTL = 5 * time.Minute
|
||||||
|
summaryStaleGraceTTL = 15 * time.Minute
|
||||||
|
statisticsCacheTTL = 5 * time.Minute
|
||||||
|
statisticsStaleGraceTTL = 15 * time.Minute
|
||||||
|
// statisticsColdLoadWait bounds how long a cold-cache request waits for a fresh
|
||||||
|
// load before falling back to a placeholder. Mongo-only loads (MinIO scope
|
||||||
|
// already cached) comfortably finish within this window; a first-ever MinIO
|
||||||
|
// bucket scan does not, so we don't block the request for that.
|
||||||
|
statisticsColdLoadWait = 4 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// Service defines dashboard business operations.
|
// Service defines dashboard business operations.
|
||||||
@@ -28,25 +44,81 @@ type Service interface {
|
|||||||
Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error)
|
Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type cacheEntry[T any] struct {
|
||||||
|
expiresAt time.Time
|
||||||
|
staleUntil time.Time
|
||||||
|
value T
|
||||||
|
}
|
||||||
|
|
||||||
type service struct {
|
type service struct {
|
||||||
repo Repository
|
repo Repository
|
||||||
logger logger.Logger
|
logger logger.Logger
|
||||||
|
redis redis.Client
|
||||||
|
summaryMu sync.Mutex
|
||||||
|
summary map[string]cacheEntry[*SummaryResponse]
|
||||||
|
summaryGroup singleflight.Group
|
||||||
|
statisticsMu sync.Mutex
|
||||||
|
statistics map[int]cacheEntry[*StatisticsReportResponse]
|
||||||
|
statisticsGroup singleflight.Group
|
||||||
|
trendCache *widgetCache[*TrendResponse]
|
||||||
|
countriesCache *widgetCache[*CountriesResponse]
|
||||||
|
noticeTypesCache *widgetCache[*NoticeTypesResponse]
|
||||||
|
closingSoonCache *widgetCache[*ClosingSoonResponse]
|
||||||
|
recentCache *widgetCache[*RecentResponse]
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewService creates a dashboard service.
|
// NewService creates a dashboard service.
|
||||||
func NewService(repo Repository, log logger.Logger) Service {
|
func NewService(repo Repository, log logger.Logger, redisClient redis.Client) Service {
|
||||||
return &service{repo: repo, logger: log}
|
s := &service{
|
||||||
|
repo: repo,
|
||||||
|
logger: log,
|
||||||
|
redis: redisClient,
|
||||||
|
summary: make(map[string]cacheEntry[*SummaryResponse]),
|
||||||
|
statistics: make(map[int]cacheEntry[*StatisticsReportResponse]),
|
||||||
|
trendCache: newWidgetCache[*TrendResponse](redisClient, log, "trend"),
|
||||||
|
countriesCache: newWidgetCache[*CountriesResponse](redisClient, log, "countries"),
|
||||||
|
noticeTypesCache: newWidgetCache[*NoticeTypesResponse](redisClient, log, "notice-types"),
|
||||||
|
closingSoonCache: newWidgetCache[*ClosingSoonResponse](redisClient, log, "closing-soon"),
|
||||||
|
recentCache: newWidgetCache[*RecentResponse](redisClient, log, "recent"),
|
||||||
|
}
|
||||||
|
go s.warmStatisticsCache()
|
||||||
|
go s.warmSummaryCache()
|
||||||
|
go s.warmWidgetCaches()
|
||||||
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *service) Summary(ctx context.Context, query SummaryQuery) (*SummaryResponse, error) {
|
func (s *service) Summary(ctx context.Context, query SummaryQuery) (*SummaryResponse, error) {
|
||||||
windowHours := closingWindowHours(query.ClosingWindow)
|
windowHours := closingWindowHours(query.ClosingWindow)
|
||||||
windowSec := int64(windowHours) * 3600
|
windowSec := int64(windowHours) * 3600
|
||||||
|
days := trendDays(query.Days)
|
||||||
|
cacheKey := summaryCacheKey(windowSec, days)
|
||||||
|
|
||||||
|
if cached, ok := s.cachedSummary(cacheKey); ok {
|
||||||
|
return cached, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if stale, ok := s.staleSummary(cacheKey); ok {
|
||||||
|
go s.refreshSummary(windowSec, days)
|
||||||
|
return stale, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if redisCached, ok := s.getRedisSummary(ctx, cacheKey); ok {
|
||||||
|
s.storeSummary(cacheKey, redisCached)
|
||||||
|
go s.refreshSummary(windowSec, days)
|
||||||
|
return redisCached, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err, _ := s.summaryGroup.Do(cacheKey, func() (interface{}, error) {
|
||||||
|
if cached, ok := s.cachedSummary(cacheKey); ok {
|
||||||
|
return cached, nil
|
||||||
|
}
|
||||||
|
|
||||||
s.logger.Info("Fetching dashboard summary", map[string]interface{}{
|
s.logger.Info("Fetching dashboard summary", map[string]interface{}{
|
||||||
"closing_window_hours": windowHours,
|
"closing_window_hours": windowHours,
|
||||||
|
"days": days,
|
||||||
})
|
})
|
||||||
|
|
||||||
out, err := s.repo.Summary(ctx, windowSec)
|
result, err := s.repo.Summary(context.WithoutCancel(ctx), windowSec, days)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to fetch dashboard summary", map[string]interface{}{
|
s.logger.Error("Failed to fetch dashboard summary", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -54,12 +126,23 @@ func (s *service) Summary(ctx context.Context, query SummaryQuery) (*SummaryResp
|
|||||||
return nil, fmt.Errorf("dashboard summary: %w", err)
|
return nil, fmt.Errorf("dashboard summary: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return out, nil
|
s.storeSummary(cacheKey, result)
|
||||||
|
s.storeRedisSummary(context.Background(), cacheKey, result)
|
||||||
|
return result, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return out.(*SummaryResponse), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *service) Trend(ctx context.Context, query TrendQuery) (*TrendResponse, error) {
|
func (s *service) Trend(ctx context.Context, query TrendQuery) (*TrendResponse, error) {
|
||||||
days := trendDays(query.Days)
|
days := trendDays(query.Days)
|
||||||
metric := normalizeTrendMetric(query.Metric)
|
metric := normalizeTrendMetric(query.Metric)
|
||||||
|
cacheKey := fmt.Sprintf("%d:%s", days, metric)
|
||||||
|
|
||||||
|
return s.trendCache.Get(ctx, cacheKey, func(loadCtx context.Context) (*TrendResponse, error) {
|
||||||
startUnix := trendStartUnix(days)
|
startUnix := trendStartUnix(days)
|
||||||
|
|
||||||
s.logger.Info("Fetching dashboard trend", map[string]interface{}{
|
s.logger.Info("Fetching dashboard trend", map[string]interface{}{
|
||||||
@@ -67,7 +150,7 @@ func (s *service) Trend(ctx context.Context, query TrendQuery) (*TrendResponse,
|
|||||||
"metric": metric,
|
"metric": metric,
|
||||||
})
|
})
|
||||||
|
|
||||||
counts, err := s.repo.Trend(ctx, days, metric, startUnix)
|
counts, err := s.repo.Trend(loadCtx, days, metric, startUnix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to fetch dashboard trend", map[string]interface{}{
|
s.logger.Error("Failed to fetch dashboard trend", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -75,23 +158,24 @@ func (s *service) Trend(ctx context.Context, query TrendQuery) (*TrendResponse,
|
|||||||
return nil, fmt.Errorf("dashboard trend: %w", err)
|
return nil, fmt.Errorf("dashboard trend: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
series := fillTrendSeries(days, counts)
|
|
||||||
|
|
||||||
return &TrendResponse{
|
return &TrendResponse{
|
||||||
Metric: metric,
|
Metric: metric,
|
||||||
Days: days,
|
Days: days,
|
||||||
Series: series,
|
Series: fillTrendSeries(days, counts),
|
||||||
}, nil
|
}, nil
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *service) Countries(ctx context.Context, query CountriesQuery) (*CountriesResponse, error) {
|
func (s *service) Countries(ctx context.Context, query CountriesQuery) (*CountriesResponse, error) {
|
||||||
limit := countriesLimit(query.Limit)
|
limit := countriesLimit(query.Limit)
|
||||||
|
cacheKey := strconv.Itoa(limit)
|
||||||
|
|
||||||
|
return s.countriesCache.Get(ctx, cacheKey, func(loadCtx context.Context) (*CountriesResponse, error) {
|
||||||
s.logger.Info("Fetching dashboard countries", map[string]interface{}{
|
s.logger.Info("Fetching dashboard countries", map[string]interface{}{
|
||||||
"limit": limit,
|
"limit": limit,
|
||||||
})
|
})
|
||||||
|
|
||||||
out, err := s.repo.Countries(ctx, limit)
|
out, err := s.repo.Countries(loadCtx, limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to fetch dashboard countries", map[string]interface{}{
|
s.logger.Error("Failed to fetch dashboard countries", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -100,12 +184,14 @@ func (s *service) Countries(ctx context.Context, query CountriesQuery) (*Countri
|
|||||||
}
|
}
|
||||||
|
|
||||||
return out, nil
|
return out, nil
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *service) NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error) {
|
func (s *service) NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error) {
|
||||||
|
return s.noticeTypesCache.Get(ctx, "all", func(loadCtx context.Context) (*NoticeTypesResponse, error) {
|
||||||
s.logger.Info("Fetching dashboard notice types", nil)
|
s.logger.Info("Fetching dashboard notice types", nil)
|
||||||
|
|
||||||
out, err := s.repo.NoticeTypes(ctx)
|
out, err := s.repo.NoticeTypes(loadCtx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to fetch dashboard notice types", map[string]interface{}{
|
s.logger.Error("Failed to fetch dashboard notice types", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -114,19 +200,22 @@ func (s *service) NoticeTypes(ctx context.Context) (*NoticeTypesResponse, error)
|
|||||||
}
|
}
|
||||||
|
|
||||||
return out, nil
|
return out, nil
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *service) ClosingSoon(ctx context.Context, query ClosingSoonQuery) (*ClosingSoonResponse, error) {
|
func (s *service) ClosingSoon(ctx context.Context, query ClosingSoonQuery) (*ClosingSoonResponse, error) {
|
||||||
limit := listLimit(query.Limit)
|
limit := listLimit(query.Limit)
|
||||||
windowHours := closingWindowHours(query.Window)
|
windowHours := closingWindowHours(query.Window)
|
||||||
windowSec := int64(windowHours) * 3600
|
windowSec := int64(windowHours) * 3600
|
||||||
|
cacheKey := fmt.Sprintf("%d:%d", limit, windowSec)
|
||||||
|
|
||||||
|
return s.closingSoonCache.Get(ctx, cacheKey, func(loadCtx context.Context) (*ClosingSoonResponse, error) {
|
||||||
s.logger.Info("Fetching dashboard closing soon", map[string]interface{}{
|
s.logger.Info("Fetching dashboard closing soon", map[string]interface{}{
|
||||||
"limit": limit,
|
"limit": limit,
|
||||||
"window": windowHours,
|
"window": windowHours,
|
||||||
})
|
})
|
||||||
|
|
||||||
items, err := s.repo.ClosingSoon(ctx, limit, windowSec)
|
items, err := s.repo.ClosingSoon(loadCtx, limit, windowSec)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to fetch dashboard closing soon", map[string]interface{}{
|
s.logger.Error("Failed to fetch dashboard closing soon", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -135,16 +224,18 @@ func (s *service) ClosingSoon(ctx context.Context, query ClosingSoonQuery) (*Clo
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &ClosingSoonResponse{Items: items}, nil
|
return &ClosingSoonResponse{Items: items}, nil
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *service) Recent(ctx context.Context, query RecentQuery) (*RecentResponse, error) {
|
func (s *service) Recent(ctx context.Context, query RecentQuery) (*RecentResponse, error) {
|
||||||
limit := listLimit(query.Limit)
|
limit := listLimit(query.Limit)
|
||||||
|
cursor := strings.TrimSpace(query.Cursor)
|
||||||
|
if cursor != "" {
|
||||||
s.logger.Info("Fetching dashboard recent tenders", map[string]interface{}{
|
s.logger.Info("Fetching dashboard recent tenders", map[string]interface{}{
|
||||||
"limit": limit,
|
"limit": limit,
|
||||||
})
|
})
|
||||||
|
|
||||||
items, nextCursor, err := s.repo.Recent(ctx, limit, strings.TrimSpace(query.Cursor))
|
items, nextCursor, err := s.repo.Recent(ctx, limit, cursor)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to fetch dashboard recent tenders", map[string]interface{}{
|
s.logger.Error("Failed to fetch dashboard recent tenders", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -158,15 +249,121 @@ func (s *service) Recent(ctx context.Context, query RecentQuery) (*RecentRespons
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
cacheKey := strconv.Itoa(limit)
|
||||||
|
return s.recentCache.Get(ctx, cacheKey, func(loadCtx context.Context) (*RecentResponse, error) {
|
||||||
|
s.logger.Info("Fetching dashboard recent tenders", map[string]interface{}{
|
||||||
|
"limit": limit,
|
||||||
|
})
|
||||||
|
|
||||||
|
items, nextCursor, err := s.repo.Recent(loadCtx, limit, "")
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to fetch dashboard recent tenders", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("dashboard recent: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &RecentResponse{
|
||||||
|
Items: items,
|
||||||
|
NextCursor: nextCursor,
|
||||||
|
}, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error) {
|
func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error) {
|
||||||
days := trendDays(query.Days)
|
days := trendDays(query.Days)
|
||||||
|
if cached, ok := s.cachedStatistics(days); ok {
|
||||||
|
return cached, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if stale, ok := s.staleStatistics(days); ok {
|
||||||
|
go s.refreshStatistics(days)
|
||||||
|
return stale, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if redisCached, ok := s.getRedisStatistics(ctx, days); ok {
|
||||||
|
s.storeStatistics(days, redisCached)
|
||||||
|
go s.refreshStatistics(days)
|
||||||
|
return redisCached, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cold cache: the underlying load performs a full MinIO bucket scan plus Mongo
|
||||||
|
// aggregations that can take minutes on a first run. Start the load in the
|
||||||
|
// background (deduped via singleflight, and detached from this request's
|
||||||
|
// context so it keeps running even if we give up waiting) and wait briefly for
|
||||||
|
// it. If it finishes in time we return real data; otherwise we serve a
|
||||||
|
// placeholder so the endpoint stays fast, and the load keeps warming the cache
|
||||||
|
// for the next request.
|
||||||
|
resultCh := make(chan *StatisticsReportResponse, 1)
|
||||||
|
go func() {
|
||||||
|
result, err := s.loadStatistics(context.Background(), days)
|
||||||
|
if err != nil {
|
||||||
|
resultCh <- nil
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resultCh <- result
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case result := <-resultCh:
|
||||||
|
if result != nil {
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
case <-time.After(statisticsColdLoadWait):
|
||||||
|
s.logger.Info("Serving placeholder dashboard statistics while cache warms", map[string]interface{}{
|
||||||
|
"days": days,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return emptyStatisticsReport(days), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) warmStatisticsCache() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
if cached, ok := s.getRedisStatistics(ctx, defaultTrendDays); ok {
|
||||||
|
s.storeStatistics(defaultTrendDays, cached)
|
||||||
|
s.logger.Info("Dashboard statistics cache warmed from Redis", map[string]interface{}{
|
||||||
|
"days": defaultTrendDays,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
go s.refreshStatistics(defaultTrendDays)
|
||||||
|
}
|
||||||
|
|
||||||
|
func emptyStatisticsReport(days int) *StatisticsReportResponse {
|
||||||
|
return &StatisticsReportResponse{
|
||||||
|
Days: days,
|
||||||
|
GeneratedAt: time.Now().UTC().Unix(),
|
||||||
|
Daily: StatisticsDailySeries{
|
||||||
|
ScrapedDocuments: fillTrendSeries(days, nil),
|
||||||
|
TranslatedNotices: fillTrendSeries(days, nil),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) refreshStatistics(days int) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
_, _ = s.loadStatistics(ctx, days)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) loadStatistics(ctx context.Context, days int) (*StatisticsReportResponse, error) {
|
||||||
|
cacheKey := strconv.Itoa(days)
|
||||||
|
out, err, _ := s.statisticsGroup.Do(cacheKey, func() (interface{}, error) {
|
||||||
|
if cached, ok := s.cachedStatistics(days); ok {
|
||||||
|
return cached, nil
|
||||||
|
}
|
||||||
|
|
||||||
startUnix := trendStartUnix(days)
|
startUnix := trendStartUnix(days)
|
||||||
|
|
||||||
s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
|
s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
|
||||||
"days": days,
|
"days": days,
|
||||||
})
|
})
|
||||||
|
|
||||||
out, err := s.repo.Statistics(ctx, days, startUnix)
|
result, err := s.repo.Statistics(context.WithoutCancel(ctx), days, startUnix)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to fetch dashboard statistics report", map[string]interface{}{
|
s.logger.Error("Failed to fetch dashboard statistics report", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -174,7 +371,349 @@ func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*Stati
|
|||||||
return nil, fmt.Errorf("dashboard statistics: %w", err)
|
return nil, fmt.Errorf("dashboard statistics: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return out, nil
|
s.storeStatistics(days, result)
|
||||||
|
s.storeRedisStatistics(context.Background(), days, result)
|
||||||
|
return result, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return out.(*StatisticsReportResponse), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) warmWidgetCaches() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
defaultWindowSec := int64(defaultClosingWindowHours) * 3600
|
||||||
|
warmed := 0
|
||||||
|
|
||||||
|
if s.trendCache.WarmFromRedis(ctx, fmt.Sprintf("%d:%s", defaultTrendDays, "created")) {
|
||||||
|
warmed++
|
||||||
|
}
|
||||||
|
if s.countriesCache.WarmFromRedis(ctx, strconv.Itoa(defaultCountriesLimit)) {
|
||||||
|
warmed++
|
||||||
|
}
|
||||||
|
if s.noticeTypesCache.WarmFromRedis(ctx, "all") {
|
||||||
|
warmed++
|
||||||
|
}
|
||||||
|
if s.closingSoonCache.WarmFromRedis(ctx, fmt.Sprintf("%d:%d", defaultListLimit, defaultWindowSec)) {
|
||||||
|
warmed++
|
||||||
|
}
|
||||||
|
if s.recentCache.WarmFromRedis(ctx, strconv.Itoa(defaultListLimit)) {
|
||||||
|
warmed++
|
||||||
|
}
|
||||||
|
|
||||||
|
if warmed > 0 {
|
||||||
|
s.logger.Info("Dashboard widget caches warmed from Redis", map[string]interface{}{
|
||||||
|
"warmed": warmed,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
go s.trendCache.Reload(fmt.Sprintf("%d:%s", defaultTrendDays, "created"), func(loadCtx context.Context) (*TrendResponse, error) {
|
||||||
|
return s.loadTrend(loadCtx, TrendQuery{Days: defaultTrendDays, Metric: "created"})
|
||||||
|
})
|
||||||
|
go s.countriesCache.Reload(strconv.Itoa(defaultCountriesLimit), func(loadCtx context.Context) (*CountriesResponse, error) {
|
||||||
|
return s.repo.Countries(loadCtx, defaultCountriesLimit)
|
||||||
|
})
|
||||||
|
go s.noticeTypesCache.Reload("all", func(loadCtx context.Context) (*NoticeTypesResponse, error) {
|
||||||
|
return s.repo.NoticeTypes(loadCtx)
|
||||||
|
})
|
||||||
|
go s.closingSoonCache.Reload(fmt.Sprintf("%d:%d", defaultListLimit, defaultWindowSec), func(loadCtx context.Context) (*ClosingSoonResponse, error) {
|
||||||
|
items, err := s.repo.ClosingSoon(loadCtx, defaultListLimit, defaultWindowSec)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &ClosingSoonResponse{Items: items}, nil
|
||||||
|
})
|
||||||
|
go s.recentCache.Reload(strconv.Itoa(defaultListLimit), func(loadCtx context.Context) (*RecentResponse, error) {
|
||||||
|
items, nextCursor, err := s.repo.Recent(loadCtx, defaultListLimit, "")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &RecentResponse{
|
||||||
|
Items: items,
|
||||||
|
NextCursor: nextCursor,
|
||||||
|
}, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) loadTrend(ctx context.Context, query TrendQuery) (*TrendResponse, error) {
|
||||||
|
days := trendDays(query.Days)
|
||||||
|
metric := normalizeTrendMetric(query.Metric)
|
||||||
|
startUnix := trendStartUnix(days)
|
||||||
|
|
||||||
|
counts, err := s.repo.Trend(ctx, days, metric, startUnix)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &TrendResponse{
|
||||||
|
Metric: metric,
|
||||||
|
Days: days,
|
||||||
|
Series: fillTrendSeries(days, counts),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) warmSummaryCache() {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
windowSec := int64(defaultClosingWindowHours) * 3600
|
||||||
|
cacheKey := summaryCacheKey(windowSec, defaultTrendDays)
|
||||||
|
|
||||||
|
if cached, ok := s.getRedisSummary(ctx, cacheKey); ok {
|
||||||
|
s.storeSummary(cacheKey, cached)
|
||||||
|
s.logger.Info("Dashboard summary cache warmed from Redis", map[string]interface{}{
|
||||||
|
"closing_window_hours": defaultClosingWindowHours,
|
||||||
|
"days": defaultTrendDays,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
go s.refreshSummary(windowSec, defaultTrendDays)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) refreshSummary(windowSec int64, days int) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
_, _ = s.loadSummary(ctx, windowSec, days)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) loadSummary(ctx context.Context, windowSec int64, days int) (*SummaryResponse, error) {
|
||||||
|
cacheKey := summaryCacheKey(windowSec, days)
|
||||||
|
out, err, _ := s.summaryGroup.Do(cacheKey, func() (interface{}, error) {
|
||||||
|
if cached, ok := s.cachedSummary(cacheKey); ok {
|
||||||
|
return cached, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Refreshing dashboard summary cache", map[string]interface{}{
|
||||||
|
"closing_window_sec": windowSec,
|
||||||
|
"days": days,
|
||||||
|
})
|
||||||
|
|
||||||
|
result, err := s.repo.Summary(context.WithoutCancel(ctx), windowSec, days)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to refresh dashboard summary", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("dashboard summary: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.storeSummary(cacheKey, result)
|
||||||
|
s.storeRedisSummary(context.Background(), cacheKey, result)
|
||||||
|
return result, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return out.(*SummaryResponse), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func summaryCacheKey(windowSec int64, days int) string {
|
||||||
|
return fmt.Sprintf("%d:%d", windowSec, days)
|
||||||
|
}
|
||||||
|
|
||||||
|
func summaryRedisKey(cacheKey string) string {
|
||||||
|
return fmt.Sprintf("dashboard:summary:%s", cacheKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) cachedSummary(cacheKey string) (*SummaryResponse, bool) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
s.summaryMu.Lock()
|
||||||
|
defer s.summaryMu.Unlock()
|
||||||
|
|
||||||
|
entry, ok := s.summary[cacheKey]
|
||||||
|
if !ok || now.After(entry.expiresAt) {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry.value, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) staleSummary(cacheKey string) (*SummaryResponse, bool) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
s.summaryMu.Lock()
|
||||||
|
defer s.summaryMu.Unlock()
|
||||||
|
|
||||||
|
entry, ok := s.summary[cacheKey]
|
||||||
|
if !ok || now.After(entry.staleUntil) {
|
||||||
|
if ok {
|
||||||
|
delete(s.summary, cacheKey)
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry.value, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) storeSummary(cacheKey string, value *SummaryResponse) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
s.summaryMu.Lock()
|
||||||
|
defer s.summaryMu.Unlock()
|
||||||
|
|
||||||
|
s.summary[cacheKey] = cacheEntry[*SummaryResponse]{
|
||||||
|
expiresAt: now.Add(summaryCacheTTL),
|
||||||
|
staleUntil: now.Add(summaryCacheTTL + summaryStaleGraceTTL),
|
||||||
|
value: value,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) getRedisSummary(ctx context.Context, cacheKey string) (*SummaryResponse, bool) {
|
||||||
|
if s.redis == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := s.redis.Get(ctx, summaryRedisKey(cacheKey))
|
||||||
|
if err != nil {
|
||||||
|
if err != goredis.Nil {
|
||||||
|
s.logger.Warn("Failed to read dashboard summary cache from Redis", map[string]interface{}{
|
||||||
|
"cache_key": cacheKey,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
var report SummaryResponse
|
||||||
|
if err := json.Unmarshal([]byte(raw), &report); err != nil {
|
||||||
|
s.logger.Warn("Failed to decode dashboard summary cache from Redis", map[string]interface{}{
|
||||||
|
"cache_key": cacheKey,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
_ = s.redis.Del(ctx, summaryRedisKey(cacheKey))
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return &report, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) storeRedisSummary(ctx context.Context, cacheKey string, value *SummaryResponse) {
|
||||||
|
if s.redis == nil || value == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
encoded, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("Failed to encode dashboard summary for Redis", map[string]interface{}{
|
||||||
|
"cache_key": cacheKey,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ttl := summaryCacheTTL + summaryStaleGraceTTL
|
||||||
|
if err := s.redis.Set(ctx, summaryRedisKey(cacheKey), string(encoded), ttl); err != nil {
|
||||||
|
s.logger.Warn("Failed to store dashboard summary in Redis", map[string]interface{}{
|
||||||
|
"cache_key": cacheKey,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) cachedStatistics(days int) (*StatisticsReportResponse, bool) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
s.statisticsMu.Lock()
|
||||||
|
defer s.statisticsMu.Unlock()
|
||||||
|
|
||||||
|
entry, ok := s.statistics[days]
|
||||||
|
if !ok || now.After(entry.expiresAt) {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry.value, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) staleStatistics(days int) (*StatisticsReportResponse, bool) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
s.statisticsMu.Lock()
|
||||||
|
defer s.statisticsMu.Unlock()
|
||||||
|
|
||||||
|
entry, ok := s.statistics[days]
|
||||||
|
if !ok || now.After(entry.staleUntil) {
|
||||||
|
if ok {
|
||||||
|
delete(s.statistics, days)
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry.value, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) storeStatistics(days int, value *StatisticsReportResponse) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
s.statisticsMu.Lock()
|
||||||
|
defer s.statisticsMu.Unlock()
|
||||||
|
|
||||||
|
s.statistics[days] = cacheEntry[*StatisticsReportResponse]{
|
||||||
|
expiresAt: now.Add(statisticsCacheTTL),
|
||||||
|
staleUntil: now.Add(statisticsCacheTTL + statisticsStaleGraceTTL),
|
||||||
|
value: value,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func statisticsRedisKey(days int) string {
|
||||||
|
return fmt.Sprintf("dashboard:statistics:%d", days)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) getRedisStatistics(ctx context.Context, days int) (*StatisticsReportResponse, bool) {
|
||||||
|
if s.redis == nil {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := s.redis.Get(ctx, statisticsRedisKey(days))
|
||||||
|
if err != nil {
|
||||||
|
if err != goredis.Nil {
|
||||||
|
s.logger.Warn("Failed to read dashboard statistics cache from Redis", map[string]interface{}{
|
||||||
|
"days": days,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
var report StatisticsReportResponse
|
||||||
|
if err := json.Unmarshal([]byte(raw), &report); err != nil {
|
||||||
|
s.logger.Warn("Failed to decode dashboard statistics cache from Redis", map[string]interface{}{
|
||||||
|
"days": days,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
_ = s.redis.Del(ctx, statisticsRedisKey(days))
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return &report, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *service) storeRedisStatistics(ctx context.Context, days int, value *StatisticsReportResponse) {
|
||||||
|
if s.redis == nil || value == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
encoded, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Warn("Failed to encode dashboard statistics for Redis", map[string]interface{}{
|
||||||
|
"days": days,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ttl := statisticsCacheTTL + statisticsStaleGraceTTL
|
||||||
|
if err := s.redis.Set(ctx, statisticsRedisKey(days), string(encoded), ttl); err != nil {
|
||||||
|
s.logger.Warn("Failed to store dashboard statistics in Redis", map[string]interface{}{
|
||||||
|
"days": days,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func closingWindowHours(hours int) int {
|
func closingWindowHours(hours int) int {
|
||||||
|
|||||||
@@ -2,114 +2,222 @@ package dashboard
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"tm/internal/notice"
|
"tm/pkg/ai_summarizer"
|
||||||
orm "tm/pkg/mongo"
|
orm "tm/pkg/mongo"
|
||||||
|
|
||||||
"go.mongodb.org/mongo-driver/v2/bson"
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
|
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
|
||||||
)
|
)
|
||||||
|
|
||||||
const noticesCollectionName = "notices"
|
// scrapedScopeCacheTTL avoids listing every MinIO object on each statistics request while
|
||||||
|
// keeping counts aligned with the tender panel (MinIO is the source of truth for scraped docs).
|
||||||
|
const scrapedScopeCacheTTL = 15 * time.Minute
|
||||||
|
|
||||||
|
const (
|
||||||
|
scrapedScopeResolveTimeout = 3 * time.Minute
|
||||||
|
statisticsMongoQueryTimeout = 90 * time.Second
|
||||||
|
maxScrapedFolderInClause = 5000
|
||||||
|
)
|
||||||
|
|
||||||
|
// scrapedTendersScope is the resolved Mongo filter for tenders with scraped documents.
|
||||||
|
// zero is set when MinIO reports no procedure folders; callers must return zero without querying.
|
||||||
|
type scrapedTendersScope struct {
|
||||||
|
match bson.M
|
||||||
|
zero bool
|
||||||
|
fromMinIO bool
|
||||||
|
dailyCounts map[string]int64 // non-nil when scope was resolved from MinIO
|
||||||
|
totalTenderCount int64 // >0 when MinIO procedure count should be used instead of Mongo Count
|
||||||
|
}
|
||||||
|
|
||||||
|
// translatedNoticesScope holds MinIO-backed translation statistics when available.
|
||||||
|
type translatedNoticesScope struct {
|
||||||
|
fromMinIO bool
|
||||||
|
dailyCounts map[string]int64
|
||||||
|
totalCount int64
|
||||||
|
}
|
||||||
|
|
||||||
func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
|
func (r *repository) Statistics(ctx context.Context, days int, startUnix int64) (*StatisticsReportResponse, error) {
|
||||||
|
_ = ctx
|
||||||
|
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
|
endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
|
||||||
startDay := endDay.AddDate(0, 0, -(days - 1))
|
startDay := endDay.AddDate(0, 0, -(days - 1))
|
||||||
|
|
||||||
scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{
|
var (
|
||||||
"source": notice.TenderSourceTEDScraper,
|
scrapedDocuments map[string]int64
|
||||||
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
|
translatedNotices map[string]int64
|
||||||
}, "processing_metadata.scraped_at")
|
totalDocuments int64
|
||||||
if err != nil {
|
totalTranslated int64
|
||||||
return nil, fmt.Errorf("scraped ted per day: %w", err)
|
totalScrapedTED int64
|
||||||
|
scrapedScope scrapedTendersScope
|
||||||
|
translatedScope translatedNoticesScope
|
||||||
|
)
|
||||||
|
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
var mu sync.Mutex
|
||||||
|
|
||||||
|
recordFailure := func(section string, err error) {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
r.logger.Warn("Dashboard statistics section failed", map[string]interface{}{
|
||||||
|
"section": section,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix)
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
total, err := r.metricsCounter.Get(qctx, orm.TEDNoticeScrapedCounterKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("scraped documents per day: %w", err)
|
recordFailure("scraped_ted_total", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
totalScrapedTED = total
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
scope, err := r.cachedTranslatedNoticesScope(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
recordFailure("translated_scope", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
translatedScope = scope
|
||||||
|
if scope.fromMinIO {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay)
|
qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
counts, err := r.translatedNoticesPerDayFromMetrics(qctx, startDay, endDay)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("translated notices per day: %w", err)
|
recordFailure("translated_notices", err)
|
||||||
|
translatedNotices = map[string]int64{}
|
||||||
|
} else {
|
||||||
|
translatedNotices = counts
|
||||||
}
|
}
|
||||||
|
|
||||||
totalDocuments, err := r.totalScrapedDocuments(ctx)
|
total, err := r.metricsCounter.Get(qctx, orm.AITranslationSuccessCounterKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("total scraped documents: %w", err)
|
recordFailure("translated_total", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
totalTranslated = total
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
scope, err := r.cachedScrapedTendersScope(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
recordFailure("scraped_scope", err)
|
||||||
|
scrapedScope = mongoScrapedTendersScope()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
scrapedScope = scope
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
|
if translatedScope.fromMinIO {
|
||||||
|
translatedNotices = filterDailyCountsSince(translatedScope.dailyCounts, startUnix)
|
||||||
|
totalTranslated = translatedScope.totalCount
|
||||||
}
|
}
|
||||||
|
|
||||||
totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey)
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
counts, err := r.scrapedDocumentsPerDay(qctx, startUnix, scrapedScope)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("total translated tenders: %w", err)
|
recordFailure("scraped_documents", err)
|
||||||
|
scrapedDocuments = map[string]int64{}
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
scrapedDocuments = counts
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
qctx, cancel := context.WithTimeout(context.Background(), statisticsMongoQueryTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
total, err := r.totalScrapedTenders(qctx, scrapedScope)
|
||||||
|
if err != nil {
|
||||||
|
recordFailure("scraped_documents_total", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
totalDocuments = total
|
||||||
|
}()
|
||||||
|
|
||||||
|
wg.Wait()
|
||||||
|
|
||||||
return &StatisticsReportResponse{
|
return &StatisticsReportResponse{
|
||||||
Days: days,
|
Days: days,
|
||||||
GeneratedAt: now.Unix(),
|
GeneratedAt: now.Unix(),
|
||||||
Daily: StatisticsDailySeries{
|
Daily: StatisticsDailySeries{
|
||||||
ScrapedTED: fillTrendSeries(days, scrapedTED),
|
|
||||||
ScrapedDocuments: fillTrendSeries(days, scrapedDocuments),
|
ScrapedDocuments: fillTrendSeries(days, scrapedDocuments),
|
||||||
TranslatedNotices: fillTrendSeries(days, translatedNotices),
|
TranslatedNotices: fillTrendSeries(days, translatedNotices),
|
||||||
},
|
},
|
||||||
Totals: StatisticsLifetimeTotals{
|
Totals: StatisticsLifetimeTotals{
|
||||||
ScrapedDocuments: totalDocuments,
|
ScrapedDocuments: totalDocuments,
|
||||||
TranslatedTenders: totalTranslated,
|
TranslatedTenders: totalTranslated,
|
||||||
|
ScrapedTEDNotices: totalScrapedTED,
|
||||||
},
|
},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *repository) dailyCountByTimestamp(ctx context.Context, collection string, match bson.M, timestampField string) (map[string]int64, error) {
|
func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64, scope scrapedTendersScope) (map[string]int64, error) {
|
||||||
|
if scope.zero {
|
||||||
|
return map[string]int64{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if scope.dailyCounts != nil {
|
||||||
|
return filterDailyCountsSince(scope.dailyCounts, startUnix), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return r.scrapedDocumentsPerDayFromMongo(ctx, startUnix, scope.match)
|
||||||
|
}
|
||||||
|
|
||||||
|
func filterDailyCountsSince(counts map[string]int64, startUnix int64) map[string]int64 {
|
||||||
|
if len(counts) == 0 {
|
||||||
|
return map[string]int64{}
|
||||||
|
}
|
||||||
|
startDay := time.Unix(startUnix, 0).UTC().Format("2006-01-02")
|
||||||
|
filtered := make(map[string]int64)
|
||||||
|
for date, count := range counts {
|
||||||
|
if date >= startDay {
|
||||||
|
filtered[date] = count
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return filtered
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *repository) scrapedDocumentsPerDayFromMongo(ctx context.Context, startUnix int64, match bson.M) (map[string]int64, error) {
|
||||||
|
combinedMatch := bson.M{
|
||||||
|
"processing_metadata.documents_scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
|
||||||
|
}
|
||||||
|
for key, value := range match {
|
||||||
|
combinedMatch[key] = value
|
||||||
|
}
|
||||||
|
|
||||||
pipeline := mongodriver.Pipeline{
|
pipeline := mongodriver.Pipeline{
|
||||||
{{Key: "$match", Value: match}},
|
{{Key: "$match", Value: combinedMatch}},
|
||||||
{{Key: "$addFields", Value: bson.M{
|
{{Key: "$addFields", Value: bson.M{
|
||||||
"metric_ts": normalizeTimestampExpr(timestampField),
|
"metric_ts": normalizeTimestampExpr("processing_metadata.documents_scraped_at"),
|
||||||
}}},
|
|
||||||
{{Key: "$group", Value: bson.M{
|
|
||||||
"_id": bson.M{
|
|
||||||
"$dateToString": bson.M{
|
|
||||||
"format": "%Y-%m-%d",
|
|
||||||
"date": bson.M{"$toDate": bson.M{"$multiply": bson.A{"$metric_ts", 1000}}},
|
|
||||||
"timezone": "UTC",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
"count": bson.M{"$sum": 1},
|
|
||||||
}}},
|
|
||||||
}
|
|
||||||
|
|
||||||
cursor, err := r.mongoManager.GetCollection(collection).Aggregate(ctx, pipeline)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer cursor.Close(ctx)
|
|
||||||
|
|
||||||
return decodeDailyCounts(ctx, cursor)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64) (map[string]int64, error) {
|
|
||||||
pipeline := mongodriver.Pipeline{
|
|
||||||
{{Key: "$match", Value: bson.M{
|
|
||||||
"processing_metadata.documents_scraped": true,
|
|
||||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
|
||||||
}}},
|
|
||||||
{{Key: "$unwind", Value: "$scraped_documents"}},
|
|
||||||
{{Key: "$addFields", Value: bson.M{
|
|
||||||
"metric_ts": bson.M{
|
|
||||||
"$cond": bson.A{
|
|
||||||
bson.M{"$gt": bson.A{"$scraped_documents.scraped_at", 0}},
|
|
||||||
"$scraped_documents.scraped_at",
|
|
||||||
"$processing_metadata.documents_scraped_at",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}}},
|
|
||||||
{{Key: "$match", Value: bson.M{
|
|
||||||
"metric_ts": bson.M{"$gte": startUnix, "$gt": 0},
|
|
||||||
}}},
|
|
||||||
{{Key: "$addFields", Value: bson.M{
|
|
||||||
"metric_ts": normalizeTimestampExpr("metric_ts"),
|
|
||||||
}}},
|
}}},
|
||||||
{{Key: "$group", Value: bson.M{
|
{{Key: "$group", Value: bson.M{
|
||||||
"_id": bson.M{
|
"_id": bson.M{
|
||||||
@@ -132,41 +240,225 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
|
|||||||
return decodeDailyCounts(ctx, cursor)
|
return decodeDailyCounts(ctx, cursor)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *repository) translatedNoticesPerDay(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
|
func (r *repository) translatedNoticesPerDayFromMetrics(ctx context.Context, startDay, endDay time.Time) (map[string]int64, error) {
|
||||||
return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
|
return r.metricsCounter.GetDailyCounts(ctx, startDay, endDay)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *repository) totalScrapedDocuments(ctx context.Context) (int64, error) {
|
func (r *repository) cachedTranslatedNoticesScope(_ context.Context) (translatedNoticesScope, error) {
|
||||||
pipeline := mongodriver.Pipeline{
|
now := time.Now()
|
||||||
{{Key: "$match", Value: bson.M{
|
|
||||||
"processing_metadata.documents_scraped": true,
|
r.translatedScopeMu.Lock()
|
||||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
if r.translatedScopeCached && now.Before(r.translatedScopeExpiry) {
|
||||||
}}},
|
scope := r.translatedScope
|
||||||
{{Key: "$project", Value: bson.M{
|
r.translatedScopeMu.Unlock()
|
||||||
"doc_count": bson.M{"$size": "$scraped_documents"},
|
return scope, nil
|
||||||
}}},
|
}
|
||||||
{{Key: "$group", Value: bson.M{
|
r.translatedScopeMu.Unlock()
|
||||||
"_id": nil,
|
|
||||||
"total": bson.M{"$sum": "$doc_count"},
|
v, err, _ := r.translatedScopeGroup.Do("translated-scope", func() (interface{}, error) {
|
||||||
}}},
|
r.translatedScopeMu.Lock()
|
||||||
|
if r.translatedScopeCached && time.Now().Before(r.translatedScopeExpiry) {
|
||||||
|
scope := r.translatedScope
|
||||||
|
r.translatedScopeMu.Unlock()
|
||||||
|
return scope, nil
|
||||||
|
}
|
||||||
|
r.translatedScopeMu.Unlock()
|
||||||
|
|
||||||
|
resolveCtx, cancel := context.WithTimeout(context.Background(), scrapedScopeResolveTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
scope, resolveErr := r.resolveTranslatedNoticesScope(resolveCtx)
|
||||||
|
if resolveErr != nil {
|
||||||
|
if r.logger != nil {
|
||||||
|
r.logger.Warn("MinIO translation scope unavailable, using metrics counter fallback for dashboard statistics", map[string]interface{}{
|
||||||
|
"error": resolveErr.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return translatedNoticesScope{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
cursor, err := r.mongoManager.GetCollection(tenderCollectionName()).Aggregate(ctx, pipeline)
|
r.translatedScopeMu.Lock()
|
||||||
|
r.translatedScope = scope
|
||||||
|
r.translatedScopeExpiry = time.Now().Add(scrapedScopeCacheTTL)
|
||||||
|
r.translatedScopeCached = true
|
||||||
|
r.translatedScopeMu.Unlock()
|
||||||
|
|
||||||
|
return scope, nil
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return translatedNoticesScope{}, err
|
||||||
}
|
}
|
||||||
defer cursor.Close(ctx)
|
|
||||||
|
|
||||||
if !cursor.Next(ctx) {
|
return v.(translatedNoticesScope), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *repository) resolveTranslatedNoticesScope(ctx context.Context) (translatedNoticesScope, error) {
|
||||||
|
if r.procedureLister == nil {
|
||||||
|
return translatedNoticesScope{}, nil
|
||||||
|
}
|
||||||
|
scanner, ok := r.procedureLister.(translatedNoticesScanner)
|
||||||
|
if !ok {
|
||||||
|
return translatedNoticesScope{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
dailyCounts, total, err := scanner.ScanTranslatedNotices(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return translatedNoticesScope{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return translatedNoticesScope{
|
||||||
|
fromMinIO: true,
|
||||||
|
dailyCounts: dailyCounts,
|
||||||
|
totalCount: total,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *repository) totalScrapedTenders(ctx context.Context, scope scrapedTendersScope) (int64, error) {
|
||||||
|
if scope.zero {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
if scope.totalTenderCount > 0 {
|
||||||
var row bson.M
|
return scope.totalTenderCount, nil
|
||||||
if err := cursor.Decode(&row); err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return aggregateCount(row, "total"), nil
|
return r.ormRepo.Count(ctx, scope.match)
|
||||||
|
}
|
||||||
|
|
||||||
|
func mongoScrapedTendersScope() scrapedTendersScope {
|
||||||
|
return scrapedTendersScope{
|
||||||
|
match: bson.M{
|
||||||
|
"processing_metadata.documents_scraped": true,
|
||||||
|
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *repository) cachedScrapedTendersScope(_ context.Context) (scrapedTendersScope, error) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
r.scrapedScopeMu.Lock()
|
||||||
|
if r.scrapedScopeCached && now.Before(r.scrapedScopeExpiry) {
|
||||||
|
scope := r.scrapedScope
|
||||||
|
r.scrapedScopeMu.Unlock()
|
||||||
|
return scope, nil
|
||||||
|
}
|
||||||
|
r.scrapedScopeMu.Unlock()
|
||||||
|
|
||||||
|
v, err, _ := r.scrapedScopeGroup.Do("scraped-scope", func() (interface{}, error) {
|
||||||
|
r.scrapedScopeMu.Lock()
|
||||||
|
if r.scrapedScopeCached && time.Now().Before(r.scrapedScopeExpiry) {
|
||||||
|
scope := r.scrapedScope
|
||||||
|
r.scrapedScopeMu.Unlock()
|
||||||
|
return scope, nil
|
||||||
|
}
|
||||||
|
r.scrapedScopeMu.Unlock()
|
||||||
|
|
||||||
|
resolveCtx, cancel := context.WithTimeout(context.Background(), scrapedScopeResolveTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
scope, resolveErr := r.resolveScrapedTendersScope(resolveCtx)
|
||||||
|
if resolveErr != nil {
|
||||||
|
if r.logger != nil {
|
||||||
|
r.logger.Warn("MinIO scraped scope unavailable, using Mongo fallback for dashboard statistics", map[string]interface{}{
|
||||||
|
"error": resolveErr.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return mongoScrapedTendersScope(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
r.scrapedScopeMu.Lock()
|
||||||
|
r.scrapedScope = scope
|
||||||
|
r.scrapedScopeExpiry = time.Now().Add(scrapedScopeCacheTTL)
|
||||||
|
r.scrapedScopeCached = true
|
||||||
|
r.scrapedScopeMu.Unlock()
|
||||||
|
|
||||||
|
return scope, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return scrapedTendersScope{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return v.(scrapedTendersScope), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *repository) resolveScrapedTendersScope(ctx context.Context) (scrapedTendersScope, error) {
|
||||||
|
procedures, dailyCounts, err := r.listScrapedProcedures(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return scrapedTendersScope{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
folderIDs := folderIDsFromProcedures(procedures)
|
||||||
|
if len(folderIDs) > 0 {
|
||||||
|
totalTenders := int64(len(folderIDs))
|
||||||
|
if len(folderIDs) > maxScrapedFolderInClause {
|
||||||
|
if r.logger != nil {
|
||||||
|
r.logger.Warn("Too many MinIO procedure folders for dashboard statistics, using MinIO counts without Mongo $in filter", map[string]interface{}{
|
||||||
|
"folder_count": len(folderIDs),
|
||||||
|
"max_in_clause": maxScrapedFolderInClause,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return scrapedTendersScope{
|
||||||
|
fromMinIO: true,
|
||||||
|
dailyCounts: dailyCounts,
|
||||||
|
totalTenderCount: totalTenders,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
return scrapedTendersScope{
|
||||||
|
match: bson.M{"contract_folder_id": bson.M{"$in": folderIDs}},
|
||||||
|
fromMinIO: true,
|
||||||
|
dailyCounts: dailyCounts,
|
||||||
|
totalTenderCount: totalTenders,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
return mongoScrapedTendersScope(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *repository) listScrapedProcedures(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, map[string]int64, error) {
|
||||||
|
if r.procedureLister == nil {
|
||||||
|
return nil, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if scanner, ok := r.procedureLister.(scrapedDocumentsScanner); ok {
|
||||||
|
return scanner.ScanScrapedDocuments(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
procedures, err := r.procedureLister.ListProceduresWithDocuments(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return procedures, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func folderIDsFromProcedures(procedures []ai_summarizer.ProcedureDocumentsSummary) []string {
|
||||||
|
folderIDs := make([]string, 0, len(procedures))
|
||||||
|
seen := make(map[string]struct{}, len(procedures))
|
||||||
|
for _, proc := range procedures {
|
||||||
|
if proc.DocumentCount <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
id := strings.TrimSpace(proc.ContractFolderID)
|
||||||
|
if id == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[id]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[id] = struct{}{}
|
||||||
|
folderIDs = append(folderIDs, id)
|
||||||
|
}
|
||||||
|
return folderIDs
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *repository) scrapedDocumentsFolderIDs(ctx context.Context) ([]string, error) {
|
||||||
|
if r.procedureLister == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
procedures, _, err := r.listScrapedProcedures(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return folderIDsFromProcedures(procedures), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) {
|
func decodeDailyCounts(ctx context.Context, cursor *mongodriver.Cursor) (map[string]int64, error) {
|
||||||
|
|||||||
@@ -0,0 +1,310 @@
|
|||||||
|
package dashboard
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
"tm/pkg/logger"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
)
|
||||||
|
|
||||||
|
type noopTestLogger struct{}
|
||||||
|
|
||||||
|
func (noopTestLogger) Debug(string, map[string]interface{}) {}
|
||||||
|
func (noopTestLogger) Info(string, map[string]interface{}) {}
|
||||||
|
func (noopTestLogger) Warn(string, map[string]interface{}) {}
|
||||||
|
func (noopTestLogger) Error(string, map[string]interface{}) {}
|
||||||
|
func (noopTestLogger) Fatal(string, map[string]interface{}) {}
|
||||||
|
func (l noopTestLogger) WithFields(map[string]interface{}) logger.Logger {
|
||||||
|
return l
|
||||||
|
}
|
||||||
|
func (noopTestLogger) Sync() error { return nil }
|
||||||
|
|
||||||
|
type mockProcedureDocumentsLister struct {
|
||||||
|
procedures []ai_summarizer.ProcedureDocumentsSummary
|
||||||
|
dailyCounts map[string]int64
|
||||||
|
err error
|
||||||
|
|
||||||
|
translatedDailyCounts map[string]int64
|
||||||
|
translatedTotal int64
|
||||||
|
translatedErr error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockProcedureDocumentsLister) ListProceduresWithDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
|
||||||
|
if m.err != nil {
|
||||||
|
return nil, m.err
|
||||||
|
}
|
||||||
|
return m.procedures, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockProcedureDocumentsLister) ScanScrapedDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, map[string]int64, error) {
|
||||||
|
if m.err != nil {
|
||||||
|
return nil, nil, m.err
|
||||||
|
}
|
||||||
|
return m.procedures, m.dailyCounts, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockProcedureDocumentsLister) ScanTranslatedNotices(context.Context) (map[string]int64, int64, error) {
|
||||||
|
if m.translatedErr != nil {
|
||||||
|
return nil, 0, m.translatedErr
|
||||||
|
}
|
||||||
|
return m.translatedDailyCounts, m.translatedTotal, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScrapedDocumentsFolderIDsUsesMinIOProcedures(t *testing.T) {
|
||||||
|
repo := &repository{
|
||||||
|
procedureLister: &mockProcedureDocumentsLister{
|
||||||
|
procedures: []ai_summarizer.ProcedureDocumentsSummary{
|
||||||
|
{ContractFolderID: "PROC-1", DocumentCount: 3},
|
||||||
|
{ContractFolderID: "PROC-2", DocumentCount: 0},
|
||||||
|
{ContractFolderID: " PROC-3 ", DocumentCount: 1},
|
||||||
|
{ContractFolderID: "PROC-1", DocumentCount: 2},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := repo.scrapedDocumentsFolderIDs(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := []string{"PROC-1", "PROC-3"}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("expected %d folder IDs, got %d: %v", len(want), len(got), got)
|
||||||
|
}
|
||||||
|
for i, id := range want {
|
||||||
|
if got[i] != id {
|
||||||
|
t.Fatalf("expected folder ID %q at index %d, got %q", id, i, got[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScrapedDocumentsFolderIDsWithoutLister(t *testing.T) {
|
||||||
|
repo := &repository{}
|
||||||
|
|
||||||
|
got, err := repo.scrapedDocumentsFolderIDs(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if got != nil {
|
||||||
|
t.Fatalf("expected nil folder IDs without lister, got %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScrapedDocumentsFolderIDsPropagatesError(t *testing.T) {
|
||||||
|
repo := &repository{
|
||||||
|
procedureLister: &mockProcedureDocumentsLister{
|
||||||
|
err: errors.New("minio unavailable"),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := repo.scrapedDocumentsFolderIDs(context.Background())
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveScrapedTendersScopeUsesMinIOFolders(t *testing.T) {
|
||||||
|
repo := &repository{
|
||||||
|
procedureLister: &mockProcedureDocumentsLister{
|
||||||
|
procedures: []ai_summarizer.ProcedureDocumentsSummary{
|
||||||
|
{ContractFolderID: "PROC-1", DocumentCount: 2},
|
||||||
|
},
|
||||||
|
dailyCounts: map[string]int64{"2026-06-28": 2},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if scope.zero {
|
||||||
|
t.Fatal("expected non-zero scope")
|
||||||
|
}
|
||||||
|
if !scope.fromMinIO {
|
||||||
|
t.Fatal("expected MinIO-backed scope")
|
||||||
|
}
|
||||||
|
if scope.totalTenderCount != 1 {
|
||||||
|
t.Fatalf("expected total tender count 1, got %d", scope.totalTenderCount)
|
||||||
|
}
|
||||||
|
if scope.dailyCounts == nil {
|
||||||
|
t.Fatal("expected MinIO daily counts")
|
||||||
|
}
|
||||||
|
in, ok := scope.match["contract_folder_id"].(bson.M)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected contract_folder_id filter, got %#v", scope.match)
|
||||||
|
}
|
||||||
|
ids, ok := in["$in"].([]string)
|
||||||
|
if !ok || len(ids) != 1 || ids[0] != "PROC-1" {
|
||||||
|
t.Fatalf("unexpected folder IDs: %#v", in["$in"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveScrapedTendersScopeKeepsMinIOCountsWhenTooManyFolders(t *testing.T) {
|
||||||
|
procedures := make([]ai_summarizer.ProcedureDocumentsSummary, maxScrapedFolderInClause+1)
|
||||||
|
for i := range procedures {
|
||||||
|
procedures[i] = ai_summarizer.ProcedureDocumentsSummary{
|
||||||
|
ContractFolderID: fmt.Sprintf("PROC-%d", i),
|
||||||
|
DocumentCount: 1,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
repo := &repository{
|
||||||
|
logger: noopTestLogger{},
|
||||||
|
procedureLister: &mockProcedureDocumentsLister{
|
||||||
|
procedures: procedures,
|
||||||
|
dailyCounts: map[string]int64{"2026-06-28": 3},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if scope.zero {
|
||||||
|
t.Fatal("expected non-zero scope")
|
||||||
|
}
|
||||||
|
if !scope.fromMinIO {
|
||||||
|
t.Fatal("expected MinIO-backed scope")
|
||||||
|
}
|
||||||
|
if scope.totalTenderCount != int64(len(procedures)) {
|
||||||
|
t.Fatalf("expected total tender count %d, got %d", len(procedures), scope.totalTenderCount)
|
||||||
|
}
|
||||||
|
if scope.dailyCounts == nil {
|
||||||
|
t.Fatal("expected MinIO daily counts to be preserved")
|
||||||
|
}
|
||||||
|
if scope.match != nil {
|
||||||
|
t.Fatalf("expected no Mongo $in filter for large MinIO scope, got %#v", scope.match)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveScrapedTendersScopeMongoFallbackWhenMinIOEmpty(t *testing.T) {
|
||||||
|
repo := &repository{
|
||||||
|
procedureLister: &mockProcedureDocumentsLister{},
|
||||||
|
}
|
||||||
|
|
||||||
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if scope.zero {
|
||||||
|
t.Fatal("expected mongo fallback scope, not zero scope")
|
||||||
|
}
|
||||||
|
if scope.match["processing_metadata.documents_scraped"] != true {
|
||||||
|
t.Fatalf("expected mongo fallback filter, got %#v", scope.match)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveScrapedTendersScopeMongoFallbackWithoutLister(t *testing.T) {
|
||||||
|
repo := &repository{}
|
||||||
|
|
||||||
|
scope, err := repo.resolveScrapedTendersScope(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if scope.zero {
|
||||||
|
t.Fatal("expected mongo fallback scope")
|
||||||
|
}
|
||||||
|
if scope.match["processing_metadata.documents_scraped"] != true {
|
||||||
|
t.Fatalf("expected mongo fallback filter, got %#v", scope.match)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFilterDailyCountsSince(t *testing.T) {
|
||||||
|
counts := map[string]int64{
|
||||||
|
"2026-06-01": 2,
|
||||||
|
"2026-06-15": 5,
|
||||||
|
"2026-06-28": 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
|
||||||
|
got := filterDailyCountsSince(counts, startUnix)
|
||||||
|
|
||||||
|
want := map[string]int64{
|
||||||
|
"2026-06-15": 5,
|
||||||
|
"2026-06-28": 1,
|
||||||
|
}
|
||||||
|
if len(got) != len(want) {
|
||||||
|
t.Fatalf("expected %d dates, got %d: %v", len(want), len(got), got)
|
||||||
|
}
|
||||||
|
for date, count := range want {
|
||||||
|
if got[date] != count {
|
||||||
|
t.Fatalf("expected %d on %s, got %d", count, date, got[date])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScrapedDocumentsPerDayUsesMinIODailyCounts(t *testing.T) {
|
||||||
|
repo := &repository{}
|
||||||
|
scope := scrapedTendersScope{
|
||||||
|
dailyCounts: map[string]int64{
|
||||||
|
"2026-06-01": 2,
|
||||||
|
"2026-06-28": 3,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
startUnix := time.Date(2026, 6, 15, 0, 0, 0, 0, time.UTC).Unix()
|
||||||
|
|
||||||
|
got, err := repo.scrapedDocumentsPerDay(context.Background(), startUnix, scope)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if got["2026-06-01"] != 0 {
|
||||||
|
t.Fatalf("expected old date filtered out, got %v", got)
|
||||||
|
}
|
||||||
|
if got["2026-06-28"] != 3 {
|
||||||
|
t.Fatalf("expected 3 on 2026-06-28, got %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMongoScrapedTendersScopeMatchesStatisticsFilter(t *testing.T) {
|
||||||
|
scope := mongoScrapedTendersScope()
|
||||||
|
if scope.zero {
|
||||||
|
t.Fatal("expected non-zero mongo statistics scope")
|
||||||
|
}
|
||||||
|
if scope.match["processing_metadata.documents_scraped"] != true {
|
||||||
|
t.Fatalf("expected documents_scraped filter, got %#v", scope.match)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveTranslatedNoticesScopeUsesMinIOCounts(t *testing.T) {
|
||||||
|
repo := &repository{
|
||||||
|
procedureLister: &mockProcedureDocumentsLister{
|
||||||
|
translatedDailyCounts: map[string]int64{
|
||||||
|
"2026-07-01": 4,
|
||||||
|
"2026-07-09": 2,
|
||||||
|
},
|
||||||
|
translatedTotal: 6,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
scope, err := repo.resolveTranslatedNoticesScope(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if !scope.fromMinIO {
|
||||||
|
t.Fatal("expected MinIO-backed translation scope")
|
||||||
|
}
|
||||||
|
if scope.totalCount != 6 {
|
||||||
|
t.Fatalf("expected total 6, got %d", scope.totalCount)
|
||||||
|
}
|
||||||
|
if scope.dailyCounts["2026-07-09"] != 2 {
|
||||||
|
t.Fatalf("expected 2 on 2026-07-09, got %d", scope.dailyCounts["2026-07-09"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolveTranslatedNoticesScopeWithoutLister(t *testing.T) {
|
||||||
|
repo := &repository{}
|
||||||
|
|
||||||
|
scope, err := repo.resolveTranslatedNoticesScope(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
if scope.fromMinIO {
|
||||||
|
t.Fatal("expected metrics fallback without lister")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,230 @@
|
|||||||
|
package dashboard
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
"tm/pkg/logger"
|
||||||
|
"tm/pkg/redis"
|
||||||
|
|
||||||
|
goredis "github.com/redis/go-redis/v9"
|
||||||
|
"golang.org/x/sync/singleflight"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// widgetCacheTTL is the fresh window before a background reload is triggered.
|
||||||
|
widgetCacheTTL = 60 * time.Second
|
||||||
|
// widgetStaleGraceTTL is how long stale entries may be served while reloading.
|
||||||
|
// Dashboard widgets have no write-side invalidation; expect up to this lag after
|
||||||
|
// data changes (e.g. a new tender may not appear in cached widgets immediately).
|
||||||
|
widgetStaleGraceTTL = 5 * time.Minute
|
||||||
|
widgetReloadTimeout = 2 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
type widgetLoader[T any] func(ctx context.Context) (T, error)
|
||||||
|
|
||||||
|
type widgetCache[T any] struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
entries map[string]cacheEntry[T]
|
||||||
|
group singleflight.Group
|
||||||
|
redis redis.Client
|
||||||
|
logger logger.Logger
|
||||||
|
keyPrefix string
|
||||||
|
ttl time.Duration
|
||||||
|
staleGrace time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
func newWidgetCache[T any](redisClient redis.Client, log logger.Logger, keyPrefix string) *widgetCache[T] {
|
||||||
|
return &widgetCache[T]{
|
||||||
|
entries: make(map[string]cacheEntry[T]),
|
||||||
|
redis: redisClient,
|
||||||
|
logger: log,
|
||||||
|
keyPrefix: keyPrefix,
|
||||||
|
ttl: widgetCacheTTL,
|
||||||
|
staleGrace: widgetStaleGraceTTL,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *widgetCache[T]) Get(ctx context.Context, key string, load widgetLoader[T]) (T, error) {
|
||||||
|
var zero T
|
||||||
|
|
||||||
|
// Returned values are shared across concurrent callers via singleflight and stale
|
||||||
|
// hits. Callers must treat them as read-only through JSON serialization.
|
||||||
|
if value, ok := c.fresh(key); ok {
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
if value, ok := c.stale(key); ok {
|
||||||
|
go c.Reload(key, load)
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
if value, ok := c.fromRedis(ctx, key); ok {
|
||||||
|
c.store(key, value)
|
||||||
|
go c.Reload(key, load)
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
out, err, _ := c.group.Do(key, func() (interface{}, error) {
|
||||||
|
if value, ok := c.fresh(key); ok {
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := load(context.WithoutCancel(ctx))
|
||||||
|
if err != nil {
|
||||||
|
return zero, err
|
||||||
|
}
|
||||||
|
|
||||||
|
c.store(key, result)
|
||||||
|
c.storeRedis(context.Background(), key, result)
|
||||||
|
return result, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return zero, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return out.(T), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *widgetCache[T]) Reload(key string, load widgetLoader[T]) {
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), widgetReloadTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
_, _, _ = c.group.Do("reload:"+key, func() (interface{}, error) {
|
||||||
|
result, err := load(ctx)
|
||||||
|
if err != nil {
|
||||||
|
if c.logger != nil {
|
||||||
|
c.logger.Warn("Failed to refresh dashboard widget cache", map[string]interface{}{
|
||||||
|
"widget": c.keyPrefix,
|
||||||
|
"key": key,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
c.store(key, result)
|
||||||
|
c.storeRedis(context.Background(), key, result)
|
||||||
|
return result, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *widgetCache[T]) WarmFromRedis(ctx context.Context, key string) bool {
|
||||||
|
value, ok := c.fromRedis(ctx, key)
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
c.store(key, value)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *widgetCache[T]) fresh(key string) (T, bool) {
|
||||||
|
var zero T
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
entry, ok := c.entries[key]
|
||||||
|
if !ok || now.After(entry.expiresAt) {
|
||||||
|
return zero, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry.value, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *widgetCache[T]) stale(key string) (T, bool) {
|
||||||
|
var zero T
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
entry, ok := c.entries[key]
|
||||||
|
if !ok || now.After(entry.staleUntil) {
|
||||||
|
if ok {
|
||||||
|
delete(c.entries, key)
|
||||||
|
}
|
||||||
|
return zero, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return entry.value, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *widgetCache[T]) store(key string, value T) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
c.entries[key] = cacheEntry[T]{
|
||||||
|
expiresAt: now.Add(c.ttl),
|
||||||
|
staleUntil: now.Add(c.ttl + c.staleGrace),
|
||||||
|
value: value,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *widgetCache[T]) redisKey(key string) string {
|
||||||
|
return fmt.Sprintf("dashboard:%s:%s", c.keyPrefix, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *widgetCache[T]) fromRedis(ctx context.Context, key string) (T, bool) {
|
||||||
|
var zero T
|
||||||
|
if c.redis == nil {
|
||||||
|
return zero, false
|
||||||
|
}
|
||||||
|
|
||||||
|
raw, err := c.redis.Get(ctx, c.redisKey(key))
|
||||||
|
if err != nil {
|
||||||
|
if err != goredis.Nil && c.logger != nil {
|
||||||
|
c.logger.Warn("Failed to read dashboard widget cache from Redis", map[string]interface{}{
|
||||||
|
"widget": c.keyPrefix,
|
||||||
|
"key": key,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return zero, false
|
||||||
|
}
|
||||||
|
|
||||||
|
var value T
|
||||||
|
if err := json.Unmarshal([]byte(raw), &value); err != nil {
|
||||||
|
if c.logger != nil {
|
||||||
|
c.logger.Warn("Failed to decode dashboard widget cache from Redis", map[string]interface{}{
|
||||||
|
"widget": c.keyPrefix,
|
||||||
|
"key": key,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
_ = c.redis.Del(ctx, c.redisKey(key))
|
||||||
|
return zero, false
|
||||||
|
}
|
||||||
|
|
||||||
|
return value, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *widgetCache[T]) storeRedis(ctx context.Context, key string, value T) {
|
||||||
|
if c.redis == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
encoded, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
if c.logger != nil {
|
||||||
|
c.logger.Warn("Failed to encode dashboard widget cache for Redis", map[string]interface{}{
|
||||||
|
"widget": c.keyPrefix,
|
||||||
|
"key": key,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ttl := c.ttl + c.staleGrace
|
||||||
|
if err := c.redis.Set(ctx, c.redisKey(key), string(encoded), ttl); err != nil && c.logger != nil {
|
||||||
|
c.logger.Warn("Failed to store dashboard widget cache in Redis", map[string]interface{}{
|
||||||
|
"widget": c.keyPrefix,
|
||||||
|
"key": key,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
package document_scraper
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrInvalidDateRange is returned when created_at_from is after created_at_to.
|
||||||
|
ErrInvalidDateRange = errors.New("invalid date range: created_at_from must be before or equal to created_at_to")
|
||||||
|
// ErrScrapePortalsUnavailable is returned when the scrape portals provider is not configured.
|
||||||
|
ErrScrapePortalsUnavailable = errors.New("scrape portals provider is not configured")
|
||||||
|
)
|
||||||
@@ -2,8 +2,24 @@ package document_scraper
|
|||||||
|
|
||||||
// DocumentScraperListRequest represents the request to list tenders pending document scraping
|
// DocumentScraperListRequest represents the request to list tenders pending document scraping
|
||||||
type DocumentScraperListRequest struct {
|
type DocumentScraperListRequest struct {
|
||||||
Limit int `query:"limit" valid:"optional,range(1|100)" default:"20"`
|
Limit int `query:"limit" valid:"optional,range(1|100)"`
|
||||||
Offset int `query:"offset" valid:"optional,range(0|1000000)" default:"0"`
|
Offset int `query:"offset" valid:"optional,range(0|1000000)"`
|
||||||
|
CreatedAtFrom *int64 `query:"created_at_from" valid:"optional"`
|
||||||
|
CreatedAtTo *int64 `query:"created_at_to" valid:"optional"`
|
||||||
|
IncludeExpired bool `query:"include_expired" valid:"optional"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalize applies default pagination bounds.
|
||||||
|
func (r *DocumentScraperListRequest) Normalize() {
|
||||||
|
if r.Limit <= 0 {
|
||||||
|
r.Limit = 20
|
||||||
|
}
|
||||||
|
if r.Limit > 100 {
|
||||||
|
r.Limit = 100
|
||||||
|
}
|
||||||
|
if r.Offset < 0 {
|
||||||
|
r.Offset = 0
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// DocumentScraperGetRequest represents the request to get a specific tender by notice publication ID
|
// DocumentScraperGetRequest represents the request to get a specific tender by notice publication ID
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
package document_scraper
|
package document_scraper
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strconv"
|
"errors"
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
"tm/pkg/response"
|
"tm/pkg/response"
|
||||||
|
|
||||||
@@ -24,36 +24,35 @@ func NewHandler(service Service, logger logger.Logger) *Handler {
|
|||||||
|
|
||||||
// ListPendingTenders retrieves tenders that need document scraping
|
// ListPendingTenders retrieves tenders that need document scraping
|
||||||
// @Summary List pending tenders for document scraping
|
// @Summary List pending tenders for document scraping
|
||||||
// @Description Retrieve tenders that have not yet been scraped for documents. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description.
|
// @Description Retrieve tenders that have not yet been scraped for documents and whose document URL matches a portal supported by the AI scraping service. Each item includes contract_folder_id, notice_publication_id, document_url, title, and description. Optionally filter by created_at range (Unix timestamps in seconds). By default only tenders with an active publication submission window are returned (submission_deadline, or 48 working hours after publication_date); set include_expired=true to include expired tenders.
|
||||||
// @Tags Document-Scraper
|
// @Tags Document-Scraper
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
||||||
// @Param offset query int false "Number of items to skip (default: 0)"
|
// @Param offset query int false "Number of items to skip (default: 0)"
|
||||||
|
// @Param created_at_from query int64 false "Filter by created_at lower bound (Unix timestamp, seconds)"
|
||||||
|
// @Param created_at_to query int64 false "Filter by created_at upper bound (Unix timestamp, seconds)"
|
||||||
|
// @Param include_expired query bool false "When true, include tenders whose deadline has passed (default: false)"
|
||||||
// @Success 200 {object} response.APIResponse{data=DocumentScraperListResponse} "List of pending tenders"
|
// @Success 200 {object} response.APIResponse{data=DocumentScraperListResponse} "List of pending tenders"
|
||||||
|
// @Failure 400 {object} response.APIResponse "Invalid query parameters"
|
||||||
// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
|
// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
|
||||||
|
// @Failure 503 {object} response.APIResponse "Scrape portals service is not configured"
|
||||||
// @Failure 500 {object} response.APIResponse "Internal server error"
|
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||||
// @Security ApiKeyAuth
|
// @Security ApiKeyAuth
|
||||||
// @Router /api/v1/scraper/tenders [get]
|
// @Router /api/v1/scraper/tenders [get]
|
||||||
func (h *Handler) ListPendingTenders(c echo.Context) error {
|
func (h *Handler) ListPendingTenders(c echo.Context) error {
|
||||||
// Parse pagination parameters
|
form, err := response.Parse[DocumentScraperListRequest](c)
|
||||||
limit, err := strconv.Atoi(c.QueryParam("limit"))
|
|
||||||
if err != nil || limit <= 0 {
|
|
||||||
limit = 20
|
|
||||||
}
|
|
||||||
if limit > 100 {
|
|
||||||
limit = 100
|
|
||||||
}
|
|
||||||
|
|
||||||
offset, err := strconv.Atoi(c.QueryParam("offset"))
|
|
||||||
if err != nil || offset < 0 {
|
|
||||||
offset = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
tenders, meta, err := h.service.ListPendingTenders(c.Request().Context(), limit, offset)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
h.logger.Error("Failed to list pending tenders", map[string]interface{}{
|
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||||
"error": err.Error(),
|
}
|
||||||
})
|
|
||||||
|
tenders, meta, err := h.service.ListPendingTenders(c.Request().Context(), *form)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrInvalidDateRange) {
|
||||||
|
return response.BadRequest(c, "Invalid date range", err.Error())
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrScrapePortalsUnavailable) {
|
||||||
|
return response.ServiceUnavailable(c, "Scrape portals service is not configured")
|
||||||
|
}
|
||||||
return response.InternalServerError(c, "Failed to list pending tenders")
|
return response.InternalServerError(c, "Failed to list pending tenders")
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,7 +61,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
|
|||||||
|
|
||||||
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
|
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
|
||||||
// @Summary Get tender by notice ID for document scraping
|
// @Summary Get tender by notice ID for document scraping
|
||||||
// @Description Retrieve a specific tender by its notice publication ID (contract_folder_id, notice_publication_id, document_url, title, description).
|
// @Description Retrieve a specific tender whose publication submission window is still open and whose document URL matches a supported scrape portal.
|
||||||
// @Tags Document-Scraper
|
// @Tags Document-Scraper
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param notice_id path string true "Notice Publication ID"
|
// @Param notice_id path string true "Notice Publication ID"
|
||||||
@@ -70,6 +69,7 @@ func (h *Handler) ListPendingTenders(c echo.Context) error {
|
|||||||
// @Failure 400 {object} response.APIResponse "Invalid notice ID"
|
// @Failure 400 {object} response.APIResponse "Invalid notice ID"
|
||||||
// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
|
// @Failure 401 {object} response.APIResponse "Invalid or missing API key"
|
||||||
// @Failure 404 {object} response.APIResponse "Tender not found"
|
// @Failure 404 {object} response.APIResponse "Tender not found"
|
||||||
|
// @Failure 503 {object} response.APIResponse "Scrape portals service is not configured"
|
||||||
// @Failure 500 {object} response.APIResponse "Internal server error"
|
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||||
// @Security ApiKeyAuth
|
// @Security ApiKeyAuth
|
||||||
// @Router /api/v1/scraper/tenders/{notice_id} [get]
|
// @Router /api/v1/scraper/tenders/{notice_id} [get]
|
||||||
@@ -81,13 +81,12 @@ func (h *Handler) GetTenderByNoticeID(c echo.Context) error {
|
|||||||
|
|
||||||
tender, err := h.service.GetTenderByNoticeID(c.Request().Context(), noticeID)
|
tender, err := h.service.GetTenderByNoticeID(c.Request().Context(), noticeID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrScrapePortalsUnavailable) {
|
||||||
|
return response.ServiceUnavailable(c, "Scrape portals service is not configured")
|
||||||
|
}
|
||||||
if err.Error() == "tender not found" {
|
if err.Error() == "tender not found" {
|
||||||
return response.NotFound(c, "Tender not found")
|
return response.NotFound(c, "Tender not found")
|
||||||
}
|
}
|
||||||
h.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
|
|
||||||
"notice_id": noticeID,
|
|
||||||
"error": err.Error(),
|
|
||||||
})
|
|
||||||
return response.InternalServerError(c, "Failed to get tender")
|
return response.InternalServerError(c, "Failed to get tender")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
package document_scraper
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"tm/internal/tender"
|
||||||
|
)
|
||||||
|
|
||||||
|
// documentURL returns the URL used for portal matching and the scraper API payload.
|
||||||
|
func documentURL(t *tender.Tender) string {
|
||||||
|
if t == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if u := strings.TrimSpace(t.DocumentURI); u != "" {
|
||||||
|
return u
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(t.TenderURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// matchesScrapePortals reports whether url contains any supported portal identifier.
|
||||||
|
func matchesScrapePortals(url string, portals []string) bool {
|
||||||
|
if url == "" || len(portals) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
urlLower := strings.ToLower(url)
|
||||||
|
for _, portal := range portals {
|
||||||
|
p := strings.TrimSpace(portal)
|
||||||
|
if p != "" && strings.Contains(urlLower, strings.ToLower(p)) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
@@ -3,55 +3,113 @@ package document_scraper
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"tm/internal/tender"
|
"tm/internal/tender"
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
"tm/pkg/response"
|
"tm/pkg/response"
|
||||||
)
|
)
|
||||||
|
|
||||||
var documentScraperCountryCodes = []string{"SWE", "FIN", "NOR"}
|
// ScrapePortalsProvider fetches supported document scraping portal identifiers.
|
||||||
|
type ScrapePortalsProvider interface {
|
||||||
func isDocumentScraperCountry(countryCode string) bool {
|
GetScrapePortals(ctx context.Context) (*ai_summarizer.ScrapePortalsResponse, error)
|
||||||
for _, code := range documentScraperCountryCodes {
|
|
||||||
if countryCode == code {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Service defines business logic for the document scraper API
|
// Service defines business logic for the document scraper API
|
||||||
type Service interface {
|
type Service interface {
|
||||||
// ListPendingTenders retrieves tenders that need document scraping
|
// ListPendingTenders retrieves tenders that need document scraping
|
||||||
ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error)
|
ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error)
|
||||||
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
|
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID
|
||||||
GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error)
|
GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type service struct {
|
type service struct {
|
||||||
tenderRepo tender.TenderRepository
|
tenderRepo tender.TenderRepository
|
||||||
|
portalsProvider ScrapePortalsProvider
|
||||||
logger logger.Logger
|
logger logger.Logger
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewService creates a new document scraper service
|
// NewService creates a new document scraper service
|
||||||
func NewService(tenderRepo tender.TenderRepository, logger logger.Logger) Service {
|
func NewService(tenderRepo tender.TenderRepository, portalsProvider ScrapePortalsProvider, logger logger.Logger) Service {
|
||||||
return &service{
|
return &service{
|
||||||
tenderRepo: tenderRepo,
|
tenderRepo: tenderRepo,
|
||||||
|
portalsProvider: portalsProvider,
|
||||||
logger: logger,
|
logger: logger,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *service) fetchScrapePortals(ctx context.Context) ([]string, error) {
|
||||||
|
if s.portalsProvider == nil {
|
||||||
|
s.logger.Warn("Scrape portals provider not configured", map[string]interface{}{})
|
||||||
|
return nil, ErrScrapePortalsUnavailable
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := s.portalsProvider.GetScrapePortals(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to fetch scrape portals", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("fetch scrape portals: %w", err)
|
||||||
|
}
|
||||||
|
if resp == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
portals := make([]string, 0, len(*resp))
|
||||||
|
for _, portal := range *resp {
|
||||||
|
if p := strings.TrimSpace(portal); p != "" {
|
||||||
|
portals = append(portals, p)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return portals, nil
|
||||||
|
}
|
||||||
|
|
||||||
// ListPendingTenders retrieves tenders that have not yet been scraped for documents.
|
// ListPendingTenders retrieves tenders that have not yet been scraped for documents.
|
||||||
// Only returns tenders from Sweden, Finland, and Norway with active deadlines (deadline not yet reached).
|
// Only returns tenders whose document URL matches a portal supported by the AI service.
|
||||||
func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*DocumentScraperListResponse, *response.Meta, error) {
|
// By default only tenders with an active publication submission window are included; set IncludeExpired to include expired tenders.
|
||||||
|
func (s *service) ListPendingTenders(ctx context.Context, req DocumentScraperListRequest) (*DocumentScraperListResponse, *response.Meta, error) {
|
||||||
|
req.Normalize()
|
||||||
|
|
||||||
|
if req.CreatedAtFrom != nil && req.CreatedAtTo != nil && *req.CreatedAtFrom > *req.CreatedAtTo {
|
||||||
|
return nil, nil, ErrInvalidDateRange
|
||||||
|
}
|
||||||
|
|
||||||
|
portals, err := s.fetchScrapePortals(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
if len(portals) == 0 {
|
||||||
|
s.logger.Info("No scrape portals configured; returning empty pending tender list", map[string]interface{}{})
|
||||||
|
meta := &response.Meta{Total: 0, Limit: req.Limit, Offset: req.Offset, HasMore: false}
|
||||||
|
if meta.Limit > 0 {
|
||||||
|
meta.Page = (req.Offset / meta.Limit) + 1
|
||||||
|
}
|
||||||
|
return &DocumentScraperListResponse{Tenders: []DocumentScraperTenderResponse{}}, meta, nil
|
||||||
|
}
|
||||||
|
|
||||||
s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{
|
s.logger.Info("Listing pending tenders for document scraper", map[string]interface{}{
|
||||||
"limit": limit,
|
"limit": req.Limit,
|
||||||
"offset": offset,
|
"offset": req.Offset,
|
||||||
|
"created_at_from": req.CreatedAtFrom,
|
||||||
|
"created_at_to": req.CreatedAtTo,
|
||||||
|
"include_expired": req.IncludeExpired,
|
||||||
|
"portal_count": len(portals),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
filter := tender.PendingDocumentScrapeFilter{
|
||||||
|
Portals: portals,
|
||||||
|
CreatedAtFrom: req.CreatedAtFrom,
|
||||||
|
CreatedAtTo: req.CreatedAtTo,
|
||||||
|
Limit: req.Limit,
|
||||||
|
Skip: req.Offset,
|
||||||
|
}
|
||||||
|
if !req.IncludeExpired {
|
||||||
now := time.Now().Unix()
|
now := time.Now().Unix()
|
||||||
|
filter.MinWindowEnd = &now
|
||||||
|
}
|
||||||
|
|
||||||
tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, documentScraperCountryCodes, now, limit, offset)
|
tenders, hasMore, err := s.tenderRepo.GetPendingDocumentScrapeTenders(ctx, filter)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to list pending tenders", map[string]interface{}{
|
s.logger.Error("Failed to list pending tenders", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -59,6 +117,11 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D
|
|||||||
return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
|
return nil, nil, fmt.Errorf("failed to list pending tenders: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !req.IncludeExpired {
|
||||||
|
now := time.Now().Unix()
|
||||||
|
tenders = filterActivePublicationWindow(tenders, now)
|
||||||
|
}
|
||||||
|
|
||||||
result := &DocumentScraperListResponse{
|
result := &DocumentScraperListResponse{
|
||||||
Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
|
Tenders: make([]DocumentScraperTenderResponse, len(tenders)),
|
||||||
}
|
}
|
||||||
@@ -69,12 +132,12 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D
|
|||||||
|
|
||||||
meta := &response.Meta{
|
meta := &response.Meta{
|
||||||
Total: len(tenders),
|
Total: len(tenders),
|
||||||
Limit: limit,
|
Limit: req.Limit,
|
||||||
Offset: offset,
|
Offset: req.Offset,
|
||||||
HasMore: hasMore,
|
HasMore: hasMore,
|
||||||
}
|
}
|
||||||
if meta.Limit > 0 {
|
if meta.Limit > 0 {
|
||||||
meta.Page = (offset / meta.Limit) + 1
|
meta.Page = (req.Offset / meta.Limit) + 1
|
||||||
}
|
}
|
||||||
|
|
||||||
s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
|
s.logger.Info("Pending tenders retrieved successfully", map[string]interface{}{
|
||||||
@@ -86,12 +149,17 @@ func (s *service) ListPendingTenders(ctx context.Context, limit, offset int) (*D
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID.
|
// GetTenderByNoticeID retrieves a specific tender by its notice publication ID.
|
||||||
// Only returns tenders from Sweden, Finland, and Norway with active deadlines (deadline not yet reached).
|
// Only returns tenders with an active publication submission window whose document URL matches a supported scrape portal.
|
||||||
func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
|
func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*DocumentScraperTenderResponse, error) {
|
||||||
s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
|
s.logger.Info("Getting tender by notice ID for document scraper", map[string]interface{}{
|
||||||
"notice_id": noticeID,
|
"notice_id": noticeID,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
portals, err := s.fetchScrapePortals(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID)
|
t, err := s.tenderRepo.GetByNoticePublicationID(ctx, noticeID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
|
s.logger.Error("Failed to get tender by notice ID", map[string]interface{}{
|
||||||
@@ -106,11 +174,12 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do
|
|||||||
}
|
}
|
||||||
|
|
||||||
now := time.Now().Unix()
|
now := time.Now().Unix()
|
||||||
if !isDocumentScraperCountry(t.CountryCode) || t.TenderDeadline <= now {
|
if !t.HasActivePublicationWindow(now) || !matchesScrapePortals(documentURL(t), portals) {
|
||||||
s.logger.Info("Tender filtered out: unsupported country or deadline passed", map[string]interface{}{
|
s.logger.Info("Tender filtered out: unsupported portal or publication window closed", map[string]interface{}{
|
||||||
"notice_id": noticeID,
|
"notice_id": noticeID,
|
||||||
"country_code": t.CountryCode,
|
"publication_date": t.PublicationDate,
|
||||||
"tender_deadline": t.TenderDeadline,
|
"submission_deadline": t.SubmissionDeadline,
|
||||||
|
"publication_deadline": t.PublicationSubmissionDeadline(),
|
||||||
"now": now,
|
"now": now,
|
||||||
})
|
})
|
||||||
return nil, fmt.Errorf("tender not found")
|
return nil, fmt.Errorf("tender not found")
|
||||||
@@ -125,3 +194,16 @@ func (s *service) GetTenderByNoticeID(ctx context.Context, noticeID string) (*Do
|
|||||||
|
|
||||||
return result, nil
|
return result, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func filterActivePublicationWindow(tenders []tender.Tender, now int64) []tender.Tender {
|
||||||
|
if len(tenders) == 0 {
|
||||||
|
return tenders
|
||||||
|
}
|
||||||
|
active := make([]tender.Tender, 0, len(tenders))
|
||||||
|
for i := range tenders {
|
||||||
|
if tenders[i].HasActivePublicationWindow(now) {
|
||||||
|
active = append(active, tenders[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return active
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,10 +1,6 @@
|
|||||||
package document_scraper
|
package document_scraper
|
||||||
|
|
||||||
import (
|
import "tm/internal/tender"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"tm/internal/tender"
|
|
||||||
)
|
|
||||||
|
|
||||||
// BuildDocumentScraperTenderResponse maps a stored tender to the scraper API contract.
|
// BuildDocumentScraperTenderResponse maps a stored tender to the scraper API contract.
|
||||||
func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTenderResponse {
|
func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTenderResponse {
|
||||||
@@ -12,15 +8,10 @@ func BuildDocumentScraperTenderResponse(t *tender.Tender) *DocumentScraperTender
|
|||||||
return &DocumentScraperTenderResponse{}
|
return &DocumentScraperTenderResponse{}
|
||||||
}
|
}
|
||||||
|
|
||||||
documentURL := strings.TrimSpace(t.DocumentURI)
|
|
||||||
if documentURL == "" {
|
|
||||||
documentURL = strings.TrimSpace(t.TenderURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &DocumentScraperTenderResponse{
|
return &DocumentScraperTenderResponse{
|
||||||
ContractFolderID: t.ContractFolderID,
|
ContractFolderID: t.ContractFolderID,
|
||||||
NoticePublicationID: t.NoticePublicationID,
|
NoticePublicationID: t.NoticePublicationID,
|
||||||
DocumentURL: documentURL,
|
DocumentURL: documentURL(t),
|
||||||
Title: t.Title,
|
Title: t.Title,
|
||||||
Description: t.Description,
|
Description: t.Description,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
package inquiry
|
package inquiry
|
||||||
|
|
||||||
import "errors"
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
// Inquiry-specific errors
|
// Inquiry-specific errors
|
||||||
var (
|
var (
|
||||||
@@ -25,6 +29,34 @@ func (e *DuplicateInquiryError) Error() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewDuplicateInquiryError creates a new duplicate inquiry error
|
// NewDuplicateInquiryError creates a new duplicate inquiry error
|
||||||
|
// InvalidStatusTransitionError is returned when an inquiry status change is not allowed.
|
||||||
|
type InvalidStatusTransitionError struct {
|
||||||
|
CurrentStatus string
|
||||||
|
NewStatus string
|
||||||
|
Allowed []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *InvalidStatusTransitionError) Error() string {
|
||||||
|
if len(e.Allowed) == 0 {
|
||||||
|
return fmt.Sprintf("Inquiry status is already %s and cannot be changed", e.CurrentStatus)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"Cannot change inquiry status from %s to %s. Allowed next statuses: %s",
|
||||||
|
e.CurrentStatus,
|
||||||
|
e.NewStatus,
|
||||||
|
strings.Join(e.Allowed, ", "),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewInvalidStatusTransitionError creates a status transition error with allowed next statuses.
|
||||||
|
func NewInvalidStatusTransitionError(currentStatus, newStatus string, allowed []string) *InvalidStatusTransitionError {
|
||||||
|
return &InvalidStatusTransitionError{
|
||||||
|
CurrentStatus: currentStatus,
|
||||||
|
NewStatus: newStatus,
|
||||||
|
Allowed: allowed,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func NewDuplicateInquiryError(inquiryType, existingID, status string, createdAt int64) *DuplicateInquiryError {
|
func NewDuplicateInquiryError(inquiryType, existingID, status string, createdAt int64) *DuplicateInquiryError {
|
||||||
var message string
|
var message string
|
||||||
if inquiryType == "email" {
|
if inquiryType == "email" {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package inquiry
|
package inquiry
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"strings"
|
||||||
"tm/pkg/response"
|
"tm/pkg/response"
|
||||||
"tm/pkg/security"
|
"tm/pkg/security"
|
||||||
|
|
||||||
@@ -21,7 +22,7 @@ type CreateInquiryForm struct {
|
|||||||
// UpdateInquiryStatusForm represents the data required to update inquiry status
|
// UpdateInquiryStatusForm represents the data required to update inquiry status
|
||||||
type UpdateInquiryStatusForm struct {
|
type UpdateInquiryStatusForm struct {
|
||||||
Status string `json:"status" valid:"required,in(pending|reviewed|approved|rejected)" example:"reviewed"` // New status
|
Status string `json:"status" valid:"required,in(pending|reviewed|approved|rejected)" example:"reviewed"` // New status
|
||||||
Reason string `json:"reason" valid:"required,length(5|500)" example:"Initial review completed"` // Reason for status change
|
Reason string `json:"reason" valid:"optional,length(1|500)" example:"Initial review completed"` // Reason for status change
|
||||||
Description string `json:"description" valid:"optional,length(0|1000)" example:"Additional notes about the review"` // Optional description
|
Description string `json:"description" valid:"optional,length(0|1000)" example:"Additional notes about the review"` // Optional description
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -125,6 +126,11 @@ func (f *CreateInquiryForm) ValidateAndSanitize(xssPolicy *security.XSSPolicy) e
|
|||||||
|
|
||||||
// ValidateAndSanitize validates and sanitizes the status update form
|
// ValidateAndSanitize validates and sanitizes the status update form
|
||||||
func (f *UpdateInquiryStatusForm) ValidateAndSanitize(xssPolicy *security.XSSPolicy) error {
|
func (f *UpdateInquiryStatusForm) ValidateAndSanitize(xssPolicy *security.XSSPolicy) error {
|
||||||
|
f.Status = strings.ToLower(strings.TrimSpace(f.Status))
|
||||||
|
if strings.TrimSpace(f.Reason) == "" {
|
||||||
|
f.Reason = "Status updated to " + f.Status
|
||||||
|
}
|
||||||
|
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
// Sanitize Reason
|
// Sanitize Reason
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
package inquiry
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"tm/pkg/security"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestUpdateInquiryStatusFormValidation(t *testing.T) {
|
||||||
|
xssPolicy := security.NewXSSPolicy()
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
body string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{"status only", `{"status":"reviewed"}`, false},
|
||||||
|
{"capitalized status", `{"status":"Reviewed"}`, false},
|
||||||
|
{"valid with reason", `{"status":"reviewed","reason":"Initial review completed"}`, false},
|
||||||
|
{"user payload", `{"status":"reviewed","reason":"test","description":"test"}`, false},
|
||||||
|
{"invalid status", `{"status":"done"}`, true},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
var form UpdateInquiryStatusForm
|
||||||
|
if err := json.Unmarshal([]byte(tc.body), &form); err != nil {
|
||||||
|
t.Fatalf("%s: unmarshal failed: %v", tc.name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err := form.ValidateAndSanitize(xssPolicy)
|
||||||
|
if (err != nil) != tc.wantErr {
|
||||||
|
t.Errorf("%s: got err=%v wantErr=%v", tc.name, err, tc.wantErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+18
-21
@@ -42,14 +42,13 @@ func NewHandler(service Service, logger logger.Logger, hcaptchaVerifier hcaptcha
|
|||||||
// @Failure 500 {object} response.APIResponse "Internal server error"
|
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||||
// @Router /api/v1/inquiries [post]
|
// @Router /api/v1/inquiries [post]
|
||||||
func (h *Handler) CreateInquiry(c echo.Context) error {
|
func (h *Handler) CreateInquiry(c echo.Context) error {
|
||||||
form, err := response.Parse[CreateInquiryForm](c)
|
var form CreateInquiryForm
|
||||||
if err != nil {
|
if err := c.Bind(&form); err != nil {
|
||||||
return response.BadRequest(c, "Invalid request format", "")
|
return handleBindError(c, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate and sanitize form
|
|
||||||
if err := form.ValidateAndSanitize(h.xssPolicy); err != nil {
|
if err := form.ValidateAndSanitize(h.xssPolicy); err != nil {
|
||||||
return response.ValidationError(c, err.Error(), "")
|
return handleValidationError(c, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify hCaptcha token
|
// Verify hCaptcha token
|
||||||
@@ -64,7 +63,7 @@ func (h *Handler) CreateInquiry(c echo.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Call service
|
// Call service
|
||||||
inquiry, err := h.service.Create(c.Request().Context(), form)
|
inquiry, err := h.service.Create(c.Request().Context(), &form)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Check if it's a duplicate inquiry error
|
// Check if it's a duplicate inquiry error
|
||||||
if duplicateErr, ok := err.(*DuplicateInquiryError); ok {
|
if duplicateErr, ok := err.(*DuplicateInquiryError); ok {
|
||||||
@@ -110,7 +109,7 @@ func (h *Handler) GetInquiryByID(c echo.Context) error {
|
|||||||
|
|
||||||
inquiry, err := h.service.GetByID(c.Request().Context(), idStr)
|
inquiry, err := h.service.GetByID(c.Request().Context(), idStr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return response.NotFound(c, "Inquiry not found")
|
return handleServiceError(c, err, "Failed to retrieve inquiry")
|
||||||
}
|
}
|
||||||
|
|
||||||
return response.Success(c, inquiry, "Inquiry retrieved successfully")
|
return response.Success(c, inquiry, "Inquiry retrieved successfully")
|
||||||
@@ -133,14 +132,13 @@ func (h *Handler) GetInquiryByID(c echo.Context) error {
|
|||||||
func (h *Handler) UpdateInquiryStatus(c echo.Context) error {
|
func (h *Handler) UpdateInquiryStatus(c echo.Context) error {
|
||||||
idStr := c.Param("id")
|
idStr := c.Param("id")
|
||||||
|
|
||||||
form, err := response.Parse[UpdateInquiryStatusForm](c)
|
var form UpdateInquiryStatusForm
|
||||||
if err != nil {
|
if err := c.Bind(&form); err != nil {
|
||||||
return response.BadRequest(c, "Invalid request format", "")
|
return handleBindError(c, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate and sanitize form
|
|
||||||
if err := form.ValidateAndSanitize(h.xssPolicy); err != nil {
|
if err := form.ValidateAndSanitize(h.xssPolicy); err != nil {
|
||||||
return response.ValidationError(c, err.Error(), "")
|
return handleValidationError(c, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get user ID from context (assuming it's set by auth middleware)
|
// Get user ID from context (assuming it's set by auth middleware)
|
||||||
@@ -148,9 +146,9 @@ func (h *Handler) UpdateInquiryStatus(c echo.Context) error {
|
|||||||
// changedBy := c.Get("user_id").(string)
|
// changedBy := c.Get("user_id").(string)
|
||||||
|
|
||||||
// Call service
|
// Call service
|
||||||
inquiry, err := h.service.UpdateStatus(c.Request().Context(), idStr, form, changedBy)
|
inquiry, err := h.service.UpdateStatus(c.Request().Context(), idStr, &form, changedBy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return response.BadRequest(c, err.Error(), "")
|
return handleServiceError(c, err, "Failed to update inquiry status")
|
||||||
}
|
}
|
||||||
|
|
||||||
return response.Success(c, inquiry, "Inquiry status updated successfully")
|
return response.Success(c, inquiry, "Inquiry status updated successfully")
|
||||||
@@ -174,14 +172,13 @@ func (h *Handler) UpdateInquiryStatus(c echo.Context) error {
|
|||||||
// @Failure 500 {object} response.APIResponse "Internal server error"
|
// @Failure 500 {object} response.APIResponse "Internal server error"
|
||||||
// @Router /admin/v1/inquiries [get]
|
// @Router /admin/v1/inquiries [get]
|
||||||
func (h *Handler) SearchInquiries(c echo.Context) error {
|
func (h *Handler) SearchInquiries(c echo.Context) error {
|
||||||
form, err := response.Parse[SearchInquiriesForm](c)
|
var form SearchInquiriesForm
|
||||||
if err != nil {
|
if err := c.Bind(&form); err != nil {
|
||||||
return response.BadRequest(c, "Invalid request format", "")
|
return handleBindError(c, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate and sanitize form
|
|
||||||
if err := form.ValidateAndSanitize(h.xssPolicy); err != nil {
|
if err := form.ValidateAndSanitize(h.xssPolicy); err != nil {
|
||||||
return response.ValidationError(c, err.Error(), "")
|
return handleValidationError(c, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Call service
|
// Call service
|
||||||
@@ -190,7 +187,7 @@ func (h *Handler) SearchInquiries(c echo.Context) error {
|
|||||||
return response.PaginationBadRequest(c, err)
|
return response.PaginationBadRequest(c, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
inquiries, err := h.service.Search(c.Request().Context(), form, pagination)
|
inquiries, err := h.service.Search(c.Request().Context(), &form, pagination)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if response.IsListPaginationError(err) {
|
if response.IsListPaginationError(err) {
|
||||||
return response.PaginationBadRequest(c, err)
|
return response.PaginationBadRequest(c, err)
|
||||||
@@ -220,7 +217,7 @@ func (h *Handler) DeleteInquiry(c echo.Context) error {
|
|||||||
|
|
||||||
err := h.service.Delete(c.Request().Context(), id)
|
err := h.service.Delete(c.Request().Context(), id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return response.BadRequest(c, err.Error(), "")
|
return handleServiceError(c, err, "Failed to delete inquiry")
|
||||||
}
|
}
|
||||||
|
|
||||||
return response.Success(c, map[string]interface{}{
|
return response.Success(c, map[string]interface{}{
|
||||||
|
|||||||
@@ -0,0 +1,38 @@
|
|||||||
|
package inquiry
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"tm/pkg/response"
|
||||||
|
|
||||||
|
"github.com/labstack/echo/v4"
|
||||||
|
)
|
||||||
|
|
||||||
|
func handleBindError(c echo.Context, err error) error {
|
||||||
|
return response.BadRequest(c, "Invalid request format", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleValidationError(c echo.Context, err error) error {
|
||||||
|
return response.ValidationError(c, "Please fix the validation errors", FormatValidationErrors(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleServiceError(c echo.Context, err error, fallback string) error {
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrInquiryNotFound) {
|
||||||
|
return response.NotFound(c, "Inquiry not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
var transitionErr *InvalidStatusTransitionError
|
||||||
|
if errors.As(err, &transitionErr) {
|
||||||
|
return response.BadRequest(c, transitionErr.Error(), "")
|
||||||
|
}
|
||||||
|
|
||||||
|
if strings.Contains(err.Error(), "cannot be updated") {
|
||||||
|
return response.BadRequest(c, err.Error(), "")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.InternalServerError(c, fallback)
|
||||||
|
}
|
||||||
@@ -105,7 +105,7 @@ func (r *inquiryRepository) GetByID(ctx context.Context, id string) (*Inquiry, e
|
|||||||
inquiry, err := r.ormRepo.FindByID(ctx, id)
|
inquiry, err := r.ormRepo.FindByID(ctx, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, orm.ErrDocumentNotFound) {
|
if errors.Is(err, orm.ErrDocumentNotFound) {
|
||||||
return nil, errors.New("inquiry not found")
|
return nil, ErrInquiryNotFound
|
||||||
}
|
}
|
||||||
r.logger.Error("Failed to get inquiry by ID", map[string]interface{}{
|
r.logger.Error("Failed to get inquiry by ID", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
|
|||||||
@@ -251,17 +251,16 @@ func (s *inquiryService) validateStatusTransition(ctx context.Context, id string
|
|||||||
|
|
||||||
allowedNextStatuses, exists := allowedTransitions[currentStatus]
|
allowedNextStatuses, exists := allowedTransitions[currentStatus]
|
||||||
if !exists {
|
if !exists {
|
||||||
return errors.New("invalid current status")
|
return fmt.Errorf("inquiry has unknown status %q and cannot be updated", currentStatus)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if transition is allowed
|
|
||||||
for _, allowedStatus := range allowedNextStatuses {
|
for _, allowedStatus := range allowedNextStatuses {
|
||||||
if allowedStatus == newStatus {
|
if allowedStatus == newStatus {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return errors.New("invalid status transition: cannot change from " + currentStatus + " to " + newStatus)
|
return NewInvalidStatusTransitionError(currentStatus, newStatus, allowedNextStatuses)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Search searches inquiries with search and filters
|
// Search searches inquiries with search and filters
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package inquiry
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/asaskevich/govalidator"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
lengthRulePattern = regexp.MustCompile(`length\((\d+)\|(\d+)\)`)
|
||||||
|
inRulePattern = regexp.MustCompile(`in\(([^)]+)\)`)
|
||||||
|
)
|
||||||
|
|
||||||
|
// FormatValidationErrors converts govalidator errors into user-facing messages.
|
||||||
|
func FormatValidationErrors(err error) string {
|
||||||
|
if err == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
errs, ok := err.(govalidator.Errors)
|
||||||
|
if !ok {
|
||||||
|
return humanizeValidationError(err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
messages := make([]string, 0, len(errs))
|
||||||
|
for _, item := range errs {
|
||||||
|
messages = append(messages, humanizeValidationError(item.Error()))
|
||||||
|
}
|
||||||
|
return strings.Join(messages, "; ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func humanizeValidationError(raw string) string {
|
||||||
|
field, rule, ok := parseGovalidatorError(raw)
|
||||||
|
if !ok {
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
label := inquiryFieldLabel(field)
|
||||||
|
lowerRule := strings.ToLower(rule)
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case strings.Contains(lowerRule, "required"):
|
||||||
|
return label + " is required"
|
||||||
|
case strings.Contains(lowerRule, "email"):
|
||||||
|
return label + " must be a valid email address"
|
||||||
|
case strings.Contains(lowerRule, "length("):
|
||||||
|
if min, max, ok := parseLengthRule(rule); ok {
|
||||||
|
if min == max {
|
||||||
|
return fmt.Sprintf("%s must be exactly %d characters", label, min)
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s must be between %d and %d characters", label, min, max)
|
||||||
|
}
|
||||||
|
return label + " has an invalid length"
|
||||||
|
case strings.Contains(lowerRule, "in("):
|
||||||
|
if values, ok := parseInRule(rule); ok {
|
||||||
|
return fmt.Sprintf("%s must be one of: %s", label, strings.Join(values, ", "))
|
||||||
|
}
|
||||||
|
return label + " has an invalid value"
|
||||||
|
case strings.Contains(lowerRule, "range("):
|
||||||
|
return label + " is out of the allowed range"
|
||||||
|
default:
|
||||||
|
return label + " is invalid"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseGovalidatorError(raw string) (field, rule string, ok bool) {
|
||||||
|
parts := strings.SplitN(raw, ": ", 2)
|
||||||
|
if len(parts) != 2 {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
return parts[0], parts[1], true
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseLengthRule(rule string) (min, max int, ok bool) {
|
||||||
|
matches := lengthRulePattern.FindStringSubmatch(rule)
|
||||||
|
if len(matches) != 3 {
|
||||||
|
return 0, 0, false
|
||||||
|
}
|
||||||
|
_, err := fmt.Sscanf(matches[1]+" "+matches[2], "%d %d", &min, &max)
|
||||||
|
return min, max, err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseInRule(rule string) (values []string, ok bool) {
|
||||||
|
matches := inRulePattern.FindStringSubmatch(rule)
|
||||||
|
if len(matches) != 2 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
for _, value := range strings.Split(matches[1], "|") {
|
||||||
|
values = append(values, value)
|
||||||
|
}
|
||||||
|
return values, len(values) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func inquiryFieldLabel(field string) string {
|
||||||
|
field = strings.TrimPrefix(field, "CreateInquiryForm.")
|
||||||
|
field = strings.TrimPrefix(field, "UpdateInquiryStatusForm.")
|
||||||
|
field = strings.TrimPrefix(field, "SearchInquiriesForm.")
|
||||||
|
|
||||||
|
labels := map[string]string{
|
||||||
|
"FullName": "Full name",
|
||||||
|
"CompanyName": "Company name",
|
||||||
|
"WorkEmail": "Work email",
|
||||||
|
"PhoneNumber": "Phone number",
|
||||||
|
"HCaptchaToken": "hCaptcha token",
|
||||||
|
"Status": "Status",
|
||||||
|
"Reason": "Reason",
|
||||||
|
"Description": "Description",
|
||||||
|
"Search": "Search query",
|
||||||
|
"Limit": "Limit",
|
||||||
|
"Offset": "Offset",
|
||||||
|
"SortBy": "Sort field",
|
||||||
|
"SortOrder": "Sort order",
|
||||||
|
}
|
||||||
|
if label, exists := labels[field]; exists {
|
||||||
|
return label
|
||||||
|
}
|
||||||
|
return field
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package inquiry
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/asaskevich/govalidator"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestFormatValidationErrors(t *testing.T) {
|
||||||
|
form := UpdateInquiryStatusForm{
|
||||||
|
Status: "done",
|
||||||
|
Reason: "",
|
||||||
|
}
|
||||||
|
_, err := govalidator.ValidateStruct(form)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected validation error")
|
||||||
|
}
|
||||||
|
|
||||||
|
message := FormatValidationErrors(err)
|
||||||
|
if message == "" {
|
||||||
|
t.Fatal("expected formatted validation message")
|
||||||
|
}
|
||||||
|
if message == err.Error() {
|
||||||
|
t.Fatalf("expected humanized message, got raw error: %s", message)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInvalidStatusTransitionErrorMessage(t *testing.T) {
|
||||||
|
err := NewInvalidStatusTransitionError(StatusPending, StatusApproved, []string{StatusReviewed, StatusRejected})
|
||||||
|
expected := "Cannot change inquiry status from pending to approved. Allowed next statuses: reviewed, rejected"
|
||||||
|
if err.Error() != expected {
|
||||||
|
t.Fatalf("got %q, want %q", err.Error(), expected)
|
||||||
|
}
|
||||||
|
|
||||||
|
finalErr := NewInvalidStatusTransitionError(StatusApproved, StatusReviewed, nil)
|
||||||
|
if finalErr.Error() != "Inquiry status is already approved and cannot be changed" {
|
||||||
|
t.Fatalf("unexpected final status message: %s", finalErr.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -39,6 +39,14 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
|
|||||||
noticeIndexes := []orm.Index{
|
noticeIndexes := []orm.Index{
|
||||||
// processing_metadata.processed
|
// processing_metadata.processed
|
||||||
*orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}),
|
*orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}),
|
||||||
|
*orm.NewIndex("source_scraped_at_idx", bson.D{
|
||||||
|
{Key: "source", Value: 1},
|
||||||
|
{Key: "processing_metadata.scraped_at", Value: 1},
|
||||||
|
}),
|
||||||
|
*orm.NewIndex("source_created_at_idx", bson.D{
|
||||||
|
{Key: "source", Value: 1},
|
||||||
|
{Key: "created_at", Value: 1},
|
||||||
|
}),
|
||||||
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
|
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
|
||||||
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
|
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
|
||||||
// One row per TED ContractNoticeID when present (parallel scrape races).
|
// One row per TED ContractNoticeID when present (parallel scrape races).
|
||||||
|
|||||||
@@ -25,8 +25,30 @@ type DeliveryChannel string
|
|||||||
const (
|
const (
|
||||||
DeliveryChannelPush DeliveryChannel = "push"
|
DeliveryChannelPush DeliveryChannel = "push"
|
||||||
DeliveryChannelEmail DeliveryChannel = "email"
|
DeliveryChannelEmail DeliveryChannel = "email"
|
||||||
|
DeliveryChannelInApp DeliveryChannel = "in_app"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Persisted event types for stored notifications (admin list / notification center).
|
||||||
|
const (
|
||||||
|
EventTypeInApp = "in_app"
|
||||||
|
EventTypeEmail = "email"
|
||||||
|
EventTypePush = "push"
|
||||||
|
)
|
||||||
|
|
||||||
|
// persistedEventType maps a delivery channel to the event_type stored in MongoDB.
|
||||||
|
func persistedEventType(channel DeliveryChannel) (string, bool) {
|
||||||
|
switch channel {
|
||||||
|
case DeliveryChannelEmail:
|
||||||
|
return EventTypeEmail, true
|
||||||
|
case DeliveryChannelPush:
|
||||||
|
return EventTypePush, true
|
||||||
|
case DeliveryChannelInApp:
|
||||||
|
return EventTypeInApp, true
|
||||||
|
default:
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// DeliveryStatus represents the delivery status of notification
|
// DeliveryStatus represents the delivery status of notification
|
||||||
type NotificationStatus string
|
type NotificationStatus string
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
package notification
|
package notification
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
"tm/pkg/response"
|
"tm/pkg/response"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -21,15 +23,37 @@ type NotificationRequest struct {
|
|||||||
|
|
||||||
// SearchForm represents the form for searching notifications
|
// SearchForm represents the form for searching notifications
|
||||||
type SearchForm struct {
|
type SearchForm struct {
|
||||||
|
Q *string `query:"q" valid:"optional"`
|
||||||
|
Search *string `query:"search" valid:"optional"`
|
||||||
Status string `query:"status" valid:"optional"`
|
Status string `query:"status" valid:"optional"`
|
||||||
Method string `query:"method" valid:"optional"`
|
Method string `query:"method" valid:"optional"`
|
||||||
EventType string `query:"event_type" valid:"optional"`
|
EventType string `query:"event_type" valid:"optional"`
|
||||||
Type string `query:"type" valid:"optional"`
|
Type string `query:"type" valid:"optional"`
|
||||||
Recipient []string `query:"recipient" json:"recipient" valid:"optional"`
|
Recipient []string `query:"recipient" json:"recipient" valid:"optional"`
|
||||||
|
UserID string `query:"user_id" valid:"optional"`
|
||||||
Seen *bool `query:"seen" valid:"optional"`
|
Seen *bool `query:"seen" valid:"optional"`
|
||||||
Priority string `query:"priority" valid:"optional"`
|
Priority string `query:"priority" valid:"optional"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResolvedSearch returns the search term from either the search or q query parameter.
|
||||||
|
func (f *SearchForm) ResolvedSearch() string {
|
||||||
|
if f.Search != nil && strings.TrimSpace(*f.Search) != "" {
|
||||||
|
return strings.TrimSpace(*f.Search)
|
||||||
|
}
|
||||||
|
if f.Q != nil {
|
||||||
|
return strings.TrimSpace(*f.Q)
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolvedRecipients returns the user IDs to filter by from user_id or recipient query params.
|
||||||
|
func (f *SearchForm) ResolvedRecipients() []string {
|
||||||
|
if id := strings.TrimSpace(f.UserID); id != "" {
|
||||||
|
return []string{id}
|
||||||
|
}
|
||||||
|
return f.Recipient
|
||||||
|
}
|
||||||
|
|
||||||
type NotificationListResponse struct {
|
type NotificationListResponse struct {
|
||||||
Notifications []*NotificationResponse `json:"notifications"`
|
Notifications []*NotificationResponse `json:"notifications"`
|
||||||
Meta *response.Meta `json:"meta"`
|
Meta *response.Meta `json:"meta"`
|
||||||
|
|||||||
@@ -49,7 +49,7 @@ func (h *NotificationHandler) Send(c echo.Context) error {
|
|||||||
// Send notification
|
// Send notification
|
||||||
go h.service.Send(context.Background(), &req)
|
go h.service.Send(context.Background(), &req)
|
||||||
|
|
||||||
return response.Created(c, nil, "Notification sent successfully")
|
return response.Created(c, nil, "Notification created successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetNotifications gets notifications
|
// GetNotifications gets notifications
|
||||||
@@ -58,11 +58,14 @@ func (h *NotificationHandler) Send(c echo.Context) error {
|
|||||||
// @Tags Admin-Notification
|
// @Tags Admin-Notification
|
||||||
// @Accept json
|
// @Accept json
|
||||||
// @Produce json
|
// @Produce json
|
||||||
|
// @Param search query string false "Search query for title and message"
|
||||||
|
// @Param q query string false "Search query for title and message (alias of search)"
|
||||||
// @Param status query string false "Status filter"
|
// @Param status query string false "Status filter"
|
||||||
// @Param method query string false "Method filter"
|
// @Param method query string false "Method filter"
|
||||||
// @Param event_type query string false "Event type filter"
|
// @Param event_type query string false "Event type filter"
|
||||||
// @Param type query string false "Type filter"
|
// @Param type query string false "Type filter"
|
||||||
// @Param recipient query string false "Recipient filter"
|
// @Param recipient query string false "Recipient filter"
|
||||||
|
// @Param user_id query string false "User ID filter"
|
||||||
// @Param seen query bool false "Seen filter"
|
// @Param seen query bool false "Seen filter"
|
||||||
// @Param priority query string false "Priority filter"
|
// @Param priority query string false "Priority filter"
|
||||||
// @Param limit query int false "Limit results"
|
// @Param limit query int false "Limit results"
|
||||||
@@ -145,6 +148,8 @@ func (h *NotificationHandler) AdminNotifications(c echo.Context) error {
|
|||||||
|
|
||||||
form.Recipient = []string{userID}
|
form.Recipient = []string{userID}
|
||||||
form.Status = "sent"
|
form.Status = "sent"
|
||||||
|
form.EventType = EventTypeInApp
|
||||||
|
form.Method = ""
|
||||||
|
|
||||||
pagination, err := response.NewPagination(c)
|
pagination, err := response.NewPagination(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -263,6 +268,8 @@ func (h *NotificationHandler) CustomerNotifications(c echo.Context) error {
|
|||||||
|
|
||||||
form.Recipient = []string{userID}
|
form.Recipient = []string{userID}
|
||||||
form.Status = "sent"
|
form.Status = "sent"
|
||||||
|
form.EventType = EventTypeInApp
|
||||||
|
form.Method = ""
|
||||||
|
|
||||||
pagination, err := response.NewPagination(c)
|
pagination, err := response.NewPagination(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ package notification
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
@@ -52,10 +54,11 @@ func (n *Notification) GetID() string {
|
|||||||
type Repository interface {
|
type Repository interface {
|
||||||
Create(ctx context.Context, notification *Notification) error
|
Create(ctx context.Context, notification *Notification) error
|
||||||
GetByID(ctx context.Context, id string) (*Notification, error)
|
GetByID(ctx context.Context, id string) (*Notification, error)
|
||||||
GetByUserID(ctx context.Context, userIDs []string, status string, seen *bool, priority, eventType, notificationType string, pagination *response.Pagination) (*orm.PaginatedResult[Notification], error)
|
GetByUserID(ctx context.Context, userIDs []string, search, status string, seen *bool, priority, eventType, notificationType string, pagination *response.Pagination) (*orm.PaginatedResult[Notification], error)
|
||||||
Update(ctx context.Context, notification *Notification) error
|
Update(ctx context.Context, notification *Notification) error
|
||||||
MarkAsSeen(ctx context.Context, notificationID, userID string) error
|
MarkAsSeen(ctx context.Context, notificationID, userID string) error
|
||||||
MarkAllAsSeen(ctx context.Context, userID string) error
|
MarkAllAsSeen(ctx context.Context, userID string) error
|
||||||
|
MarkDueScheduledAsSent(ctx context.Context, dueBefore int64) (int64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type notificationRepository struct {
|
type notificationRepository struct {
|
||||||
@@ -72,6 +75,11 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
|
|||||||
*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
|
*orm.NewIndex("status_idx", bson.D{{Key: "status", Value: 1}}),
|
||||||
*orm.NewIndex("seen_idx", bson.D{{Key: "seen", Value: 1}}),
|
*orm.NewIndex("seen_idx", bson.D{{Key: "seen", Value: 1}}),
|
||||||
*orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}),
|
*orm.NewIndex("created_at_idx", bson.D{{Key: "created_at", Value: -1}}),
|
||||||
|
*orm.NewIndex("scheduled_delivery_idx", bson.D{
|
||||||
|
{Key: "status", Value: 1},
|
||||||
|
{Key: "is_scheduled", Value: 1},
|
||||||
|
{Key: "schedule_at", Value: 1},
|
||||||
|
}),
|
||||||
}
|
}
|
||||||
|
|
||||||
err := mongoManager.CreateIndexes("notifications", indexes)
|
err := mongoManager.CreateIndexes("notifications", indexes)
|
||||||
@@ -115,8 +123,19 @@ func (r *notificationRepository) GetByID(ctx context.Context, id string) (*Notif
|
|||||||
}
|
}
|
||||||
|
|
||||||
// GetByUserID retrieves notifications with optional user filters
|
// GetByUserID retrieves notifications with optional user filters
|
||||||
func (r *notificationRepository) GetByUserID(ctx context.Context, userIDs []string, status string, seen *bool, priority, eventType, notificationType string, pagination *response.Pagination) (*orm.PaginatedResult[Notification], error) {
|
func (r *notificationRepository) GetByUserID(ctx context.Context, userIDs []string, search, status string, seen *bool, priority, eventType, notificationType string, pagination *response.Pagination) (*orm.PaginatedResult[Notification], error) {
|
||||||
filter := bson.M{}
|
filter := bson.M{}
|
||||||
|
andConditions := bson.A{}
|
||||||
|
|
||||||
|
if q := strings.TrimSpace(search); q != "" {
|
||||||
|
pattern := regexp.QuoteMeta(q)
|
||||||
|
andConditions = append(andConditions, bson.M{
|
||||||
|
"$or": bson.A{
|
||||||
|
bson.M{"title": bson.M{"$regex": pattern, "$options": "i"}},
|
||||||
|
bson.M{"message": bson.M{"$regex": pattern, "$options": "i"}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
if len(userIDs) == 1 {
|
if len(userIDs) == 1 {
|
||||||
filter["user_id"] = userIDs[0]
|
filter["user_id"] = userIDs[0]
|
||||||
@@ -137,13 +156,31 @@ func (r *notificationRepository) GetByUserID(ctx context.Context, userIDs []stri
|
|||||||
}
|
}
|
||||||
|
|
||||||
if eventType != "" {
|
if eventType != "" {
|
||||||
|
if eventType == EventTypeInApp {
|
||||||
|
andConditions = append(andConditions, bson.M{
|
||||||
|
"$or": bson.A{
|
||||||
|
bson.M{"event_type": EventTypeInApp},
|
||||||
|
bson.M{"event_type": ""},
|
||||||
|
bson.M{"event_type": bson.M{"$exists": false}},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
} else {
|
||||||
filter["event_type"] = eventType
|
filter["event_type"] = eventType
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if notificationType != "" {
|
if notificationType != "" {
|
||||||
filter["type"] = notificationType
|
filter["type"] = notificationType
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if len(andConditions) == 1 {
|
||||||
|
for key, value := range andConditions[0].(bson.M) {
|
||||||
|
filter[key] = value
|
||||||
|
}
|
||||||
|
} else if len(andConditions) > 1 {
|
||||||
|
filter["$and"] = andConditions
|
||||||
|
}
|
||||||
|
|
||||||
mongoPagination, err := orm.BuildListPagination(
|
mongoPagination, err := orm.BuildListPagination(
|
||||||
pagination.Limit,
|
pagination.Limit,
|
||||||
pagination.Offset,
|
pagination.Offset,
|
||||||
@@ -248,3 +285,33 @@ func (r *notificationRepository) MarkAllAsSeen(ctx context.Context, userID strin
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// MarkDueScheduledAsSent updates pending scheduled notifications whose delivery time has passed.
|
||||||
|
func (r *notificationRepository) MarkDueScheduledAsSent(ctx context.Context, dueBefore int64) (int64, error) {
|
||||||
|
filter := bson.M{
|
||||||
|
"status": string(DeliveryStatusPending),
|
||||||
|
"is_scheduled": true,
|
||||||
|
"schedule_at": bson.M{
|
||||||
|
"$gt": 0,
|
||||||
|
"$lte": dueBefore,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
update := bson.M{
|
||||||
|
"$set": bson.M{
|
||||||
|
"status": string(DeliveryStatusSent),
|
||||||
|
"updated_at": time.Now().Unix(),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := r.collection.UpdateMany(ctx, filter, update)
|
||||||
|
if err != nil {
|
||||||
|
r.logger.Error("Failed to mark due scheduled notifications as sent", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"due_before": dueBefore,
|
||||||
|
})
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.ModifiedCount, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"tm/internal/customer"
|
"tm/internal/customer"
|
||||||
"tm/internal/user"
|
"tm/internal/user"
|
||||||
@@ -98,13 +99,27 @@ func (s *notificationService) Send(ctx context.Context, req *NotificationRequest
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *notificationService) sendToRecipient(ctx context.Context, recipient notificationRecipient, req *NotificationRequest, link, image string) {
|
func (s *notificationService) sendToRecipient(ctx context.Context, recipient notificationRecipient, req *NotificationRequest, link, image string) {
|
||||||
if err := s.persistNotification(ctx, recipient, req, link, image); err != nil {
|
// Always persist an in-app record so every send appears in the notification center.
|
||||||
|
if err := s.persistNotification(ctx, recipient, req, link, image, DeliveryChannelInApp); err != nil {
|
||||||
s.logger.Error("Failed to persist in-app notification", map[string]interface{}{
|
s.logger.Error("Failed to persist in-app notification", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
"user_id": recipient.UserID,
|
"user_id": recipient.UserID,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, channel := range req.Channels {
|
||||||
|
if channel == DeliveryChannelInApp {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := s.persistNotification(ctx, recipient, req, link, image, channel); err != nil {
|
||||||
|
s.logger.Error("Failed to persist notification", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
"user_id": recipient.UserID,
|
||||||
|
"channel": channel,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
metadata := map[string]any{
|
metadata := map[string]any{
|
||||||
"tender": strings.TrimSpace(req.Tender),
|
"tender": strings.TrimSpace(req.Tender),
|
||||||
}
|
}
|
||||||
@@ -148,13 +163,26 @@ func (s *notificationService) sendToRecipient(ctx context.Context, recipient not
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *notificationService) persistNotification(ctx context.Context, recipient notificationRecipient, req *NotificationRequest, link, image string) error {
|
func (s *notificationService) persistNotification(ctx context.Context, recipient notificationRecipient, req *NotificationRequest, link, image string, channel DeliveryChannel) error {
|
||||||
methods := make(map[string]string)
|
eventType, ok := persistedEventType(channel)
|
||||||
if slices.Contains(req.Channels, DeliveryChannelEmail) && recipient.Email != "" {
|
if !ok {
|
||||||
methods["email"] = recipient.Email
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
methods := map[string]string{}
|
||||||
|
switch channel {
|
||||||
|
case DeliveryChannelEmail:
|
||||||
|
if recipient.Email == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
methods["email"] = recipient.Email
|
||||||
|
case DeliveryChannelPush:
|
||||||
|
if len(recipient.DeviceTokens) == 0 {
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
if slices.Contains(req.Channels, DeliveryChannelPush) && len(recipient.DeviceTokens) > 0 {
|
|
||||||
methods["push"] = recipient.DeviceTokens[0]
|
methods["push"] = recipient.DeviceTokens[0]
|
||||||
|
case DeliveryChannelInApp:
|
||||||
|
methods["in_app"] = "true"
|
||||||
}
|
}
|
||||||
|
|
||||||
status := string(DeliveryStatusSent)
|
status := string(DeliveryStatusSent)
|
||||||
@@ -170,6 +198,7 @@ func (s *notificationService) persistNotification(ctx context.Context, recipient
|
|||||||
Image: image,
|
Image: image,
|
||||||
Priority: string(req.Priority),
|
Priority: string(req.Priority),
|
||||||
Type: string(req.Type),
|
Type: string(req.Type),
|
||||||
|
EventType: eventType,
|
||||||
Status: status,
|
Status: status,
|
||||||
Methods: methods,
|
Methods: methods,
|
||||||
Metadata: map[string]any{
|
Metadata: map[string]any{
|
||||||
@@ -183,19 +212,24 @@ func (s *notificationService) persistNotification(ctx context.Context, recipient
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *notificationService) GetNotifications(ctx context.Context, req *SearchForm, pagination *response.Pagination) (*NotificationListResponse, error) {
|
func (s *notificationService) GetNotifications(ctx context.Context, req *SearchForm, pagination *response.Pagination) (*NotificationListResponse, error) {
|
||||||
|
s.processDueScheduled(ctx)
|
||||||
|
|
||||||
|
search := req.ResolvedSearch()
|
||||||
|
|
||||||
s.logger.Info("Getting notifications", map[string]interface{}{
|
s.logger.Info("Getting notifications", map[string]interface{}{
|
||||||
|
"search": search,
|
||||||
"status": req.Status,
|
"status": req.Status,
|
||||||
"method": req.Method,
|
"method": req.Method,
|
||||||
"event_type": req.EventType,
|
"event_type": req.EventType,
|
||||||
"type": req.Type,
|
"type": req.Type,
|
||||||
"recipient": req.Recipient,
|
"recipient": req.ResolvedRecipients(),
|
||||||
"seen": req.Seen,
|
"seen": req.Seen,
|
||||||
"priority": req.Priority,
|
"priority": req.Priority,
|
||||||
"limit": pagination.Limit,
|
"limit": pagination.Limit,
|
||||||
"offset": pagination.Offset,
|
"offset": pagination.Offset,
|
||||||
})
|
})
|
||||||
|
|
||||||
page, err := s.repository.GetByUserID(ctx, req.Recipient, req.Status, req.Seen, req.Priority, req.EventType, req.Type, pagination)
|
page, err := s.repository.GetByUserID(ctx, req.ResolvedRecipients(), search, req.Status, req.Seen, req.Priority, req.EventType, req.Type, pagination)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to get notifications", map[string]interface{}{
|
s.logger.Error("Failed to get notifications", map[string]interface{}{
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
@@ -258,6 +292,8 @@ func (s *notificationService) GetNotifications(ctx context.Context, req *SearchF
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *notificationService) GetNotification(ctx context.Context, notificationID string) (*NotificationResponse, error) {
|
func (s *notificationService) GetNotification(ctx context.Context, notificationID string) (*NotificationResponse, error) {
|
||||||
|
s.processDueScheduled(ctx)
|
||||||
|
|
||||||
s.logger.Info("Getting notification", map[string]interface{}{
|
s.logger.Info("Getting notification", map[string]interface{}{
|
||||||
"notification_id": notificationID,
|
"notification_id": notificationID,
|
||||||
})
|
})
|
||||||
@@ -461,3 +497,19 @@ func (s *notificationService) getCustomers(ctx context.Context, values []string,
|
|||||||
|
|
||||||
return recipients, nil
|
return recipients, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *notificationService) processDueScheduled(ctx context.Context) {
|
||||||
|
count, err := s.repository.MarkDueScheduledAsSent(ctx, time.Now().Unix())
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to process due scheduled notifications", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if count > 0 {
|
||||||
|
s.logger.Info("Processed due scheduled notifications", map[string]interface{}{
|
||||||
|
"count": count,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,78 @@
|
|||||||
|
package tender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
)
|
||||||
|
|
||||||
|
const aiProcedurePrefix = "PROC_"
|
||||||
|
|
||||||
|
// FormatAIProcedureRef builds the AI service tender reference:
|
||||||
|
// PROC_{contract_folder_id}/{notice_publication_id}
|
||||||
|
func FormatAIProcedureRef(contractFolderID, noticePublicationID string) string {
|
||||||
|
contractFolderID = strings.TrimSpace(contractFolderID)
|
||||||
|
noticePublicationID = strings.TrimSpace(noticePublicationID)
|
||||||
|
if contractFolderID == "" || noticePublicationID == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return aiProcedurePrefix + contractFolderID + "/" + noticePublicationID
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProcedureReference identifies a tender using AI procedure coordinates.
|
||||||
|
type ProcedureReference struct {
|
||||||
|
Ref string
|
||||||
|
ContractFolderID string
|
||||||
|
NoticePublicationID string
|
||||||
|
}
|
||||||
|
|
||||||
|
// ParseAIProcedureRef parses PROC_{contract_folder_id}/{notice_publication_id}.
|
||||||
|
func ParseAIProcedureRef(ref string) (contractFolderID, noticePublicationID string, ok bool) {
|
||||||
|
ref = strings.TrimSpace(ref)
|
||||||
|
if ref == "" {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
withoutPrefix := ref
|
||||||
|
if len(ref) >= len(aiProcedurePrefix) && strings.EqualFold(ref[:len(aiProcedurePrefix)], aiProcedurePrefix) {
|
||||||
|
withoutPrefix = ref[len(aiProcedurePrefix):]
|
||||||
|
}
|
||||||
|
|
||||||
|
slash := strings.Index(withoutPrefix, "/")
|
||||||
|
if slash <= 0 || slash >= len(withoutPrefix)-1 {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
contractFolderID = strings.TrimSpace(withoutPrefix[:slash])
|
||||||
|
noticePublicationID = strings.TrimSpace(withoutPrefix[slash+1:])
|
||||||
|
if contractFolderID == "" || noticePublicationID == "" {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
return contractFolderID, noticePublicationID, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func tenderMatchesProcedureRef(t Tender, contractFolderID, noticePublicationID string) bool {
|
||||||
|
if strings.TrimSpace(t.ContractFolderID) != strings.TrimSpace(contractFolderID) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if t.NoticePublicationID == noticePublicationID {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, related := range t.RelatedNoticePublicationIDs {
|
||||||
|
if related == noticePublicationID {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsMongoObjectIDRef reports whether ref is a 24-char hex MongoDB ObjectId.
|
||||||
|
func IsMongoObjectIDRef(ref string) bool {
|
||||||
|
ref = strings.TrimSpace(ref)
|
||||||
|
if len(ref) != 24 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
_, err := bson.ObjectIDFromHex(ref)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,124 @@
|
|||||||
|
package tender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseAIProcedureRef(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
ref string
|
||||||
|
folder string
|
||||||
|
notice string
|
||||||
|
ok bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
ref: "PROC_055329e0-3d8f-4eeb-946a-85a451f2e36b/00423458-2026",
|
||||||
|
folder: "055329e0-3d8f-4eeb-946a-85a451f2e36b",
|
||||||
|
notice: "00423458-2026",
|
||||||
|
ok: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ref: "proc_d3d34785-e10f-4947-8d00-701816777bf5/00332053-2026",
|
||||||
|
folder: "d3d34785-e10f-4947-8d00-701816777bf5",
|
||||||
|
notice: "00332053-2026",
|
||||||
|
ok: true,
|
||||||
|
},
|
||||||
|
{ref: "", ok: false},
|
||||||
|
{ref: "507f1f77bcf86cd799439011", ok: false},
|
||||||
|
{ref: "PROC_no-slash", ok: false},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
folder, notice, ok := ParseAIProcedureRef(tt.ref)
|
||||||
|
if ok != tt.ok {
|
||||||
|
t.Fatalf("ParseAIProcedureRef(%q) ok=%v want %v", tt.ref, ok, tt.ok)
|
||||||
|
}
|
||||||
|
if !tt.ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if folder != tt.folder || notice != tt.notice {
|
||||||
|
t.Fatalf("ParseAIProcedureRef(%q) = (%q, %q) want (%q, %q)", tt.ref, folder, notice, tt.folder, tt.notice)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatAIProcedureRef(t *testing.T) {
|
||||||
|
got := FormatAIProcedureRef("055329e0-3d8f-4eeb-946a-85a451f2e36b", "00423458-2026")
|
||||||
|
want := "PROC_055329e0-3d8f-4eeb-946a-85a451f2e36b/00423458-2026"
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("FormatAIProcedureRef() = %q want %q", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapProcedureReferencesToTenders(t *testing.T) {
|
||||||
|
candidates := []Tender{
|
||||||
|
{
|
||||||
|
ContractFolderID: "folder-1",
|
||||||
|
NoticePublicationID: "notice-new",
|
||||||
|
RelatedNoticePublicationIDs: []string{"notice-old"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ContractFolderID: "folder-1",
|
||||||
|
NoticePublicationID: "notice-legacy",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ContractFolderID: "folder-2",
|
||||||
|
NoticePublicationID: "notice-2",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
refs := []ProcedureReference{
|
||||||
|
{Ref: "PROC_folder-1/notice-old", ContractFolderID: "folder-1", NoticePublicationID: "notice-old"},
|
||||||
|
{Ref: "PROC_folder-1/notice-legacy", ContractFolderID: "folder-1", NoticePublicationID: "notice-legacy"},
|
||||||
|
{Ref: "PROC_folder-2/notice-2", ContractFolderID: "folder-2", NoticePublicationID: "notice-2"},
|
||||||
|
}
|
||||||
|
|
||||||
|
got := mapProcedureReferencesToTenders(candidates, refs)
|
||||||
|
if len(got) != 3 {
|
||||||
|
t.Fatalf("mapProcedureReferencesToTenders() len = %d, want 3", len(got))
|
||||||
|
}
|
||||||
|
if got["PROC_folder-1/notice-old"].NoticePublicationID != "notice-new" {
|
||||||
|
t.Fatalf("expected notice-old ref to resolve to latest merged tender, got %+v", got["PROC_folder-1/notice-old"])
|
||||||
|
}
|
||||||
|
if got["PROC_folder-1/notice-legacy"].NoticePublicationID != "notice-legacy" {
|
||||||
|
t.Fatalf("expected legacy ref to resolve to legacy tender, got %+v", got["PROC_folder-1/notice-legacy"])
|
||||||
|
}
|
||||||
|
if got["PROC_folder-2/notice-2"].NoticePublicationID != "notice-2" {
|
||||||
|
t.Fatalf("expected folder-2 ref to resolve, got %+v", got["PROC_folder-2/notice-2"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMapProcedureReferencesToTendersUsesFolderCandidatesWhenNoticeLookupMisses(t *testing.T) {
|
||||||
|
// Notice-old is only present on the merged tender loaded via contract_folder_id.
|
||||||
|
candidates := []Tender{
|
||||||
|
{
|
||||||
|
ContractFolderID: "folder-1",
|
||||||
|
NoticePublicationID: "notice-new",
|
||||||
|
RelatedNoticePublicationIDs: []string{"notice-old"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
refs := []ProcedureReference{
|
||||||
|
{Ref: "PROC_folder-1/notice-old", ContractFolderID: "folder-1", NoticePublicationID: "notice-old"},
|
||||||
|
}
|
||||||
|
|
||||||
|
got := mapProcedureReferencesToTenders(candidates, refs)
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("mapProcedureReferencesToTenders() len = %d, want 1", len(got))
|
||||||
|
}
|
||||||
|
if got["PROC_folder-1/notice-old"].NoticePublicationID != "notice-new" {
|
||||||
|
t.Fatalf("expected folder candidate to resolve notice-old ref, got %+v", got["PROC_folder-1/notice-old"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMergeTenderCandidatesDedupesByMongoID(t *testing.T) {
|
||||||
|
first := Tender{ContractFolderID: "folder-1", NoticePublicationID: "notice-a"}
|
||||||
|
first.ID, _ = bson.ObjectIDFromHex("507f1f77bcf86cd799439011")
|
||||||
|
second := Tender{ContractFolderID: "folder-1", NoticePublicationID: "notice-b"}
|
||||||
|
second.ID = first.ID
|
||||||
|
|
||||||
|
got := mergeTenderCandidates([]Tender{first}, []Tender{second})
|
||||||
|
if len(got) != 1 {
|
||||||
|
t.Fatalf("mergeTenderCandidates() len = %d, want 1", len(got))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package tender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// contractFoldersWithDocsCacheTTL avoids scanning MinIO on every documents_scraped list request.
|
||||||
|
// Kept >= resolve timeout so a slow scan is not immediately invalidated for waiters.
|
||||||
|
contractFoldersWithDocsCacheTTL = 15 * time.Minute
|
||||||
|
// contractFoldersWithDocsResolveTimeout bounds a single MinIO procedure scan.
|
||||||
|
contractFoldersWithDocsResolveTimeout = 3 * time.Minute
|
||||||
|
)
|
||||||
|
|
||||||
|
type contractFoldersWithDocsCacheEntry struct {
|
||||||
|
expiresAt time.Time
|
||||||
|
folderIDs []string
|
||||||
|
valid bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// enrichDocumentsScrapedFilter resolves contract_folder_id values that currently have
|
||||||
|
// documents in MinIO and stores them on the search form for the repository filter.
|
||||||
|
func (s *tenderService) enrichDocumentsScrapedFilter(ctx context.Context, form *SearchForm) error {
|
||||||
|
if form == nil || !form.DocumentsScraped {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
folderIDs, err := s.cachedContractFolderIDsWithDocuments(ctx)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to resolve tenders with scraped documents from MinIO", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
if errors.Is(err, ai_summarizer.ErrMinIOUnavailable) {
|
||||||
|
return fmt.Errorf("failed to resolve tenders with scraped documents: MinIO connection unavailable: %w", err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("failed to resolve tenders with scraped documents: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
form.ContractFolderIDsWithDocuments = folderIDs
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *tenderService) cachedContractFolderIDsWithDocuments(ctx context.Context) ([]string, error) {
|
||||||
|
now := time.Now()
|
||||||
|
|
||||||
|
s.contractFoldersCacheMu.Lock()
|
||||||
|
if s.contractFoldersCache.valid && now.Before(s.contractFoldersCache.expiresAt) {
|
||||||
|
ids := append([]string(nil), s.contractFoldersCache.folderIDs...)
|
||||||
|
s.contractFoldersCacheMu.Unlock()
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
s.contractFoldersCacheMu.Unlock()
|
||||||
|
|
||||||
|
v, err, _ := s.contractFoldersCacheGroup.Do("contract-folders-with-docs", func() (interface{}, error) {
|
||||||
|
s.contractFoldersCacheMu.Lock()
|
||||||
|
if s.contractFoldersCache.valid && time.Now().Before(s.contractFoldersCache.expiresAt) {
|
||||||
|
ids := append([]string(nil), s.contractFoldersCache.folderIDs...)
|
||||||
|
s.contractFoldersCacheMu.Unlock()
|
||||||
|
return ids, nil
|
||||||
|
}
|
||||||
|
s.contractFoldersCacheMu.Unlock()
|
||||||
|
|
||||||
|
// Detach from the caller's cancellation so a client disconnect or proxy timeout
|
||||||
|
// does not abort the shared singleflight scan for every waiter.
|
||||||
|
resolveCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), contractFoldersWithDocsResolveTimeout)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
procedures, listErr := s.listProceduresWithDocuments(resolveCtx)
|
||||||
|
if listErr != nil {
|
||||||
|
return nil, listErr
|
||||||
|
}
|
||||||
|
|
||||||
|
folderIDs := contractFolderIDsFromProcedures(procedures)
|
||||||
|
|
||||||
|
s.contractFoldersCacheMu.Lock()
|
||||||
|
s.contractFoldersCache = contractFoldersWithDocsCacheEntry{
|
||||||
|
expiresAt: time.Now().Add(contractFoldersWithDocsCacheTTL),
|
||||||
|
folderIDs: folderIDs,
|
||||||
|
valid: true,
|
||||||
|
}
|
||||||
|
s.contractFoldersCacheMu.Unlock()
|
||||||
|
|
||||||
|
return append([]string(nil), folderIDs...), nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
ids, ok := v.([]string)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("unexpected contract folder cache value type %T", v)
|
||||||
|
}
|
||||||
|
return append([]string(nil), ids...), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func contractFolderIDsFromProcedures(procedures []ai_summarizer.ProcedureDocumentsSummary) []string {
|
||||||
|
folderIDs := make([]string, 0, len(procedures))
|
||||||
|
seen := make(map[string]struct{}, len(procedures))
|
||||||
|
for _, proc := range procedures {
|
||||||
|
if proc.DocumentCount <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
id := strings.TrimSpace(proc.ContractFolderID)
|
||||||
|
if id == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[id]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[id] = struct{}{}
|
||||||
|
folderIDs = append(folderIDs, id)
|
||||||
|
}
|
||||||
|
return folderIDs
|
||||||
|
}
|
||||||
@@ -0,0 +1,241 @@
|
|||||||
|
package tender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"go.mongodb.org/mongo-driver/v2/bson"
|
||||||
|
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
)
|
||||||
|
|
||||||
|
type mockProcedureDocumentLister struct {
|
||||||
|
procedures []ai_summarizer.ProcedureDocumentsSummary
|
||||||
|
err error
|
||||||
|
calls atomic.Int32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mockProcedureDocumentLister) ListProceduresWithDocuments(context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
|
||||||
|
m.calls.Add(1)
|
||||||
|
if m.err != nil {
|
||||||
|
return nil, m.err
|
||||||
|
}
|
||||||
|
return m.procedures, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type stubProcedureDocumentStorage struct {
|
||||||
|
lister interface {
|
||||||
|
ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s stubProcedureDocumentStorage) GetSummaryFromStorage(context.Context, string, string) (string, error) {
|
||||||
|
return "", ai_summarizer.ErrObjectNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s stubProcedureDocumentStorage) GetDocumentSummariesFromStorage(context.Context, string, string) ([]ai_summarizer.DocumentSummary, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s stubProcedureDocumentStorage) GetTranslationFromStorage(context.Context, string, string, string) (ai_summarizer.StoredTranslation, error) {
|
||||||
|
return ai_summarizer.StoredTranslation{}, ai_summarizer.ErrTranslationNotReady
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s stubProcedureDocumentStorage) ListDocuments(context.Context, string, string) ([]ai_summarizer.StoredDocument, error) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s stubProcedureDocumentStorage) DownloadDocument(context.Context, string) (io.ReadCloser, error) {
|
||||||
|
return nil, ai_summarizer.ErrObjectNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s stubProcedureDocumentStorage) ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
|
||||||
|
return s.lister.ListProceduresWithDocuments(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBuildSearchFilterDocumentsScrapedUsesMinIOFolderIDs(t *testing.T) {
|
||||||
|
repo := &tenderRepository{}
|
||||||
|
filter := repo.buildSearchFilter(&SearchForm{
|
||||||
|
DocumentsScraped: true,
|
||||||
|
ContractFolderIDsWithDocuments: []string{"folder-a", "folder-b"},
|
||||||
|
})
|
||||||
|
|
||||||
|
in, ok := filter["contract_folder_id"].(bson.M)["$in"].([]string)
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected contract_folder_id $in filter, got %#v", filter)
|
||||||
|
}
|
||||||
|
if len(in) != 2 || in[0] != "folder-a" || in[1] != "folder-b" {
|
||||||
|
t.Fatalf("unexpected folder ids: %#v", in)
|
||||||
|
}
|
||||||
|
if filter["processing_metadata.documents_scraped"] != nil {
|
||||||
|
t.Fatalf("expected no stale mongo documents_scraped flag, got %#v", filter)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContractFolderIDsSearchClauseChunksLargeLists(t *testing.T) {
|
||||||
|
folderIDs := make([]string, maxContractFolderInClause+2)
|
||||||
|
for i := range folderIDs {
|
||||||
|
folderIDs[i] = fmt.Sprintf("folder-%d", i)
|
||||||
|
}
|
||||||
|
|
||||||
|
clause := contractFolderIDsSearchClause(folderIDs)
|
||||||
|
or, ok := clause["$or"].([]bson.M)
|
||||||
|
if !ok || len(or) != 2 {
|
||||||
|
t.Fatalf("expected chunked $or filter, got %#v", clause)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCachedContractFolderIDsWithDocumentsUsesCache(t *testing.T) {
|
||||||
|
lister := &mockProcedureDocumentLister{
|
||||||
|
procedures: []ai_summarizer.ProcedureDocumentsSummary{
|
||||||
|
{ContractFolderID: "folder-1", DocumentCount: 2},
|
||||||
|
{ContractFolderID: "folder-2", DocumentCount: 0},
|
||||||
|
{ContractFolderID: "folder-3", DocumentCount: 1},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &tenderService{
|
||||||
|
aiSummarizerStorage: stubProcedureDocumentStorage{lister: lister},
|
||||||
|
}
|
||||||
|
|
||||||
|
first, err := svc.cachedContractFolderIDsWithDocuments(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first lookup failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(first) != 2 || first[0] != "folder-1" || first[1] != "folder-3" {
|
||||||
|
t.Fatalf("unexpected folder ids: %#v", first)
|
||||||
|
}
|
||||||
|
if lister.calls.Load() != 1 {
|
||||||
|
t.Fatalf("expected one MinIO scan, got %d", lister.calls.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
second, err := svc.cachedContractFolderIDsWithDocuments(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second lookup failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(second) != 2 {
|
||||||
|
t.Fatalf("unexpected cached folder ids: %#v", second)
|
||||||
|
}
|
||||||
|
if lister.calls.Load() != 1 {
|
||||||
|
t.Fatalf("expected cache hit without extra MinIO scan, got %d calls", lister.calls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEnrichDocumentsScrapedFilterPropagatesMinIOError(t *testing.T) {
|
||||||
|
svc := &tenderService{
|
||||||
|
aiSummarizerStorage: stubProcedureDocumentStorage{
|
||||||
|
lister: &mockProcedureDocumentLister{
|
||||||
|
err: ai_summarizer.ErrMinIOUnavailable,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
logger: noopTenderTestLogger{},
|
||||||
|
}
|
||||||
|
|
||||||
|
form := &SearchForm{DocumentsScraped: true}
|
||||||
|
err := svc.enrichDocumentsScrapedFilter(context.Background(), form)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected MinIO error")
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ai_summarizer.ErrMinIOUnavailable) {
|
||||||
|
t.Fatalf("expected wrapped MinIO unavailable error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContractFolderIDsFromProceduresDedupesAndSkipsEmpty(t *testing.T) {
|
||||||
|
ids := contractFolderIDsFromProcedures([]ai_summarizer.ProcedureDocumentsSummary{
|
||||||
|
{ContractFolderID: "a", DocumentCount: 1},
|
||||||
|
{ContractFolderID: "a", DocumentCount: 2},
|
||||||
|
{ContractFolderID: " ", DocumentCount: 1},
|
||||||
|
{ContractFolderID: "b", DocumentCount: 0},
|
||||||
|
})
|
||||||
|
if len(ids) != 1 || ids[0] != "a" {
|
||||||
|
t.Fatalf("unexpected ids: %#v", ids)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type scanStartedProcedureLister struct {
|
||||||
|
mockProcedureDocumentLister
|
||||||
|
started chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *scanStartedProcedureLister) ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
|
||||||
|
if m.started != nil {
|
||||||
|
select {
|
||||||
|
case <-m.started:
|
||||||
|
default:
|
||||||
|
close(m.started)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return m.mockProcedureDocumentLister.ListProceduresWithDocuments(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCachedContractFolderIDsSurvivesCallerCancellation(t *testing.T) {
|
||||||
|
started := make(chan struct{})
|
||||||
|
lister := &scanStartedProcedureLister{
|
||||||
|
mockProcedureDocumentLister: mockProcedureDocumentLister{
|
||||||
|
procedures: []ai_summarizer.ProcedureDocumentsSummary{
|
||||||
|
{ContractFolderID: "folder-1", DocumentCount: 1},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
started: started,
|
||||||
|
}
|
||||||
|
svc := &tenderService{aiSummarizerStorage: stubProcedureDocumentStorage{lister: lister}}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
errCh := make(chan error, 1)
|
||||||
|
var ids []string
|
||||||
|
go func() {
|
||||||
|
var err error
|
||||||
|
ids, err = svc.cachedContractFolderIDsWithDocuments(ctx)
|
||||||
|
errCh <- err
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-started:
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("MinIO scan did not start")
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
if err := <-errCh; err != nil {
|
||||||
|
t.Fatalf("expected shared scan to finish despite caller cancel: %v", err)
|
||||||
|
}
|
||||||
|
if len(ids) != 1 || ids[0] != "folder-1" {
|
||||||
|
t.Fatalf("unexpected folder ids: %#v", ids)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestContractFoldersWithDocsCacheExpires(t *testing.T) {
|
||||||
|
lister := &mockProcedureDocumentLister{
|
||||||
|
procedures: []ai_summarizer.ProcedureDocumentsSummary{
|
||||||
|
{ContractFolderID: "folder-1", DocumentCount: 1},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
svc := &tenderService{aiSummarizerStorage: stubProcedureDocumentStorage{lister: lister}}
|
||||||
|
|
||||||
|
if _, err := svc.cachedContractFolderIDsWithDocuments(context.Background()); err != nil {
|
||||||
|
t.Fatalf("lookup failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
svc.contractFoldersCacheMu.Lock()
|
||||||
|
svc.contractFoldersCache.expiresAt = time.Now().Add(-time.Second)
|
||||||
|
svc.contractFoldersCacheMu.Unlock()
|
||||||
|
|
||||||
|
if _, err := svc.cachedContractFolderIDsWithDocuments(context.Background()); err != nil {
|
||||||
|
t.Fatalf("lookup after expiry failed: %v", err)
|
||||||
|
}
|
||||||
|
if lister.calls.Load() != 2 {
|
||||||
|
t.Fatalf("expected cache refresh after expiry, got %d calls", lister.calls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,7 +5,9 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"tm/pkg/mongo"
|
"tm/pkg/mongo"
|
||||||
|
"tm/ted"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DocumentSummary represents a summary of a tender document
|
// DocumentSummary represents a summary of a tender document
|
||||||
@@ -86,7 +88,6 @@ type Tender struct {
|
|||||||
Source TenderSource `bson:"source" json:"source"`
|
Source TenderSource `bson:"source" json:"source"`
|
||||||
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
|
SourceFileURL string `bson:"source_file_url" json:"source_file_url"`
|
||||||
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
|
SourceFileName string `bson:"source_file_name" json:"source_file_name"`
|
||||||
ContentXML string `bson:"content_xml,omitempty" json:"content_xml,omitempty"`
|
|
||||||
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
|
ProcessingMetadata ProcessingMetadata `bson:"processing_metadata" json:"processing_metadata"`
|
||||||
// AIOverallSummary optional denormalized overall summary for search/list (e.g. synced from AI pipeline storage).
|
// AIOverallSummary optional denormalized overall summary for search/list (e.g. synced from AI pipeline storage).
|
||||||
AIOverallSummary string `bson:"ai_overall_summary,omitempty" json:"ai_overall_summary,omitempty"`
|
AIOverallSummary string `bson:"ai_overall_summary,omitempty" json:"ai_overall_summary,omitempty"`
|
||||||
@@ -262,11 +263,38 @@ func (t *Tender) IsExpired() bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if deadline (in milliseconds) is in the past
|
|
||||||
now := time.Now().Unix()
|
now := time.Now().Unix()
|
||||||
return t.TenderDeadline < now
|
return t.TenderDeadline < now
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PublicationSubmissionDeadline returns the submission window end derived from publication_date.
|
||||||
|
// Stored submission_deadline is preferred; otherwise it is calculated as 48 working hours after publication.
|
||||||
|
func (t *Tender) PublicationSubmissionDeadline() int64 {
|
||||||
|
if t.SubmissionDeadline > 0 {
|
||||||
|
return t.SubmissionDeadline
|
||||||
|
}
|
||||||
|
if t.PublicationDate <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return ted.CalculateSubmissionDeadline(time.Unix(t.PublicationDate, 0)).Unix()
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasActivePublicationWindow reports whether the publication-based submission window is still open.
|
||||||
|
func (t *Tender) HasActivePublicationWindow(now int64) bool {
|
||||||
|
deadline := t.PublicationSubmissionDeadline()
|
||||||
|
return deadline > 0 && deadline > now
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsRecommendable reports whether a tender should appear in customer AI recommendation results.
|
||||||
|
// Recommendations use the publication-based submission window, not tender_deadline.
|
||||||
|
func (t *Tender) IsRecommendable(now int64) bool {
|
||||||
|
switch t.Status {
|
||||||
|
case TenderStatusExpired, TenderStatusCancelled, TenderStatusAwarded, TenderStatusClosed:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return t.HasActivePublicationWindow(now)
|
||||||
|
}
|
||||||
|
|
||||||
// GetTenderURL returns the TED tender URL
|
// GetTenderURL returns the TED tender URL
|
||||||
func (t *Tender) GetTenderURL() string {
|
func (t *Tender) GetTenderURL() string {
|
||||||
if t.TenderURL != "" {
|
if t.TenderURL != "" {
|
||||||
@@ -305,20 +333,74 @@ func (t *Tender) GetMainOrganization() *Organization {
|
|||||||
|
|
||||||
// HasEstimatedValue returns true if the tender has an estimated value
|
// HasEstimatedValue returns true if the tender has an estimated value
|
||||||
func (t *Tender) HasEstimatedValue() bool {
|
func (t *Tender) HasEstimatedValue() bool {
|
||||||
return t.EstimatedValue > 0
|
value, _ := t.ResolvedEstimatedValueAndCurrency()
|
||||||
|
return value > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// AggregateProcurementLotEstimatedValue sums lot-level estimated values when all non-empty
|
||||||
|
// lot currencies match. Conflicting currencies are not aggregated.
|
||||||
|
func AggregateProcurementLotEstimatedValue(lots []ProcurementLot) (value float64, currency string) {
|
||||||
|
var refCurrency string
|
||||||
|
for _, lot := range lots {
|
||||||
|
if lot.EstimatedValue <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
c := strings.TrimSpace(lot.Currency)
|
||||||
|
if c == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if refCurrency == "" {
|
||||||
|
refCurrency = c
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c != refCurrency {
|
||||||
|
return 0, ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, lot := range lots {
|
||||||
|
if lot.EstimatedValue <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
value += lot.EstimatedValue
|
||||||
|
}
|
||||||
|
if value <= 0 {
|
||||||
|
return 0, ""
|
||||||
|
}
|
||||||
|
return value, refCurrency
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolvedEstimatedValueAndCurrency returns the tender-level estimated value when set,
|
||||||
|
// otherwise aggregates procurement lot values for API and display use.
|
||||||
|
func (t *Tender) ResolvedEstimatedValueAndCurrency() (float64, string) {
|
||||||
|
if t == nil {
|
||||||
|
return 0, ""
|
||||||
|
}
|
||||||
|
if t.EstimatedValue > 0 {
|
||||||
|
return t.EstimatedValue, strings.TrimSpace(t.Currency)
|
||||||
|
}
|
||||||
|
value, currency := AggregateProcurementLotEstimatedValue(t.ProcurementLots)
|
||||||
|
if value > 0 {
|
||||||
|
if currency == "" {
|
||||||
|
currency = strings.TrimSpace(t.Currency)
|
||||||
|
}
|
||||||
|
return value, currency
|
||||||
|
}
|
||||||
|
return 0, strings.TrimSpace(t.Currency)
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetFormattedEstimatedValue returns formatted estimated value with currency
|
// GetFormattedEstimatedValue returns formatted estimated value with currency
|
||||||
func (t *Tender) GetFormattedEstimatedValue() string {
|
func (t *Tender) GetFormattedEstimatedValue() string {
|
||||||
if !t.HasEstimatedValue() {
|
value, currency := t.ResolvedEstimatedValueAndCurrency()
|
||||||
|
if value <= 0 {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
if t.Currency != "" {
|
if currency != "" {
|
||||||
return fmt.Sprintf("%.2f %s", t.EstimatedValue, t.Currency)
|
return fmt.Sprintf("%.2f %s", value, currency)
|
||||||
}
|
}
|
||||||
|
|
||||||
return fmt.Sprintf("%.2f", t.EstimatedValue)
|
return fmt.Sprintf("%.2f", value)
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateStatus updates the tender status and updated timestamp
|
// UpdateStatus updates the tender status and updated timestamp
|
||||||
|
|||||||
@@ -0,0 +1,197 @@
|
|||||||
|
package tender
|
||||||
|
|
||||||
|
import "testing"
|
||||||
|
|
||||||
|
func TestAggregateProcurementLotEstimatedValue(t *testing.T) {
|
||||||
|
value, currency := AggregateProcurementLotEstimatedValue([]ProcurementLot{
|
||||||
|
{EstimatedValue: 100000, Currency: "EUR"},
|
||||||
|
{EstimatedValue: 250000, Currency: "EUR"},
|
||||||
|
})
|
||||||
|
if value != 350000 {
|
||||||
|
t.Fatalf("expected total 350000, got %v", value)
|
||||||
|
}
|
||||||
|
if currency != "EUR" {
|
||||||
|
t.Fatalf("expected currency EUR, got %q", currency)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAggregateProcurementLotEstimatedValueRejectsMixedCurrencies(t *testing.T) {
|
||||||
|
value, currency := AggregateProcurementLotEstimatedValue([]ProcurementLot{
|
||||||
|
{EstimatedValue: 100000, Currency: "EUR"},
|
||||||
|
{EstimatedValue: 250000, Currency: "USD"},
|
||||||
|
})
|
||||||
|
if value != 0 || currency != "" {
|
||||||
|
t.Fatalf("expected no aggregation for mixed currencies, got value=%v currency=%q", value, currency)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAggregateProcurementLotEstimatedValueAllowsEmptyLotCurrency(t *testing.T) {
|
||||||
|
value, currency := AggregateProcurementLotEstimatedValue([]ProcurementLot{
|
||||||
|
{EstimatedValue: 100000, Currency: "EUR"},
|
||||||
|
{EstimatedValue: 250000},
|
||||||
|
})
|
||||||
|
if value != 350000 {
|
||||||
|
t.Fatalf("expected total 350000, got %v", value)
|
||||||
|
}
|
||||||
|
if currency != "EUR" {
|
||||||
|
t.Fatalf("expected currency EUR, got %q", currency)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvedEstimatedValueAndCurrencyUsesLotsWhenTopLevelMissing(t *testing.T) {
|
||||||
|
tender := &Tender{
|
||||||
|
Currency: "EUR",
|
||||||
|
ProcurementLots: []ProcurementLot{
|
||||||
|
{LotID: "LOT-0001", EstimatedValue: 500000, Currency: "EUR"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
value, currency := tender.ResolvedEstimatedValueAndCurrency()
|
||||||
|
if value != 500000 {
|
||||||
|
t.Fatalf("expected 500000, got %v", value)
|
||||||
|
}
|
||||||
|
if currency != "EUR" {
|
||||||
|
t.Fatalf("expected EUR, got %q", currency)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvedEstimatedValueAndCurrencyPrefersTopLevel(t *testing.T) {
|
||||||
|
tender := &Tender{
|
||||||
|
EstimatedValue: 900000,
|
||||||
|
Currency: "USD",
|
||||||
|
ProcurementLots: []ProcurementLot{
|
||||||
|
{EstimatedValue: 100000, Currency: "EUR"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
value, currency := tender.ResolvedEstimatedValueAndCurrency()
|
||||||
|
if value != 900000 {
|
||||||
|
t.Fatalf("expected 900000, got %v", value)
|
||||||
|
}
|
||||||
|
if currency != "USD" {
|
||||||
|
t.Fatalf("expected USD, got %q", currency)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTenderPublicationSubmissionDeadline(t *testing.T) {
|
||||||
|
pubDate := int64(1_700_000_000)
|
||||||
|
tender := Tender{PublicationDate: pubDate}
|
||||||
|
|
||||||
|
got := tender.PublicationSubmissionDeadline()
|
||||||
|
if got <= pubDate {
|
||||||
|
t.Fatalf("expected submission deadline after publication_date, got %d (publication_date=%d)", got, pubDate)
|
||||||
|
}
|
||||||
|
|
||||||
|
withStored := Tender{PublicationDate: pubDate, SubmissionDeadline: pubDate + 999}
|
||||||
|
if deadline := withStored.PublicationSubmissionDeadline(); deadline != pubDate+999 {
|
||||||
|
t.Fatalf("expected stored submission_deadline, got %d", deadline)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTenderHasActivePublicationWindow(t *testing.T) {
|
||||||
|
now := int64(1_700_000_000)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
tender Tender
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "future submission deadline",
|
||||||
|
tender: Tender{SubmissionDeadline: now + 3600},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "past submission deadline",
|
||||||
|
tender: Tender{SubmissionDeadline: now - 3600},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "future tender deadline only",
|
||||||
|
tender: Tender{TenderDeadline: now + 86400},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "publication date only with open window",
|
||||||
|
tender: Tender{PublicationDate: now - 3600},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no publication or submission deadline",
|
||||||
|
tender: Tender{},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := tt.tender.HasActivePublicationWindow(now); got != tt.want {
|
||||||
|
t.Fatalf("HasActivePublicationWindow() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTenderIsRecommendable(t *testing.T) {
|
||||||
|
now := int64(1_700_000_000)
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
tender Tender
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "active with future submission deadline",
|
||||||
|
tender: Tender{Status: TenderStatusActive, SubmissionDeadline: now + 3600},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "active with past submission deadline",
|
||||||
|
tender: Tender{Status: TenderStatusActive, SubmissionDeadline: now - 3600},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "active with future tender deadline but past submission deadline",
|
||||||
|
tender: Tender{Status: TenderStatusActive, TenderDeadline: now + 86400, SubmissionDeadline: now - 3600},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "active with future tender deadline only",
|
||||||
|
tender: Tender{Status: TenderStatusActive, TenderDeadline: now + 86400},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "active with submission deadline at now",
|
||||||
|
tender: Tender{Status: TenderStatusActive, SubmissionDeadline: now},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "active without publication or submission deadline",
|
||||||
|
tender: Tender{Status: TenderStatusActive},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "active with publication date only and open submission window",
|
||||||
|
tender: Tender{Status: TenderStatusActive, PublicationDate: now - 3600},
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "expired status with future submission deadline",
|
||||||
|
tender: Tender{Status: TenderStatusExpired, SubmissionDeadline: now + 3600},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "awarded with future submission deadline",
|
||||||
|
tender: Tender{Status: TenderStatusAwarded, SubmissionDeadline: now + 3600},
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
if got := tt.tender.IsRecommendable(now); got != tt.want {
|
||||||
|
t.Fatalf("IsRecommendable() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -9,5 +9,6 @@ var (
|
|||||||
ErrTenderNotFound = errors.New("tender: not found")
|
ErrTenderNotFound = errors.New("tender: not found")
|
||||||
ErrTenderMissingNoticeID = errors.New("tender: missing notice publication id")
|
ErrTenderMissingNoticeID = errors.New("tender: missing notice publication id")
|
||||||
ErrTenderMissingContractFolderID = errors.New("tender: missing contract folder id")
|
ErrTenderMissingContractFolderID = errors.New("tender: missing contract folder id")
|
||||||
|
ErrTenderDocumentsNotScraped = errors.New("tender: documents not scraped")
|
||||||
ErrAINotConfigured = errors.New("tender: ai service not configured")
|
ErrAINotConfigured = errors.New("tender: ai service not configured")
|
||||||
)
|
)
|
||||||
|
|||||||
+88
-3
@@ -32,6 +32,7 @@ type SearchForm struct {
|
|||||||
CountryCodes []string `query:"country_codes" valid:"optional"`
|
CountryCodes []string `query:"country_codes" valid:"optional"`
|
||||||
RegionCodes []string `query:"region_codes" valid:"optional"`
|
RegionCodes []string `query:"region_codes" valid:"optional"`
|
||||||
CompanyID *string `query:"company_id" valid:"optional"`
|
CompanyID *string `query:"company_id" valid:"optional"`
|
||||||
|
CompanyIDs []string `query:"-" valid:"optional"`
|
||||||
CustomerID *string `query:"customer_id" valid:"optional"`
|
CustomerID *string `query:"customer_id" valid:"optional"`
|
||||||
Status []string `query:"status" valid:"optional"`
|
Status []string `query:"status" valid:"optional"`
|
||||||
MainClassification *string `query:"main_classification" valid:"optional"`
|
MainClassification *string `query:"main_classification" valid:"optional"`
|
||||||
@@ -58,10 +59,88 @@ type SearchForm struct {
|
|||||||
SubmissionDateTo *int64 `query:"submission_date_to" valid:"optional"`
|
SubmissionDateTo *int64 `query:"submission_date_to" valid:"optional"`
|
||||||
Source []string `query:"source" valid:"optional"`
|
Source []string `query:"source" valid:"optional"`
|
||||||
BuyerOrganizationID *string `query:"buyer_organization_id" valid:"optional"`
|
BuyerOrganizationID *string `query:"buyer_organization_id" valid:"optional"`
|
||||||
|
NoticePublicationID *string `query:"notice_publication_id" valid:"optional"`
|
||||||
Languages []string `query:"languages" valid:"optional"`
|
Languages []string `query:"languages" valid:"optional"`
|
||||||
OnlyActiveDeadlines bool `query:"-" valid:"optional"`
|
OnlyActiveDeadlines bool `query:"only_active_deadlines" valid:"optional"`
|
||||||
Language *string `query:"lang" valid:"optional"`
|
Language *string `query:"lang" valid:"optional"`
|
||||||
DocumentsScraped bool `query:"documents_scraped" valid:"optional"`
|
DocumentsScraped bool `query:"documents_scraped" valid:"optional"`
|
||||||
|
// ContractFolderIDsWithDocuments is populated by the service from a cached MinIO scan when DocumentsScraped is true.
|
||||||
|
ContractFolderIDsWithDocuments []string `query:"-" valid:"optional"`
|
||||||
|
// ExcludeRejectedTenders hides company-rejected tenders from recommendation results.
|
||||||
|
ExcludeRejectedTenders bool `query:"-" valid:"optional"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasListFilters reports whether the form applies any MongoDB list filters.
|
||||||
|
func (f *SearchForm) HasListFilters() bool {
|
||||||
|
if f == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if f.DocumentsScraped || f.OnlyActiveDeadlines || f.ExcludeRejectedTenders {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.Search != nil && strings.TrimSpace(*f.Search) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.Title != nil && strings.TrimSpace(*f.Title) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.Description != nil && strings.TrimSpace(*f.Description) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.NoticeType != nil && strings.TrimSpace(*f.NoticeType) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if len(f.NoticeTypes) > 0 || len(f.FormTypes) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.ProcurementType != nil && strings.TrimSpace(*f.ProcurementType) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.Country != nil && strings.TrimSpace(*f.Country) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.CountryCode != nil && strings.TrimSpace(*f.CountryCode) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if len(f.CountryCodes) > 0 || len(f.RegionCodes) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.MainClassification != nil && strings.TrimSpace(*f.MainClassification) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if len(f.Classifications) > 0 || len(f.CpvCodes) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.MinEstimatedValue != nil || f.MaxEstimatedValue != nil || f.Currency != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.CreatedAt != nil || f.CreatedAtFrom != nil || f.CreatedAtTo != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.TenderDeadline != nil || f.TenderDeadlineFrom != nil || f.TenderDeadlineTo != nil ||
|
||||||
|
f.DeadlineFrom != nil || f.DeadlineTo != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.PublicationDate != nil || f.PublicationDateFrom != nil || f.PublicationDateTo != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.SubmissionDeadline != nil || f.SubmissionDateAliasFrom != nil || f.SubmissionDateAliasTo != nil ||
|
||||||
|
f.SubmissionDateFrom != nil || f.SubmissionDateTo != nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if len(f.Status) > 0 || len(f.Source) > 0 || len(f.Languages) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.BuyerOrganizationID != nil && strings.TrimSpace(*f.BuyerOrganizationID) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if f.NoticePublicationID != nil && strings.TrimSpace(*f.NoticePublicationID) != "" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if len(f.ContractFolderIDsWithDocuments) > 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// SearchResponse represents the response for listing tenders
|
// SearchResponse represents the response for listing tenders
|
||||||
@@ -230,6 +309,10 @@ type TenderResponse struct {
|
|||||||
|
|
||||||
OverallSummary string `json:"overall_summary,omitempty"`
|
OverallSummary string `json:"overall_summary,omitempty"`
|
||||||
Language string `json:"language,omitempty"`
|
Language string `json:"language,omitempty"`
|
||||||
|
|
||||||
|
Rank int `json:"rank,omitempty"`
|
||||||
|
Analysis string `json:"analysis,omitempty"`
|
||||||
|
ProcedureRef string `json:"procedure_ref,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TenderDocumentResponse represents a scraped tender document available for download.
|
// TenderDocumentResponse represents a scraped tender document available for download.
|
||||||
@@ -274,6 +357,8 @@ func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse {
|
|||||||
documentURL = strings.TrimSpace(t.TenderURL)
|
documentURL = strings.TrimSpace(t.TenderURL)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
estimatedValue, currency := t.ResolvedEstimatedValueAndCurrency()
|
||||||
|
|
||||||
response := &TenderResponse{
|
response := &TenderResponse{
|
||||||
ID: t.ID.Hex(),
|
ID: t.ID.Hex(),
|
||||||
NoticePublicationID: t.NoticePublicationID,
|
NoticePublicationID: t.NoticePublicationID,
|
||||||
@@ -304,8 +389,8 @@ func (t *Tender) ToResponseWithLanguage(language string) *TenderResponse {
|
|||||||
MainClassificationDescription: mainDesc,
|
MainClassificationDescription: mainDesc,
|
||||||
MainClassificationDisplay: classificationDisplay(mainCode, mainDesc),
|
MainClassificationDisplay: classificationDisplay(mainCode, mainDesc),
|
||||||
AdditionalClassifications: append([]string(nil), t.AdditionalClassifications...),
|
AdditionalClassifications: append([]string(nil), t.AdditionalClassifications...),
|
||||||
EstimatedValue: t.EstimatedValue,
|
EstimatedValue: estimatedValue,
|
||||||
Currency: strings.TrimSpace(t.Currency),
|
Currency: currency,
|
||||||
EstimatedValueVATBasis: vatBasisUnspecified,
|
EstimatedValueVATBasis: vatBasisUnspecified,
|
||||||
|
|
||||||
Duration: t.Duration,
|
Duration: t.Duration,
|
||||||
|
|||||||
+88
-48
@@ -7,6 +7,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
"tm/internal/company"
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
orm "tm/pkg/mongo"
|
orm "tm/pkg/mongo"
|
||||||
"tm/pkg/response"
|
"tm/pkg/response"
|
||||||
@@ -158,10 +159,12 @@ func (h *TenderHandler) Delete(c echo.Context) error {
|
|||||||
// @Param submission_date_to query number false "Submission deadline to (Unix timestamp, seconds)"
|
// @Param submission_date_to query number false "Submission deadline to (Unix timestamp, seconds)"
|
||||||
// @Param languages query []string false "Filter by languages (comma-separated)"
|
// @Param languages query []string false "Filter by languages (comma-separated)"
|
||||||
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
|
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
|
||||||
// @Param documents_scraped query bool false "When true, only tenders with scraped documents"
|
// @Param notice_publication_id query string false "Filter by TED notice publication ID"
|
||||||
|
// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO (cached scan)"
|
||||||
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
||||||
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
|
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
|
||||||
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
|
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
|
||||||
|
// @Param include_total query bool false "When true, include meta.total and meta.pages (slower on large collections)"
|
||||||
// @Param sort_by query string false "Sort by field"
|
// @Param sort_by query string false "Sort by field"
|
||||||
// @Param sort_order query string false "Sort order (asc or desc)"
|
// @Param sort_order query string false "Sort order (asc or desc)"
|
||||||
// @Success 200 {object} response.APIResponse{data=map[string]interface{}}
|
// @Success 200 {object} response.APIResponse{data=map[string]interface{}}
|
||||||
@@ -196,6 +199,44 @@ func (h *TenderHandler) Search(c echo.Context) error {
|
|||||||
return response.SuccessWithMeta(c, tenders, tenders.Metadata, "Tenders retrieved successfully")
|
return response.SuccessWithMeta(c, tenders, tenders.Metadata, "Tenders retrieved successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AdminRecommendTenders returns AI-ranked tender recommendations for a company (Web Panel)
|
||||||
|
// @Summary Get recommended tenders for a company
|
||||||
|
// @Description Retrieve AI-ranked tenders with full tender details for the given company. Unlike the customer API, active-only filters are not applied unless requested via query parameters.
|
||||||
|
// @Tags Admin-Companies
|
||||||
|
// @Produce json
|
||||||
|
// @Param id path string true "Company ID"
|
||||||
|
// @Param limit query int false "Number of items per page (default: 10, max: 100)"
|
||||||
|
// @Param offset query int false "Number of items to skip (default: 0)"
|
||||||
|
// @Param lang query string false "Response language code (defaults to en)"
|
||||||
|
// @Param status query []string false "Filter by status (comma-separated)"
|
||||||
|
// @Param only_active_deadlines query bool false "When true, exclude tenders past their deadline"
|
||||||
|
// @Success 200 {object} response.APIResponse{data=SearchResponse}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 404 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Security BearerAuth
|
||||||
|
// @Router /admin/v1/companies/{id}/recommended-tenders [get]
|
||||||
|
func (h *TenderHandler) AdminRecommendTenders(c echo.Context) error {
|
||||||
|
companyID := strings.TrimSpace(c.Param("id"))
|
||||||
|
if companyID == "" {
|
||||||
|
return response.BadRequest(c, "Company ID is required", "")
|
||||||
|
}
|
||||||
|
|
||||||
|
form, err := response.Parse[SearchForm](c)
|
||||||
|
if err != nil {
|
||||||
|
return response.BadRequest(c, "Invalid request format", "")
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := govalidator.ValidateStruct(form); err != nil {
|
||||||
|
return response.ValidationError(c, err.Error(), "")
|
||||||
|
}
|
||||||
|
|
||||||
|
form.CompanyID = &companyID
|
||||||
|
|
||||||
|
return h.serveRecommendTenders(c, form)
|
||||||
|
}
|
||||||
|
|
||||||
// *****************************************************
|
// *****************************************************
|
||||||
// * PUBLIC HANDLERS *
|
// * PUBLIC HANDLERS *
|
||||||
// *****************************************************
|
// *****************************************************
|
||||||
@@ -229,10 +270,12 @@ func (h *TenderHandler) Search(c echo.Context) error {
|
|||||||
// @Param submission_date_to query number false "Submission deadline to (Unix seconds)"
|
// @Param submission_date_to query number false "Submission deadline to (Unix seconds)"
|
||||||
// @Param languages query []string false "Filter by languages (comma-separated)"
|
// @Param languages query []string false "Filter by languages (comma-separated)"
|
||||||
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
|
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
|
||||||
// @Param documents_scraped query bool false "When true, only tenders with scraped documents"
|
// @Param notice_publication_id query string false "Filter by TED notice publication ID"
|
||||||
|
// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO (cached scan)"
|
||||||
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
||||||
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
|
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
|
||||||
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
|
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
|
||||||
|
// @Param include_total query bool false "When true, include meta.total and meta.pages (slower on large collections)"
|
||||||
// @Param sort_by query string false "Sort by field"
|
// @Param sort_by query string false "Sort by field"
|
||||||
// @Param sort_order query string false "Sort order (asc or desc)"
|
// @Param sort_order query string false "Sort order (asc or desc)"
|
||||||
// @Success 200 {object} response.APIResponse{data=SearchResponse}
|
// @Success 200 {object} response.APIResponse{data=SearchResponse}
|
||||||
@@ -283,43 +326,18 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error {
|
|||||||
return response.SuccessWithMeta(c, result, result.Metadata, "Tenders retrieved successfully")
|
return response.SuccessWithMeta(c, result, result.Metadata, "Tenders retrieved successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPublicTenders retrieves public tenders for mobile app
|
// RecommendTenders returns AI-ranked tender recommendations for the authenticated customer's company
|
||||||
// @Summary Get public tenders
|
// @Summary Get recommended tenders
|
||||||
// @Description Retrieve active tenders for mobile application users with automatic company-based matching from customer profile
|
// @Description Retrieve AI-ranked tenders with full tender details. Only active tenders whose publication-based submission window has not passed are returned (submission_deadline, or 48 working hours after publication_date). Company-rejected tenders are excluded. AI tender_id values use PROC_{contract_folder_id}/{notice_publication_id}; each item includes rank, analysis, procedure_ref, and the standard tender fields.
|
||||||
// @Tags Tenders
|
// @Tags Tenders
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param q query string false "Keyword search across title, description, AI document summaries, optional stored overall summary"
|
// @Param limit query int false "Number of items per page (default: 10, max: 100)"
|
||||||
// @Param notice_type query string false "Filter by TED notice type code (notice_type_code)"
|
// @Param offset query int false "Number of items to skip (default: 0)"
|
||||||
// @Param notice_types query []string false "Multiple notice type codes"
|
// @Param lang query string false "Response language code (defaults to en)"
|
||||||
// @Param form_types query []string false "TED form types: competition, result, planning, contract-award, prior-information, etc."
|
|
||||||
// @Param procurement_type query string false "Filter by procurement type code"
|
|
||||||
// @Param country query string false "Single ISO country code"
|
|
||||||
// @Param country_codes query []string false "Filter by country codes"
|
|
||||||
// @Param status query []string false "Filter by status (comma-separated)"
|
|
||||||
// @Param source query []string false "Filter by source (comma-separated)"
|
|
||||||
// @Param classifications query []string false "CPV / classification codes"
|
|
||||||
// @Param cpv_codes query []string false "One or more CPV codes"
|
|
||||||
// @Param min_estimated_value query number false "Minimum estimated value"
|
|
||||||
// @Param max_estimated_value query number false "Maximum estimated value"
|
|
||||||
// @Param currency query string false "Filter by currency"
|
|
||||||
// @Param created_at_from query number false "Created at from (Unix seconds)"
|
|
||||||
// @Param created_at_to query number false "Created at to (Unix seconds)"
|
|
||||||
// @Param deadline_from query number false "Tender deadline from (Unix seconds)"
|
|
||||||
// @Param deadline_to query number false "Tender deadline to (Unix seconds)"
|
|
||||||
// @Param publication_date_from query number false "Publication date from (Unix seconds)"
|
|
||||||
// @Param publication_date_to query number false "Publication date to (Unix seconds)"
|
|
||||||
// @Param submission_date_from query number false "Submission deadline from (Unix seconds)"
|
|
||||||
// @Param submission_date_to query number false "Submission deadline to (Unix seconds)"
|
|
||||||
// @Param languages query []string false "Filter by languages (comma-separated)"
|
|
||||||
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
|
|
||||||
// @Param documents_scraped query bool false "When true, only tenders with scraped documents"
|
|
||||||
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
|
||||||
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
|
|
||||||
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
|
|
||||||
// @Param sort_by query string false "Sort by field"
|
|
||||||
// @Param sort_order query string false "Sort order (asc or desc)"
|
|
||||||
// @Success 200 {object} response.APIResponse{data=SearchResponse}
|
// @Success 200 {object} response.APIResponse{data=SearchResponse}
|
||||||
// @Failure 400 {object} response.APIResponse
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 404 {object} response.APIResponse
|
||||||
|
// @Failure 503 {object} response.APIResponse
|
||||||
// @Failure 500 {object} response.APIResponse
|
// @Failure 500 {object} response.APIResponse
|
||||||
// @Security BearerAuth
|
// @Security BearerAuth
|
||||||
// @Router /api/v1/tenders/recommend [get]
|
// @Router /api/v1/tenders/recommend [get]
|
||||||
@@ -336,19 +354,31 @@ func (h *TenderHandler) RecommendTenders(c echo.Context) error {
|
|||||||
|
|
||||||
form.Status = []string{string(TenderStatusActive)}
|
form.Status = []string{string(TenderStatusActive)}
|
||||||
form.OnlyActiveDeadlines = true
|
form.OnlyActiveDeadlines = true
|
||||||
|
form.ExcludeRejectedTenders = true
|
||||||
|
|
||||||
var companyID *string
|
if customerIDStr, ok := c.Get("customer_id").(string); ok && customerIDStr != "" {
|
||||||
if customerCompanyID, ok := c.Get("company_id").(string); ok {
|
form.CustomerID = &customerIDStr
|
||||||
companyID = &customerCompanyID
|
|
||||||
}
|
}
|
||||||
form.CompanyID = companyID
|
|
||||||
|
|
||||||
var customerID *string
|
requestedCompanyID := ""
|
||||||
if customerIDStr, ok := c.Get("customer_id").(string); ok {
|
if form.CompanyID != nil {
|
||||||
customerID = &customerIDStr
|
requestedCompanyID = strings.TrimSpace(*form.CompanyID)
|
||||||
}
|
}
|
||||||
form.CustomerID = customerID
|
|
||||||
|
|
||||||
|
if requestedCompanyID != "" {
|
||||||
|
if customerCompanyID, ok := c.Get("company_id").(string); ok && customerCompanyID != "" {
|
||||||
|
form.CompanyID = &customerCompanyID
|
||||||
|
}
|
||||||
|
} else if customerCompanyIDs, ok := c.Get("company_ids").([]string); ok && len(customerCompanyIDs) > 0 {
|
||||||
|
form.CompanyIDs = append([]string(nil), customerCompanyIDs...)
|
||||||
|
} else if customerCompanyID, ok := c.Get("company_id").(string); ok && customerCompanyID != "" {
|
||||||
|
form.CompanyID = &customerCompanyID
|
||||||
|
}
|
||||||
|
|
||||||
|
return h.serveRecommendTenders(c, form)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *TenderHandler) serveRecommendTenders(c echo.Context, form *SearchForm) error {
|
||||||
pagination, err := response.NewPagination(c)
|
pagination, err := response.NewPagination(c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return response.PaginationBadRequest(c, err)
|
return response.PaginationBadRequest(c, err)
|
||||||
@@ -359,10 +389,19 @@ func (h *TenderHandler) RecommendTenders(c echo.Context) error {
|
|||||||
if response.IsListPaginationError(err) {
|
if response.IsListPaginationError(err) {
|
||||||
return response.PaginationBadRequest(c, err)
|
return response.PaginationBadRequest(c, err)
|
||||||
}
|
}
|
||||||
return response.InternalServerError(c, "Failed to retrieve tenders")
|
if errors.Is(err, company.ErrAIRecommendationNotConfigured) {
|
||||||
|
return response.ServiceUnavailable(c, "AI recommendation service is not configured")
|
||||||
|
}
|
||||||
|
if err.Error() == "company ID is required" {
|
||||||
|
return response.BadRequest(c, "Company ID is required", "")
|
||||||
|
}
|
||||||
|
if err.Error() == "company not found" {
|
||||||
|
return response.NotFound(c, "Company not found")
|
||||||
|
}
|
||||||
|
return response.InternalServerError(c, "Failed to retrieve recommended tenders")
|
||||||
}
|
}
|
||||||
|
|
||||||
return response.SuccessWithMeta(c, result, result.Metadata, "Tenders retrieved successfully")
|
return response.SuccessWithMeta(c, result, result.Metadata, "Recommended tenders retrieved successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPublicTenderDetails retrieves public tender details for mobile app
|
// GetPublicTenderDetails retrieves public tender details for mobile app
|
||||||
@@ -534,7 +573,7 @@ func (h *TenderHandler) streamDocumentDownload(c echo.Context, tenderID, filenam
|
|||||||
|
|
||||||
// GetAISummary retrieves the AI-generated overall summary for a tender
|
// GetAISummary retrieves the AI-generated overall summary for a tender
|
||||||
// @Summary Get AI-generated tender summary
|
// @Summary Get AI-generated tender summary
|
||||||
// @Description Retrieve the AI-generated overall summary for a tender from the AI pipeline storage
|
// @Description Retrieve the AI-generated overall summary for a tender. When documents are scraped but no summary exists yet, on-demand summarization is triggered automatically.
|
||||||
// @Tags Admin-Tenders
|
// @Tags Admin-Tenders
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param id path string true "Tender ID"
|
// @Param id path string true "Tender ID"
|
||||||
@@ -634,6 +673,8 @@ func mapAITriggerHTTPError(c echo.Context, err error, action string) error {
|
|||||||
return response.NotFound(c, "Tender not found")
|
return response.NotFound(c, "Tender not found")
|
||||||
case errors.Is(err, ErrTenderMissingNoticeID), errors.Is(err, ErrTenderMissingContractFolderID):
|
case errors.Is(err, ErrTenderMissingNoticeID), errors.Is(err, ErrTenderMissingContractFolderID):
|
||||||
return response.BadRequest(c, "Tender not eligible", "Tender is missing required identifiers for AI processing")
|
return response.BadRequest(c, "Tender not eligible", "Tender is missing required identifiers for AI processing")
|
||||||
|
case errors.Is(err, ErrTenderDocumentsNotScraped):
|
||||||
|
return response.BadRequest(c, "Tender not eligible", "Tender documents have not been scraped yet")
|
||||||
case errors.Is(err, ErrAINotConfigured):
|
case errors.Is(err, ErrAINotConfigured):
|
||||||
return response.InternalServerError(c, "AI service is not available")
|
return response.InternalServerError(c, "AI service is not available")
|
||||||
default:
|
default:
|
||||||
@@ -680,7 +721,7 @@ func (h *TenderHandler) TriggerAITranslate(c echo.Context) error {
|
|||||||
|
|
||||||
// GetPublicAISummary retrieves the AI-generated overall summary for a tender (public endpoint)
|
// GetPublicAISummary retrieves the AI-generated overall summary for a tender (public endpoint)
|
||||||
// @Summary Get AI-generated tender summary (public)
|
// @Summary Get AI-generated tender summary (public)
|
||||||
// @Description Retrieve the AI-generated overall summary for a tender from the AI pipeline storage
|
// @Description Retrieve the AI-generated overall summary for a tender. When documents are scraped but no summary exists yet, on-demand summarization is triggered automatically.
|
||||||
// @Tags Tenders
|
// @Tags Tenders
|
||||||
// @Produce json
|
// @Produce json
|
||||||
// @Param id path string true "Tender ID"
|
// @Param id path string true "Tender ID"
|
||||||
@@ -796,4 +837,3 @@ func (h *TenderHandler) GetAllAISummaries(c echo.Context) error {
|
|||||||
|
|
||||||
return response.Success(c, summaries, "AI summaries retrieved successfully")
|
return response.Success(c, summaries, "AI summaries retrieved successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,158 @@
|
|||||||
|
package tender
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"golang.org/x/sync/errgroup"
|
||||||
|
)
|
||||||
|
|
||||||
|
// CompanyRejectedTenderLister returns tender IDs the company has rejected.
|
||||||
|
type CompanyRejectedTenderLister interface {
|
||||||
|
ListRejectedTenderIDsByCompanyID(ctx context.Context, companyID string) ([]string, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeRecommendationCompanyScope(companyIDs []string) []string {
|
||||||
|
normalized := make([]string, 0, len(companyIDs))
|
||||||
|
seen := make(map[string]struct{}, len(companyIDs))
|
||||||
|
for _, companyID := range companyIDs {
|
||||||
|
clean := strings.TrimSpace(companyID)
|
||||||
|
if clean == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := seen[clean]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[clean] = struct{}{}
|
||||||
|
normalized = append(normalized, clean)
|
||||||
|
}
|
||||||
|
return normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *tenderService) buildRejectedTenderIDSet(ctx context.Context, companyID string) (map[string]struct{}, error) {
|
||||||
|
excluded := make(map[string]struct{})
|
||||||
|
|
||||||
|
if s.rejectedTenderLister == nil {
|
||||||
|
return excluded, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ids, err := s.rejectedTenderLister.ListRejectedTenderIDsByCompanyID(ctx, companyID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to list rejected tender exclusions: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
addTenderIDsToSet(excluded, ids)
|
||||||
|
return excluded, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *tenderService) buildRejectedTenderIDSetsForCompanies(ctx context.Context, companyIDs []string) (map[string]map[string]struct{}, error) {
|
||||||
|
excludedByCompany := make(map[string]map[string]struct{})
|
||||||
|
if s.rejectedTenderLister == nil {
|
||||||
|
return excludedByCompany, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized := normalizeRecommendationCompanyScope(companyIDs)
|
||||||
|
if len(normalized) == 0 {
|
||||||
|
return excludedByCompany, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
results := make([][]string, len(normalized))
|
||||||
|
group, groupCtx := errgroup.WithContext(ctx)
|
||||||
|
for i, companyID := range normalized {
|
||||||
|
i := i
|
||||||
|
companyID := companyID
|
||||||
|
group.Go(func() error {
|
||||||
|
ids, err := s.rejectedTenderLister.ListRejectedTenderIDsByCompanyID(groupCtx, companyID)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to list rejected tender exclusions: %w", err)
|
||||||
|
}
|
||||||
|
results[i] = ids
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if err := group.Wait(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, companyID := range normalized {
|
||||||
|
excluded := make(map[string]struct{})
|
||||||
|
addTenderIDsToSet(excluded, results[i])
|
||||||
|
excludedByCompany[companyID] = excluded
|
||||||
|
}
|
||||||
|
|
||||||
|
return excludedByCompany, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func addTenderIDsToSet(set map[string]struct{}, ids []string) {
|
||||||
|
for _, id := range ids {
|
||||||
|
id = strings.TrimSpace(id)
|
||||||
|
if id == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
set[id] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// markResolvedRecommendedTenderSeen records a resolved tender and reports whether it was already seen.
|
||||||
|
// Recommendations may reference the same tender via different AI refs (procedure ref vs Mongo ID,
|
||||||
|
// or multiple notice publication IDs for one merged tender).
|
||||||
|
func markResolvedRecommendedTenderSeen(seen map[string]struct{}, tender Tender, tenderRef string) bool {
|
||||||
|
if seen == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
key := strings.TrimSpace(tender.GetID())
|
||||||
|
if key == "" {
|
||||||
|
key = strings.TrimSpace(tenderRef)
|
||||||
|
}
|
||||||
|
if key == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if _, exists := seen[key]; exists {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isRecommendedTenderExcluded(tender Tender, tenderRef string, excluded map[string]struct{}) bool {
|
||||||
|
if len(excluded) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, ok := excluded[tender.GetID()]; ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
tenderRef = strings.TrimSpace(tenderRef)
|
||||||
|
if tenderRef != "" {
|
||||||
|
if _, ok := excluded[tenderRef]; ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
procRef := FormatAIProcedureRef(tender.ContractFolderID, tender.NoticePublicationID)
|
||||||
|
if procRef != "" {
|
||||||
|
if _, ok := excluded[procRef]; ok {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isRecommendedTenderExcludedForAllCompanies(tender Tender, tenderRef string, excludedByCompany map[string]map[string]struct{}, companyIDs []string) bool {
|
||||||
|
normalized := normalizeRecommendationCompanyScope(companyIDs)
|
||||||
|
if len(normalized) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, companyID := range normalized {
|
||||||
|
if !isRecommendedTenderExcluded(tender, tenderRef, excludedByCompany[companyID]) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return true
|
||||||
|
}
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user