List and download tender documents - worker auto translation to english job

This commit is contained in:
Mazyar
2026-05-10 02:14:53 +03:30
parent 567e20b19c
commit e136f0eaa7
11 changed files with 652 additions and 2 deletions
+4
View File
@@ -109,6 +109,8 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
tendersGP.GET("/:id", tenderHandler.Get) tendersGP.GET("/:id", tenderHandler.Get)
tendersGP.PUT("/:id", tenderHandler.Update) tendersGP.PUT("/:id", tenderHandler.Update)
tendersGP.DELETE("/:id", tenderHandler.Delete) tendersGP.DELETE("/:id", tenderHandler.Delete)
tendersGP.GET("/:id/documents", tenderHandler.GetDocuments)
tendersGP.GET("/:id/documents/download", tenderHandler.DownloadDocuments)
tendersGP.GET("/:id/ai-summary", tenderHandler.GetAISummary) tendersGP.GET("/:id/ai-summary", tenderHandler.GetAISummary)
tendersGP.POST("/:id/ai-summarize", tenderHandler.TriggerAISummarize) tendersGP.POST("/:id/ai-summarize", tenderHandler.TriggerAISummarize)
tendersGP.POST("/:id/ai-translate", tenderHandler.TriggerAITranslate) tendersGP.POST("/:id/ai-translate", tenderHandler.TriggerAITranslate)
@@ -236,6 +238,8 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende
tendersGP.GET("", tenderHandler.GetPublicTenders) tendersGP.GET("", tenderHandler.GetPublicTenders)
tendersGP.GET("/recommend", tenderHandler.RecommendTenders) tendersGP.GET("/recommend", tenderHandler.RecommendTenders)
tendersGP.GET("/details/:id", tenderHandler.GetPublicTenderDetails) tendersGP.GET("/details/:id", tenderHandler.GetPublicTenderDetails)
tendersGP.GET("/:id/documents", tenderHandler.GetDocuments)
tendersGP.GET("/:id/documents/download", tenderHandler.DownloadDocuments)
tendersGP.GET("/:id/document-summaries", tenderHandler.GetDocumentSummaries) tendersGP.GET("/:id/document-summaries", tenderHandler.GetDocumentSummaries)
tendersGP.GET("/:id/ai-summary", tenderHandler.GetPublicAISummary) tendersGP.GET("/:id/ai-summary", tenderHandler.GetPublicAISummary)
tendersGP.POST("/:id/ai-translate", tenderHandler.TriggerPublicAITranslate) tendersGP.POST("/:id/ai-translate", tenderHandler.TriggerPublicAITranslate)
+59 -1
View File
@@ -6,6 +6,7 @@ import (
"tm/cmd/worker/workers" "tm/cmd/worker/workers"
"tm/internal/notice" "tm/internal/notice"
"tm/internal/tender" "tm/internal/tender"
ai_summarizer "tm/pkg/ai_summarizer"
"tm/pkg/config" "tm/pkg/config"
"tm/pkg/glm" "tm/pkg/glm"
"tm/pkg/logger" "tm/pkg/logger"
@@ -77,7 +78,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
} }
// Init Worker // Init Worker
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, minioService *minio.Service) *schedule.CronScheduler { func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client) *schedule.CronScheduler {
// Debug: Log worker config // Debug: Log worker config
appLogger.Info("Worker configuration", map[string]interface{}{ appLogger.Info("Worker configuration", map[string]interface{}{
"worker_interval": config.Worker.Interval, "worker_interval": config.Worker.Interval,
@@ -186,6 +187,33 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
Expr: workerInterval, Expr: workerInterval,
}) })
// Initialize translation backfill worker for default language
if aiClient != nil {
targetLanguage := config.AISummarizer.DefaultLanguage
scheduler.AddJob(schedule.Job{
Name: "Tender Translation Worker Job",
Func: func() {
worker := workers.NewTranslationWorker(
mongoManager,
appLogger,
tenderRepo,
aiClient,
targetLanguage,
config.Worker.TranslationBatchSize,
)
worker.Run()
},
Expr: config.Worker.TranslationInterval,
})
appLogger.Info("Scheduled tender translation worker", map[string]interface{}{
"target_language": targetLanguage,
"interval": config.Worker.TranslationInterval,
"batch_size": config.Worker.TranslationBatchSize,
})
} else {
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
}
// Start the cron scheduler so all recurring jobs actually run // Start the cron scheduler so all recurring jobs actually run
scheduler.Start() scheduler.Start()
appLogger.Info("Cron scheduler started with all worker jobs", map[string]interface{}{}) appLogger.Info("Cron scheduler started with all worker jobs", map[string]interface{}{})
@@ -193,6 +221,36 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
return scheduler return scheduler
} }
// InitAISummarizerClient initializes the AI summarizer HTTP client used by worker jobs.
func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.Client {
if conf.APIBaseURL == "" {
log.Warn("AI Summarizer API base URL not configured, translation worker will be unavailable", map[string]interface{}{})
return nil
}
cfg := &ai_summarizer.Config{
APIBaseURL: conf.APIBaseURL,
APITimeout: conf.APITimeout,
APIRetryCount: conf.APIRetryCount,
APIRetryDelay: conf.APIRetryDelay,
}
client, err := ai_summarizer.NewClient(cfg, log)
if err != nil {
log.Error("Failed to initialize AI Summarizer client for worker", map[string]interface{}{
"error": err.Error(),
})
return nil
}
log.Info("AI Summarizer client initialized for worker", map[string]interface{}{
"base_url": conf.APIBaseURL,
"timeout": conf.APITimeout,
})
return client
}
// Init Notification Service // Init Notification Service
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK { func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
notificationConfig := &notification.Config{ notificationConfig := &notification.Config{
+12
View File
@@ -15,6 +15,7 @@ type Config struct {
GLM GLMConfig GLM GLMConfig
Scraper config.ScraperConfig Scraper config.ScraperConfig
MinIO config.MinIOConfig MinIO config.MinIOConfig
AISummarizer AISummarizerConfig
Queue QueueConfig `env:"QUEUE_" envPrefix:"QUEUE_"` Queue QueueConfig `env:"QUEUE_" envPrefix:"QUEUE_"`
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"` AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
} }
@@ -37,6 +38,17 @@ type QueueConfig struct {
type WorkerConfig struct { type WorkerConfig struct {
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"` Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"` NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
}
// AISummarizerConfig holds configuration for the external AI summarizer service.
type AISummarizerConfig struct {
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
} }
type GLMConfig struct { type GLMConfig struct {
+4 -1
View File
@@ -54,8 +54,11 @@ func main() {
appLogger.Info("MinIO service initialized successfully", map[string]interface{}{}) appLogger.Info("MinIO service initialized successfully", map[string]interface{}{})
} }
// Initialize AI summarizer client (used by translation worker)
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, appLogger)
// Initialize Worker // Initialize Worker
scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, glmService, minioService) scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, glmService, minioService, aiSummarizerClient)
// Set up signal handling for graceful shutdown // Set up signal handling for graceful shutdown
signalChan := make(chan os.Signal, 1) signalChan := make(chan os.Signal, 1)
+149
View File
@@ -0,0 +1,149 @@
package workers
import (
"context"
"strings"
"time"
"tm/internal/tender"
"tm/pkg/ai_summarizer"
"tm/pkg/logger"
"tm/pkg/mongo"
)
// TranslationWorker backfills missing tender translations using AI summarizer API.
type TranslationWorker struct {
Mongo *mongo.ConnectionManager
Logger logger.Logger
TenderRepo tender.TenderRepository
AIClient *ai_summarizer.Client
TargetLanguage string
BatchSize int
}
// NewTranslationWorker creates a translation worker for a target language.
func NewTranslationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, aiClient *ai_summarizer.Client, targetLanguage string, batchSize int) *TranslationWorker {
if strings.TrimSpace(targetLanguage) == "" {
targetLanguage = "en"
}
if batchSize <= 0 {
batchSize = 20
}
return &TranslationWorker{
Mongo: mongo,
Logger: logger,
TenderRepo: tenderRepo,
AIClient: aiClient,
TargetLanguage: strings.ToLower(strings.TrimSpace(targetLanguage)),
BatchSize: batchSize,
}
}
// Run starts backfilling translations for tenders missing target-language content.
func (w *TranslationWorker) Run() {
if w.AIClient == nil {
w.Logger.Warn("Translation worker skipped: AI client is not configured", map[string]interface{}{
"target_language": w.TargetLanguage,
})
return
}
w.Logger.Info("Translation worker started", map[string]interface{}{
"target_language": w.TargetLanguage,
"batch_size": w.BatchSize,
})
for {
// Always use skip=0 because successfully translated tenders are excluded by filter.
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), w.TargetLanguage, w.BatchSize, 0)
if err != nil {
w.Logger.Error("Failed to fetch untranslated tenders", map[string]interface{}{
"target_language": w.TargetLanguage,
"error": err.Error(),
})
break
}
w.Logger.Info("Fetched untranslated tenders batch", map[string]interface{}{
"target_language": w.TargetLanguage,
"batch_count": len(tenders),
"total_count": totalCount,
})
if len(tenders) == 0 {
break
}
for _, t := range tenders {
w.translateTender(&t)
}
}
w.Logger.Info("Translation worker completed", map[string]interface{}{
"target_language": w.TargetLanguage,
})
}
func (w *TranslationWorker) translateTender(t *tender.Tender) {
if t.NoticePublicationID == "" {
return
}
if existing, ok := t.Translations[w.TargetLanguage]; ok && strings.TrimSpace(existing.Title) != "" {
w.Logger.Debug("Skipping tender translation because target language already exists", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
})
return
}
resp, err := w.AIClient.FetchTranslationOnDemand(context.Background(), ai_summarizer.TranslateRequest{
NoticeID: t.NoticePublicationID,
Title: t.Title,
Description: t.Description,
Language: w.TargetLanguage,
})
if err != nil {
w.Logger.Error("Failed to translate tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"translation_error": err.Error(),
})
return
}
if t.Translations == nil {
t.Translations = make(map[string]tender.TranslationEntry)
}
now := time.Now().Unix()
// Only update the target-language slot. Existing translations in other
// languages (e.g., "de", "fr") are preserved and never modified.
t.Translations[w.TargetLanguage] = tender.TranslationEntry{
Title: resp.TranslatedTitle,
Description: resp.TranslatedDescription,
Language: w.TargetLanguage,
TranslatedAt: now,
}
if t.ProcessingMetadata.TranslatedData == nil {
t.ProcessingMetadata.TranslatedData = make(map[string]string)
}
t.ProcessingMetadata.TranslatedData[w.TargetLanguage] = "done"
t.ProcessingMetadata.TranslatedAt = now
if err := w.TenderRepo.Update(context.Background(), t); err != nil {
w.Logger.Error("Failed to persist translated tender", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
"persistence_error": err.Error(),
})
return
}
w.Logger.Info("Tender translated successfully", map[string]interface{}{
"tender_id": t.ID.Hex(),
"notice_id": t.NoticePublicationID,
"target_language": w.TargetLanguage,
})
}
+14
View File
@@ -70,6 +70,20 @@ type TenderResponse struct {
Language string `json:"language,omitempty"` Language string `json:"language,omitempty"`
} }
// TenderDocumentResponse represents a scraped tender document available for download.
type TenderDocumentResponse struct {
Filename string `json:"filename"`
Size int64 `json:"size"`
LastModified int64 `json:"last_modified"`
DocumentType string `json:"document_type,omitempty"`
}
// TenderDocumentsResponse represents scraped documents for a tender.
type TenderDocumentsResponse struct {
TenderID string `json:"tender_id"`
Documents []TenderDocumentResponse `json:"documents"`
}
type OrganizationResponse struct { type OrganizationResponse struct {
Name string `json:"name"` Name string `json:"name"`
} }
+69
View File
@@ -1,6 +1,10 @@
package tender package tender
import ( import (
"errors"
"mime"
"net/http"
"strconv"
"time" "time"
"tm/pkg/logger" "tm/pkg/logger"
"tm/pkg/response" "tm/pkg/response"
@@ -361,6 +365,71 @@ func (h *TenderHandler) GetDocumentSummaries(c echo.Context) error {
return response.Success(c, summaries, "Document summaries retrieved successfully") return response.Success(c, summaries, "Document summaries retrieved successfully")
} }
// GetDocuments retrieves scraped documents for a tender
// @Summary Get scraped documents for a tender
// @Description Retrieve scraped document metadata for a specific tender
// @Tags Tenders
// @Produce json
// @Param id path string true "Tender ID"
// @Success 200 {object} response.APIResponse{data=TenderDocumentsResponse}
// @Failure 400 {object} response.APIResponse
// @Failure 404 {object} response.APIResponse
// @Failure 500 {object} response.APIResponse
// @Router /api/v1/tenders/{id}/documents [get]
func (h *TenderHandler) GetDocuments(c echo.Context) error {
id := c.Param("id")
if id == "" {
return response.BadRequest(c, "Tender ID is required", "ID parameter cannot be empty")
}
documents, err := h.service.GetDocuments(c.Request().Context(), id)
if err != nil {
return response.NotFound(c, "Tender documents not found")
}
return response.Success(c, documents, "Tender documents retrieved successfully")
}
// DownloadDocuments downloads all scraped documents for a tender
// @Summary Download scraped documents for a tender
// @Description Stream all scraped tender documents from storage as a ZIP archive
// @Tags Tenders
// @Produce application/zip
// @Param id path string true "Tender ID"
// @Success 200 {file} file
// @Failure 400 {object} response.APIResponse
// @Failure 404 {object} response.APIResponse
// @Failure 500 {object} response.APIResponse
// @Router /api/v1/tenders/{id}/documents/download [get]
func (h *TenderHandler) DownloadDocuments(c echo.Context) error {
id := c.Param("id")
if id == "" {
return response.BadRequest(c, "Tender ID is required", "ID parameter cannot be empty")
}
download, err := h.service.DownloadDocuments(c.Request().Context(), id)
if err != nil {
if errors.Is(err, ErrTenderDocumentNotFound) {
return response.NotFound(c, "Tender documents not found")
}
if errors.Is(err, ErrTenderDocumentStorageUnavailable) {
return response.InternalServerError(c, "Tender document storage is unavailable")
}
return response.InternalServerError(c, "Failed to download tender documents")
}
defer download.Reader.Close()
headers := c.Response().Header()
headers.Set(echo.HeaderContentDisposition, mime.FormatMediaType("attachment", map[string]string{
"filename": download.Filename,
}))
if download.Size > 0 {
headers.Set(echo.HeaderContentLength, strconv.FormatInt(download.Size, 10))
}
return c.Stream(http.StatusOK, download.ContentType, download.Reader)
}
// GetAISummary retrieves the AI-generated overall summary for a tender // GetAISummary retrieves the AI-generated overall summary for a tender
// @Summary Get AI-generated tender summary // @Summary Get AI-generated tender summary
// @Description Retrieve the AI-generated overall summary for a tender from the AI pipeline storage // @Description Retrieve the AI-generated overall summary for a tender from the AI pipeline storage
+47
View File
@@ -25,6 +25,7 @@ type TenderRepository interface {
GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error) GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error) GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error)
Update(ctx context.Context, tender *Tender) error Update(ctx context.Context, tender *Tender) error
Delete(ctx context.Context, id string) error Delete(ctx context.Context, id string) error
GetStatistics(ctx context.Context) (*TenderStatistics, error) GetStatistics(ctx context.Context) (*TenderStatistics, error)
@@ -337,6 +338,52 @@ func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, sk
return result.Items, result.TotalCount, nil return result.Items, result.TotalCount, nil
} }
// GetUnTranslatedTenders retrieves tenders that don't have a translation for the given language yet.
func (r *tenderRepository) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) {
translationKey := "translations." + language
metadataKey := "processing_metadata.translated_data." + language
filter := bson.M{
"notice_publication_id": bson.M{"$ne": ""},
"title": bson.M{"$ne": ""},
"description": bson.M{"$ne": ""},
"$and": []bson.M{
{
"$or": []bson.M{
{translationKey: bson.M{"$exists": false}},
{translationKey: nil},
},
},
{
"$or": []bson.M{
{metadataKey: bson.M{"$exists": false}},
{metadataKey: bson.M{"$ne": "done"}},
},
},
},
}
pagination := orm.Pagination{
Limit: limit,
Skip: skip,
SortField: "created_at",
SortOrder: 1, // Oldest first
}
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
if err != nil {
r.logger.Error("Failed to get untranslated tenders", map[string]interface{}{
"language": language,
"limit": limit,
"skip": skip,
"error": err.Error(),
})
return nil, 0, err
}
return result.Items, result.TotalCount, nil
}
// GetStatistics calculates tender statistics using aggregation // GetStatistics calculates tender statistics using aggregation
func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) { func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
stats := &TenderStatistics{ stats := &TenderStatistics{
+193
View File
@@ -1,9 +1,13 @@
package tender package tender
import ( import (
"archive/zip"
"context" "context"
"errors" "errors"
"fmt" "fmt"
"io"
"path"
"path/filepath"
"strings" "strings"
"time" "time"
"tm/internal/company" "tm/internal/company"
@@ -22,6 +26,16 @@ type AISummarizerClient interface {
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage // AISummarizerStorage defines the interface for fetching summaries from MinIO storage
type AISummarizerStorage interface { type AISummarizerStorage interface {
GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error) GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error)
ListDocuments(ctx context.Context, noticeID string) ([]ai_summarizer.StoredDocument, error)
DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error)
}
// TenderDocumentDownload contains an authorized document stream.
type TenderDocumentDownload struct {
Reader io.ReadCloser
Filename string
ContentType string
Size int64
} }
// TenderService interface defines tender business logic operations // TenderService interface defines tender business logic operations
@@ -33,6 +47,8 @@ type Service interface {
Search(ctx context.Context, req *SearchForm, pagination *response.Pagination) (*SearchResponse, error) Search(ctx context.Context, req *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
GetDocumentSummaries(ctx context.Context, id string) ([]DocumentSummary, error) GetDocumentSummaries(ctx context.Context, id string) ([]DocumentSummary, error)
GetDocuments(ctx context.Context, id string) (*TenderDocumentsResponse, error)
DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error)
// AI summarization operations // AI summarization operations
GetAISummary(ctx context.Context, id string) (*AISummaryResponse, error) GetAISummary(ctx context.Context, id string) (*AISummaryResponse, error)
@@ -58,6 +74,11 @@ type tenderService struct {
const aiSummaryEnrichmentTimeout = 2 * time.Second const aiSummaryEnrichmentTimeout = 2 * time.Second
var (
ErrTenderDocumentStorageUnavailable = errors.New("tender document storage is unavailable")
ErrTenderDocumentNotFound = errors.New("tender document not found")
)
// NewService creates a new tender service // NewService creates a new tender service
func NewService(repository TenderRepository, companyService company.Service, customerService customer.Service, logger logger.Logger, aiClient AISummarizerClient, aiStorage AISummarizerStorage, defaultLanguage string) Service { func NewService(repository TenderRepository, companyService company.Service, customerService customer.Service, logger logger.Logger, aiClient AISummarizerClient, aiStorage AISummarizerStorage, defaultLanguage string) Service {
if strings.TrimSpace(defaultLanguage) == "" { if strings.TrimSpace(defaultLanguage) == "" {
@@ -241,6 +262,178 @@ func (s *tenderService) GetDocumentSummaries(ctx context.Context, id string) ([]
return tender.DocumentSummaries, nil return tender.DocumentSummaries, nil
} }
// GetDocuments retrieves scraped document metadata for a tender.
func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDocumentsResponse, error) {
if id == "" {
return nil, fmt.Errorf("tender ID cannot be empty")
}
if s.aiSummarizerStorage == nil {
return nil, ErrTenderDocumentStorageUnavailable
}
tender, err := s.repository.GetByID(ctx, id)
if err != nil {
s.logger.Error("Failed to get tender for documents", map[string]interface{}{
"tender_id": id,
"error": err.Error(),
})
return nil, fmt.Errorf("failed to get tender: %w", err)
}
if tender == nil {
return nil, fmt.Errorf("tender not found")
}
if strings.TrimSpace(tender.NoticePublicationID) == "" {
return nil, fmt.Errorf("tender notice publication ID is empty")
}
storedDocuments, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
if err != nil {
s.logger.Error("Failed to list tender documents from storage", map[string]interface{}{
"tender_id": id,
"notice_id": tender.NoticePublicationID,
"error": err.Error(),
})
return nil, fmt.Errorf("failed to list tender documents: %w", err)
}
documents := make([]TenderDocumentResponse, 0, len(storedDocuments))
for _, doc := range storedDocuments {
documents = append(documents, TenderDocumentResponse{
Filename: doc.DocumentName,
Size: doc.Size,
LastModified: doc.LastModified,
DocumentType: doc.DocumentType,
})
}
s.logger.Info("Retrieved scraped documents for tender", map[string]interface{}{
"tender_id": id,
"notice_id": tender.NoticePublicationID,
"documents_count": len(documents),
})
return &TenderDocumentsResponse{
TenderID: id,
Documents: documents,
}, nil
}
// DownloadDocuments returns a ZIP stream containing all documents for a tender.
func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error) {
if id == "" {
return nil, fmt.Errorf("tender ID cannot be empty")
}
if s.aiSummarizerStorage == nil {
return nil, ErrTenderDocumentStorageUnavailable
}
tender, err := s.repository.GetByID(ctx, id)
if err != nil {
s.logger.Error("Failed to get tender for document download", map[string]interface{}{
"tender_id": id,
"error": err.Error(),
})
return nil, fmt.Errorf("failed to get tender: %w", err)
}
if tender == nil {
return nil, fmt.Errorf("tender not found")
}
if strings.TrimSpace(tender.NoticePublicationID) == "" {
return nil, ErrTenderDocumentNotFound
}
documents, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
if err != nil {
s.logger.Error("Failed to list tender documents for archive", map[string]interface{}{
"tender_id": id,
"notice_id": tender.NoticePublicationID,
"error": err.Error(),
})
return nil, fmt.Errorf("failed to list tender documents: %w", err)
}
if len(documents) == 0 {
return nil, ErrTenderDocumentNotFound
}
reader, writer := io.Pipe()
go s.writeDocumentsArchive(ctx, writer, documents)
filename := fmt.Sprintf("%s-documents.zip", tender.NoticePublicationID)
s.logger.Info("Tender documents archive download prepared", map[string]interface{}{
"tender_id": id,
"notice_id": tender.NoticePublicationID,
"documents_count": len(documents),
})
return &TenderDocumentDownload{
Reader: reader,
Filename: filename,
ContentType: "application/zip",
}, nil
}
func (s *tenderService) writeDocumentsArchive(ctx context.Context, writer *io.PipeWriter, documents []ai_summarizer.StoredDocument) {
zipWriter := zip.NewWriter(writer)
usedNames := make(map[string]int, len(documents))
for _, doc := range documents {
select {
case <-ctx.Done():
zipWriter.Close()
writer.CloseWithError(ctx.Err())
return
default:
}
if err := s.addDocumentToArchive(ctx, zipWriter, usedNames, doc); err != nil {
zipWriter.Close()
writer.CloseWithError(err)
return
}
}
if err := zipWriter.Close(); err != nil {
writer.CloseWithError(err)
return
}
writer.Close()
}
func (s *tenderService) addDocumentToArchive(ctx context.Context, zipWriter *zip.Writer, usedNames map[string]int, doc ai_summarizer.StoredDocument) error {
reader, err := s.aiSummarizerStorage.DownloadDocument(ctx, doc.ObjectName)
if err != nil {
return fmt.Errorf("failed to open document %s: %w", doc.DocumentName, err)
}
defer reader.Close()
entry, err := zipWriter.Create(uniqueArchiveName(usedNames, doc.DocumentName))
if err != nil {
return fmt.Errorf("failed to create archive entry for %s: %w", doc.DocumentName, err)
}
if _, err := io.Copy(entry, reader); err != nil {
return fmt.Errorf("failed to write document %s to archive: %w", doc.DocumentName, err)
}
return nil
}
func uniqueArchiveName(usedNames map[string]int, filename string) string {
cleanName := path.Base(filename)
if cleanName == "." || cleanName == "/" || cleanName == "" {
cleanName = "document"
}
count := usedNames[cleanName]
usedNames[cleanName] = count + 1
if count == 0 {
return cleanName
}
extension := filepath.Ext(cleanName)
base := strings.TrimSuffix(cleanName, extension)
return fmt.Sprintf("%s-%d%s", base, count+1, extension)
}
// ListTenders retrieves tenders with pagination and filtering // ListTenders retrieves tenders with pagination and filtering
func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) { func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) {
s.logger.Info("Listing tenders", map[string]interface{}{ s.logger.Info("Listing tenders", map[string]interface{}{
+9
View File
@@ -30,6 +30,15 @@ type DocumentSummary struct {
Error string `json:"error"` // empty string when no error Error string `json:"error"` // empty string when no error
} }
// StoredDocument represents a document object stored for a tender in MinIO.
type StoredDocument struct {
ObjectName string `json:"-"`
DocumentName string `json:"document_name"`
Size int64 `json:"size"`
LastModified int64 `json:"last_modified"`
DocumentType string `json:"document_type,omitempty"`
}
// TranslateRequest represents the request payload sent to the AI service // TranslateRequest represents the request payload sent to the AI service
// POST /ai/translate endpoint. // POST /ai/translate endpoint.
type TranslateRequest struct { type TranslateRequest struct {
+92
View File
@@ -7,6 +7,7 @@ import (
"fmt" "fmt"
"io" "io"
"net/http" "net/http"
"path/filepath"
"strings" "strings"
"github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7"
@@ -153,6 +154,89 @@ func (s *StorageClient) IsSummaryReady(ctx context.Context, noticeID string) (bo
return tenderJSON.SummarizeStatus.Done, nil return tenderJSON.SummarizeStatus.Done, nil
} }
// ListDocuments lists document files stored under <noticeID>/.
func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]StoredDocument, error) {
noticeID = strings.TrimSpace(noticeID)
if noticeID == "" {
return nil, fmt.Errorf("ai_summarizer: noticeID is required")
}
prefix := fmt.Sprintf("%s/", noticeID)
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: true,
})
documents := make([]StoredDocument, 0)
for object := range objectCh {
if object.Err != nil {
return nil, fmt.Errorf("ai_summarizer: failed to list documents for notice %s: %w", noticeID, object.Err)
}
if !isTenderDocumentObject(noticeID, object.Key) {
continue
}
documentName := filepath.Base(object.Key)
documents = append(documents, StoredDocument{
ObjectName: object.Key,
DocumentName: documentName,
Size: object.Size,
LastModified: object.LastModified.Unix(),
DocumentType: documentType(documentName),
})
}
s.logger.Info("Listed tender documents from storage", map[string]interface{}{
"notice_id": noticeID,
"documents_count": len(documents),
})
return documents, nil
}
func isTenderDocumentObject(noticeID, objectName string) bool {
if objectName == "" || strings.HasSuffix(objectName, "/") {
return false
}
relativePath := strings.TrimPrefix(objectName, fmt.Sprintf("%s/", noticeID))
if relativePath == "" {
return false
}
filename := strings.ToLower(filepath.Base(relativePath))
switch filename {
case "tender.json", "ai_cache.db":
return false
}
switch strings.ToLower(filepath.Ext(filename)) {
case ".json", ".db", ".sqlite", ".sqlite3":
return false
default:
return true
}
}
// DownloadDocument opens a document object from the configured MinIO bucket.
func (s *StorageClient) DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error) {
objectName = strings.TrimSpace(objectName)
if objectName == "" {
return nil, fmt.Errorf("ai_summarizer: objectName is required")
}
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectName, minio.GetObjectOptions{})
if err != nil {
var minioErr minio.ErrorResponse
if errors.As(err, &minioErr) && minioErr.StatusCode == http.StatusNotFound {
return nil, ErrObjectNotFound
}
return nil, fmt.Errorf("ai_summarizer: failed to get document object: %w", err)
}
return object, nil
}
// GetAllSummaries lists all tender.json files in the MinIO bucket and returns // GetAllSummaries lists all tender.json files in the MinIO bucket and returns
// the overall_summary for each tender where summarize_status.done is true. // the overall_summary for each tender where summarize_status.done is true.
// This is useful for batch-retrieving all summaries that the AI pipeline has // This is useful for batch-retrieving all summaries that the AI pipeline has
@@ -233,3 +317,11 @@ func countDone(items []TenderSummaryItem) int {
} }
return count return count
} }
func documentType(filename string) string {
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".")
if extension == "" {
return "unknown"
}
return extension
}