List and download tender documents - worker auto translation to english job
This commit is contained in:
@@ -109,6 +109,8 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
|
|||||||
tendersGP.GET("/:id", tenderHandler.Get)
|
tendersGP.GET("/:id", tenderHandler.Get)
|
||||||
tendersGP.PUT("/:id", tenderHandler.Update)
|
tendersGP.PUT("/:id", tenderHandler.Update)
|
||||||
tendersGP.DELETE("/:id", tenderHandler.Delete)
|
tendersGP.DELETE("/:id", tenderHandler.Delete)
|
||||||
|
tendersGP.GET("/:id/documents", tenderHandler.GetDocuments)
|
||||||
|
tendersGP.GET("/:id/documents/download", tenderHandler.DownloadDocuments)
|
||||||
tendersGP.GET("/:id/ai-summary", tenderHandler.GetAISummary)
|
tendersGP.GET("/:id/ai-summary", tenderHandler.GetAISummary)
|
||||||
tendersGP.POST("/:id/ai-summarize", tenderHandler.TriggerAISummarize)
|
tendersGP.POST("/:id/ai-summarize", tenderHandler.TriggerAISummarize)
|
||||||
tendersGP.POST("/:id/ai-translate", tenderHandler.TriggerAITranslate)
|
tendersGP.POST("/:id/ai-translate", tenderHandler.TriggerAITranslate)
|
||||||
@@ -236,6 +238,8 @@ func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tende
|
|||||||
tendersGP.GET("", tenderHandler.GetPublicTenders)
|
tendersGP.GET("", tenderHandler.GetPublicTenders)
|
||||||
tendersGP.GET("/recommend", tenderHandler.RecommendTenders)
|
tendersGP.GET("/recommend", tenderHandler.RecommendTenders)
|
||||||
tendersGP.GET("/details/:id", tenderHandler.GetPublicTenderDetails)
|
tendersGP.GET("/details/:id", tenderHandler.GetPublicTenderDetails)
|
||||||
|
tendersGP.GET("/:id/documents", tenderHandler.GetDocuments)
|
||||||
|
tendersGP.GET("/:id/documents/download", tenderHandler.DownloadDocuments)
|
||||||
tendersGP.GET("/:id/document-summaries", tenderHandler.GetDocumentSummaries)
|
tendersGP.GET("/:id/document-summaries", tenderHandler.GetDocumentSummaries)
|
||||||
tendersGP.GET("/:id/ai-summary", tenderHandler.GetPublicAISummary)
|
tendersGP.GET("/:id/ai-summary", tenderHandler.GetPublicAISummary)
|
||||||
tendersGP.POST("/:id/ai-translate", tenderHandler.TriggerPublicAITranslate)
|
tendersGP.POST("/:id/ai-translate", tenderHandler.TriggerPublicAITranslate)
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"tm/cmd/worker/workers"
|
"tm/cmd/worker/workers"
|
||||||
"tm/internal/notice"
|
"tm/internal/notice"
|
||||||
"tm/internal/tender"
|
"tm/internal/tender"
|
||||||
|
ai_summarizer "tm/pkg/ai_summarizer"
|
||||||
"tm/pkg/config"
|
"tm/pkg/config"
|
||||||
"tm/pkg/glm"
|
"tm/pkg/glm"
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
@@ -77,7 +78,7 @@ func InitMongoDB(conf config.MongoConfig, log logger.Logger) *mongo.ConnectionMa
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Init Worker
|
// Init Worker
|
||||||
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, minioService *minio.Service) *schedule.CronScheduler {
|
func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger logger.Logger, notify notification.SDK, glmService *glm.SDK, minioService *minio.Service, aiClient *ai_summarizer.Client) *schedule.CronScheduler {
|
||||||
// Debug: Log worker config
|
// Debug: Log worker config
|
||||||
appLogger.Info("Worker configuration", map[string]interface{}{
|
appLogger.Info("Worker configuration", map[string]interface{}{
|
||||||
"worker_interval": config.Worker.Interval,
|
"worker_interval": config.Worker.Interval,
|
||||||
@@ -186,6 +187,33 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
|||||||
Expr: workerInterval,
|
Expr: workerInterval,
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Initialize translation backfill worker for default language
|
||||||
|
if aiClient != nil {
|
||||||
|
targetLanguage := config.AISummarizer.DefaultLanguage
|
||||||
|
scheduler.AddJob(schedule.Job{
|
||||||
|
Name: "Tender Translation Worker Job",
|
||||||
|
Func: func() {
|
||||||
|
worker := workers.NewTranslationWorker(
|
||||||
|
mongoManager,
|
||||||
|
appLogger,
|
||||||
|
tenderRepo,
|
||||||
|
aiClient,
|
||||||
|
targetLanguage,
|
||||||
|
config.Worker.TranslationBatchSize,
|
||||||
|
)
|
||||||
|
worker.Run()
|
||||||
|
},
|
||||||
|
Expr: config.Worker.TranslationInterval,
|
||||||
|
})
|
||||||
|
appLogger.Info("Scheduled tender translation worker", map[string]interface{}{
|
||||||
|
"target_language": targetLanguage,
|
||||||
|
"interval": config.Worker.TranslationInterval,
|
||||||
|
"batch_size": config.Worker.TranslationBatchSize,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
appLogger.Warn("AI summarizer client not available, tender translation worker is disabled", map[string]interface{}{})
|
||||||
|
}
|
||||||
|
|
||||||
// Start the cron scheduler so all recurring jobs actually run
|
// Start the cron scheduler so all recurring jobs actually run
|
||||||
scheduler.Start()
|
scheduler.Start()
|
||||||
appLogger.Info("Cron scheduler started with all worker jobs", map[string]interface{}{})
|
appLogger.Info("Cron scheduler started with all worker jobs", map[string]interface{}{})
|
||||||
@@ -193,6 +221,36 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
|||||||
return scheduler
|
return scheduler
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InitAISummarizerClient initializes the AI summarizer HTTP client used by worker jobs.
|
||||||
|
func InitAISummarizerClient(conf AISummarizerConfig, log logger.Logger) *ai_summarizer.Client {
|
||||||
|
if conf.APIBaseURL == "" {
|
||||||
|
log.Warn("AI Summarizer API base URL not configured, translation worker will be unavailable", map[string]interface{}{})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg := &ai_summarizer.Config{
|
||||||
|
APIBaseURL: conf.APIBaseURL,
|
||||||
|
APITimeout: conf.APITimeout,
|
||||||
|
APIRetryCount: conf.APIRetryCount,
|
||||||
|
APIRetryDelay: conf.APIRetryDelay,
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := ai_summarizer.NewClient(cfg, log)
|
||||||
|
if err != nil {
|
||||||
|
log.Error("Failed to initialize AI Summarizer client for worker", map[string]interface{}{
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("AI Summarizer client initialized for worker", map[string]interface{}{
|
||||||
|
"base_url": conf.APIBaseURL,
|
||||||
|
"timeout": conf.APITimeout,
|
||||||
|
})
|
||||||
|
|
||||||
|
return client
|
||||||
|
}
|
||||||
|
|
||||||
// Init Notification Service
|
// Init Notification Service
|
||||||
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
|
func InitNotificationService(conf config.NotificationConfig, log logger.Logger) notification.SDK {
|
||||||
notificationConfig := ¬ification.Config{
|
notificationConfig := ¬ification.Config{
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ type Config struct {
|
|||||||
GLM GLMConfig
|
GLM GLMConfig
|
||||||
Scraper config.ScraperConfig
|
Scraper config.ScraperConfig
|
||||||
MinIO config.MinIOConfig
|
MinIO config.MinIOConfig
|
||||||
|
AISummarizer AISummarizerConfig
|
||||||
Queue QueueConfig `env:"QUEUE_" envPrefix:"QUEUE_"`
|
Queue QueueConfig `env:"QUEUE_" envPrefix:"QUEUE_"`
|
||||||
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
|
AlertMail string `env:"ALERT_MAIL" envDefault:"alerts@opplens.com"`
|
||||||
}
|
}
|
||||||
@@ -37,6 +38,17 @@ type QueueConfig struct {
|
|||||||
type WorkerConfig struct {
|
type WorkerConfig struct {
|
||||||
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
|
Interval string `env:"WORKER_INTERVAL" envDefault:"* 10 * * * *"`
|
||||||
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
|
NoticeProcessingLimit int `env:"WORKER_NOTICE_PROCESSING_LIMIT" envDefault:"0"`
|
||||||
|
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
|
||||||
|
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// AISummarizerConfig holds configuration for the external AI summarizer service.
|
||||||
|
type AISummarizerConfig struct {
|
||||||
|
APIBaseURL string `env:"AI_SUMMARIZER_API_BASE_URL" envDefault:""`
|
||||||
|
APITimeout time.Duration `env:"AI_SUMMARIZER_API_TIMEOUT" envDefault:"120s"`
|
||||||
|
APIRetryCount int `env:"AI_SUMMARIZER_API_RETRY_COUNT" envDefault:"2"`
|
||||||
|
APIRetryDelay time.Duration `env:"AI_SUMMARIZER_API_RETRY_DELAY" envDefault:"3s"`
|
||||||
|
DefaultLanguage string `env:"AI_SUMMARIZER_DEFAULT_LANGUAGE" envDefault:"en"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type GLMConfig struct {
|
type GLMConfig struct {
|
||||||
|
|||||||
+4
-1
@@ -54,8 +54,11 @@ func main() {
|
|||||||
appLogger.Info("MinIO service initialized successfully", map[string]interface{}{})
|
appLogger.Info("MinIO service initialized successfully", map[string]interface{}{})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Initialize AI summarizer client (used by translation worker)
|
||||||
|
aiSummarizerClient := bootstrap.InitAISummarizerClient(config.AISummarizer, appLogger)
|
||||||
|
|
||||||
// Initialize Worker
|
// Initialize Worker
|
||||||
scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, glmService, minioService)
|
scheduler := bootstrap.InitWorker(*config, mongoManager, appLogger, notificationService, glmService, minioService, aiSummarizerClient)
|
||||||
|
|
||||||
// Set up signal handling for graceful shutdown
|
// Set up signal handling for graceful shutdown
|
||||||
signalChan := make(chan os.Signal, 1)
|
signalChan := make(chan os.Signal, 1)
|
||||||
|
|||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package workers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
"tm/internal/tender"
|
||||||
|
"tm/pkg/ai_summarizer"
|
||||||
|
"tm/pkg/logger"
|
||||||
|
"tm/pkg/mongo"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TranslationWorker backfills missing tender translations using AI summarizer API.
|
||||||
|
type TranslationWorker struct {
|
||||||
|
Mongo *mongo.ConnectionManager
|
||||||
|
Logger logger.Logger
|
||||||
|
TenderRepo tender.TenderRepository
|
||||||
|
AIClient *ai_summarizer.Client
|
||||||
|
TargetLanguage string
|
||||||
|
BatchSize int
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewTranslationWorker creates a translation worker for a target language.
|
||||||
|
func NewTranslationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, aiClient *ai_summarizer.Client, targetLanguage string, batchSize int) *TranslationWorker {
|
||||||
|
if strings.TrimSpace(targetLanguage) == "" {
|
||||||
|
targetLanguage = "en"
|
||||||
|
}
|
||||||
|
if batchSize <= 0 {
|
||||||
|
batchSize = 20
|
||||||
|
}
|
||||||
|
|
||||||
|
return &TranslationWorker{
|
||||||
|
Mongo: mongo,
|
||||||
|
Logger: logger,
|
||||||
|
TenderRepo: tenderRepo,
|
||||||
|
AIClient: aiClient,
|
||||||
|
TargetLanguage: strings.ToLower(strings.TrimSpace(targetLanguage)),
|
||||||
|
BatchSize: batchSize,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run starts backfilling translations for tenders missing target-language content.
|
||||||
|
func (w *TranslationWorker) Run() {
|
||||||
|
if w.AIClient == nil {
|
||||||
|
w.Logger.Warn("Translation worker skipped: AI client is not configured", map[string]interface{}{
|
||||||
|
"target_language": w.TargetLanguage,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Logger.Info("Translation worker started", map[string]interface{}{
|
||||||
|
"target_language": w.TargetLanguage,
|
||||||
|
"batch_size": w.BatchSize,
|
||||||
|
})
|
||||||
|
|
||||||
|
for {
|
||||||
|
// Always use skip=0 because successfully translated tenders are excluded by filter.
|
||||||
|
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(context.Background(), w.TargetLanguage, w.BatchSize, 0)
|
||||||
|
if err != nil {
|
||||||
|
w.Logger.Error("Failed to fetch untranslated tenders", map[string]interface{}{
|
||||||
|
"target_language": w.TargetLanguage,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Logger.Info("Fetched untranslated tenders batch", map[string]interface{}{
|
||||||
|
"target_language": w.TargetLanguage,
|
||||||
|
"batch_count": len(tenders),
|
||||||
|
"total_count": totalCount,
|
||||||
|
})
|
||||||
|
|
||||||
|
if len(tenders) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, t := range tenders {
|
||||||
|
w.translateTender(&t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Logger.Info("Translation worker completed", map[string]interface{}{
|
||||||
|
"target_language": w.TargetLanguage,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *TranslationWorker) translateTender(t *tender.Tender) {
|
||||||
|
if t.NoticePublicationID == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if existing, ok := t.Translations[w.TargetLanguage]; ok && strings.TrimSpace(existing.Title) != "" {
|
||||||
|
w.Logger.Debug("Skipping tender translation because target language already exists", map[string]interface{}{
|
||||||
|
"tender_id": t.ID.Hex(),
|
||||||
|
"notice_id": t.NoticePublicationID,
|
||||||
|
"target_language": w.TargetLanguage,
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := w.AIClient.FetchTranslationOnDemand(context.Background(), ai_summarizer.TranslateRequest{
|
||||||
|
NoticeID: t.NoticePublicationID,
|
||||||
|
Title: t.Title,
|
||||||
|
Description: t.Description,
|
||||||
|
Language: w.TargetLanguage,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
w.Logger.Error("Failed to translate tender", map[string]interface{}{
|
||||||
|
"tender_id": t.ID.Hex(),
|
||||||
|
"notice_id": t.NoticePublicationID,
|
||||||
|
"target_language": w.TargetLanguage,
|
||||||
|
"translation_error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if t.Translations == nil {
|
||||||
|
t.Translations = make(map[string]tender.TranslationEntry)
|
||||||
|
}
|
||||||
|
now := time.Now().Unix()
|
||||||
|
// Only update the target-language slot. Existing translations in other
|
||||||
|
// languages (e.g., "de", "fr") are preserved and never modified.
|
||||||
|
t.Translations[w.TargetLanguage] = tender.TranslationEntry{
|
||||||
|
Title: resp.TranslatedTitle,
|
||||||
|
Description: resp.TranslatedDescription,
|
||||||
|
Language: w.TargetLanguage,
|
||||||
|
TranslatedAt: now,
|
||||||
|
}
|
||||||
|
if t.ProcessingMetadata.TranslatedData == nil {
|
||||||
|
t.ProcessingMetadata.TranslatedData = make(map[string]string)
|
||||||
|
}
|
||||||
|
t.ProcessingMetadata.TranslatedData[w.TargetLanguage] = "done"
|
||||||
|
t.ProcessingMetadata.TranslatedAt = now
|
||||||
|
|
||||||
|
if err := w.TenderRepo.Update(context.Background(), t); err != nil {
|
||||||
|
w.Logger.Error("Failed to persist translated tender", map[string]interface{}{
|
||||||
|
"tender_id": t.ID.Hex(),
|
||||||
|
"notice_id": t.NoticePublicationID,
|
||||||
|
"target_language": w.TargetLanguage,
|
||||||
|
"persistence_error": err.Error(),
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Logger.Info("Tender translated successfully", map[string]interface{}{
|
||||||
|
"tender_id": t.ID.Hex(),
|
||||||
|
"notice_id": t.NoticePublicationID,
|
||||||
|
"target_language": w.TargetLanguage,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -70,6 +70,20 @@ type TenderResponse struct {
|
|||||||
Language string `json:"language,omitempty"`
|
Language string `json:"language,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TenderDocumentResponse represents a scraped tender document available for download.
|
||||||
|
type TenderDocumentResponse struct {
|
||||||
|
Filename string `json:"filename"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
LastModified int64 `json:"last_modified"`
|
||||||
|
DocumentType string `json:"document_type,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// TenderDocumentsResponse represents scraped documents for a tender.
|
||||||
|
type TenderDocumentsResponse struct {
|
||||||
|
TenderID string `json:"tender_id"`
|
||||||
|
Documents []TenderDocumentResponse `json:"documents"`
|
||||||
|
}
|
||||||
|
|
||||||
type OrganizationResponse struct {
|
type OrganizationResponse struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
package tender
|
package tender
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
|
"mime"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
"tm/pkg/response"
|
"tm/pkg/response"
|
||||||
@@ -361,6 +365,71 @@ func (h *TenderHandler) GetDocumentSummaries(c echo.Context) error {
|
|||||||
return response.Success(c, summaries, "Document summaries retrieved successfully")
|
return response.Success(c, summaries, "Document summaries retrieved successfully")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetDocuments retrieves scraped documents for a tender
|
||||||
|
// @Summary Get scraped documents for a tender
|
||||||
|
// @Description Retrieve scraped document metadata for a specific tender
|
||||||
|
// @Tags Tenders
|
||||||
|
// @Produce json
|
||||||
|
// @Param id path string true "Tender ID"
|
||||||
|
// @Success 200 {object} response.APIResponse{data=TenderDocumentsResponse}
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 404 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Router /api/v1/tenders/{id}/documents [get]
|
||||||
|
func (h *TenderHandler) GetDocuments(c echo.Context) error {
|
||||||
|
id := c.Param("id")
|
||||||
|
if id == "" {
|
||||||
|
return response.BadRequest(c, "Tender ID is required", "ID parameter cannot be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
documents, err := h.service.GetDocuments(c.Request().Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
return response.NotFound(c, "Tender documents not found")
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.Success(c, documents, "Tender documents retrieved successfully")
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadDocuments downloads all scraped documents for a tender
|
||||||
|
// @Summary Download scraped documents for a tender
|
||||||
|
// @Description Stream all scraped tender documents from storage as a ZIP archive
|
||||||
|
// @Tags Tenders
|
||||||
|
// @Produce application/zip
|
||||||
|
// @Param id path string true "Tender ID"
|
||||||
|
// @Success 200 {file} file
|
||||||
|
// @Failure 400 {object} response.APIResponse
|
||||||
|
// @Failure 404 {object} response.APIResponse
|
||||||
|
// @Failure 500 {object} response.APIResponse
|
||||||
|
// @Router /api/v1/tenders/{id}/documents/download [get]
|
||||||
|
func (h *TenderHandler) DownloadDocuments(c echo.Context) error {
|
||||||
|
id := c.Param("id")
|
||||||
|
if id == "" {
|
||||||
|
return response.BadRequest(c, "Tender ID is required", "ID parameter cannot be empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
download, err := h.service.DownloadDocuments(c.Request().Context(), id)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrTenderDocumentNotFound) {
|
||||||
|
return response.NotFound(c, "Tender documents not found")
|
||||||
|
}
|
||||||
|
if errors.Is(err, ErrTenderDocumentStorageUnavailable) {
|
||||||
|
return response.InternalServerError(c, "Tender document storage is unavailable")
|
||||||
|
}
|
||||||
|
return response.InternalServerError(c, "Failed to download tender documents")
|
||||||
|
}
|
||||||
|
defer download.Reader.Close()
|
||||||
|
|
||||||
|
headers := c.Response().Header()
|
||||||
|
headers.Set(echo.HeaderContentDisposition, mime.FormatMediaType("attachment", map[string]string{
|
||||||
|
"filename": download.Filename,
|
||||||
|
}))
|
||||||
|
if download.Size > 0 {
|
||||||
|
headers.Set(echo.HeaderContentLength, strconv.FormatInt(download.Size, 10))
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.Stream(http.StatusOK, download.ContentType, download.Reader)
|
||||||
|
}
|
||||||
|
|
||||||
// GetAISummary retrieves the AI-generated overall summary for a tender
|
// GetAISummary retrieves the AI-generated overall summary for a tender
|
||||||
// @Summary Get AI-generated tender summary
|
// @Summary Get AI-generated tender summary
|
||||||
// @Description Retrieve the AI-generated overall summary for a tender from the AI pipeline storage
|
// @Description Retrieve the AI-generated overall summary for a tender from the AI pipeline storage
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ type TenderRepository interface {
|
|||||||
GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
|
GetExpiredTenders(ctx context.Context, beforeDate int64) ([]Tender, error)
|
||||||
GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
|
GetUnScrapedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
|
||||||
GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
|
GetUnSummarizedTenders(ctx context.Context, limit, skip int) ([]Tender, int64, error)
|
||||||
|
GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error)
|
||||||
Update(ctx context.Context, tender *Tender) error
|
Update(ctx context.Context, tender *Tender) error
|
||||||
Delete(ctx context.Context, id string) error
|
Delete(ctx context.Context, id string) error
|
||||||
GetStatistics(ctx context.Context) (*TenderStatistics, error)
|
GetStatistics(ctx context.Context) (*TenderStatistics, error)
|
||||||
@@ -337,6 +338,52 @@ func (r *tenderRepository) GetUnSummarizedTenders(ctx context.Context, limit, sk
|
|||||||
return result.Items, result.TotalCount, nil
|
return result.Items, result.TotalCount, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetUnTranslatedTenders retrieves tenders that don't have a translation for the given language yet.
|
||||||
|
func (r *tenderRepository) GetUnTranslatedTenders(ctx context.Context, language string, limit, skip int) ([]Tender, int64, error) {
|
||||||
|
translationKey := "translations." + language
|
||||||
|
metadataKey := "processing_metadata.translated_data." + language
|
||||||
|
|
||||||
|
filter := bson.M{
|
||||||
|
"notice_publication_id": bson.M{"$ne": ""},
|
||||||
|
"title": bson.M{"$ne": ""},
|
||||||
|
"description": bson.M{"$ne": ""},
|
||||||
|
"$and": []bson.M{
|
||||||
|
{
|
||||||
|
"$or": []bson.M{
|
||||||
|
{translationKey: bson.M{"$exists": false}},
|
||||||
|
{translationKey: nil},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"$or": []bson.M{
|
||||||
|
{metadataKey: bson.M{"$exists": false}},
|
||||||
|
{metadataKey: bson.M{"$ne": "done"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
pagination := orm.Pagination{
|
||||||
|
Limit: limit,
|
||||||
|
Skip: skip,
|
||||||
|
SortField: "created_at",
|
||||||
|
SortOrder: 1, // Oldest first
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := r.ormRepo.FindAll(ctx, filter, pagination)
|
||||||
|
if err != nil {
|
||||||
|
r.logger.Error("Failed to get untranslated tenders", map[string]interface{}{
|
||||||
|
"language": language,
|
||||||
|
"limit": limit,
|
||||||
|
"skip": skip,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.Items, result.TotalCount, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetStatistics calculates tender statistics using aggregation
|
// GetStatistics calculates tender statistics using aggregation
|
||||||
func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
|
func (r *tenderRepository) GetStatistics(ctx context.Context) (*TenderStatistics, error) {
|
||||||
stats := &TenderStatistics{
|
stats := &TenderStatistics{
|
||||||
|
|||||||
@@ -1,9 +1,13 @@
|
|||||||
package tender
|
package tender
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"archive/zip"
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"path"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"tm/internal/company"
|
"tm/internal/company"
|
||||||
@@ -22,6 +26,16 @@ type AISummarizerClient interface {
|
|||||||
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage
|
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage
|
||||||
type AISummarizerStorage interface {
|
type AISummarizerStorage interface {
|
||||||
GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error)
|
GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error)
|
||||||
|
ListDocuments(ctx context.Context, noticeID string) ([]ai_summarizer.StoredDocument, error)
|
||||||
|
DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TenderDocumentDownload contains an authorized document stream.
|
||||||
|
type TenderDocumentDownload struct {
|
||||||
|
Reader io.ReadCloser
|
||||||
|
Filename string
|
||||||
|
ContentType string
|
||||||
|
Size int64
|
||||||
}
|
}
|
||||||
|
|
||||||
// TenderService interface defines tender business logic operations
|
// TenderService interface defines tender business logic operations
|
||||||
@@ -33,6 +47,8 @@ type Service interface {
|
|||||||
Search(ctx context.Context, req *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
|
Search(ctx context.Context, req *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
|
||||||
Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
|
Recommend(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error)
|
||||||
GetDocumentSummaries(ctx context.Context, id string) ([]DocumentSummary, error)
|
GetDocumentSummaries(ctx context.Context, id string) ([]DocumentSummary, error)
|
||||||
|
GetDocuments(ctx context.Context, id string) (*TenderDocumentsResponse, error)
|
||||||
|
DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error)
|
||||||
|
|
||||||
// AI summarization operations
|
// AI summarization operations
|
||||||
GetAISummary(ctx context.Context, id string) (*AISummaryResponse, error)
|
GetAISummary(ctx context.Context, id string) (*AISummaryResponse, error)
|
||||||
@@ -58,6 +74,11 @@ type tenderService struct {
|
|||||||
|
|
||||||
const aiSummaryEnrichmentTimeout = 2 * time.Second
|
const aiSummaryEnrichmentTimeout = 2 * time.Second
|
||||||
|
|
||||||
|
var (
|
||||||
|
ErrTenderDocumentStorageUnavailable = errors.New("tender document storage is unavailable")
|
||||||
|
ErrTenderDocumentNotFound = errors.New("tender document not found")
|
||||||
|
)
|
||||||
|
|
||||||
// NewService creates a new tender service
|
// NewService creates a new tender service
|
||||||
func NewService(repository TenderRepository, companyService company.Service, customerService customer.Service, logger logger.Logger, aiClient AISummarizerClient, aiStorage AISummarizerStorage, defaultLanguage string) Service {
|
func NewService(repository TenderRepository, companyService company.Service, customerService customer.Service, logger logger.Logger, aiClient AISummarizerClient, aiStorage AISummarizerStorage, defaultLanguage string) Service {
|
||||||
if strings.TrimSpace(defaultLanguage) == "" {
|
if strings.TrimSpace(defaultLanguage) == "" {
|
||||||
@@ -241,6 +262,178 @@ func (s *tenderService) GetDocumentSummaries(ctx context.Context, id string) ([]
|
|||||||
return tender.DocumentSummaries, nil
|
return tender.DocumentSummaries, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetDocuments retrieves scraped document metadata for a tender.
|
||||||
|
func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDocumentsResponse, error) {
|
||||||
|
if id == "" {
|
||||||
|
return nil, fmt.Errorf("tender ID cannot be empty")
|
||||||
|
}
|
||||||
|
if s.aiSummarizerStorage == nil {
|
||||||
|
return nil, ErrTenderDocumentStorageUnavailable
|
||||||
|
}
|
||||||
|
|
||||||
|
tender, err := s.repository.GetByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to get tender for documents", map[string]interface{}{
|
||||||
|
"tender_id": id,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to get tender: %w", err)
|
||||||
|
}
|
||||||
|
if tender == nil {
|
||||||
|
return nil, fmt.Errorf("tender not found")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(tender.NoticePublicationID) == "" {
|
||||||
|
return nil, fmt.Errorf("tender notice publication ID is empty")
|
||||||
|
}
|
||||||
|
|
||||||
|
storedDocuments, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to list tender documents from storage", map[string]interface{}{
|
||||||
|
"tender_id": id,
|
||||||
|
"notice_id": tender.NoticePublicationID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to list tender documents: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
documents := make([]TenderDocumentResponse, 0, len(storedDocuments))
|
||||||
|
for _, doc := range storedDocuments {
|
||||||
|
documents = append(documents, TenderDocumentResponse{
|
||||||
|
Filename: doc.DocumentName,
|
||||||
|
Size: doc.Size,
|
||||||
|
LastModified: doc.LastModified,
|
||||||
|
DocumentType: doc.DocumentType,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Retrieved scraped documents for tender", map[string]interface{}{
|
||||||
|
"tender_id": id,
|
||||||
|
"notice_id": tender.NoticePublicationID,
|
||||||
|
"documents_count": len(documents),
|
||||||
|
})
|
||||||
|
|
||||||
|
return &TenderDocumentsResponse{
|
||||||
|
TenderID: id,
|
||||||
|
Documents: documents,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadDocuments returns a ZIP stream containing all documents for a tender.
|
||||||
|
func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error) {
|
||||||
|
if id == "" {
|
||||||
|
return nil, fmt.Errorf("tender ID cannot be empty")
|
||||||
|
}
|
||||||
|
if s.aiSummarizerStorage == nil {
|
||||||
|
return nil, ErrTenderDocumentStorageUnavailable
|
||||||
|
}
|
||||||
|
|
||||||
|
tender, err := s.repository.GetByID(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to get tender for document download", map[string]interface{}{
|
||||||
|
"tender_id": id,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to get tender: %w", err)
|
||||||
|
}
|
||||||
|
if tender == nil {
|
||||||
|
return nil, fmt.Errorf("tender not found")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(tender.NoticePublicationID) == "" {
|
||||||
|
return nil, ErrTenderDocumentNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
documents, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
|
||||||
|
if err != nil {
|
||||||
|
s.logger.Error("Failed to list tender documents for archive", map[string]interface{}{
|
||||||
|
"tender_id": id,
|
||||||
|
"notice_id": tender.NoticePublicationID,
|
||||||
|
"error": err.Error(),
|
||||||
|
})
|
||||||
|
return nil, fmt.Errorf("failed to list tender documents: %w", err)
|
||||||
|
}
|
||||||
|
if len(documents) == 0 {
|
||||||
|
return nil, ErrTenderDocumentNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, writer := io.Pipe()
|
||||||
|
go s.writeDocumentsArchive(ctx, writer, documents)
|
||||||
|
|
||||||
|
filename := fmt.Sprintf("%s-documents.zip", tender.NoticePublicationID)
|
||||||
|
s.logger.Info("Tender documents archive download prepared", map[string]interface{}{
|
||||||
|
"tender_id": id,
|
||||||
|
"notice_id": tender.NoticePublicationID,
|
||||||
|
"documents_count": len(documents),
|
||||||
|
})
|
||||||
|
|
||||||
|
return &TenderDocumentDownload{
|
||||||
|
Reader: reader,
|
||||||
|
Filename: filename,
|
||||||
|
ContentType: "application/zip",
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *tenderService) writeDocumentsArchive(ctx context.Context, writer *io.PipeWriter, documents []ai_summarizer.StoredDocument) {
|
||||||
|
zipWriter := zip.NewWriter(writer)
|
||||||
|
usedNames := make(map[string]int, len(documents))
|
||||||
|
|
||||||
|
for _, doc := range documents {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
zipWriter.Close()
|
||||||
|
writer.CloseWithError(ctx.Err())
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.addDocumentToArchive(ctx, zipWriter, usedNames, doc); err != nil {
|
||||||
|
zipWriter.Close()
|
||||||
|
writer.CloseWithError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := zipWriter.Close(); err != nil {
|
||||||
|
writer.CloseWithError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writer.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *tenderService) addDocumentToArchive(ctx context.Context, zipWriter *zip.Writer, usedNames map[string]int, doc ai_summarizer.StoredDocument) error {
|
||||||
|
reader, err := s.aiSummarizerStorage.DownloadDocument(ctx, doc.ObjectName)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open document %s: %w", doc.DocumentName, err)
|
||||||
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
entry, err := zipWriter.Create(uniqueArchiveName(usedNames, doc.DocumentName))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to create archive entry for %s: %w", doc.DocumentName, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := io.Copy(entry, reader); err != nil {
|
||||||
|
return fmt.Errorf("failed to write document %s to archive: %w", doc.DocumentName, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func uniqueArchiveName(usedNames map[string]int, filename string) string {
|
||||||
|
cleanName := path.Base(filename)
|
||||||
|
if cleanName == "." || cleanName == "/" || cleanName == "" {
|
||||||
|
cleanName = "document"
|
||||||
|
}
|
||||||
|
|
||||||
|
count := usedNames[cleanName]
|
||||||
|
usedNames[cleanName] = count + 1
|
||||||
|
if count == 0 {
|
||||||
|
return cleanName
|
||||||
|
}
|
||||||
|
|
||||||
|
extension := filepath.Ext(cleanName)
|
||||||
|
base := strings.TrimSuffix(cleanName, extension)
|
||||||
|
return fmt.Sprintf("%s-%d%s", base, count+1, extension)
|
||||||
|
}
|
||||||
|
|
||||||
// ListTenders retrieves tenders with pagination and filtering
|
// ListTenders retrieves tenders with pagination and filtering
|
||||||
func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) {
|
func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination *response.Pagination) (*SearchResponse, error) {
|
||||||
s.logger.Info("Listing tenders", map[string]interface{}{
|
s.logger.Info("Listing tenders", map[string]interface{}{
|
||||||
|
|||||||
@@ -30,6 +30,15 @@ type DocumentSummary struct {
|
|||||||
Error string `json:"error"` // empty string when no error
|
Error string `json:"error"` // empty string when no error
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StoredDocument represents a document object stored for a tender in MinIO.
|
||||||
|
type StoredDocument struct {
|
||||||
|
ObjectName string `json:"-"`
|
||||||
|
DocumentName string `json:"document_name"`
|
||||||
|
Size int64 `json:"size"`
|
||||||
|
LastModified int64 `json:"last_modified"`
|
||||||
|
DocumentType string `json:"document_type,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
// TranslateRequest represents the request payload sent to the AI service
|
// TranslateRequest represents the request payload sent to the AI service
|
||||||
// POST /ai/translate endpoint.
|
// POST /ai/translate endpoint.
|
||||||
type TranslateRequest struct {
|
type TranslateRequest struct {
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/minio/minio-go/v7"
|
"github.com/minio/minio-go/v7"
|
||||||
@@ -153,6 +154,89 @@ func (s *StorageClient) IsSummaryReady(ctx context.Context, noticeID string) (bo
|
|||||||
return tenderJSON.SummarizeStatus.Done, nil
|
return tenderJSON.SummarizeStatus.Done, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListDocuments lists document files stored under <noticeID>/.
|
||||||
|
func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]StoredDocument, error) {
|
||||||
|
noticeID = strings.TrimSpace(noticeID)
|
||||||
|
if noticeID == "" {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: noticeID is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
prefix := fmt.Sprintf("%s/", noticeID)
|
||||||
|
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
|
||||||
|
Prefix: prefix,
|
||||||
|
Recursive: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
documents := make([]StoredDocument, 0)
|
||||||
|
for object := range objectCh {
|
||||||
|
if object.Err != nil {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: failed to list documents for notice %s: %w", noticeID, object.Err)
|
||||||
|
}
|
||||||
|
if !isTenderDocumentObject(noticeID, object.Key) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
documentName := filepath.Base(object.Key)
|
||||||
|
documents = append(documents, StoredDocument{
|
||||||
|
ObjectName: object.Key,
|
||||||
|
DocumentName: documentName,
|
||||||
|
Size: object.Size,
|
||||||
|
LastModified: object.LastModified.Unix(),
|
||||||
|
DocumentType: documentType(documentName),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
s.logger.Info("Listed tender documents from storage", map[string]interface{}{
|
||||||
|
"notice_id": noticeID,
|
||||||
|
"documents_count": len(documents),
|
||||||
|
})
|
||||||
|
|
||||||
|
return documents, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isTenderDocumentObject(noticeID, objectName string) bool {
|
||||||
|
if objectName == "" || strings.HasSuffix(objectName, "/") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
relativePath := strings.TrimPrefix(objectName, fmt.Sprintf("%s/", noticeID))
|
||||||
|
if relativePath == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := strings.ToLower(filepath.Base(relativePath))
|
||||||
|
switch filename {
|
||||||
|
case "tender.json", "ai_cache.db":
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
switch strings.ToLower(filepath.Ext(filename)) {
|
||||||
|
case ".json", ".db", ".sqlite", ".sqlite3":
|
||||||
|
return false
|
||||||
|
default:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// DownloadDocument opens a document object from the configured MinIO bucket.
|
||||||
|
func (s *StorageClient) DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error) {
|
||||||
|
objectName = strings.TrimSpace(objectName)
|
||||||
|
if objectName == "" {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: objectName is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectName, minio.GetObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
var minioErr minio.ErrorResponse
|
||||||
|
if errors.As(err, &minioErr) && minioErr.StatusCode == http.StatusNotFound {
|
||||||
|
return nil, ErrObjectNotFound
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: failed to get document object: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return object, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetAllSummaries lists all tender.json files in the MinIO bucket and returns
|
// GetAllSummaries lists all tender.json files in the MinIO bucket and returns
|
||||||
// the overall_summary for each tender where summarize_status.done is true.
|
// the overall_summary for each tender where summarize_status.done is true.
|
||||||
// This is useful for batch-retrieving all summaries that the AI pipeline has
|
// This is useful for batch-retrieving all summaries that the AI pipeline has
|
||||||
@@ -233,3 +317,11 @@ func countDone(items []TenderSummaryItem) int {
|
|||||||
}
|
}
|
||||||
return count
|
return count
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func documentType(filename string) string {
|
||||||
|
extension := strings.TrimPrefix(strings.ToLower(filepath.Ext(filename)), ".")
|
||||||
|
if extension == "" {
|
||||||
|
return "unknown"
|
||||||
|
}
|
||||||
|
return extension
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user