Mazyar
270 lines
8.2 KiB
Go
270 lines
8.2 KiB
Go
package workers
|
|
|
|
import (
|
|
"context"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
"tm/internal/tender"
|
|
ai_summarizer "tm/pkg/ai_summarizer"
|
|
"tm/pkg/logger"
|
|
"tm/pkg/minio"
|
|
"tm/pkg/mongo"
|
|
)
|
|
|
|
const presignedDocumentURLSeconds int64 = 7200
|
|
|
|
// DocumentSummarizationWorker asks the external AI service to summarize scraped tender documents.
|
|
type DocumentSummarizationWorker struct {
|
|
Mongo *mongo.ConnectionManager
|
|
Logger logger.Logger
|
|
TenderRepo tender.TenderRepository
|
|
MinioSDK *minio.Service
|
|
AIClient *ai_summarizer.Client
|
|
}
|
|
|
|
// NewDocumentSummarizationWorker creates a document summarization worker.
|
|
func NewDocumentSummarizationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, minioSDK *minio.Service, aiClient *ai_summarizer.Client) *DocumentSummarizationWorker {
|
|
return &DocumentSummarizationWorker{
|
|
Mongo: mongo,
|
|
Logger: logger,
|
|
TenderRepo: tenderRepo,
|
|
MinioSDK: minioSDK,
|
|
AIClient: aiClient,
|
|
}
|
|
}
|
|
|
|
// Run starts the document summarization process.
|
|
func (w *DocumentSummarizationWorker) Run() {
|
|
w.Logger.Info("Document summarization worker started", map[string]interface{}{})
|
|
|
|
if w.AIClient == nil {
|
|
w.Logger.Warn("AI summarizer client not configured, skipping document summarization run", map[string]interface{}{})
|
|
return
|
|
}
|
|
if w.MinioSDK == nil {
|
|
w.Logger.Warn("MinIO not configured, skipping document summarization run", map[string]interface{}{})
|
|
return
|
|
}
|
|
|
|
limit := 5
|
|
for {
|
|
tenders, totalCount, err := w.TenderRepo.GetUnSummarizedTenders(context.Background(), limit, 0)
|
|
if err != nil {
|
|
w.Logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
|
|
w.Logger.Info("Found tenders for document summarization", map[string]interface{}{
|
|
"count": len(tenders),
|
|
"total_count": totalCount,
|
|
})
|
|
|
|
if len(tenders) == 0 {
|
|
w.Logger.Info("No more tenders to process for document summarization", map[string]interface{}{})
|
|
break
|
|
}
|
|
|
|
for i := range tenders {
|
|
t := &tenders[i]
|
|
w.Logger.Info("Processing tender for document summarization", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"document_count": len(t.ScrapedDocuments),
|
|
})
|
|
w.summarizeDocumentsForTender(t)
|
|
}
|
|
}
|
|
|
|
w.Logger.Info("Document summarization worker completed", map[string]interface{}{})
|
|
}
|
|
|
|
func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tender) {
|
|
if len(t.ScrapedDocuments) == 0 {
|
|
w.Logger.Warn("No scraped documents found for tender", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
})
|
|
t.ProcessingMetadata.DocumentsSummarized = true
|
|
t.ProcessingMetadata.DocumentsSummarizedAt = time.Now().Unix()
|
|
_ = w.updateTenderSummarizationStatus(t)
|
|
return
|
|
}
|
|
|
|
summaries := make([]tender.DocumentSummary, 0, len(t.ScrapedDocuments))
|
|
summarizedAt := time.Now().Unix()
|
|
|
|
for _, doc := range t.ScrapedDocuments {
|
|
summaryText, model, errStr := w.summarizeDocumentViaAI(context.Background(), t, doc)
|
|
if errStr != "" {
|
|
w.Logger.Error("Failed to summarize document", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"document_name": doc.Filename,
|
|
"object_name": doc.ObjectName,
|
|
"error": errStr,
|
|
})
|
|
summaries = append(summaries, tender.DocumentSummary{
|
|
DocumentName: doc.Filename,
|
|
ObjectName: doc.ObjectName,
|
|
BucketName: doc.BucketName,
|
|
Summary: "",
|
|
SummaryLanguage: "en",
|
|
DocumentType: w.getDocumentType(doc.Filename),
|
|
SummarizedAt: summarizedAt,
|
|
SummaryModel: model,
|
|
Error: errStr,
|
|
})
|
|
continue
|
|
}
|
|
|
|
summaries = append(summaries, tender.DocumentSummary{
|
|
DocumentName: doc.Filename,
|
|
ObjectName: doc.ObjectName,
|
|
BucketName: doc.BucketName,
|
|
Summary: summaryText,
|
|
SummaryLanguage: "en",
|
|
DocumentType: w.getDocumentType(doc.Filename),
|
|
SummarizedAt: summarizedAt,
|
|
SummaryModel: model,
|
|
})
|
|
|
|
w.Logger.Info("Document summarized successfully", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"document_name": doc.Filename,
|
|
"summary_length": len(summaryText),
|
|
})
|
|
}
|
|
|
|
t.DocumentSummaries = summaries
|
|
t.ProcessingMetadata.DocumentsSummarized = true
|
|
t.ProcessingMetadata.DocumentsSummarizedAt = summarizedAt
|
|
|
|
if err := w.updateTenderSummarizationStatus(t); err != nil {
|
|
w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"error": err.Error(),
|
|
})
|
|
return
|
|
}
|
|
|
|
w.Logger.Info("Document summarization completed for tender", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"summarized_count": len(summaries),
|
|
"total_documents": len(t.ScrapedDocuments),
|
|
})
|
|
}
|
|
|
|
func (w *DocumentSummarizationWorker) summarizeDocumentViaAI(ctx context.Context, t *tender.Tender, doc tender.ScrapedDocument) (summaryText, model, errStr string) {
|
|
docURL, err := w.MinioSDK.GetPresignedURL(ctx, doc.BucketName, doc.ObjectName, presignedDocumentURLSeconds)
|
|
if err != nil {
|
|
return "", "", err.Error()
|
|
}
|
|
|
|
noticeID := strings.TrimSpace(t.NoticePublicationID)
|
|
if noticeID == "" {
|
|
return "", "", "tender has no notice_publication_id for AI summarize request"
|
|
}
|
|
|
|
req := ai_summarizer.SummarizeRequest{
|
|
NoticeID: noticeID,
|
|
DocumentURL: docURL,
|
|
Title: t.Title,
|
|
Description: t.Description,
|
|
Country: t.CountryCode,
|
|
Currency: t.Currency,
|
|
SubmissionDeadline: "",
|
|
AuthorityName: authorityNameFromTender(t),
|
|
}
|
|
if t.EstimatedValue > 0 {
|
|
v := t.EstimatedValue
|
|
req.EstimatedValue = &v
|
|
}
|
|
|
|
resp, err := w.AIClient.FetchSummaryOnDemand(ctx, req)
|
|
if err != nil {
|
|
return "", "", err.Error()
|
|
}
|
|
if resp == nil {
|
|
return "", "", "AI summarize returned nil response"
|
|
}
|
|
|
|
summaryText, model = pickDocumentSummary(resp.Summaries, doc.Filename)
|
|
if summaryText == "" && len(resp.Summaries) > 0 {
|
|
// Fall back to first summary if filename did not match
|
|
summaryText = resp.Summaries[0].Summary
|
|
model = resp.Summaries[0].SummaryModel
|
|
}
|
|
if summaryText == "" {
|
|
return "", model, "AI summarize returned no summary text for document"
|
|
}
|
|
return summaryText, model, ""
|
|
}
|
|
|
|
func authorityNameFromTender(t *tender.Tender) string {
|
|
if t == nil || t.BuyerOrganization == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(t.BuyerOrganization.Name)
|
|
}
|
|
|
|
func pickDocumentSummary(items []ai_summarizer.DocumentSummary, filename string) (text, model string) {
|
|
filename = strings.TrimSpace(filename)
|
|
for _, s := range items {
|
|
if strings.EqualFold(strings.TrimSpace(s.DocumentName), filename) {
|
|
return strings.TrimSpace(s.Summary), strings.TrimSpace(s.SummaryModel)
|
|
}
|
|
}
|
|
for _, s := range items {
|
|
if strings.TrimSpace(s.Summary) != "" {
|
|
return strings.TrimSpace(s.Summary), strings.TrimSpace(s.SummaryModel)
|
|
}
|
|
}
|
|
return "", ""
|
|
}
|
|
|
|
func (w *DocumentSummarizationWorker) getDocumentType(filename string) string {
|
|
ext := strings.ToLower(filepath.Ext(filename))
|
|
switch ext {
|
|
case ".xlsx":
|
|
return "xlsx"
|
|
case ".pdf":
|
|
return "pdf"
|
|
case ".docx":
|
|
return "docx"
|
|
case ".doc":
|
|
return "doc"
|
|
case ".txt":
|
|
return "txt"
|
|
case ".html", ".htm":
|
|
return "html"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
func (w *DocumentSummarizationWorker) updateTenderSummarizationStatus(tender *tender.Tender) error {
|
|
err := w.TenderRepo.Update(context.Background(), tender)
|
|
if err != nil {
|
|
w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{
|
|
"tender_id": tender.ID.Hex(),
|
|
"error": err.Error(),
|
|
})
|
|
return err
|
|
}
|
|
|
|
w.Logger.Debug("Updated tender summarization status", map[string]interface{}{
|
|
"tender_id": tender.ID.Hex(),
|
|
"documents_summarized": tender.ProcessingMetadata.DocumentsSummarized,
|
|
"documents_summarized_at": tender.ProcessingMetadata.DocumentsSummarizedAt,
|
|
"summary_count": len(tender.DocumentSummaries),
|
|
})
|
|
|
|
return nil
|
|
}
|