Mazyar
364 lines
12 KiB
Go
364 lines
12 KiB
Go
package workers
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
"tm/internal/tender"
|
|
"tm/pkg/glm"
|
|
"tm/pkg/logger"
|
|
"tm/pkg/minio"
|
|
"tm/pkg/mongo"
|
|
)
|
|
|
|
// DocumentSummarizationWorker handles document summarization using GLM AI
|
|
type DocumentSummarizationWorker struct {
|
|
Mongo *mongo.ConnectionManager
|
|
Logger logger.Logger
|
|
TenderRepo tender.TenderRepository
|
|
MinioSDK *minio.Service
|
|
GLM *glm.SDK
|
|
}
|
|
|
|
// NewDocumentSummarizationWorker creates a new document summarization worker
|
|
func NewDocumentSummarizationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, minioSDK *minio.Service, glmSDK *glm.SDK) *DocumentSummarizationWorker {
|
|
return &DocumentSummarizationWorker{
|
|
Mongo: mongo,
|
|
Logger: logger,
|
|
TenderRepo: tenderRepo,
|
|
MinioSDK: minioSDK,
|
|
GLM: glmSDK,
|
|
}
|
|
}
|
|
|
|
// Run starts the document summarization process
|
|
func (w *DocumentSummarizationWorker) Run() {
|
|
w.Logger.Info("Document summarization worker started", map[string]interface{}{})
|
|
|
|
limit := 5 // Process fewer tenders at a time since summarization is resource-intensive
|
|
for {
|
|
// Always use skip=0 because processed items are excluded by the filter,
|
|
// so incrementing skip would cause unprocessed items to be skipped.
|
|
tenders, totalCount, err := w.TenderRepo.GetUnSummarizedTenders(context.Background(), limit, 0)
|
|
if err != nil {
|
|
w.Logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
|
|
w.Logger.Info("Found tenders for document summarization", map[string]interface{}{
|
|
"count": len(tenders),
|
|
"total_count": totalCount,
|
|
})
|
|
|
|
if len(tenders) == 0 {
|
|
w.Logger.Info("No more tenders to process for document summarization", map[string]interface{}{})
|
|
break
|
|
}
|
|
|
|
for _, t := range tenders {
|
|
w.Logger.Info("Processing tender for document summarization", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"document_count": len(t.ScrapedDocuments),
|
|
})
|
|
|
|
w.summarizeDocumentsForTender(&t)
|
|
}
|
|
}
|
|
|
|
w.Logger.Info("Document summarization worker completed", map[string]interface{}{})
|
|
}
|
|
|
|
// summarizeDocumentsForTender summarizes all documents for a single tender
|
|
func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tender) {
|
|
if len(t.ScrapedDocuments) == 0 {
|
|
w.Logger.Warn("No scraped documents found for tender", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
})
|
|
|
|
// Mark as processed even if no documents (nothing to summarize)
|
|
t.ProcessingMetadata.DocumentsSummarized = true
|
|
t.ProcessingMetadata.DocumentsSummarizedAt = time.Now().Unix()
|
|
w.updateTenderSummarizationStatus(t)
|
|
return
|
|
}
|
|
|
|
summaries := make([]tender.DocumentSummary, 0, len(t.ScrapedDocuments))
|
|
summarizedAt := time.Now().Unix()
|
|
|
|
for _, doc := range t.ScrapedDocuments {
|
|
summary, err := w.summarizeDocument(doc, t.NoticeLanguageCode)
|
|
if err != nil {
|
|
w.Logger.Error("Failed to summarize document", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"document_name": doc.Filename,
|
|
"object_name": doc.ObjectName,
|
|
"error": err.Error(),
|
|
})
|
|
|
|
// Add summary with error
|
|
summaries = append(summaries, tender.DocumentSummary{
|
|
DocumentName: doc.Filename,
|
|
ObjectName: doc.ObjectName,
|
|
BucketName: doc.BucketName,
|
|
Summary: "",
|
|
SummaryLanguage: "en",
|
|
DocumentType: w.getDocumentType(doc.Filename),
|
|
SummarizedAt: summarizedAt,
|
|
SummaryModel: w.getGLMModelName(),
|
|
Error: err.Error(),
|
|
})
|
|
continue
|
|
}
|
|
|
|
summaries = append(summaries, tender.DocumentSummary{
|
|
DocumentName: doc.Filename,
|
|
ObjectName: doc.ObjectName,
|
|
BucketName: doc.BucketName,
|
|
Summary: summary,
|
|
SummaryLanguage: "en",
|
|
DocumentType: w.getDocumentType(doc.Filename),
|
|
SummarizedAt: summarizedAt,
|
|
SummaryModel: w.getGLMModelName(),
|
|
})
|
|
|
|
w.Logger.Info("Document summarized successfully", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"document_name": doc.Filename,
|
|
"summary_length": len(summary),
|
|
})
|
|
}
|
|
|
|
// Update tender with summaries
|
|
t.DocumentSummaries = summaries
|
|
t.ProcessingMetadata.DocumentsSummarized = true
|
|
t.ProcessingMetadata.DocumentsSummarizedAt = summarizedAt
|
|
|
|
err := w.updateTenderSummarizationStatus(t)
|
|
if err != nil {
|
|
w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"error": err.Error(),
|
|
})
|
|
return
|
|
}
|
|
|
|
w.Logger.Info("Document summarization completed for tender", map[string]interface{}{
|
|
"tender_id": t.ID.Hex(),
|
|
"notice_publication_id": t.NoticePublicationID,
|
|
"summarized_count": len(summaries),
|
|
"total_documents": len(t.ScrapedDocuments),
|
|
})
|
|
}
|
|
|
|
// summarizeDocument downloads a document from MinIO and summarizes it using GLM
|
|
func (w *DocumentSummarizationWorker) summarizeDocument(doc tender.ScrapedDocument, sourceLanguage string) (string, error) {
|
|
// Download document from MinIO
|
|
documentData, err := w.MinioSDK.Download(context.Background(), doc.BucketName, doc.ObjectName, minio.DownloadOptions{})
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to download document from MinIO: %w", err)
|
|
}
|
|
defer documentData.Close()
|
|
|
|
// Read document content
|
|
documentBytes, err := io.ReadAll(documentData)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to read document content: %w", err)
|
|
}
|
|
|
|
// Extract text from document based on type
|
|
textContent, err := w.extractTextFromDocument(documentBytes, doc.Filename)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to extract text from document: %w", err)
|
|
}
|
|
|
|
if len(textContent) == 0 {
|
|
return "", fmt.Errorf("no text content extracted from document")
|
|
}
|
|
|
|
// Limit text length for summarization (GLM has token limits)
|
|
maxLength := 10000 // Adjust based on GLM model limits
|
|
if len(textContent) > maxLength {
|
|
textContent = textContent[:maxLength] + "..."
|
|
}
|
|
|
|
// Create summarization prompt
|
|
prompt := w.createSummarizationPrompt(textContent, sourceLanguage)
|
|
|
|
// Use GLM to summarize
|
|
summary, err := w.GLM.ChatCompletion(context.Background(), &glm.ChatCompletionRequest{
|
|
Messages: []glm.Message{
|
|
{
|
|
Role: glm.MessageRoleUser,
|
|
Content: prompt,
|
|
},
|
|
},
|
|
Temperature: 0.3, // Lower temperature for more consistent summaries
|
|
MaxTokens: 1000,
|
|
})
|
|
if err != nil {
|
|
// Check if GLM is configured (has API key)
|
|
if strings.Contains(err.Error(), "token expired") || strings.Contains(err.Error(), "AUTHENTICATION_ERROR") {
|
|
// GLM is configured but API key is invalid - return error instead of mock
|
|
return "", fmt.Errorf("GLM API authentication failed: %w", err)
|
|
}
|
|
|
|
// GLM is not configured or other error - use mock summary for development
|
|
w.Logger.Warn("GLM API failed, using mock summary for development", map[string]interface{}{
|
|
"error": err.Error(),
|
|
})
|
|
return fmt.Sprintf("Mock summary for document: %s (GLM API not configured)", doc.Filename), nil
|
|
}
|
|
|
|
if len(summary.Choices) == 0 || summary.Choices[0].Message.Content == "" {
|
|
return "", fmt.Errorf("GLM API returned empty response for document: %s", doc.Filename)
|
|
}
|
|
|
|
return strings.TrimSpace(summary.Choices[0].Message.Content), nil
|
|
}
|
|
|
|
// extractTextFromDocument extracts text content from various document formats
|
|
func (w *DocumentSummarizationWorker) extractTextFromDocument(data []byte, filename string) (string, error) {
|
|
ext := strings.ToLower(filepath.Ext(filename))
|
|
|
|
switch ext {
|
|
case ".xlsx":
|
|
return w.extractTextFromExcel(data)
|
|
case ".pdf":
|
|
return w.extractTextFromPDF(data)
|
|
case ".docx":
|
|
return w.extractTextFromDOCX(data)
|
|
case ".txt":
|
|
return string(data), nil
|
|
case ".html", ".htm":
|
|
return w.extractTextFromHTML(data)
|
|
default:
|
|
// Try to extract as plain text for unknown formats
|
|
return string(data), nil
|
|
}
|
|
}
|
|
|
|
// extractTextFromPDF extracts text from PDF documents
|
|
func (w *DocumentSummarizationWorker) extractTextFromPDF(data []byte) (string, error) {
|
|
// For now, return a placeholder. In production, you'd use a PDF parsing library
|
|
// like github.com/unidoc/unipdf or github.com/rsc/pdf
|
|
return fmt.Sprintf("PDF content extraction not implemented. File size: %d bytes", len(data)), nil
|
|
}
|
|
|
|
// extractTextFromDOCX extracts text from DOCX documents
|
|
func (w *DocumentSummarizationWorker) extractTextFromDOCX(data []byte) (string, error) {
|
|
// For now, return a placeholder. In production, you'd use a DOCX parsing library
|
|
// like github.com/lukasjarosch/go-docx or github.com/fumiama/go-docx
|
|
return fmt.Sprintf("DOCX content extraction not implemented. File size: %d bytes", len(data)), nil
|
|
}
|
|
|
|
// extractTextFromExcel extracts text from Excel documents (.xlsx)
|
|
func (w *DocumentSummarizationWorker) extractTextFromExcel(data []byte) (string, error) {
|
|
// For now, return a placeholder. In production, you'd use an Excel parsing library
|
|
// like github.com/tealeg/xlsx/v3 or github.com/360EntSecGroup-Skylar/excelize
|
|
return fmt.Sprintf("Excel content extraction not implemented. File size: %d bytes", len(data)), nil
|
|
}
|
|
|
|
// extractTextFromHTML extracts text from HTML documents
|
|
func (w *DocumentSummarizationWorker) extractTextFromHTML(data []byte) (string, error) {
|
|
// Simple HTML text extraction (remove tags)
|
|
htmlContent := string(data)
|
|
|
|
// Basic HTML tag removal (very simple implementation)
|
|
// In production, use a proper HTML parser like goquery
|
|
htmlContent = strings.ReplaceAll(htmlContent, "<script", "</script>")
|
|
htmlContent = strings.ReplaceAll(htmlContent, "<style", "</style>")
|
|
|
|
// Remove HTML tags using simple regex-like approach
|
|
var result strings.Builder
|
|
inTag := false
|
|
for _, r := range htmlContent {
|
|
switch r {
|
|
case '<':
|
|
inTag = true
|
|
case '>':
|
|
inTag = false
|
|
default:
|
|
if !inTag {
|
|
result.WriteRune(r)
|
|
}
|
|
}
|
|
}
|
|
|
|
return strings.TrimSpace(result.String()), nil
|
|
}
|
|
|
|
// createSummarizationPrompt creates a prompt for GLM to summarize tender documents
|
|
func (w *DocumentSummarizationWorker) createSummarizationPrompt(textContent, sourceLanguage string) string {
|
|
return fmt.Sprintf(`Please provide a concise summary of the following tender document in English. Focus on key information such as:
|
|
|
|
1. Project/Tender name and description
|
|
2. Contracting authority/organization details
|
|
3. Key requirements and specifications
|
|
4. Important dates (deadline, award date, etc.)
|
|
5. Estimated value and currency
|
|
6. Selection criteria
|
|
7. Contact information
|
|
|
|
Document content:
|
|
%s
|
|
|
|
Summary:`, textContent)
|
|
}
|
|
|
|
// getDocumentType returns the document type based on file extension
|
|
func (w *DocumentSummarizationWorker) getDocumentType(filename string) string {
|
|
ext := strings.ToLower(filepath.Ext(filename))
|
|
switch ext {
|
|
case ".xlsx":
|
|
return "xlsx"
|
|
case ".pdf":
|
|
return "pdf"
|
|
case ".docx":
|
|
return "docx"
|
|
case ".doc":
|
|
return "doc"
|
|
case ".txt":
|
|
return "txt"
|
|
case ".html", ".htm":
|
|
return "html"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
// getGLMModelName returns the GLM model name being used
|
|
func (w *DocumentSummarizationWorker) getGLMModelName() string {
|
|
return "glm-4.5"
|
|
}
|
|
|
|
// updateTenderSummarizationStatus updates the tender's document summarization metadata
|
|
func (w *DocumentSummarizationWorker) updateTenderSummarizationStatus(tender *tender.Tender) error {
|
|
err := w.TenderRepo.Update(context.Background(), tender)
|
|
if err != nil {
|
|
w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{
|
|
"tender_id": tender.ID.Hex(),
|
|
"error": err.Error(),
|
|
})
|
|
return err
|
|
}
|
|
|
|
w.Logger.Debug("Updated tender summarization status", map[string]interface{}{
|
|
"tender_id": tender.ID.Hex(),
|
|
"documents_summarized": tender.ProcessingMetadata.DocumentsSummarized,
|
|
"documents_summarized_at": tender.ProcessingMetadata.DocumentsSummarizedAt,
|
|
"summary_count": len(tender.DocumentSummaries),
|
|
})
|
|
|
|
return nil
|
|
}
|