merge tenders which have the same procedure id - worker performance fix
This commit is contained in:
Mazyar
@@ -2,46 +2,53 @@ package workers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
"tm/internal/tender"
|
||||
"tm/pkg/glm"
|
||||
ai_summarizer "tm/pkg/ai_summarizer"
|
||||
"tm/pkg/logger"
|
||||
"tm/pkg/minio"
|
||||
"tm/pkg/mongo"
|
||||
)
|
||||
|
||||
// DocumentSummarizationWorker handles document summarization using GLM AI
|
||||
const presignedDocumentURLSeconds int64 = 7200
|
||||
|
||||
// DocumentSummarizationWorker asks the external AI service to summarize scraped tender documents.
|
||||
type DocumentSummarizationWorker struct {
|
||||
Mongo *mongo.ConnectionManager
|
||||
Logger logger.Logger
|
||||
TenderRepo tender.TenderRepository
|
||||
MinioSDK *minio.Service
|
||||
GLM *glm.SDK
|
||||
AIClient *ai_summarizer.Client
|
||||
}
|
||||
|
||||
// NewDocumentSummarizationWorker creates a new document summarization worker
|
||||
func NewDocumentSummarizationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, minioSDK *minio.Service, glmSDK *glm.SDK) *DocumentSummarizationWorker {
|
||||
// NewDocumentSummarizationWorker creates a document summarization worker.
|
||||
func NewDocumentSummarizationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, minioSDK *minio.Service, aiClient *ai_summarizer.Client) *DocumentSummarizationWorker {
|
||||
return &DocumentSummarizationWorker{
|
||||
Mongo: mongo,
|
||||
Logger: logger,
|
||||
TenderRepo: tenderRepo,
|
||||
MinioSDK: minioSDK,
|
||||
GLM: glmSDK,
|
||||
AIClient: aiClient,
|
||||
}
|
||||
}
|
||||
|
||||
// Run starts the document summarization process
|
||||
// Run starts the document summarization process.
|
||||
func (w *DocumentSummarizationWorker) Run() {
|
||||
w.Logger.Info("Document summarization worker started", map[string]interface{}{})
|
||||
|
||||
limit := 5 // Process fewer tenders at a time since summarization is resource-intensive
|
||||
if w.AIClient == nil {
|
||||
w.Logger.Warn("AI summarizer client not configured, skipping document summarization run", map[string]interface{}{})
|
||||
return
|
||||
}
|
||||
if w.MinioSDK == nil {
|
||||
w.Logger.Warn("MinIO not configured, skipping document summarization run", map[string]interface{}{})
|
||||
return
|
||||
}
|
||||
|
||||
limit := 5
|
||||
for {
|
||||
// Always use skip=0 because processed items are excluded by the filter,
|
||||
// so incrementing skip would cause unprocessed items to be skipped.
|
||||
tenders, totalCount, err := w.TenderRepo.GetUnSummarizedTenders(context.Background(), limit, 0)
|
||||
if err != nil {
|
||||
w.Logger.Error("Failed to get un-summarized tenders", map[string]interface{}{
|
||||
@@ -60,32 +67,29 @@ func (w *DocumentSummarizationWorker) Run() {
|
||||
break
|
||||
}
|
||||
|
||||
for _, t := range tenders {
|
||||
for i := range tenders {
|
||||
t := &tenders[i]
|
||||
w.Logger.Info("Processing tender for document summarization", map[string]interface{}{
|
||||
"tender_id": t.ID.Hex(),
|
||||
"notice_publication_id": t.NoticePublicationID,
|
||||
"document_count": len(t.ScrapedDocuments),
|
||||
})
|
||||
|
||||
w.summarizeDocumentsForTender(&t)
|
||||
w.summarizeDocumentsForTender(t)
|
||||
}
|
||||
}
|
||||
|
||||
w.Logger.Info("Document summarization worker completed", map[string]interface{}{})
|
||||
}
|
||||
|
||||
// summarizeDocumentsForTender summarizes all documents for a single tender
|
||||
func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tender) {
|
||||
if len(t.ScrapedDocuments) == 0 {
|
||||
w.Logger.Warn("No scraped documents found for tender", map[string]interface{}{
|
||||
"tender_id": t.ID.Hex(),
|
||||
"notice_publication_id": t.NoticePublicationID,
|
||||
})
|
||||
|
||||
// Mark as processed even if no documents (nothing to summarize)
|
||||
t.ProcessingMetadata.DocumentsSummarized = true
|
||||
t.ProcessingMetadata.DocumentsSummarizedAt = time.Now().Unix()
|
||||
w.updateTenderSummarizationStatus(t)
|
||||
_ = w.updateTenderSummarizationStatus(t)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -93,17 +97,15 @@ func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tend
|
||||
summarizedAt := time.Now().Unix()
|
||||
|
||||
for _, doc := range t.ScrapedDocuments {
|
||||
summary, err := w.summarizeDocument(doc, t.NoticeLanguageCode)
|
||||
if err != nil {
|
||||
summaryText, model, errStr := w.summarizeDocumentViaAI(context.Background(), t, doc)
|
||||
if errStr != "" {
|
||||
w.Logger.Error("Failed to summarize document", map[string]interface{}{
|
||||
"tender_id": t.ID.Hex(),
|
||||
"notice_publication_id": t.NoticePublicationID,
|
||||
"document_name": doc.Filename,
|
||||
"object_name": doc.ObjectName,
|
||||
"error": err.Error(),
|
||||
"error": errStr,
|
||||
})
|
||||
|
||||
// Add summary with error
|
||||
summaries = append(summaries, tender.DocumentSummary{
|
||||
DocumentName: doc.Filename,
|
||||
ObjectName: doc.ObjectName,
|
||||
@@ -112,8 +114,8 @@ func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tend
|
||||
SummaryLanguage: "en",
|
||||
DocumentType: w.getDocumentType(doc.Filename),
|
||||
SummarizedAt: summarizedAt,
|
||||
SummaryModel: w.getGLMModelName(),
|
||||
Error: err.Error(),
|
||||
SummaryModel: model,
|
||||
Error: errStr,
|
||||
})
|
||||
continue
|
||||
}
|
||||
@@ -122,28 +124,26 @@ func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tend
|
||||
DocumentName: doc.Filename,
|
||||
ObjectName: doc.ObjectName,
|
||||
BucketName: doc.BucketName,
|
||||
Summary: summary,
|
||||
Summary: summaryText,
|
||||
SummaryLanguage: "en",
|
||||
DocumentType: w.getDocumentType(doc.Filename),
|
||||
SummarizedAt: summarizedAt,
|
||||
SummaryModel: w.getGLMModelName(),
|
||||
SummaryModel: model,
|
||||
})
|
||||
|
||||
w.Logger.Info("Document summarized successfully", map[string]interface{}{
|
||||
"tender_id": t.ID.Hex(),
|
||||
"notice_publication_id": t.NoticePublicationID,
|
||||
"document_name": doc.Filename,
|
||||
"summary_length": len(summary),
|
||||
"summary_length": len(summaryText),
|
||||
})
|
||||
}
|
||||
|
||||
// Update tender with summaries
|
||||
t.DocumentSummaries = summaries
|
||||
t.ProcessingMetadata.DocumentsSummarized = true
|
||||
t.ProcessingMetadata.DocumentsSummarizedAt = summarizedAt
|
||||
|
||||
err := w.updateTenderSummarizationStatus(t)
|
||||
if err != nil {
|
||||
if err := w.updateTenderSummarizationStatus(t); err != nil {
|
||||
w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{
|
||||
"tender_id": t.ID.Hex(),
|
||||
"notice_publication_id": t.NoticePublicationID,
|
||||
@@ -160,162 +160,74 @@ func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tend
|
||||
})
|
||||
}
|
||||
|
||||
// summarizeDocument downloads a document from MinIO and summarizes it using GLM
|
||||
func (w *DocumentSummarizationWorker) summarizeDocument(doc tender.ScrapedDocument, sourceLanguage string) (string, error) {
|
||||
// Download document from MinIO
|
||||
documentData, err := w.MinioSDK.Download(context.Background(), doc.BucketName, doc.ObjectName, minio.DownloadOptions{})
|
||||
func (w *DocumentSummarizationWorker) summarizeDocumentViaAI(ctx context.Context, t *tender.Tender, doc tender.ScrapedDocument) (summaryText, model, errStr string) {
|
||||
docURL, err := w.MinioSDK.GetPresignedURL(ctx, doc.BucketName, doc.ObjectName, presignedDocumentURLSeconds)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to download document from MinIO: %w", err)
|
||||
return "", "", err.Error()
|
||||
}
|
||||
defer documentData.Close()
|
||||
|
||||
// Read document content
|
||||
documentBytes, err := io.ReadAll(documentData)
|
||||
noticeID := strings.TrimSpace(t.NoticePublicationID)
|
||||
if noticeID == "" {
|
||||
return "", "", "tender has no notice_publication_id for AI summarize request"
|
||||
}
|
||||
|
||||
req := ai_summarizer.SummarizeRequest{
|
||||
NoticeID: noticeID,
|
||||
DocumentURL: docURL,
|
||||
Title: t.Title,
|
||||
Description: t.Description,
|
||||
Country: t.CountryCode,
|
||||
Currency: t.Currency,
|
||||
SubmissionDeadline: "",
|
||||
AuthorityName: authorityNameFromTender(t),
|
||||
}
|
||||
if t.EstimatedValue > 0 {
|
||||
v := t.EstimatedValue
|
||||
req.EstimatedValue = &v
|
||||
}
|
||||
|
||||
resp, err := w.AIClient.FetchSummaryOnDemand(ctx, req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read document content: %w", err)
|
||||
return "", "", err.Error()
|
||||
}
|
||||
if resp == nil {
|
||||
return "", "", "AI summarize returned nil response"
|
||||
}
|
||||
|
||||
// Extract text from document based on type
|
||||
textContent, err := w.extractTextFromDocument(documentBytes, doc.Filename)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to extract text from document: %w", err)
|
||||
summaryText, model = pickDocumentSummary(resp.Summaries, doc.Filename)
|
||||
if summaryText == "" && len(resp.Summaries) > 0 {
|
||||
// Fall back to first summary if filename did not match
|
||||
summaryText = resp.Summaries[0].Summary
|
||||
model = resp.Summaries[0].SummaryModel
|
||||
}
|
||||
|
||||
if len(textContent) == 0 {
|
||||
return "", fmt.Errorf("no text content extracted from document")
|
||||
if summaryText == "" {
|
||||
return "", model, "AI summarize returned no summary text for document"
|
||||
}
|
||||
|
||||
// Limit text length for summarization (GLM has token limits)
|
||||
maxLength := 10000 // Adjust based on GLM model limits
|
||||
if len(textContent) > maxLength {
|
||||
textContent = textContent[:maxLength] + "..."
|
||||
}
|
||||
|
||||
// Create summarization prompt
|
||||
prompt := w.createSummarizationPrompt(textContent, sourceLanguage)
|
||||
|
||||
// Use GLM to summarize
|
||||
summary, err := w.GLM.ChatCompletion(context.Background(), &glm.ChatCompletionRequest{
|
||||
Messages: []glm.Message{
|
||||
{
|
||||
Role: glm.MessageRoleUser,
|
||||
Content: prompt,
|
||||
},
|
||||
},
|
||||
Temperature: 0.3, // Lower temperature for more consistent summaries
|
||||
MaxTokens: 1000,
|
||||
})
|
||||
if err != nil {
|
||||
// Check if GLM is configured (has API key)
|
||||
if strings.Contains(err.Error(), "token expired") || strings.Contains(err.Error(), "AUTHENTICATION_ERROR") {
|
||||
// GLM is configured but API key is invalid - return error instead of mock
|
||||
return "", fmt.Errorf("GLM API authentication failed: %w", err)
|
||||
}
|
||||
|
||||
// GLM is not configured or other error - use mock summary for development
|
||||
w.Logger.Warn("GLM API failed, using mock summary for development", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Sprintf("Mock summary for document: %s (GLM API not configured)", doc.Filename), nil
|
||||
}
|
||||
|
||||
if len(summary.Choices) == 0 || summary.Choices[0].Message.Content == "" {
|
||||
return "", fmt.Errorf("GLM API returned empty response for document: %s", doc.Filename)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(summary.Choices[0].Message.Content), nil
|
||||
return summaryText, model, ""
|
||||
}
|
||||
|
||||
// extractTextFromDocument extracts text content from various document formats
|
||||
func (w *DocumentSummarizationWorker) extractTextFromDocument(data []byte, filename string) (string, error) {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
|
||||
switch ext {
|
||||
case ".xlsx":
|
||||
return w.extractTextFromExcel(data)
|
||||
case ".pdf":
|
||||
return w.extractTextFromPDF(data)
|
||||
case ".docx":
|
||||
return w.extractTextFromDOCX(data)
|
||||
case ".txt":
|
||||
return string(data), nil
|
||||
case ".html", ".htm":
|
||||
return w.extractTextFromHTML(data)
|
||||
default:
|
||||
// Try to extract as plain text for unknown formats
|
||||
return string(data), nil
|
||||
func authorityNameFromTender(t *tender.Tender) string {
|
||||
if t == nil || t.BuyerOrganization == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(t.BuyerOrganization.Name)
|
||||
}
|
||||
|
||||
// extractTextFromPDF extracts text from PDF documents
|
||||
func (w *DocumentSummarizationWorker) extractTextFromPDF(data []byte) (string, error) {
|
||||
// For now, return a placeholder. In production, you'd use a PDF parsing library
|
||||
// like github.com/unidoc/unipdf or github.com/rsc/pdf
|
||||
return fmt.Sprintf("PDF content extraction not implemented. File size: %d bytes", len(data)), nil
|
||||
}
|
||||
|
||||
// extractTextFromDOCX extracts text from DOCX documents
|
||||
func (w *DocumentSummarizationWorker) extractTextFromDOCX(data []byte) (string, error) {
|
||||
// For now, return a placeholder. In production, you'd use a DOCX parsing library
|
||||
// like github.com/lukasjarosch/go-docx or github.com/fumiama/go-docx
|
||||
return fmt.Sprintf("DOCX content extraction not implemented. File size: %d bytes", len(data)), nil
|
||||
}
|
||||
|
||||
// extractTextFromExcel extracts text from Excel documents (.xlsx)
|
||||
func (w *DocumentSummarizationWorker) extractTextFromExcel(data []byte) (string, error) {
|
||||
// For now, return a placeholder. In production, you'd use an Excel parsing library
|
||||
// like github.com/tealeg/xlsx/v3 or github.com/360EntSecGroup-Skylar/excelize
|
||||
return fmt.Sprintf("Excel content extraction not implemented. File size: %d bytes", len(data)), nil
|
||||
}
|
||||
|
||||
// extractTextFromHTML extracts text from HTML documents
|
||||
func (w *DocumentSummarizationWorker) extractTextFromHTML(data []byte) (string, error) {
|
||||
// Simple HTML text extraction (remove tags)
|
||||
htmlContent := string(data)
|
||||
|
||||
// Basic HTML tag removal (very simple implementation)
|
||||
// In production, use a proper HTML parser like goquery
|
||||
htmlContent = strings.ReplaceAll(htmlContent, "<script", "</script>")
|
||||
htmlContent = strings.ReplaceAll(htmlContent, "<style", "</style>")
|
||||
|
||||
// Remove HTML tags using simple regex-like approach
|
||||
var result strings.Builder
|
||||
inTag := false
|
||||
for _, r := range htmlContent {
|
||||
switch r {
|
||||
case '<':
|
||||
inTag = true
|
||||
case '>':
|
||||
inTag = false
|
||||
default:
|
||||
if !inTag {
|
||||
result.WriteRune(r)
|
||||
}
|
||||
func pickDocumentSummary(items []ai_summarizer.DocumentSummary, filename string) (text, model string) {
|
||||
filename = strings.TrimSpace(filename)
|
||||
for _, s := range items {
|
||||
if strings.EqualFold(strings.TrimSpace(s.DocumentName), filename) {
|
||||
return strings.TrimSpace(s.Summary), strings.TrimSpace(s.SummaryModel)
|
||||
}
|
||||
}
|
||||
|
||||
return strings.TrimSpace(result.String()), nil
|
||||
for _, s := range items {
|
||||
if strings.TrimSpace(s.Summary) != "" {
|
||||
return strings.TrimSpace(s.Summary), strings.TrimSpace(s.SummaryModel)
|
||||
}
|
||||
}
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// createSummarizationPrompt creates a prompt for GLM to summarize tender documents
|
||||
func (w *DocumentSummarizationWorker) createSummarizationPrompt(textContent, sourceLanguage string) string {
|
||||
return fmt.Sprintf(`Please provide a concise summary of the following tender document in English. Focus on key information such as:
|
||||
|
||||
1. Project/Tender name and description
|
||||
2. Contracting authority/organization details
|
||||
3. Key requirements and specifications
|
||||
4. Important dates (deadline, award date, etc.)
|
||||
5. Estimated value and currency
|
||||
6. Selection criteria
|
||||
7. Contact information
|
||||
|
||||
Document content:
|
||||
%s
|
||||
|
||||
Summary:`, textContent)
|
||||
}
|
||||
|
||||
// getDocumentType returns the document type based on file extension
|
||||
func (w *DocumentSummarizationWorker) getDocumentType(filename string) string {
|
||||
ext := strings.ToLower(filepath.Ext(filename))
|
||||
switch ext {
|
||||
@@ -336,12 +248,6 @@ func (w *DocumentSummarizationWorker) getDocumentType(filename string) string {
|
||||
}
|
||||
}
|
||||
|
||||
// getGLMModelName returns the GLM model name being used
|
||||
func (w *DocumentSummarizationWorker) getGLMModelName() string {
|
||||
return "glm-4.5"
|
||||
}
|
||||
|
||||
// updateTenderSummarizationStatus updates the tender's document summarization metadata
|
||||
func (w *DocumentSummarizationWorker) updateTenderSummarizationStatus(tender *tender.Tender) error {
|
||||
err := w.TenderRepo.Update(context.Background(), tender)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user