package workers import ( "context" "fmt" "io" "path/filepath" "strings" "time" "tm/internal/tender" "tm/pkg/glm" "tm/pkg/logger" "tm/pkg/minio" "tm/pkg/mongo" ) // DocumentSummarizationWorker handles document summarization using GLM AI type DocumentSummarizationWorker struct { Mongo *mongo.ConnectionManager Logger logger.Logger TenderRepo tender.TenderRepository MinioSDK *minio.Service GLM *glm.SDK } // NewDocumentSummarizationWorker creates a new document summarization worker func NewDocumentSummarizationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, minioSDK *minio.Service, glmSDK *glm.SDK) *DocumentSummarizationWorker { return &DocumentSummarizationWorker{ Mongo: mongo, Logger: logger, TenderRepo: tenderRepo, MinioSDK: minioSDK, GLM: glmSDK, } } // Run starts the document summarization process func (w *DocumentSummarizationWorker) Run() { w.Logger.Info("Document summarization worker started", map[string]interface{}{}) limit := 5 // Process fewer tenders at a time since summarization is resource-intensive skip := 0 for { tenders, totalCount, err := w.TenderRepo.GetUnSummarizedTenders(context.Background(), limit, skip) if err != nil { w.Logger.Error("Failed to get un-summarized tenders", map[string]interface{}{ "error": err.Error(), }) continue } w.Logger.Info("Found tenders for document summarization", map[string]interface{}{ "count": len(tenders), "total_count": totalCount, "skip": skip, }) if len(tenders) == 0 { w.Logger.Info("No more tenders to process for document summarization", map[string]interface{}{}) break } for _, t := range tenders { w.Logger.Info("Processing tender for document summarization", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "document_count": len(t.ScrapedDocuments), }) w.summarizeDocumentsForTender(&t) } skip += limit } w.Logger.Info("Document summarization worker completed", map[string]interface{}{}) } // summarizeDocumentsForTender summarizes all documents for a single tender func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tender) { if len(t.ScrapedDocuments) == 0 { w.Logger.Warn("No scraped documents found for tender", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, }) // Mark as processed even if no documents (nothing to summarize) t.ProcessingMetadata.DocumentsSummarized = true t.ProcessingMetadata.DocumentsSummarizedAt = time.Now().Unix() w.updateTenderSummarizationStatus(t) return } summaries := make([]tender.DocumentSummary, 0, len(t.ScrapedDocuments)) summarizedAt := time.Now().Unix() for _, doc := range t.ScrapedDocuments { summary, err := w.summarizeDocument(doc, t.NoticeLanguageCode) if err != nil { w.Logger.Error("Failed to summarize document", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "document_name": doc.Filename, "object_name": doc.ObjectName, "error": err.Error(), }) // Add summary with error summaries = append(summaries, tender.DocumentSummary{ DocumentName: doc.Filename, ObjectName: doc.ObjectName, BucketName: doc.BucketName, Summary: "", SummaryLanguage: "en", DocumentType: w.getDocumentType(doc.Filename), SummarizedAt: summarizedAt, SummaryModel: w.getGLMModelName(), Error: err.Error(), }) continue } summaries = append(summaries, tender.DocumentSummary{ DocumentName: doc.Filename, ObjectName: doc.ObjectName, BucketName: doc.BucketName, Summary: summary, SummaryLanguage: "en", DocumentType: w.getDocumentType(doc.Filename), SummarizedAt: summarizedAt, SummaryModel: w.getGLMModelName(), }) w.Logger.Info("Document summarized successfully", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "document_name": doc.Filename, "summary_length": len(summary), }) } // Update tender with summaries t.DocumentSummaries = summaries t.ProcessingMetadata.DocumentsSummarized = true t.ProcessingMetadata.DocumentsSummarizedAt = summarizedAt err := w.updateTenderSummarizationStatus(t) if err != nil { w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "error": err.Error(), }) return } w.Logger.Info("Document summarization completed for tender", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "summarized_count": len(summaries), "total_documents": len(t.ScrapedDocuments), }) } // summarizeDocument downloads a document from MinIO and summarizes it using GLM func (w *DocumentSummarizationWorker) summarizeDocument(doc tender.ScrapedDocument, sourceLanguage string) (string, error) { // Download document from MinIO documentData, err := w.MinioSDK.Download(context.Background(), doc.BucketName, doc.ObjectName, minio.DownloadOptions{}) if err != nil { return "", fmt.Errorf("failed to download document from MinIO: %w", err) } defer documentData.Close() // Read document content documentBytes, err := io.ReadAll(documentData) if err != nil { return "", fmt.Errorf("failed to read document content: %w", err) } // Extract text from document based on type textContent, err := w.extractTextFromDocument(documentBytes, doc.Filename) if err != nil { return "", fmt.Errorf("failed to extract text from document: %w", err) } if len(textContent) == 0 { return "", fmt.Errorf("no text content extracted from document") } // Limit text length for summarization (GLM has token limits) maxLength := 10000 // Adjust based on GLM model limits if len(textContent) > maxLength { textContent = textContent[:maxLength] + "..." } // Create summarization prompt prompt := w.createSummarizationPrompt(textContent, sourceLanguage) // Use GLM to summarize summary, err := w.GLM.ChatCompletion(context.Background(), &glm.ChatCompletionRequest{ Messages: []glm.Message{ { Role: glm.MessageRoleUser, Content: prompt, }, }, Temperature: 0.3, // Lower temperature for more consistent summaries MaxTokens: 1000, }) if err != nil { // Check if GLM is configured (has API key) if strings.Contains(err.Error(), "token expired") || strings.Contains(err.Error(), "AUTHENTICATION_ERROR") { // GLM is configured but API key is invalid - return error instead of mock return "", fmt.Errorf("GLM API authentication failed: %w", err) } // GLM is not configured or other error - use mock summary for development w.Logger.Warn("GLM API failed, using mock summary for development", map[string]interface{}{ "error": err.Error(), }) return fmt.Sprintf("Mock summary for document: %s (GLM API not configured)", doc.Filename), nil } if len(summary.Choices) == 0 || summary.Choices[0].Message.Content == "" { return "", fmt.Errorf("GLM API returned empty response for document: %s", doc.Filename) } return strings.TrimSpace(summary.Choices[0].Message.Content), nil } // extractTextFromDocument extracts text content from various document formats func (w *DocumentSummarizationWorker) extractTextFromDocument(data []byte, filename string) (string, error) { ext := strings.ToLower(filepath.Ext(filename)) switch ext { case ".xlsx": return w.extractTextFromExcel(data) case ".pdf": return w.extractTextFromPDF(data) case ".docx": return w.extractTextFromDOCX(data) case ".txt": return string(data), nil case ".html", ".htm": return w.extractTextFromHTML(data) default: // Try to extract as plain text for unknown formats return string(data), nil } } // extractTextFromPDF extracts text from PDF documents func (w *DocumentSummarizationWorker) extractTextFromPDF(data []byte) (string, error) { // For now, return a placeholder. In production, you'd use a PDF parsing library // like github.com/unidoc/unipdf or github.com/rsc/pdf return fmt.Sprintf("PDF content extraction not implemented. File size: %d bytes", len(data)), nil } // extractTextFromDOCX extracts text from DOCX documents func (w *DocumentSummarizationWorker) extractTextFromDOCX(data []byte) (string, error) { // For now, return a placeholder. In production, you'd use a DOCX parsing library // like github.com/lukasjarosch/go-docx or github.com/fumiama/go-docx return fmt.Sprintf("DOCX content extraction not implemented. File size: %d bytes", len(data)), nil } // extractTextFromExcel extracts text from Excel documents (.xlsx) func (w *DocumentSummarizationWorker) extractTextFromExcel(data []byte) (string, error) { // For now, return a placeholder. In production, you'd use an Excel parsing library // like github.com/tealeg/xlsx/v3 or github.com/360EntSecGroup-Skylar/excelize return fmt.Sprintf("Excel content extraction not implemented. File size: %d bytes", len(data)), nil } // extractTextFromHTML extracts text from HTML documents func (w *DocumentSummarizationWorker) extractTextFromHTML(data []byte) (string, error) { // Simple HTML text extraction (remove tags) htmlContent := string(data) // Basic HTML tag removal (very simple implementation) // In production, use a proper HTML parser like goquery htmlContent = strings.ReplaceAll(htmlContent, "") htmlContent = strings.ReplaceAll(htmlContent, "") // Remove HTML tags using simple regex-like approach var result strings.Builder inTag := false for _, r := range htmlContent { switch r { case '<': inTag = true case '>': inTag = false default: if !inTag { result.WriteRune(r) } } } return strings.TrimSpace(result.String()), nil } // createSummarizationPrompt creates a prompt for GLM to summarize tender documents func (w *DocumentSummarizationWorker) createSummarizationPrompt(textContent, sourceLanguage string) string { return fmt.Sprintf(`Please provide a concise summary of the following tender document in English. Focus on key information such as: 1. Project/Tender name and description 2. Contracting authority/organization details 3. Key requirements and specifications 4. Important dates (deadline, award date, etc.) 5. Estimated value and currency 6. Selection criteria 7. Contact information Document content: %s Summary:`, textContent) } // getDocumentType returns the document type based on file extension func (w *DocumentSummarizationWorker) getDocumentType(filename string) string { ext := strings.ToLower(filepath.Ext(filename)) switch ext { case ".xlsx": return "xlsx" case ".pdf": return "pdf" case ".docx": return "docx" case ".doc": return "doc" case ".txt": return "txt" case ".html", ".htm": return "html" default: return "unknown" } } // getGLMModelName returns the GLM model name being used func (w *DocumentSummarizationWorker) getGLMModelName() string { return "glm-4.5" } // updateTenderSummarizationStatus updates the tender's document summarization metadata func (w *DocumentSummarizationWorker) updateTenderSummarizationStatus(tender *tender.Tender) error { err := w.TenderRepo.Update(context.Background(), tender) if err != nil { w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{ "tender_id": tender.ID.Hex(), "error": err.Error(), }) return err } w.Logger.Debug("Updated tender summarization status", map[string]interface{}{ "tender_id": tender.ID.Hex(), "documents_summarized": tender.ProcessingMetadata.DocumentsSummarized, "documents_summarized_at": tender.ProcessingMetadata.DocumentsSummarizedAt, "summary_count": len(tender.DocumentSummaries), }) return nil }