package workers import ( "context" "path/filepath" "strings" "time" "tm/internal/tender" ai_summarizer "tm/pkg/ai_summarizer" "tm/pkg/logger" "tm/pkg/minio" "tm/pkg/mongo" ) const presignedDocumentURLSeconds int64 = 7200 // DocumentSummarizationWorker asks the external AI service to summarize scraped tender documents. type DocumentSummarizationWorker struct { Mongo *mongo.ConnectionManager Logger logger.Logger TenderRepo tender.TenderRepository MinioSDK *minio.Service AIClient *ai_summarizer.Client } // NewDocumentSummarizationWorker creates a document summarization worker. func NewDocumentSummarizationWorker(mongo *mongo.ConnectionManager, logger logger.Logger, tenderRepo tender.TenderRepository, minioSDK *minio.Service, aiClient *ai_summarizer.Client) *DocumentSummarizationWorker { return &DocumentSummarizationWorker{ Mongo: mongo, Logger: logger, TenderRepo: tenderRepo, MinioSDK: minioSDK, AIClient: aiClient, } } // Run starts the document summarization process. func (w *DocumentSummarizationWorker) Run() { w.Logger.Info("Document summarization worker started", map[string]interface{}{}) if w.AIClient == nil { w.Logger.Warn("AI summarizer client not configured, skipping document summarization run", map[string]interface{}{}) return } if w.MinioSDK == nil { w.Logger.Warn("MinIO not configured, skipping document summarization run", map[string]interface{}{}) return } limit := 5 for { tenders, totalCount, err := w.TenderRepo.GetUnSummarizedTenders(context.Background(), limit, 0) if err != nil { w.Logger.Error("Failed to get un-summarized tenders", map[string]interface{}{ "error": err.Error(), }) continue } w.Logger.Info("Found tenders for document summarization", map[string]interface{}{ "count": len(tenders), "total_count": totalCount, }) if len(tenders) == 0 { w.Logger.Info("No more tenders to process for document summarization", map[string]interface{}{}) break } for i := range tenders { t := &tenders[i] w.Logger.Info("Processing tender for document summarization", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "document_count": len(t.ScrapedDocuments), }) w.summarizeDocumentsForTender(t) } } w.Logger.Info("Document summarization worker completed", map[string]interface{}{}) } func (w *DocumentSummarizationWorker) summarizeDocumentsForTender(t *tender.Tender) { if len(t.ScrapedDocuments) == 0 { w.Logger.Warn("No scraped documents found for tender", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, }) t.ProcessingMetadata.DocumentsSummarized = true t.ProcessingMetadata.DocumentsSummarizedAt = time.Now().Unix() _ = w.updateTenderSummarizationStatus(t) return } summaries := make([]tender.DocumentSummary, 0, len(t.ScrapedDocuments)) summarizedAt := time.Now().Unix() for _, doc := range t.ScrapedDocuments { summaryText, model, errStr := w.summarizeDocumentViaAI(context.Background(), t, doc) if errStr != "" { w.Logger.Error("Failed to summarize document", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "document_name": doc.Filename, "object_name": doc.ObjectName, "error": errStr, }) summaries = append(summaries, tender.DocumentSummary{ DocumentName: doc.Filename, ObjectName: doc.ObjectName, BucketName: doc.BucketName, Summary: "", SummaryLanguage: "en", DocumentType: w.getDocumentType(doc.Filename), SummarizedAt: summarizedAt, SummaryModel: model, Error: errStr, }) continue } summaries = append(summaries, tender.DocumentSummary{ DocumentName: doc.Filename, ObjectName: doc.ObjectName, BucketName: doc.BucketName, Summary: summaryText, SummaryLanguage: "en", DocumentType: w.getDocumentType(doc.Filename), SummarizedAt: summarizedAt, SummaryModel: model, }) w.Logger.Info("Document summarized successfully", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "document_name": doc.Filename, "summary_length": len(summaryText), }) } t.DocumentSummaries = summaries t.ProcessingMetadata.DocumentsSummarized = true t.ProcessingMetadata.DocumentsSummarizedAt = summarizedAt if err := w.updateTenderSummarizationStatus(t); err != nil { w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "error": err.Error(), }) return } w.Logger.Info("Document summarization completed for tender", map[string]interface{}{ "tender_id": t.ID.Hex(), "notice_publication_id": t.NoticePublicationID, "summarized_count": len(summaries), "total_documents": len(t.ScrapedDocuments), }) } func (w *DocumentSummarizationWorker) summarizeDocumentViaAI(ctx context.Context, t *tender.Tender, doc tender.ScrapedDocument) (summaryText, model, errStr string) { docURL, err := w.MinioSDK.GetPresignedURL(ctx, doc.BucketName, doc.ObjectName, presignedDocumentURLSeconds) if err != nil { return "", "", err.Error() } noticeID := strings.TrimSpace(t.NoticePublicationID) if noticeID == "" { return "", "", "tender has no notice_publication_id for AI summarize request" } req := ai_summarizer.SummarizeRequest{ NoticeID: noticeID, DocumentURL: docURL, Title: t.Title, Description: t.Description, Country: t.CountryCode, Currency: t.Currency, SubmissionDeadline: "", AuthorityName: authorityNameFromTender(t), } if t.EstimatedValue > 0 { v := t.EstimatedValue req.EstimatedValue = &v } resp, err := w.AIClient.FetchSummaryOnDemand(ctx, req) if err != nil { return "", "", err.Error() } if resp == nil { return "", "", "AI summarize returned nil response" } summaryText, model = pickDocumentSummary(resp.Summaries, doc.Filename) if summaryText == "" && len(resp.Summaries) > 0 { // Fall back to first summary if filename did not match summaryText = resp.Summaries[0].Summary model = resp.Summaries[0].SummaryModel } if summaryText == "" { return "", model, "AI summarize returned no summary text for document" } return summaryText, model, "" } func authorityNameFromTender(t *tender.Tender) string { if t == nil || t.BuyerOrganization == nil { return "" } return strings.TrimSpace(t.BuyerOrganization.Name) } func pickDocumentSummary(items []ai_summarizer.DocumentSummary, filename string) (text, model string) { filename = strings.TrimSpace(filename) for _, s := range items { if strings.EqualFold(strings.TrimSpace(s.DocumentName), filename) { return strings.TrimSpace(s.Summary), strings.TrimSpace(s.SummaryModel) } } for _, s := range items { if strings.TrimSpace(s.Summary) != "" { return strings.TrimSpace(s.Summary), strings.TrimSpace(s.SummaryModel) } } return "", "" } func (w *DocumentSummarizationWorker) getDocumentType(filename string) string { ext := strings.ToLower(filepath.Ext(filename)) switch ext { case ".xlsx": return "xlsx" case ".pdf": return "pdf" case ".docx": return "docx" case ".doc": return "doc" case ".txt": return "txt" case ".html", ".htm": return "html" default: return "unknown" } } func (w *DocumentSummarizationWorker) updateTenderSummarizationStatus(tender *tender.Tender) error { err := w.TenderRepo.Update(context.Background(), tender) if err != nil { w.Logger.Error("Failed to update tender summarization status", map[string]interface{}{ "tender_id": tender.ID.Hex(), "error": err.Error(), }) return err } w.Logger.Debug("Updated tender summarization status", map[string]interface{}{ "tender_id": tender.ID.Hex(), "documents_summarized": tender.ProcessingMetadata.DocumentsSummarized, "documents_summarized_at": tender.ProcessingMetadata.DocumentsSummarizedAt, "summary_count": len(tender.DocumentSummaries), }) return nil }