get scraped documents list performance update

internal/tender/service.go

@@ -59,7 +59,9 @@ type Service interface {
 	TriggerAISummarize(ctx context.Context, id string) (*AISummarizeResponse, error)
 	TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error)
 	GetAllAISummaries(ctx context.Context) ([]ai_summarizer.TenderSummaryItem, error)
-	ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error)
+	// ListTendersWithScrapedDocuments lists tenders that have documents in MinIO. When syncMongoMetadata is true,
+	// each tender is re-listed from MinIO and scraped document metadata is persisted to Mongo (slow).
+	ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination, syncMongoMetadata bool) (*TendersWithScrapedDocumentsResponse, error)
 
 	// Maintenance operations
 	UpdateExpiredTenders(ctx context.Context) error
@@ -1161,8 +1163,9 @@ func (s *tenderService) GetAllAISummaries(ctx context.Context) ([]ai_summarizer.
 }
 
 // ListTendersWithScrapedDocuments lists database tenders whose contract_folder_id has
-// documents in MinIO, and syncs scraped document metadata onto each tender.
-func (s *tenderService) ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error) {
+// documents in MinIO. By default uses counts from the initial MinIO procedure scan only (fast).
+// Set syncMongoMetadata=true to list every notice folder again and persist ScrapedDocuments to Mongo (slow).
+func (s *tenderService) ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination, syncMongoMetadata bool) (*TendersWithScrapedDocumentsResponse, error) {
 	if s.aiSummarizerStorage == nil {
 		return nil, fmt.Errorf("document storage is not configured")
 	}
@@ -1176,8 +1179,9 @@ func (s *tenderService) ListTendersWithScrapedDocuments(ctx context.Context, pag
 	}
 
 	s.logger.Info("Listing tenders with scraped documents from MinIO", map[string]interface{}{
-		"limit":  pagination.Limit,
-		"offset": pagination.Offset,
+		"limit":              pagination.Limit,
+		"offset":             pagination.Offset,
+		"sync_mongo_metadata": syncMongoMetadata,
 	})
 
 	procedures, err := storage.ListProceduresWithDocuments(ctx)
@@ -1191,33 +1195,53 @@ func (s *tenderService) ListTendersWithScrapedDocuments(ctx context.Context, pag
 		return nil, fmt.Errorf("failed to list tenders with scraped documents: %w", err)
 	}
 
+	folderIDs := make([]string, 0, len(procedures))
+	for _, proc := range procedures {
+		folderIDs = append(folderIDs, proc.ContractFolderID)
+	}
+
+	tenderByFolder, err := s.repository.GetLatestByContractFolderIDs(ctx, folderIDs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to load tenders by contract_folder_id: %w", err)
+	}
+
 	matched := make([]TenderScrapedDocumentsItem, 0, len(procedures))
 	for _, proc := range procedures {
-		tender, err := s.repository.GetByContractFolderID(ctx, proc.ContractFolderID)
-		if err != nil || tender == nil {
+		if proc.DocumentCount <= 0 {
 			continue
 		}
-
-		docs, listErr := s.listDocumentsForTender(ctx, tender)
-		if listErr != nil {
-			if errors.Is(listErr, ai_summarizer.ErrMinIOUnavailable) {
-				return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", listErr)
-			}
-			s.logger.Debug("Skipping tender: could not list documents from MinIO", map[string]interface{}{
-				"tender_id":          tender.ID.Hex(),
-				"contract_folder_id": proc.ContractFolderID,
-				"error":              listErr.Error(),
-			})
+		tender, ok := tenderByFolder[strings.TrimSpace(proc.ContractFolderID)]
+		if tender == nil || !ok {
 			continue
 		}
-		if len(docs) == 0 {
-			continue
-		}
-
-		s.persistScrapedDocuments(ctx, tender, docs)
 
 		item := tender.ToTenderScrapedDocumentsItem()
-		item.DocumentCount = len(docs)
+		item.DocumentCount = proc.DocumentCount
+		item.DocumentsScrapedAt = proc.LatestModified
+
+		if syncMongoMetadata {
+			docs, listErr := s.listDocumentsForTender(ctx, tender)
+			if listErr != nil {
+				if errors.Is(listErr, ai_summarizer.ErrMinIOUnavailable) {
+					return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", listErr)
+				}
+				s.logger.Debug("Skipping tender: could not list documents from MinIO", map[string]interface{}{
+					"tender_id":          tender.ID.Hex(),
+					"contract_folder_id": proc.ContractFolderID,
+					"error":              listErr.Error(),
+				})
+				continue
+			}
+			if len(docs) == 0 {
+				continue
+			}
+			s.persistScrapedDocuments(ctx, tender, docs)
+			item.DocumentCount = len(docs)
+			if tender.ProcessingMetadata.DocumentsScrapedAt != 0 {
+				item.DocumentsScrapedAt = tender.ProcessingMetadata.DocumentsScrapedAt
+			}
+		}
+
 		matched = append(matched, item)
 	}