get all tenders with documeents scraped API

internal/tender/service.go

@@ -59,6 +59,7 @@ type Service interface {
 	TriggerAISummarize(ctx context.Context, id string) (*AISummarizeResponse, error)
 	TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error)
 	GetAllAISummaries(ctx context.Context) ([]ai_summarizer.TenderSummaryItem, error)
+	ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error)
 
 	// Maintenance operations
 	UpdateExpiredTenders(ctx context.Context) error
@@ -1069,6 +1070,142 @@ func (s *tenderService) GetAllAISummaries(ctx context.Context) ([]ai_summarizer.
 	return results, nil
 }
 
+// ListTendersWithScrapedDocuments lists database tenders whose contract_folder_id has
+// documents in MinIO, and syncs scraped document metadata onto each tender.
+func (s *tenderService) ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error) {
+	if s.aiSummarizerStorage == nil {
+		return nil, fmt.Errorf("document storage is not configured")
+	}
+
+	type procedureLister interface {
+		ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error)
+	}
+	storage, ok := s.aiSummarizerStorage.(procedureLister)
+	if !ok {
+		return nil, fmt.Errorf("document storage does not support procedure listing")
+	}
+
+	s.logger.Info("Listing tenders with scraped documents from MinIO", map[string]interface{}{
+		"limit":  pagination.Limit,
+		"offset": pagination.Offset,
+	})
+
+	procedures, err := storage.ListProceduresWithDocuments(ctx)
+	if err != nil {
+		fields := map[string]interface{}{"error": err.Error()}
+		if errors.Is(err, ai_summarizer.ErrMinIOUnavailable) {
+			s.logger.Error("Failed to list scraped documents: MinIO unavailable", fields)
+			return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", err)
+		}
+		s.logger.Error("Failed to list procedure documents from MinIO", fields)
+		return nil, fmt.Errorf("failed to list tenders with scraped documents: %w", err)
+	}
+
+	matched := make([]TenderScrapedDocumentsItem, 0, len(procedures))
+	for _, proc := range procedures {
+		tender, err := s.repository.GetByContractFolderID(ctx, proc.ContractFolderID)
+		if err != nil || tender == nil {
+			continue
+		}
+
+		docs, listErr := s.listDocumentsForTender(ctx, tender)
+		if listErr != nil {
+			if errors.Is(listErr, ai_summarizer.ErrMinIOUnavailable) {
+				return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", listErr)
+			}
+			s.logger.Debug("Skipping tender: could not list documents from MinIO", map[string]interface{}{
+				"tender_id":          tender.ID.Hex(),
+				"contract_folder_id": proc.ContractFolderID,
+				"error":              listErr.Error(),
+			})
+			continue
+		}
+		if len(docs) == 0 {
+			continue
+		}
+
+		s.persistScrapedDocuments(ctx, tender, docs)
+
+		item := tender.ToTenderScrapedDocumentsItem()
+		item.DocumentCount = len(docs)
+		matched = append(matched, item)
+	}
+
+	total := int64(len(matched))
+	start := pagination.Offset
+	if start > len(matched) {
+		start = len(matched)
+	}
+	end := start + pagination.Limit
+	if end > len(matched) {
+		end = len(matched)
+	}
+	page := matched[start:end]
+	hasMore := int64(end) < total
+
+	s.logger.Info("Listed tenders with scraped documents", map[string]interface{}{
+		"count":       len(page),
+		"total_count": total,
+	})
+
+	return &TendersWithScrapedDocumentsResponse{
+		Tenders:  page,
+		Metadata: pagination.ListMeta(total, "", hasMore, pagination.Offset),
+	}, nil
+}
+
+// persistScrapedDocuments updates the tender when MinIO has scraped documents for its procedure.
+func (s *tenderService) persistScrapedDocuments(ctx context.Context, t *Tender, docs []ai_summarizer.StoredDocument) {
+	if t == nil || len(docs) == 0 {
+		return
+	}
+
+	bucket := ""
+	if named, ok := s.aiSummarizerStorage.(interface{ BucketName() string }); ok {
+		bucket = named.BucketName()
+	}
+
+	scraped := make([]ScrapedDocument, len(docs))
+	var latestModified int64
+	for i, doc := range docs {
+		scraped[i] = ScrapedDocument{
+			Filename:   doc.DocumentName,
+			ObjectName: doc.ObjectName,
+			BucketName: bucket,
+			Size:       doc.Size,
+			ScrapedAt:  doc.LastModified,
+		}
+		if doc.LastModified > latestModified {
+			latestModified = doc.LastModified
+		}
+	}
+
+	unchanged := t.ProcessingMetadata.DocumentsScraped &&
+		len(t.ScrapedDocuments) == len(scraped)
+	if unchanged {
+		return
+	}
+
+	t.ProcessingMetadata.DocumentsScraped = true
+	if t.ProcessingMetadata.DocumentsScrapedAt == 0 {
+		if latestModified > 0 {
+			t.ProcessingMetadata.DocumentsScrapedAt = latestModified
+		} else {
+			t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix()
+		}
+	}
+	t.ScrapedDocuments = scraped
+	t.UpdatedAt = time.Now().Unix()
+
+	if err := s.repository.Update(ctx, t); err != nil {
+		s.logger.Warn("Failed to persist scraped documents on tender", map[string]interface{}{
+			"tender_id":          t.ID.Hex(),
+			"contract_folder_id": t.ContractFolderID,
+			"error":              err.Error(),
+		})
+	}
+}
+
 // TriggerAITranslate triggers on-demand AI translation for a tender and stores
 // translated result per language in MongoDB for future reuse.
 func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error) {