get all tenders with documeents scraped API

pkg/ai_summarizer/storage.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -180,6 +181,14 @@ func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error)
 	return storage, nil
 }
 
+// BucketName returns the configured MinIO bucket used for AI pipeline artefacts.
+func (s *StorageClient) BucketName() string {
+	if s == nil || s.config == nil {
+		return ""
+	}
+	return s.config.MinioBucket
+}
+
 // verifyConnection checks that MinIO is reachable and the configured bucket exists.
 func (s *StorageClient) verifyConnection(ctx context.Context) error {
 	exists, err := s.client.BucketExists(ctx, s.config.MinioBucket)
@@ -522,6 +531,99 @@ func (s *StorageClient) ListDocuments(ctx context.Context, contractFolderID, not
 	return documents, nil
 }
 
+// ListProceduresWithDocuments scans MinIO for PROC_<contractFolderID>/…/documents/* keys
+// and returns one entry per contract folder id that has at least one document file.
+func (s *StorageClient) ListProceduresWithDocuments(ctx context.Context) ([]ProcedureDocumentsSummary, error) {
+	s.logger.Debug("Listing procedure folders with documents from MinIO", map[string]interface{}{
+		"bucket": s.config.MinioBucket,
+		"prefix": procedurePrefix,
+	})
+
+	byFolder := make(map[string]*ProcedureDocumentsSummary)
+	var fatalErr error
+
+	appendFromPrefix := func(prefix string) bool {
+		objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
+			Prefix:    prefix,
+			Recursive: true,
+		})
+		for object := range objectCh {
+			if object.Err != nil {
+				fatalErr = s.logMinIOFailure("list_procedure_documents", object.Err, map[string]interface{}{
+					"prefix": prefix,
+				})
+				return false
+			}
+			contractFolderID, noticePrefix, ok := contractFolderIDFromDocumentKey(object.Key)
+			if !ok {
+				continue
+			}
+			if !isTenderDocumentObject(noticePrefix, object.Key) {
+				continue
+			}
+			entry := byFolder[contractFolderID]
+			if entry == nil {
+				entry = &ProcedureDocumentsSummary{ContractFolderID: contractFolderID}
+				byFolder[contractFolderID] = entry
+			}
+			entry.DocumentCount++
+			modified := object.LastModified.Unix()
+			if modified > entry.LatestModified {
+				entry.LatestModified = modified
+			}
+		}
+		return true
+	}
+
+	if !appendFromPrefix(procedurePrefix) {
+		return nil, fatalErr
+	}
+	if len(byFolder) == 0 {
+		if !appendFromPrefix("") {
+			return nil, fatalErr
+		}
+	}
+	if fatalErr != nil {
+		return nil, fatalErr
+	}
+
+	results := make([]ProcedureDocumentsSummary, 0, len(byFolder))
+	for _, entry := range byFolder {
+		results = append(results, *entry)
+	}
+	sort.Slice(results, func(i, j int) bool {
+		if results[i].LatestModified == results[j].LatestModified {
+			return results[i].ContractFolderID < results[j].ContractFolderID
+		}
+		return results[i].LatestModified > results[j].LatestModified
+	})
+
+	s.logger.Info("Listed procedure folders with documents from storage", map[string]interface{}{
+		"bucket":        s.config.MinioBucket,
+		"procedure_count": len(results),
+	})
+
+	return results, nil
+}
+
+// contractFolderIDFromDocumentKey parses PROC_<contractFolderID>/<notice>/documents/<file>.
+func contractFolderIDFromDocumentKey(key string) (contractFolderID, noticePrefix string, ok bool) {
+	key = strings.TrimPrefix(strings.TrimSpace(key), "/")
+	parts := strings.Split(key, "/")
+	if len(parts) < 4 {
+		return "", "", false
+	}
+	if !strings.EqualFold(parts[2], "documents") {
+		return "", "", false
+	}
+	contractFolderID = stripLeadingProcPrefix(parts[0])
+	if contractFolderID == "" {
+		return "", "", false
+	}
+	noticePrefix = strings.Join(parts[:3], "/") + "/"
+	return contractFolderID, noticePrefix, true
+}
+
 // isTenderDocumentObject reports whether an object key under the given prefix
 // represents a downloadable tender document (PDF/DOCX/...) rather than an
 // internal artefact such as tender.json or a cache file.