get all tenders with documeents scraped API

cmd/web/router/routes.go

@@ -107,6 +107,7 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
 		tendersGP.Use(userHandler.AuthMiddleware())
 		tendersGP.GET("", tenderHandler.Search)
 		tendersGP.GET("/ai-summaries", tenderHandler.GetAllAISummaries)
+		tendersGP.GET("/scraped-documents", tenderHandler.ListTendersWithScrapedDocuments)
 		tendersGP.GET("/:id", tenderHandler.Get)
 		tendersGP.PUT("/:id", tenderHandler.Update)
 		tendersGP.DELETE("/:id", tenderHandler.Delete)

internal/tender/entity.go

@@ -175,9 +175,9 @@ type ProcessingMetadata struct {
 	TranslatedAt          int64             `bson:"translated_at,omitempty" json:"translated_at,omitempty"`
 	Processed             bool              `bson:"processed" json:"processed"`
 	DocumentsScraped      bool              `bson:"documents_scraped" json:"documents_scraped"`             // Whether documents have been scraped
-	DocumentsScrapedAt    int64             `bson:"documents_scraped_at" json:"documents_scraped_at"`       // When documents were scraped
+	DocumentsScrapedAt    int64             `bson:"documents_scraped_at" json:"documents_scraped_at"`       // Unix timestamp (seconds) when documents were scraped
 	DocumentsSummarized   bool              `bson:"documents_summarized" json:"documents_summarized"`       // Whether documents have been summarized
-	DocumentsSummarizedAt int64             `bson:"documents_summarized_at" json:"documents_summarized_at"` // When documents were summarized
+	DocumentsSummarizedAt int64             `bson:"documents_summarized_at" json:"documents_summarized_at"` // Unix timestamp (seconds) when documents were summarized
 }
 
 // TenderModification represents a modification made to a tender

internal/tender/form.go

@@ -69,6 +69,37 @@ type SearchResponse struct {
 	Metadata *response.Meta   `json:"-"`
 }
 
+// TenderScrapedDocumentsItem is a tender that has scraped documents stored in the database.
+type TenderScrapedDocumentsItem struct {
+	TenderID            string `json:"tender_id"`
+	NoticePublicationID string `json:"notice_publication_id"`
+	ContractFolderID    string `json:"contract_folder_id"`
+	Title               string `json:"title"`
+	DocumentsScrapedAt  int64  `json:"documents_scraped_at,omitempty"` // Unix timestamp (seconds)
+	DocumentCount       int    `json:"document_count"`
+}
+
+// TendersWithScrapedDocumentsResponse is the paginated list of tenders with scraped documents.
+type TendersWithScrapedDocumentsResponse struct {
+	Tenders  []TenderScrapedDocumentsItem `json:"tenders"`
+	Metadata *response.Meta               `json:"-"`
+}
+
+// ToTenderScrapedDocumentsItem maps a tender entity to a list item.
+func (t *Tender) ToTenderScrapedDocumentsItem() TenderScrapedDocumentsItem {
+	if t == nil {
+		return TenderScrapedDocumentsItem{}
+	}
+	return TenderScrapedDocumentsItem{
+		TenderID:            t.ID.Hex(),
+		NoticePublicationID: strings.TrimSpace(t.NoticePublicationID),
+		ContractFolderID:    strings.TrimSpace(t.ContractFolderID),
+		Title:               strings.TrimSpace(t.Title),
+		DocumentsScrapedAt:  t.ProcessingMetadata.DocumentsScrapedAt,
+		DocumentCount:       len(t.ScrapedDocuments),
+	}
+}
+
 // AddressResponse is postal / NUTS address data for API consumers.
 type AddressResponse struct {
 	StreetName           string `json:"street_name,omitempty"`

internal/tender/handler.go

@@ -657,6 +657,32 @@ func (h *TenderHandler) TriggerPublicAITranslate(c echo.Context) error {
 	return response.Success(c, result, "AI translation completed successfully")
 }
 
+// ListTendersWithScrapedDocuments lists tenders that have scraped documents in the database
+// @Summary List tenders with scraped documents
+// @Description Lists database tenders whose contract_folder_id has documents in MinIO (PROC_<contract_folder_id>/…) and syncs scraped document metadata
+// @Tags Admin-Tenders
+// @Produce json
+// @Param limit query int false "Number of items per page (default: 10, max: 100)"
+// @Param offset query int false "Number of items to skip (default: 0)"
+// @Success 200 {object} response.APIResponse{data=TendersWithScrapedDocumentsResponse}
+// @Failure 400 {object} response.APIResponse
+// @Failure 500 {object} response.APIResponse
+// @Router /admin/v1/tenders/scraped-documents [get]
+// @Security BearerAuth
+func (h *TenderHandler) ListTendersWithScrapedDocuments(c echo.Context) error {
+	pagination, err := response.NewPagination(c)
+	if err != nil {
+		return response.PaginationBadRequest(c, err)
+	}
+
+	result, err := h.service.ListTendersWithScrapedDocuments(c.Request().Context(), pagination)
+	if err != nil {
+		return response.InternalServerError(c, "Failed to list tenders with scraped documents")
+	}
+
+	return response.SuccessWithMeta(c, result, result.Metadata, "Tenders with scraped documents retrieved successfully")
+}
+
 // GetAllAISummaries retrieves all AI-generated summaries from storage
 // @Summary Get all AI-generated tender summaries
 // @Description Retrieve all AI-generated summaries from the AI pipeline storage, including pending and completed ones

internal/tender/service.go

@@ -59,6 +59,7 @@ type Service interface {
 	TriggerAISummarize(ctx context.Context, id string) (*AISummarizeResponse, error)
 	TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error)
 	GetAllAISummaries(ctx context.Context) ([]ai_summarizer.TenderSummaryItem, error)
+	ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error)
 
 	// Maintenance operations
 	UpdateExpiredTenders(ctx context.Context) error
@@ -1069,6 +1070,142 @@ func (s *tenderService) GetAllAISummaries(ctx context.Context) ([]ai_summarizer.
 	return results, nil
 }
 
+// ListTendersWithScrapedDocuments lists database tenders whose contract_folder_id has
+// documents in MinIO, and syncs scraped document metadata onto each tender.
+func (s *tenderService) ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error) {
+	if s.aiSummarizerStorage == nil {
+		return nil, fmt.Errorf("document storage is not configured")
+	}
+
+	type procedureLister interface {
+		ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error)
+	}
+	storage, ok := s.aiSummarizerStorage.(procedureLister)
+	if !ok {
+		return nil, fmt.Errorf("document storage does not support procedure listing")
+	}
+
+	s.logger.Info("Listing tenders with scraped documents from MinIO", map[string]interface{}{
+		"limit":  pagination.Limit,
+		"offset": pagination.Offset,
+	})
+
+	procedures, err := storage.ListProceduresWithDocuments(ctx)
+	if err != nil {
+		fields := map[string]interface{}{"error": err.Error()}
+		if errors.Is(err, ai_summarizer.ErrMinIOUnavailable) {
+			s.logger.Error("Failed to list scraped documents: MinIO unavailable", fields)
+			return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", err)
+		}
+		s.logger.Error("Failed to list procedure documents from MinIO", fields)
+		return nil, fmt.Errorf("failed to list tenders with scraped documents: %w", err)
+	}
+
+	matched := make([]TenderScrapedDocumentsItem, 0, len(procedures))
+	for _, proc := range procedures {
+		tender, err := s.repository.GetByContractFolderID(ctx, proc.ContractFolderID)
+		if err != nil || tender == nil {
+			continue
+		}
+
+		docs, listErr := s.listDocumentsForTender(ctx, tender)
+		if listErr != nil {
+			if errors.Is(listErr, ai_summarizer.ErrMinIOUnavailable) {
+				return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", listErr)
+			}
+			s.logger.Debug("Skipping tender: could not list documents from MinIO", map[string]interface{}{
+				"tender_id":          tender.ID.Hex(),
+				"contract_folder_id": proc.ContractFolderID,
+				"error":              listErr.Error(),
+			})
+			continue
+		}
+		if len(docs) == 0 {
+			continue
+		}
+
+		s.persistScrapedDocuments(ctx, tender, docs)
+
+		item := tender.ToTenderScrapedDocumentsItem()
+		item.DocumentCount = len(docs)
+		matched = append(matched, item)
+	}
+
+	total := int64(len(matched))
+	start := pagination.Offset
+	if start > len(matched) {
+		start = len(matched)
+	}
+	end := start + pagination.Limit
+	if end > len(matched) {
+		end = len(matched)
+	}
+	page := matched[start:end]
+	hasMore := int64(end) < total
+
+	s.logger.Info("Listed tenders with scraped documents", map[string]interface{}{
+		"count":       len(page),
+		"total_count": total,
+	})
+
+	return &TendersWithScrapedDocumentsResponse{
+		Tenders:  page,
+		Metadata: pagination.ListMeta(total, "", hasMore, pagination.Offset),
+	}, nil
+}
+
+// persistScrapedDocuments updates the tender when MinIO has scraped documents for its procedure.
+func (s *tenderService) persistScrapedDocuments(ctx context.Context, t *Tender, docs []ai_summarizer.StoredDocument) {
+	if t == nil || len(docs) == 0 {
+		return
+	}
+
+	bucket := ""
+	if named, ok := s.aiSummarizerStorage.(interface{ BucketName() string }); ok {
+		bucket = named.BucketName()
+	}
+
+	scraped := make([]ScrapedDocument, len(docs))
+	var latestModified int64
+	for i, doc := range docs {
+		scraped[i] = ScrapedDocument{
+			Filename:   doc.DocumentName,
+			ObjectName: doc.ObjectName,
+			BucketName: bucket,
+			Size:       doc.Size,
+			ScrapedAt:  doc.LastModified,
+		}
+		if doc.LastModified > latestModified {
+			latestModified = doc.LastModified
+		}
+	}
+
+	unchanged := t.ProcessingMetadata.DocumentsScraped &&
+		len(t.ScrapedDocuments) == len(scraped)
+	if unchanged {
+		return
+	}
+
+	t.ProcessingMetadata.DocumentsScraped = true
+	if t.ProcessingMetadata.DocumentsScrapedAt == 0 {
+		if latestModified > 0 {
+			t.ProcessingMetadata.DocumentsScrapedAt = latestModified
+		} else {
+			t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix()
+		}
+	}
+	t.ScrapedDocuments = scraped
+	t.UpdatedAt = time.Now().Unix()
+
+	if err := s.repository.Update(ctx, t); err != nil {
+		s.logger.Warn("Failed to persist scraped documents on tender", map[string]interface{}{
+			"tender_id":          t.ID.Hex(),
+			"contract_folder_id": t.ContractFolderID,
+			"error":              err.Error(),
+		})
+	}
+}
+
 // TriggerAITranslate triggers on-demand AI translation for a tender and stores
 // translated result per language in MongoDB for future reuse.
 func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error) {

pkg/ai_summarizer/entities.go

@@ -85,6 +85,13 @@ type DocumentSummary struct {
 	Error        string `json:"error"` // empty string when no error
 }
 
+// ProcedureDocumentsSummary reports scraped documents stored under one procedure folder in MinIO.
+type ProcedureDocumentsSummary struct {
+	ContractFolderID string `json:"contract_folder_id"`
+	DocumentCount    int    `json:"document_count"`
+	LatestModified   int64  `json:"latest_modified"` // Unix seconds
+}
+
 // StoredDocument represents a document object stored for a tender in MinIO.
 type StoredDocument struct {
 	ObjectName   string `json:"-"`

pkg/ai_summarizer/storage.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"net/http"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -180,6 +181,14 @@ func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error)
 	return storage, nil
 }
 
+// BucketName returns the configured MinIO bucket used for AI pipeline artefacts.
+func (s *StorageClient) BucketName() string {
+	if s == nil || s.config == nil {
+		return ""
+	}
+	return s.config.MinioBucket
+}
+
 // verifyConnection checks that MinIO is reachable and the configured bucket exists.
 func (s *StorageClient) verifyConnection(ctx context.Context) error {
 	exists, err := s.client.BucketExists(ctx, s.config.MinioBucket)
@@ -522,6 +531,99 @@ func (s *StorageClient) ListDocuments(ctx context.Context, contractFolderID, not
 	return documents, nil
 }
 
+// ListProceduresWithDocuments scans MinIO for PROC_<contractFolderID>/…/documents/* keys
+// and returns one entry per contract folder id that has at least one document file.
+func (s *StorageClient) ListProceduresWithDocuments(ctx context.Context) ([]ProcedureDocumentsSummary, error) {
+	s.logger.Debug("Listing procedure folders with documents from MinIO", map[string]interface{}{
+		"bucket": s.config.MinioBucket,
+		"prefix": procedurePrefix,
+	})
+
+	byFolder := make(map[string]*ProcedureDocumentsSummary)
+	var fatalErr error
+
+	appendFromPrefix := func(prefix string) bool {
+		objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
+			Prefix:    prefix,
+			Recursive: true,
+		})
+		for object := range objectCh {
+			if object.Err != nil {
+				fatalErr = s.logMinIOFailure("list_procedure_documents", object.Err, map[string]interface{}{
+					"prefix": prefix,
+				})
+				return false
+			}
+			contractFolderID, noticePrefix, ok := contractFolderIDFromDocumentKey(object.Key)
+			if !ok {
+				continue
+			}
+			if !isTenderDocumentObject(noticePrefix, object.Key) {
+				continue
+			}
+			entry := byFolder[contractFolderID]
+			if entry == nil {
+				entry = &ProcedureDocumentsSummary{ContractFolderID: contractFolderID}
+				byFolder[contractFolderID] = entry
+			}
+			entry.DocumentCount++
+			modified := object.LastModified.Unix()
+			if modified > entry.LatestModified {
+				entry.LatestModified = modified
+			}
+		}
+		return true
+	}
+
+	if !appendFromPrefix(procedurePrefix) {
+		return nil, fatalErr
+	}
+	if len(byFolder) == 0 {
+		if !appendFromPrefix("") {
+			return nil, fatalErr
+		}
+	}
+	if fatalErr != nil {
+		return nil, fatalErr
+	}
+
+	results := make([]ProcedureDocumentsSummary, 0, len(byFolder))
+	for _, entry := range byFolder {
+		results = append(results, *entry)
+	}
+	sort.Slice(results, func(i, j int) bool {
+		if results[i].LatestModified == results[j].LatestModified {
+			return results[i].ContractFolderID < results[j].ContractFolderID
+		}
+		return results[i].LatestModified > results[j].LatestModified
+	})
+
+	s.logger.Info("Listed procedure folders with documents from storage", map[string]interface{}{
+		"bucket":        s.config.MinioBucket,
+		"procedure_count": len(results),
+	})
+
+	return results, nil
+}
+
+// contractFolderIDFromDocumentKey parses PROC_<contractFolderID>/<notice>/documents/<file>.
+func contractFolderIDFromDocumentKey(key string) (contractFolderID, noticePrefix string, ok bool) {
+	key = strings.TrimPrefix(strings.TrimSpace(key), "/")
+	parts := strings.Split(key, "/")
+	if len(parts) < 4 {
+		return "", "", false
+	}
+	if !strings.EqualFold(parts[2], "documents") {
+		return "", "", false
+	}
+	contractFolderID = stripLeadingProcPrefix(parts[0])
+	if contractFolderID == "" {
+		return "", "", false
+	}
+	noticePrefix = strings.Join(parts[:3], "/") + "/"
+	return contractFolderID, noticePrefix, true
+}
+
 // isTenderDocumentObject reports whether an object key under the given prefix
 // represents a downloadable tender document (PDF/DOCX/...) rather than an
 // internal artefact such as tender.json or a cache file.