get all tenders with documeents scraped API

2026-05-19 19:23:18 +03:30
parent fcf2cae6bc
commit f7763cf997
7 changed files with 306 additions and 2 deletions
@@ -175,9 +175,9 @@ type ProcessingMetadata struct {
 	TranslatedAt          int64             `bson:"translated_at,omitempty" json:"translated_at,omitempty"`
 	Processed             bool              `bson:"processed" json:"processed"`
 	DocumentsScraped      bool              `bson:"documents_scraped" json:"documents_scraped"`             // Whether documents have been scraped
-	DocumentsScrapedAt    int64             `bson:"documents_scraped_at" json:"documents_scraped_at"`       // When documents were scraped
+	DocumentsScrapedAt    int64             `bson:"documents_scraped_at" json:"documents_scraped_at"`       // Unix timestamp (seconds) when documents were scraped
 	DocumentsSummarized   bool              `bson:"documents_summarized" json:"documents_summarized"`       // Whether documents have been summarized
-	DocumentsSummarizedAt int64             `bson:"documents_summarized_at" json:"documents_summarized_at"` // When documents were summarized
+	DocumentsSummarizedAt int64             `bson:"documents_summarized_at" json:"documents_summarized_at"` // Unix timestamp (seconds) when documents were summarized
 }

 // TenderModification represents a modification made to a tender
@@ -69,6 +69,37 @@ type SearchResponse struct {
 	Metadata *response.Meta   `json:"-"`
 }

+// TenderScrapedDocumentsItem is a tender that has scraped documents stored in the database.
+type TenderScrapedDocumentsItem struct {
+	TenderID            string `json:"tender_id"`
+	NoticePublicationID string `json:"notice_publication_id"`
+	ContractFolderID    string `json:"contract_folder_id"`
+	Title               string `json:"title"`
+	DocumentsScrapedAt  int64  `json:"documents_scraped_at,omitempty"` // Unix timestamp (seconds)
+	DocumentCount       int    `json:"document_count"`
+}
+
+// TendersWithScrapedDocumentsResponse is the paginated list of tenders with scraped documents.
+type TendersWithScrapedDocumentsResponse struct {
+	Tenders  []TenderScrapedDocumentsItem `json:"tenders"`
+	Metadata *response.Meta               `json:"-"`
+}
+
+// ToTenderScrapedDocumentsItem maps a tender entity to a list item.
+func (t *Tender) ToTenderScrapedDocumentsItem() TenderScrapedDocumentsItem {
+	if t == nil {
+		return TenderScrapedDocumentsItem{}
+	}
+	return TenderScrapedDocumentsItem{
+		TenderID:            t.ID.Hex(),
+		NoticePublicationID: strings.TrimSpace(t.NoticePublicationID),
+		ContractFolderID:    strings.TrimSpace(t.ContractFolderID),
+		Title:               strings.TrimSpace(t.Title),
+		DocumentsScrapedAt:  t.ProcessingMetadata.DocumentsScrapedAt,
+		DocumentCount:       len(t.ScrapedDocuments),
+	}
+}
+
 // AddressResponse is postal / NUTS address data for API consumers.
 type AddressResponse struct {
 	StreetName           string `json:"street_name,omitempty"`
@@ -657,6 +657,32 @@ func (h *TenderHandler) TriggerPublicAITranslate(c echo.Context) error {
 	return response.Success(c, result, "AI translation completed successfully")
 }

+// ListTendersWithScrapedDocuments lists tenders that have scraped documents in the database
+// @Summary List tenders with scraped documents
+// @Description Lists database tenders whose contract_folder_id has documents in MinIO (PROC_<contract_folder_id>/…) and syncs scraped document metadata
+// @Tags Admin-Tenders
+// @Produce json
+// @Param limit query int false "Number of items per page (default: 10, max: 100)"
+// @Param offset query int false "Number of items to skip (default: 0)"
+// @Success 200 {object} response.APIResponse{data=TendersWithScrapedDocumentsResponse}
+// @Failure 400 {object} response.APIResponse
+// @Failure 500 {object} response.APIResponse
+// @Router /admin/v1/tenders/scraped-documents [get]
+// @Security BearerAuth
+func (h *TenderHandler) ListTendersWithScrapedDocuments(c echo.Context) error {
+	pagination, err := response.NewPagination(c)
+	if err != nil {
+		return response.PaginationBadRequest(c, err)
+	}
+
+	result, err := h.service.ListTendersWithScrapedDocuments(c.Request().Context(), pagination)
+	if err != nil {
+		return response.InternalServerError(c, "Failed to list tenders with scraped documents")
+	}
+
+	return response.SuccessWithMeta(c, result, result.Metadata, "Tenders with scraped documents retrieved successfully")
+}
+
 // GetAllAISummaries retrieves all AI-generated summaries from storage
 // @Summary Get all AI-generated tender summaries
 // @Description Retrieve all AI-generated summaries from the AI pipeline storage, including pending and completed ones
@@ -59,6 +59,7 @@ type Service interface {
 	TriggerAISummarize(ctx context.Context, id string) (*AISummarizeResponse, error)
 	TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error)
 	GetAllAISummaries(ctx context.Context) ([]ai_summarizer.TenderSummaryItem, error)
+	ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error)

 	// Maintenance operations
 	UpdateExpiredTenders(ctx context.Context) error
@@ -1069,6 +1070,142 @@ func (s *tenderService) GetAllAISummaries(ctx context.Context) ([]ai_summarizer.
 	return results, nil
 }

+// ListTendersWithScrapedDocuments lists database tenders whose contract_folder_id has
+// documents in MinIO, and syncs scraped document metadata onto each tender.
+func (s *tenderService) ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination) (*TendersWithScrapedDocumentsResponse, error) {
+	if s.aiSummarizerStorage == nil {
+		return nil, fmt.Errorf("document storage is not configured")
+	}
+
+	type procedureLister interface {
+		ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error)
+	}
+	storage, ok := s.aiSummarizerStorage.(procedureLister)
+	if !ok {
+		return nil, fmt.Errorf("document storage does not support procedure listing")
+	}
+
+	s.logger.Info("Listing tenders with scraped documents from MinIO", map[string]interface{}{
+		"limit":  pagination.Limit,
+		"offset": pagination.Offset,
+	})
+
+	procedures, err := storage.ListProceduresWithDocuments(ctx)
+	if err != nil {
+		fields := map[string]interface{}{"error": err.Error()}
+		if errors.Is(err, ai_summarizer.ErrMinIOUnavailable) {
+			s.logger.Error("Failed to list scraped documents: MinIO unavailable", fields)
+			return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", err)
+		}
+		s.logger.Error("Failed to list procedure documents from MinIO", fields)
+		return nil, fmt.Errorf("failed to list tenders with scraped documents: %w", err)
+	}
+
+	matched := make([]TenderScrapedDocumentsItem, 0, len(procedures))
+	for _, proc := range procedures {
+		tender, err := s.repository.GetByContractFolderID(ctx, proc.ContractFolderID)
+		if err != nil || tender == nil {
+			continue
+		}
+
+		docs, listErr := s.listDocumentsForTender(ctx, tender)
+		if listErr != nil {
+			if errors.Is(listErr, ai_summarizer.ErrMinIOUnavailable) {
+				return nil, fmt.Errorf("failed to list tenders with scraped documents: MinIO connection unavailable: %w", listErr)
+			}
+			s.logger.Debug("Skipping tender: could not list documents from MinIO", map[string]interface{}{
+				"tender_id":          tender.ID.Hex(),
+				"contract_folder_id": proc.ContractFolderID,
+				"error":              listErr.Error(),
+			})
+			continue
+		}
+		if len(docs) == 0 {
+			continue
+		}
+
+		s.persistScrapedDocuments(ctx, tender, docs)
+
+		item := tender.ToTenderScrapedDocumentsItem()
+		item.DocumentCount = len(docs)
+		matched = append(matched, item)
+	}
+
+	total := int64(len(matched))
+	start := pagination.Offset
+	if start > len(matched) {
+		start = len(matched)
+	}
+	end := start + pagination.Limit
+	if end > len(matched) {
+		end = len(matched)
+	}
+	page := matched[start:end]
+	hasMore := int64(end) < total
+
+	s.logger.Info("Listed tenders with scraped documents", map[string]interface{}{
+		"count":       len(page),
+		"total_count": total,
+	})
+
+	return &TendersWithScrapedDocumentsResponse{
+		Tenders:  page,
+		Metadata: pagination.ListMeta(total, "", hasMore, pagination.Offset),
+	}, nil
+}
+
+// persistScrapedDocuments updates the tender when MinIO has scraped documents for its procedure.
+func (s *tenderService) persistScrapedDocuments(ctx context.Context, t *Tender, docs []ai_summarizer.StoredDocument) {
+	if t == nil || len(docs) == 0 {
+		return
+	}
+
+	bucket := ""
+	if named, ok := s.aiSummarizerStorage.(interface{ BucketName() string }); ok {
+		bucket = named.BucketName()
+	}
+
+	scraped := make([]ScrapedDocument, len(docs))
+	var latestModified int64
+	for i, doc := range docs {
+		scraped[i] = ScrapedDocument{
+			Filename:   doc.DocumentName,
+			ObjectName: doc.ObjectName,
+			BucketName: bucket,
+			Size:       doc.Size,
+			ScrapedAt:  doc.LastModified,
+		}
+		if doc.LastModified > latestModified {
+			latestModified = doc.LastModified
+		}
+	}
+
+	unchanged := t.ProcessingMetadata.DocumentsScraped &&
+		len(t.ScrapedDocuments) == len(scraped)
+	if unchanged {
+		return
+	}
+
+	t.ProcessingMetadata.DocumentsScraped = true
+	if t.ProcessingMetadata.DocumentsScrapedAt == 0 {
+		if latestModified > 0 {
+			t.ProcessingMetadata.DocumentsScrapedAt = latestModified
+		} else {
+			t.ProcessingMetadata.DocumentsScrapedAt = time.Now().Unix()
+		}
+	}
+	t.ScrapedDocuments = scraped
+	t.UpdatedAt = time.Now().Unix()
+
+	if err := s.repository.Update(ctx, t); err != nil {
+		s.logger.Warn("Failed to persist scraped documents on tender", map[string]interface{}{
+			"tender_id":          t.ID.Hex(),
+			"contract_folder_id": t.ContractFolderID,
+			"error":              err.Error(),
+		})
+	}
+}
+
 // TriggerAITranslate triggers on-demand AI translation for a tender and stores
 // translated result per language in MongoDB for future reuse.
 func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error) {