document scraper logic and API refactor

2026-05-11 15:52:32 +03:30
parent b4efb97f47
commit 7c42b343ce
4 changed files with 318 additions and 97 deletions
@@ -11,9 +11,14 @@ type DocumentScraperGetRequest struct {
 	NoticeID string `param:"notice_id" valid:"required~Notice ID is required"`
 }

-// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service
+// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service.
+//
+// ContractFolderID is the TED procedure identifier (BT-04 / ContractFolderID),
+// included so the scraper can build the MinIO upload path for this notice:
+//
+//	PROC_<contract_folder_id>/<notice_publication_id>/<file>
 type DocumentScraperTenderResponse struct {
-	ID                  string `json:"id"`
+	ContractFolderID    string `json:"contract_folder_id"`
 	NoticePublicationID string `json:"notice_publication_id"`
 	DocumentURL         string `json:"document_url"`
 	Title               string `json:"title"`
@@ -41,7 +41,7 @@ func (s *service) buildTenderResponse(_ context.Context, t *tender.Tender) *Docu
 	}

 	return &DocumentScraperTenderResponse{
-		ID:                  t.ID.Hex(),
+		ContractFolderID:    t.ContractFolderID,
 		NoticePublicationID: t.NoticePublicationID,
 		DocumentURL:         documentURL,
 		Title:               t.Title,
@@ -23,10 +23,12 @@ type AISummarizerClient interface {
 	FetchTranslationOnDemand(ctx context.Context, reqBody ai_summarizer.TranslateRequest) (*ai_summarizer.TranslateResponse, error)
 }

-// AISummarizerStorage defines the interface for fetching summaries from MinIO storage
+// AISummarizerStorage defines the interface for fetching summaries from MinIO storage.
+// Paths follow the AI team's procedure-scoped layout: artefacts for a notice live under
+// PROC_<contractFolderID>/<noticePublicationID>/ in the shared bucket.
 type AISummarizerStorage interface {
-	GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error)
-	ListDocuments(ctx context.Context, noticeID string) ([]ai_summarizer.StoredDocument, error)
+	GetSummaryFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) (string, error)
+	ListDocuments(ctx context.Context, contractFolderID, noticePublicationID string) ([]ai_summarizer.StoredDocument, error)
 	DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error)
 }

@@ -119,7 +121,7 @@ func (s *tenderService) GetByID(ctx context.Context, id string) (*TenderResponse
 	}

 	// Enrich with AI summary from MinIO storage (best-effort, non-blocking)
-	s.enrichWithAISummary(ctx, resp)
+	s.enrichWithAISummary(ctx, tender, resp)

 	return resp, nil
 }
@@ -127,8 +129,13 @@ func (s *tenderService) GetByID(ctx context.Context, id string) (*TenderResponse
 // enrichWithAISummary attempts to fetch the AI-generated overall summary from
 // MinIO storage and attaches it to the response. Failures are logged but do not
 // cause the request to fail — the field will simply be omitted.
-func (s *tenderService) enrichWithAISummary(ctx context.Context, resp *TenderResponse) {
-	if s.aiSummarizerStorage == nil || resp.NoticePublicationID == "" {
+//
+// Because one tender record can span several TED notices (we merge by
+// contract_folder_id), we try the latest publication id first and fall back to
+// older RelatedNoticePublicationIDs if the AI pipeline only finished an
+// earlier notice in the procedure.
+func (s *tenderService) enrichWithAISummary(ctx context.Context, t *Tender, resp *TenderResponse) {
+	if s.aiSummarizerStorage == nil || t == nil || t.ContractFolderID == "" {
 		return
 	}

@@ -136,17 +143,89 @@ func (s *tenderService) enrichWithAISummary(ctx context.Context, resp *TenderRes
 	enrichCtx, cancel := context.WithTimeout(ctx, aiSummaryEnrichmentTimeout)
 	defer cancel()

-	summary, err := s.aiSummarizerStorage.GetSummaryFromStorage(enrichCtx, resp.NoticePublicationID)
+	summary, usedNotice, err := s.lookupSummaryForTender(enrichCtx, t)
 	if err != nil {
 		s.logger.Debug("AI summary not available for tender", map[string]interface{}{
-			"tender_id": resp.ID,
-			"notice_id": resp.NoticePublicationID,
-			"error":     err.Error(),
+			"tender_id":          resp.ID,
+			"contract_folder_id": t.ContractFolderID,
+			"notice_id":          t.NoticePublicationID,
+			"error":              err.Error(),
 		})
 		return
 	}

 	resp.OverallSummary = summary
+	if usedNotice != t.NoticePublicationID {
+		s.logger.Debug("AI summary resolved from related notice", map[string]interface{}{
+			"tender_id":          resp.ID,
+			"contract_folder_id": t.ContractFolderID,
+			"primary_notice_id":  t.NoticePublicationID,
+			"used_notice_id":     usedNotice,
+		})
+	}
+}
+
+// lookupSummaryForTender returns the overall_summary for a tender by trying its
+// primary notice publication id first, then each related publication id in
+// reverse (newest-first) order. The notice id that actually served the summary
+// is returned alongside.
+func (s *tenderService) lookupSummaryForTender(ctx context.Context, t *Tender) (string, string, error) {
+	if s.aiSummarizerStorage == nil {
+		return "", "", fmt.Errorf("AI summarizer storage is not configured")
+	}
+	candidates := orderedNoticeCandidates(t)
+	if len(candidates) == 0 {
+		return "", "", ai_summarizer.ErrObjectNotFound
+	}
+
+	var lastErr error
+	for _, noticeID := range candidates {
+		summary, err := s.aiSummarizerStorage.GetSummaryFromStorage(ctx, t.ContractFolderID, noticeID)
+		if err == nil {
+			return summary, noticeID, nil
+		}
+		lastErr = err
+		// Only "not found" / "not ready" justify trying the next candidate.
+		// Anything else (network/storage error) is propagated immediately.
+		if !errors.Is(err, ai_summarizer.ErrObjectNotFound) &&
+			!errors.Is(err, ai_summarizer.ErrSummaryNotReady) &&
+			!errors.Is(err, ai_summarizer.ErrNoOverallSummary) {
+			return "", noticeID, err
+		}
+	}
+	if lastErr == nil {
+		lastErr = ai_summarizer.ErrObjectNotFound
+	}
+	return "", "", lastErr
+}
+
+// orderedNoticeCandidates returns the publication ids to try when resolving
+// AI-side artefacts for a (possibly merged) tender, newest-first and deduped.
+// The primary NoticePublicationID is tried first, then any older ids stored in
+// RelatedNoticePublicationIDs (which we append oldest-first; reverse so newest
+// wins).
+func orderedNoticeCandidates(t *Tender) []string {
+	if t == nil {
+		return nil
+	}
+	seen := make(map[string]struct{}, len(t.RelatedNoticePublicationIDs)+1)
+	out := make([]string, 0, len(t.RelatedNoticePublicationIDs)+1)
+	add := func(id string) {
+		id = strings.TrimSpace(id)
+		if id == "" {
+			return
+		}
+		if _, ok := seen[id]; ok {
+			return
+		}
+		seen[id] = struct{}{}
+		out = append(out, id)
+	}
+	add(t.NoticePublicationID)
+	for i := len(t.RelatedNoticePublicationIDs) - 1; i >= 0; i-- {
+		add(t.RelatedNoticePublicationIDs[i])
+	}
+	return out
 }

 // UpdateTender updates an existing tender
@@ -282,16 +361,20 @@ func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDoc
 	if tender == nil {
 		return nil, fmt.Errorf("tender not found")
 	}
+	if strings.TrimSpace(tender.ContractFolderID) == "" {
+		return nil, fmt.Errorf("tender has no contract folder id")
+	}
 	if strings.TrimSpace(tender.NoticePublicationID) == "" {
 		return nil, fmt.Errorf("tender notice publication ID is empty")
 	}

-	storedDocuments, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
+	storedDocuments, err := s.listDocumentsForTender(ctx, tender)
 	if err != nil {
 		s.logger.Error("Failed to list tender documents from storage", map[string]interface{}{
-			"tender_id": id,
-			"notice_id": tender.NoticePublicationID,
-			"error":     err.Error(),
+			"tender_id":          id,
+			"contract_folder_id": tender.ContractFolderID,
+			"notice_id":          tender.NoticePublicationID,
+			"error":              err.Error(),
 		})
 		return nil, fmt.Errorf("failed to list tender documents: %w", err)
 	}
@@ -307,9 +390,10 @@ func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDoc
 	}

 	s.logger.Info("Retrieved scraped documents for tender", map[string]interface{}{
-		"tender_id":       id,
-		"notice_id":       tender.NoticePublicationID,
-		"documents_count": len(documents),
+		"tender_id":          id,
+		"contract_folder_id": tender.ContractFolderID,
+		"notice_id":          tender.NoticePublicationID,
+		"documents_count":    len(documents),
 	})

 	return &TenderDocumentsResponse{
@@ -318,6 +402,48 @@ func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDoc
 	}, nil
 }

+// listDocumentsForTender unions document listings across every notice tied to a
+// tender (primary + RelatedNoticePublicationIDs), deduplicating by object name.
+// This way, a merged tender (e.g. CN + Result for the same procedure) exposes
+// the documents from every notice that contributed to it.
+func (s *tenderService) listDocumentsForTender(ctx context.Context, t *Tender) ([]ai_summarizer.StoredDocument, error) {
+	if s.aiSummarizerStorage == nil {
+		return nil, ErrTenderDocumentStorageUnavailable
+	}
+
+	candidates := orderedNoticeCandidates(t)
+	if len(candidates) == 0 {
+		return nil, nil
+	}
+
+	seen := make(map[string]struct{})
+	var aggregated []ai_summarizer.StoredDocument
+	var firstErr error
+	for _, noticeID := range candidates {
+		docs, err := s.aiSummarizerStorage.ListDocuments(ctx, t.ContractFolderID, noticeID)
+		if err != nil {
+			// Remember the first error, but keep trying other notices so a
+			// missing folder for an older notice does not hide newer docs.
+			if firstErr == nil {
+				firstErr = err
+			}
+			continue
+		}
+		for _, doc := range docs {
+			if _, ok := seen[doc.ObjectName]; ok {
+				continue
+			}
+			seen[doc.ObjectName] = struct{}{}
+			aggregated = append(aggregated, doc)
+		}
+	}
+
+	if len(aggregated) == 0 && firstErr != nil {
+		return nil, firstErr
+	}
+	return aggregated, nil
+}
+
 // DownloadDocuments returns a ZIP stream containing all documents for a tender.
 func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error) {
 	if id == "" {
@@ -338,16 +464,20 @@ func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*Tend
 	if tender == nil {
 		return nil, fmt.Errorf("tender not found")
 	}
+	if strings.TrimSpace(tender.ContractFolderID) == "" {
+		return nil, ErrTenderDocumentNotFound
+	}
 	if strings.TrimSpace(tender.NoticePublicationID) == "" {
 		return nil, ErrTenderDocumentNotFound
 	}

-	documents, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
+	documents, err := s.listDocumentsForTender(ctx, tender)
 	if err != nil {
 		s.logger.Error("Failed to list tender documents for archive", map[string]interface{}{
-			"tender_id": id,
-			"notice_id": tender.NoticePublicationID,
-			"error":     err.Error(),
+			"tender_id":          id,
+			"contract_folder_id": tender.ContractFolderID,
+			"notice_id":          tender.NoticePublicationID,
+			"error":              err.Error(),
 		})
 		return nil, fmt.Errorf("failed to list tender documents: %w", err)
 	}
@@ -722,37 +852,37 @@ func (s *tenderService) GetAISummary(ctx context.Context, id string) (*AISummary
 	if noticeID == "" {
 		return nil, fmt.Errorf("tender has no notice publication ID")
 	}
-
-	// Try to get summary from MinIO storage (async pipeline)
-	if s.aiSummarizerStorage != nil {
-		summary, err := s.aiSummarizerStorage.GetSummaryFromStorage(ctx, noticeID)
-		if err == nil {
-			return &AISummaryResponse{
-				NoticeID:       noticeID,
-				OverallSummary: summary,
-				Source:         "storage",
-			}, nil
-		}
-
-		// Log but don't fail — the summary might just not be ready yet
-		s.logger.Info("AI summary not available from storage", map[string]interface{}{
-			"tender_id": id,
-			"notice_id": noticeID,
-			"error":     err.Error(),
-		})
-
-		// Return specific error if summary not ready
-		if errors.Is(err, ai_summarizer.ErrSummaryNotReady) {
-			return nil, fmt.Errorf("AI summary is not ready yet, the pipeline is still processing")
-		}
-		if errors.Is(err, ai_summarizer.ErrObjectNotFound) {
-			return nil, fmt.Errorf("AI summary not found — this tender has not been processed by the AI pipeline yet")
-		}
-
-		return nil, fmt.Errorf("failed to get AI summary: %w", err)
+	if t.ContractFolderID == "" {
+		return nil, fmt.Errorf("tender has no contract folder id")
 	}

-	return nil, fmt.Errorf("AI summarizer storage is not configured")
+	if s.aiSummarizerStorage == nil {
+		return nil, fmt.Errorf("AI summarizer storage is not configured")
+	}
+
+	summary, usedNotice, err := s.lookupSummaryForTender(ctx, t)
+	if err == nil {
+		return &AISummaryResponse{
+			NoticeID:       usedNotice,
+			OverallSummary: summary,
+			Source:         "storage",
+		}, nil
+	}
+
+	s.logger.Info("AI summary not available from storage", map[string]interface{}{
+		"tender_id":          id,
+		"contract_folder_id": t.ContractFolderID,
+		"notice_id":          noticeID,
+		"error":              err.Error(),
+	})
+
+	if errors.Is(err, ai_summarizer.ErrSummaryNotReady) {
+		return nil, fmt.Errorf("AI summary is not ready yet, the pipeline is still processing")
+	}
+	if errors.Is(err, ai_summarizer.ErrObjectNotFound) || errors.Is(err, ai_summarizer.ErrNoOverallSummary) {
+		return nil, fmt.Errorf("AI summary not found — this tender has not been processed by the AI pipeline yet")
+	}
+	return nil, fmt.Errorf("failed to get AI summary: %w", err)
 }

 // TriggerAISummarize triggers on-demand AI summarization for a tender.