document scraper logic and API refactor
This commit is contained in:
@@ -11,9 +11,14 @@ type DocumentScraperGetRequest struct {
|
||||
NoticeID string `param:"notice_id" valid:"required~Notice ID is required"`
|
||||
}
|
||||
|
||||
// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service
|
||||
// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service.
|
||||
//
|
||||
// ContractFolderID is the TED procedure identifier (BT-04 / ContractFolderID),
|
||||
// included so the scraper can build the MinIO upload path for this notice:
|
||||
//
|
||||
// PROC_<contract_folder_id>/<notice_publication_id>/<file>
|
||||
type DocumentScraperTenderResponse struct {
|
||||
ID string `json:"id"`
|
||||
ContractFolderID string `json:"contract_folder_id"`
|
||||
NoticePublicationID string `json:"notice_publication_id"`
|
||||
DocumentURL string `json:"document_url"`
|
||||
Title string `json:"title"`
|
||||
|
||||
@@ -41,7 +41,7 @@ func (s *service) buildTenderResponse(_ context.Context, t *tender.Tender) *Docu
|
||||
}
|
||||
|
||||
return &DocumentScraperTenderResponse{
|
||||
ID: t.ID.Hex(),
|
||||
ContractFolderID: t.ContractFolderID,
|
||||
NoticePublicationID: t.NoticePublicationID,
|
||||
DocumentURL: documentURL,
|
||||
Title: t.Title,
|
||||
|
||||
+180
-50
@@ -23,10 +23,12 @@ type AISummarizerClient interface {
|
||||
FetchTranslationOnDemand(ctx context.Context, reqBody ai_summarizer.TranslateRequest) (*ai_summarizer.TranslateResponse, error)
|
||||
}
|
||||
|
||||
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage
|
||||
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage.
|
||||
// Paths follow the AI team's procedure-scoped layout: artefacts for a notice live under
|
||||
// PROC_<contractFolderID>/<noticePublicationID>/ in the shared bucket.
|
||||
type AISummarizerStorage interface {
|
||||
GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error)
|
||||
ListDocuments(ctx context.Context, noticeID string) ([]ai_summarizer.StoredDocument, error)
|
||||
GetSummaryFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) (string, error)
|
||||
ListDocuments(ctx context.Context, contractFolderID, noticePublicationID string) ([]ai_summarizer.StoredDocument, error)
|
||||
DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error)
|
||||
}
|
||||
|
||||
@@ -119,7 +121,7 @@ func (s *tenderService) GetByID(ctx context.Context, id string) (*TenderResponse
|
||||
}
|
||||
|
||||
// Enrich with AI summary from MinIO storage (best-effort, non-blocking)
|
||||
s.enrichWithAISummary(ctx, resp)
|
||||
s.enrichWithAISummary(ctx, tender, resp)
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
@@ -127,8 +129,13 @@ func (s *tenderService) GetByID(ctx context.Context, id string) (*TenderResponse
|
||||
// enrichWithAISummary attempts to fetch the AI-generated overall summary from
|
||||
// MinIO storage and attaches it to the response. Failures are logged but do not
|
||||
// cause the request to fail — the field will simply be omitted.
|
||||
func (s *tenderService) enrichWithAISummary(ctx context.Context, resp *TenderResponse) {
|
||||
if s.aiSummarizerStorage == nil || resp.NoticePublicationID == "" {
|
||||
//
|
||||
// Because one tender record can span several TED notices (we merge by
|
||||
// contract_folder_id), we try the latest publication id first and fall back to
|
||||
// older RelatedNoticePublicationIDs if the AI pipeline only finished an
|
||||
// earlier notice in the procedure.
|
||||
func (s *tenderService) enrichWithAISummary(ctx context.Context, t *Tender, resp *TenderResponse) {
|
||||
if s.aiSummarizerStorage == nil || t == nil || t.ContractFolderID == "" {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -136,17 +143,89 @@ func (s *tenderService) enrichWithAISummary(ctx context.Context, resp *TenderRes
|
||||
enrichCtx, cancel := context.WithTimeout(ctx, aiSummaryEnrichmentTimeout)
|
||||
defer cancel()
|
||||
|
||||
summary, err := s.aiSummarizerStorage.GetSummaryFromStorage(enrichCtx, resp.NoticePublicationID)
|
||||
summary, usedNotice, err := s.lookupSummaryForTender(enrichCtx, t)
|
||||
if err != nil {
|
||||
s.logger.Debug("AI summary not available for tender", map[string]interface{}{
|
||||
"tender_id": resp.ID,
|
||||
"notice_id": resp.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
"tender_id": resp.ID,
|
||||
"contract_folder_id": t.ContractFolderID,
|
||||
"notice_id": t.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
resp.OverallSummary = summary
|
||||
if usedNotice != t.NoticePublicationID {
|
||||
s.logger.Debug("AI summary resolved from related notice", map[string]interface{}{
|
||||
"tender_id": resp.ID,
|
||||
"contract_folder_id": t.ContractFolderID,
|
||||
"primary_notice_id": t.NoticePublicationID,
|
||||
"used_notice_id": usedNotice,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// lookupSummaryForTender returns the overall_summary for a tender by trying its
|
||||
// primary notice publication id first, then each related publication id in
|
||||
// reverse (newest-first) order. The notice id that actually served the summary
|
||||
// is returned alongside.
|
||||
func (s *tenderService) lookupSummaryForTender(ctx context.Context, t *Tender) (string, string, error) {
|
||||
if s.aiSummarizerStorage == nil {
|
||||
return "", "", fmt.Errorf("AI summarizer storage is not configured")
|
||||
}
|
||||
candidates := orderedNoticeCandidates(t)
|
||||
if len(candidates) == 0 {
|
||||
return "", "", ai_summarizer.ErrObjectNotFound
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, noticeID := range candidates {
|
||||
summary, err := s.aiSummarizerStorage.GetSummaryFromStorage(ctx, t.ContractFolderID, noticeID)
|
||||
if err == nil {
|
||||
return summary, noticeID, nil
|
||||
}
|
||||
lastErr = err
|
||||
// Only "not found" / "not ready" justify trying the next candidate.
|
||||
// Anything else (network/storage error) is propagated immediately.
|
||||
if !errors.Is(err, ai_summarizer.ErrObjectNotFound) &&
|
||||
!errors.Is(err, ai_summarizer.ErrSummaryNotReady) &&
|
||||
!errors.Is(err, ai_summarizer.ErrNoOverallSummary) {
|
||||
return "", noticeID, err
|
||||
}
|
||||
}
|
||||
if lastErr == nil {
|
||||
lastErr = ai_summarizer.ErrObjectNotFound
|
||||
}
|
||||
return "", "", lastErr
|
||||
}
|
||||
|
||||
// orderedNoticeCandidates returns the publication ids to try when resolving
|
||||
// AI-side artefacts for a (possibly merged) tender, newest-first and deduped.
|
||||
// The primary NoticePublicationID is tried first, then any older ids stored in
|
||||
// RelatedNoticePublicationIDs (which we append oldest-first; reverse so newest
|
||||
// wins).
|
||||
func orderedNoticeCandidates(t *Tender) []string {
|
||||
if t == nil {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(t.RelatedNoticePublicationIDs)+1)
|
||||
out := make([]string, 0, len(t.RelatedNoticePublicationIDs)+1)
|
||||
add := func(id string) {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
return
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
out = append(out, id)
|
||||
}
|
||||
add(t.NoticePublicationID)
|
||||
for i := len(t.RelatedNoticePublicationIDs) - 1; i >= 0; i-- {
|
||||
add(t.RelatedNoticePublicationIDs[i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// UpdateTender updates an existing tender
|
||||
@@ -282,16 +361,20 @@ func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDoc
|
||||
if tender == nil {
|
||||
return nil, fmt.Errorf("tender not found")
|
||||
}
|
||||
if strings.TrimSpace(tender.ContractFolderID) == "" {
|
||||
return nil, fmt.Errorf("tender has no contract folder id")
|
||||
}
|
||||
if strings.TrimSpace(tender.NoticePublicationID) == "" {
|
||||
return nil, fmt.Errorf("tender notice publication ID is empty")
|
||||
}
|
||||
|
||||
storedDocuments, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
|
||||
storedDocuments, err := s.listDocumentsForTender(ctx, tender)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to list tender documents from storage", map[string]interface{}{
|
||||
"tender_id": id,
|
||||
"notice_id": tender.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
"tender_id": id,
|
||||
"contract_folder_id": tender.ContractFolderID,
|
||||
"notice_id": tender.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to list tender documents: %w", err)
|
||||
}
|
||||
@@ -307,9 +390,10 @@ func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDoc
|
||||
}
|
||||
|
||||
s.logger.Info("Retrieved scraped documents for tender", map[string]interface{}{
|
||||
"tender_id": id,
|
||||
"notice_id": tender.NoticePublicationID,
|
||||
"documents_count": len(documents),
|
||||
"tender_id": id,
|
||||
"contract_folder_id": tender.ContractFolderID,
|
||||
"notice_id": tender.NoticePublicationID,
|
||||
"documents_count": len(documents),
|
||||
})
|
||||
|
||||
return &TenderDocumentsResponse{
|
||||
@@ -318,6 +402,48 @@ func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDoc
|
||||
}, nil
|
||||
}
|
||||
|
||||
// listDocumentsForTender unions document listings across every notice tied to a
|
||||
// tender (primary + RelatedNoticePublicationIDs), deduplicating by object name.
|
||||
// This way, a merged tender (e.g. CN + Result for the same procedure) exposes
|
||||
// the documents from every notice that contributed to it.
|
||||
func (s *tenderService) listDocumentsForTender(ctx context.Context, t *Tender) ([]ai_summarizer.StoredDocument, error) {
|
||||
if s.aiSummarizerStorage == nil {
|
||||
return nil, ErrTenderDocumentStorageUnavailable
|
||||
}
|
||||
|
||||
candidates := orderedNoticeCandidates(t)
|
||||
if len(candidates) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
seen := make(map[string]struct{})
|
||||
var aggregated []ai_summarizer.StoredDocument
|
||||
var firstErr error
|
||||
for _, noticeID := range candidates {
|
||||
docs, err := s.aiSummarizerStorage.ListDocuments(ctx, t.ContractFolderID, noticeID)
|
||||
if err != nil {
|
||||
// Remember the first error, but keep trying other notices so a
|
||||
// missing folder for an older notice does not hide newer docs.
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
continue
|
||||
}
|
||||
for _, doc := range docs {
|
||||
if _, ok := seen[doc.ObjectName]; ok {
|
||||
continue
|
||||
}
|
||||
seen[doc.ObjectName] = struct{}{}
|
||||
aggregated = append(aggregated, doc)
|
||||
}
|
||||
}
|
||||
|
||||
if len(aggregated) == 0 && firstErr != nil {
|
||||
return nil, firstErr
|
||||
}
|
||||
return aggregated, nil
|
||||
}
|
||||
|
||||
// DownloadDocuments returns a ZIP stream containing all documents for a tender.
|
||||
func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error) {
|
||||
if id == "" {
|
||||
@@ -338,16 +464,20 @@ func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*Tend
|
||||
if tender == nil {
|
||||
return nil, fmt.Errorf("tender not found")
|
||||
}
|
||||
if strings.TrimSpace(tender.ContractFolderID) == "" {
|
||||
return nil, ErrTenderDocumentNotFound
|
||||
}
|
||||
if strings.TrimSpace(tender.NoticePublicationID) == "" {
|
||||
return nil, ErrTenderDocumentNotFound
|
||||
}
|
||||
|
||||
documents, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
|
||||
documents, err := s.listDocumentsForTender(ctx, tender)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to list tender documents for archive", map[string]interface{}{
|
||||
"tender_id": id,
|
||||
"notice_id": tender.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
"tender_id": id,
|
||||
"contract_folder_id": tender.ContractFolderID,
|
||||
"notice_id": tender.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("failed to list tender documents: %w", err)
|
||||
}
|
||||
@@ -722,37 +852,37 @@ func (s *tenderService) GetAISummary(ctx context.Context, id string) (*AISummary
|
||||
if noticeID == "" {
|
||||
return nil, fmt.Errorf("tender has no notice publication ID")
|
||||
}
|
||||
|
||||
// Try to get summary from MinIO storage (async pipeline)
|
||||
if s.aiSummarizerStorage != nil {
|
||||
summary, err := s.aiSummarizerStorage.GetSummaryFromStorage(ctx, noticeID)
|
||||
if err == nil {
|
||||
return &AISummaryResponse{
|
||||
NoticeID: noticeID,
|
||||
OverallSummary: summary,
|
||||
Source: "storage",
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Log but don't fail — the summary might just not be ready yet
|
||||
s.logger.Info("AI summary not available from storage", map[string]interface{}{
|
||||
"tender_id": id,
|
||||
"notice_id": noticeID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
|
||||
// Return specific error if summary not ready
|
||||
if errors.Is(err, ai_summarizer.ErrSummaryNotReady) {
|
||||
return nil, fmt.Errorf("AI summary is not ready yet, the pipeline is still processing")
|
||||
}
|
||||
if errors.Is(err, ai_summarizer.ErrObjectNotFound) {
|
||||
return nil, fmt.Errorf("AI summary not found — this tender has not been processed by the AI pipeline yet")
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("failed to get AI summary: %w", err)
|
||||
if t.ContractFolderID == "" {
|
||||
return nil, fmt.Errorf("tender has no contract folder id")
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("AI summarizer storage is not configured")
|
||||
if s.aiSummarizerStorage == nil {
|
||||
return nil, fmt.Errorf("AI summarizer storage is not configured")
|
||||
}
|
||||
|
||||
summary, usedNotice, err := s.lookupSummaryForTender(ctx, t)
|
||||
if err == nil {
|
||||
return &AISummaryResponse{
|
||||
NoticeID: usedNotice,
|
||||
OverallSummary: summary,
|
||||
Source: "storage",
|
||||
}, nil
|
||||
}
|
||||
|
||||
s.logger.Info("AI summary not available from storage", map[string]interface{}{
|
||||
"tender_id": id,
|
||||
"contract_folder_id": t.ContractFolderID,
|
||||
"notice_id": noticeID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
|
||||
if errors.Is(err, ai_summarizer.ErrSummaryNotReady) {
|
||||
return nil, fmt.Errorf("AI summary is not ready yet, the pipeline is still processing")
|
||||
}
|
||||
if errors.Is(err, ai_summarizer.ErrObjectNotFound) || errors.Is(err, ai_summarizer.ErrNoOverallSummary) {
|
||||
return nil, fmt.Errorf("AI summary not found — this tender has not been processed by the AI pipeline yet")
|
||||
}
|
||||
return nil, fmt.Errorf("failed to get AI summary: %w", err)
|
||||
}
|
||||
|
||||
// TriggerAISummarize triggers on-demand AI summarization for a tender.
|
||||
|
||||
+130
-44
@@ -16,6 +16,12 @@ import (
|
||||
"tm/pkg/logger"
|
||||
)
|
||||
|
||||
// procedurePrefix is prepended to every contract folder id in the MinIO key.
|
||||
// The AI team's storage layout is:
|
||||
//
|
||||
// PROC_<contractFolderID>/<noticePublicationID>/{tender.json, *.pdf, *.docx, ...}
|
||||
const procedurePrefix = "PROC_"
|
||||
|
||||
// StorageClient wraps a MinIO client configured to connect to the AI service's
|
||||
// MinIO bucket and retrieve tender.json files.
|
||||
type StorageClient struct {
|
||||
@@ -26,9 +32,47 @@ type StorageClient struct {
|
||||
|
||||
// TenderSummaryItem represents a single tender's summary retrieved from storage.
|
||||
type TenderSummaryItem struct {
|
||||
NoticeID string `json:"notice_id"`
|
||||
OverallSummary string `json:"overall_summary"`
|
||||
Done bool `json:"done"`
|
||||
ContractFolderID string `json:"contract_folder_id"`
|
||||
NoticeID string `json:"notice_id"`
|
||||
OverallSummary string `json:"overall_summary"`
|
||||
Done bool `json:"done"`
|
||||
}
|
||||
|
||||
// noticeObjectPrefix returns the MinIO key prefix that contains all artefacts
|
||||
// (tender.json + scraped documents) for a single notice within a procedure.
|
||||
// The trailing slash is intentional so it can be passed directly to
|
||||
// minio.ListObjectsOptions.Prefix and also concatenated with a filename.
|
||||
func noticeObjectPrefix(contractFolderID, noticePublicationID string) string {
|
||||
return fmt.Sprintf(
|
||||
"%s%s/%s/",
|
||||
procedurePrefix,
|
||||
strings.TrimSpace(contractFolderID),
|
||||
normalizeNoticeKey(noticePublicationID),
|
||||
)
|
||||
}
|
||||
|
||||
// tenderJSONKey returns the MinIO key for a notice's tender.json (overall summary + status).
|
||||
func tenderJSONKey(contractFolderID, noticePublicationID string) string {
|
||||
return noticeObjectPrefix(contractFolderID, noticePublicationID) + "tender.json"
|
||||
}
|
||||
|
||||
// normalizeNoticeKey trims the leading zeros that we add when persisting the
|
||||
// notice publication id (e.g. "00322263-2026") so it matches the AI team's
|
||||
// folder convention ("322263-2026"). Empty input is preserved.
|
||||
func normalizeNoticeKey(noticePublicationID string) string {
|
||||
id := strings.TrimSpace(noticePublicationID)
|
||||
if id == "" {
|
||||
return ""
|
||||
}
|
||||
// Strip leading zeros from the numeric portion only; keep the year suffix
|
||||
// (and any non-numeric tail) intact.
|
||||
out := strings.TrimLeft(id, "0")
|
||||
if out == "" || strings.HasPrefix(out, "-") {
|
||||
// All-zero numeric portion: keep a single "0" so the key is never empty
|
||||
// and never starts with "-".
|
||||
return "0" + out
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NewStorageClient creates a new MinIO storage client for the AI service.
|
||||
@@ -58,7 +102,8 @@ func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetSummaryFromStorage fetches the <noticeID>/tender.json file from the AI
|
||||
// GetSummaryFromStorage fetches the
|
||||
// PROC_<contractFolderID>/<noticePublicationID>/tender.json file from the AI
|
||||
// service's MinIO bucket, parses it, and returns the overall_summary string.
|
||||
//
|
||||
// It returns:
|
||||
@@ -67,8 +112,14 @@ func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error)
|
||||
// - ("", ErrSummaryNotReady) if the file exists but summarize_status.done is false
|
||||
// - ("", ErrNoOverallSummary) if done is true but overall_summary is empty/null
|
||||
// - ("", error) for any other failure
|
||||
func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error) {
|
||||
objectPath := fmt.Sprintf("%s/tender.json", noticeID)
|
||||
func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) (string, error) {
|
||||
if strings.TrimSpace(contractFolderID) == "" {
|
||||
return "", fmt.Errorf("ai_summarizer: contractFolderID is required")
|
||||
}
|
||||
if strings.TrimSpace(noticePublicationID) == "" {
|
||||
return "", fmt.Errorf("ai_summarizer: noticePublicationID is required")
|
||||
}
|
||||
objectPath := tenderJSONKey(contractFolderID, noticePublicationID)
|
||||
|
||||
s.logger.Debug("Fetching tender.json from MinIO", map[string]interface{}{
|
||||
"bucket": s.config.MinioBucket,
|
||||
@@ -118,8 +169,9 @@ func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID stri
|
||||
}
|
||||
|
||||
s.logger.Info("Retrieved overall_summary from storage", map[string]interface{}{
|
||||
"notice_id": noticeID,
|
||||
"summary_length": len(*tenderJSON.OverallSummary),
|
||||
"contract_folder_id": contractFolderID,
|
||||
"notice_id": noticePublicationID,
|
||||
"summary_length": len(*tenderJSON.OverallSummary),
|
||||
})
|
||||
|
||||
return *tenderJSON.OverallSummary, nil
|
||||
@@ -128,8 +180,11 @@ func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID stri
|
||||
// IsSummaryReady is a convenience method that checks whether the tender.json
|
||||
// exists and the summarize_status.done flag is true, without returning the
|
||||
// actual summary text.
|
||||
func (s *StorageClient) IsSummaryReady(ctx context.Context, noticeID string) (bool, error) {
|
||||
objectPath := fmt.Sprintf("%s/tender.json", noticeID)
|
||||
func (s *StorageClient) IsSummaryReady(ctx context.Context, contractFolderID, noticePublicationID string) (bool, error) {
|
||||
if strings.TrimSpace(contractFolderID) == "" || strings.TrimSpace(noticePublicationID) == "" {
|
||||
return false, fmt.Errorf("ai_summarizer: contractFolderID and noticePublicationID are required")
|
||||
}
|
||||
objectPath := tenderJSONKey(contractFolderID, noticePublicationID)
|
||||
|
||||
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectPath, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
@@ -154,14 +209,18 @@ func (s *StorageClient) IsSummaryReady(ctx context.Context, noticeID string) (bo
|
||||
return tenderJSON.SummarizeStatus.Done, nil
|
||||
}
|
||||
|
||||
// ListDocuments lists document files stored under <noticeID>/.
|
||||
func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]StoredDocument, error) {
|
||||
noticeID = strings.TrimSpace(noticeID)
|
||||
if noticeID == "" {
|
||||
return nil, fmt.Errorf("ai_summarizer: noticeID is required")
|
||||
// ListDocuments lists document files stored under
|
||||
// PROC_<contractFolderID>/<noticePublicationID>/. Non-document artefacts
|
||||
// (tender.json, AI cache files, etc.) are filtered out.
|
||||
func (s *StorageClient) ListDocuments(ctx context.Context, contractFolderID, noticePublicationID string) ([]StoredDocument, error) {
|
||||
if strings.TrimSpace(contractFolderID) == "" {
|
||||
return nil, fmt.Errorf("ai_summarizer: contractFolderID is required")
|
||||
}
|
||||
if strings.TrimSpace(noticePublicationID) == "" {
|
||||
return nil, fmt.Errorf("ai_summarizer: noticePublicationID is required")
|
||||
}
|
||||
|
||||
prefix := fmt.Sprintf("%s/", noticeID)
|
||||
prefix := noticeObjectPrefix(contractFolderID, noticePublicationID)
|
||||
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
|
||||
Prefix: prefix,
|
||||
Recursive: true,
|
||||
@@ -170,9 +229,9 @@ func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]S
|
||||
documents := make([]StoredDocument, 0)
|
||||
for object := range objectCh {
|
||||
if object.Err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to list documents for notice %s: %w", noticeID, object.Err)
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to list documents under %s: %w", prefix, object.Err)
|
||||
}
|
||||
if !isTenderDocumentObject(noticeID, object.Key) {
|
||||
if !isTenderDocumentObject(prefix, object.Key) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -187,20 +246,26 @@ func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]S
|
||||
}
|
||||
|
||||
s.logger.Info("Listed tender documents from storage", map[string]interface{}{
|
||||
"notice_id": noticeID,
|
||||
"documents_count": len(documents),
|
||||
"contract_folder_id": contractFolderID,
|
||||
"notice_id": noticePublicationID,
|
||||
"prefix": prefix,
|
||||
"documents_count": len(documents),
|
||||
})
|
||||
|
||||
return documents, nil
|
||||
}
|
||||
|
||||
func isTenderDocumentObject(noticeID, objectName string) bool {
|
||||
// isTenderDocumentObject reports whether an object key under the given prefix
|
||||
// represents a downloadable tender document (PDF/DOCX/...) rather than an
|
||||
// internal artefact such as tender.json or a cache file.
|
||||
func isTenderDocumentObject(prefix, objectName string) bool {
|
||||
if objectName == "" || strings.HasSuffix(objectName, "/") {
|
||||
return false
|
||||
}
|
||||
|
||||
relativePath := strings.TrimPrefix(objectName, fmt.Sprintf("%s/", noticeID))
|
||||
if relativePath == "" {
|
||||
relativePath := strings.TrimPrefix(objectName, prefix)
|
||||
if relativePath == "" || relativePath == objectName {
|
||||
// objectName did not start with prefix; ignore.
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -237,18 +302,18 @@ func (s *StorageClient) DownloadDocument(ctx context.Context, objectName string)
|
||||
return object, nil
|
||||
}
|
||||
|
||||
// GetAllSummaries lists all tender.json files in the MinIO bucket and returns
|
||||
// the overall_summary for each tender where summarize_status.done is true.
|
||||
// This is useful for batch-retrieving all summaries that the AI pipeline has
|
||||
// already processed.
|
||||
// GetAllSummaries walks the MinIO bucket for the AI team's procedure-scoped
|
||||
// layout (PROC_<contractFolderID>/<noticePublicationID>/tender.json) and
|
||||
// returns one TenderSummaryItem per notice. Tenders whose summarize_status.done
|
||||
// is true are returned with their overall_summary; tenders still in progress
|
||||
// are returned with Done=false and an empty summary.
|
||||
func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryItem, error) {
|
||||
s.logger.Debug("Listing all tender.json files from MinIO", map[string]interface{}{
|
||||
"bucket": s.config.MinioBucket,
|
||||
})
|
||||
|
||||
// List all objects ending with /tender.json
|
||||
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
|
||||
Prefix: "",
|
||||
Prefix: procedurePrefix,
|
||||
Recursive: true,
|
||||
})
|
||||
|
||||
@@ -262,40 +327,39 @@ func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryIte
|
||||
continue
|
||||
}
|
||||
|
||||
// Only process tender.json files (path pattern: <notice_id>/tender.json)
|
||||
// Expected key shape: PROC_<contractFolderID>/<noticePublicationID>/tender.json
|
||||
if !strings.HasSuffix(object.Key, "/tender.json") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract notice_id from the path
|
||||
noticeID := strings.TrimSuffix(object.Key, "/tender.json")
|
||||
if noticeID == "" {
|
||||
contractFolderID, noticePublicationID, ok := parseTenderJSONKey(object.Key)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
// Fetch and parse the tender.json
|
||||
summary, err := s.GetSummaryFromStorage(ctx, noticeID)
|
||||
summary, err := s.GetSummaryFromStorage(ctx, contractFolderID, noticePublicationID)
|
||||
if err != nil {
|
||||
// Log but skip tenders that aren't ready or have errors
|
||||
s.logger.Debug("Skipping tender summary", map[string]interface{}{
|
||||
"notice_id": noticeID,
|
||||
"error": err.Error(),
|
||||
"contract_folder_id": contractFolderID,
|
||||
"notice_id": noticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
|
||||
// Still include items that aren't done so the caller knows they exist
|
||||
if errors.Is(err, ErrSummaryNotReady) {
|
||||
results = append(results, TenderSummaryItem{
|
||||
NoticeID: noticeID,
|
||||
Done: false,
|
||||
ContractFolderID: contractFolderID,
|
||||
NoticeID: noticePublicationID,
|
||||
Done: false,
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
results = append(results, TenderSummaryItem{
|
||||
NoticeID: noticeID,
|
||||
OverallSummary: summary,
|
||||
Done: true,
|
||||
ContractFolderID: contractFolderID,
|
||||
NoticeID: noticePublicationID,
|
||||
OverallSummary: summary,
|
||||
Done: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -307,6 +371,28 @@ func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryIte
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// parseTenderJSONKey splits a key of the form
|
||||
// PROC_<contractFolderID>/<noticePublicationID>/tender.json into its components.
|
||||
// The boolean is false if the key does not match the expected layout.
|
||||
func parseTenderJSONKey(key string) (contractFolderID, noticePublicationID string, ok bool) {
|
||||
if !strings.HasPrefix(key, procedurePrefix) {
|
||||
return "", "", false
|
||||
}
|
||||
trimmed := strings.TrimSuffix(key, "/tender.json")
|
||||
if trimmed == key {
|
||||
return "", "", false
|
||||
}
|
||||
parts := strings.SplitN(trimmed, "/", 3)
|
||||
if len(parts) != 2 {
|
||||
return "", "", false
|
||||
}
|
||||
cfid := strings.TrimPrefix(parts[0], procedurePrefix)
|
||||
if cfid == "" || parts[1] == "" {
|
||||
return "", "", false
|
||||
}
|
||||
return cfid, parts[1], true
|
||||
}
|
||||
|
||||
// countDone counts how many items have Done=true
|
||||
func countDone(items []TenderSummaryItem) int {
|
||||
count := 0
|
||||
|
||||
Reference in New Issue
Block a user