document scraper logic and API refactor
This commit is contained in:
@@ -11,9 +11,14 @@ type DocumentScraperGetRequest struct {
|
|||||||
NoticeID string `param:"notice_id" valid:"required~Notice ID is required"`
|
NoticeID string `param:"notice_id" valid:"required~Notice ID is required"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service
|
// DocumentScraperTenderResponse represents the response for a tender returned to the document scraper service.
|
||||||
|
//
|
||||||
|
// ContractFolderID is the TED procedure identifier (BT-04 / ContractFolderID),
|
||||||
|
// included so the scraper can build the MinIO upload path for this notice:
|
||||||
|
//
|
||||||
|
// PROC_<contract_folder_id>/<notice_publication_id>/<file>
|
||||||
type DocumentScraperTenderResponse struct {
|
type DocumentScraperTenderResponse struct {
|
||||||
ID string `json:"id"`
|
ContractFolderID string `json:"contract_folder_id"`
|
||||||
NoticePublicationID string `json:"notice_publication_id"`
|
NoticePublicationID string `json:"notice_publication_id"`
|
||||||
DocumentURL string `json:"document_url"`
|
DocumentURL string `json:"document_url"`
|
||||||
Title string `json:"title"`
|
Title string `json:"title"`
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ func (s *service) buildTenderResponse(_ context.Context, t *tender.Tender) *Docu
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &DocumentScraperTenderResponse{
|
return &DocumentScraperTenderResponse{
|
||||||
ID: t.ID.Hex(),
|
ContractFolderID: t.ContractFolderID,
|
||||||
NoticePublicationID: t.NoticePublicationID,
|
NoticePublicationID: t.NoticePublicationID,
|
||||||
DocumentURL: documentURL,
|
DocumentURL: documentURL,
|
||||||
Title: t.Title,
|
Title: t.Title,
|
||||||
|
|||||||
+151
-21
@@ -23,10 +23,12 @@ type AISummarizerClient interface {
|
|||||||
FetchTranslationOnDemand(ctx context.Context, reqBody ai_summarizer.TranslateRequest) (*ai_summarizer.TranslateResponse, error)
|
FetchTranslationOnDemand(ctx context.Context, reqBody ai_summarizer.TranslateRequest) (*ai_summarizer.TranslateResponse, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage
|
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage.
|
||||||
|
// Paths follow the AI team's procedure-scoped layout: artefacts for a notice live under
|
||||||
|
// PROC_<contractFolderID>/<noticePublicationID>/ in the shared bucket.
|
||||||
type AISummarizerStorage interface {
|
type AISummarizerStorage interface {
|
||||||
GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error)
|
GetSummaryFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) (string, error)
|
||||||
ListDocuments(ctx context.Context, noticeID string) ([]ai_summarizer.StoredDocument, error)
|
ListDocuments(ctx context.Context, contractFolderID, noticePublicationID string) ([]ai_summarizer.StoredDocument, error)
|
||||||
DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error)
|
DownloadDocument(ctx context.Context, objectName string) (io.ReadCloser, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -119,7 +121,7 @@ func (s *tenderService) GetByID(ctx context.Context, id string) (*TenderResponse
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Enrich with AI summary from MinIO storage (best-effort, non-blocking)
|
// Enrich with AI summary from MinIO storage (best-effort, non-blocking)
|
||||||
s.enrichWithAISummary(ctx, resp)
|
s.enrichWithAISummary(ctx, tender, resp)
|
||||||
|
|
||||||
return resp, nil
|
return resp, nil
|
||||||
}
|
}
|
||||||
@@ -127,8 +129,13 @@ func (s *tenderService) GetByID(ctx context.Context, id string) (*TenderResponse
|
|||||||
// enrichWithAISummary attempts to fetch the AI-generated overall summary from
|
// enrichWithAISummary attempts to fetch the AI-generated overall summary from
|
||||||
// MinIO storage and attaches it to the response. Failures are logged but do not
|
// MinIO storage and attaches it to the response. Failures are logged but do not
|
||||||
// cause the request to fail — the field will simply be omitted.
|
// cause the request to fail — the field will simply be omitted.
|
||||||
func (s *tenderService) enrichWithAISummary(ctx context.Context, resp *TenderResponse) {
|
//
|
||||||
if s.aiSummarizerStorage == nil || resp.NoticePublicationID == "" {
|
// Because one tender record can span several TED notices (we merge by
|
||||||
|
// contract_folder_id), we try the latest publication id first and fall back to
|
||||||
|
// older RelatedNoticePublicationIDs if the AI pipeline only finished an
|
||||||
|
// earlier notice in the procedure.
|
||||||
|
func (s *tenderService) enrichWithAISummary(ctx context.Context, t *Tender, resp *TenderResponse) {
|
||||||
|
if s.aiSummarizerStorage == nil || t == nil || t.ContractFolderID == "" {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -136,17 +143,89 @@ func (s *tenderService) enrichWithAISummary(ctx context.Context, resp *TenderRes
|
|||||||
enrichCtx, cancel := context.WithTimeout(ctx, aiSummaryEnrichmentTimeout)
|
enrichCtx, cancel := context.WithTimeout(ctx, aiSummaryEnrichmentTimeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
summary, err := s.aiSummarizerStorage.GetSummaryFromStorage(enrichCtx, resp.NoticePublicationID)
|
summary, usedNotice, err := s.lookupSummaryForTender(enrichCtx, t)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Debug("AI summary not available for tender", map[string]interface{}{
|
s.logger.Debug("AI summary not available for tender", map[string]interface{}{
|
||||||
"tender_id": resp.ID,
|
"tender_id": resp.ID,
|
||||||
"notice_id": resp.NoticePublicationID,
|
"contract_folder_id": t.ContractFolderID,
|
||||||
|
"notice_id": t.NoticePublicationID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
resp.OverallSummary = summary
|
resp.OverallSummary = summary
|
||||||
|
if usedNotice != t.NoticePublicationID {
|
||||||
|
s.logger.Debug("AI summary resolved from related notice", map[string]interface{}{
|
||||||
|
"tender_id": resp.ID,
|
||||||
|
"contract_folder_id": t.ContractFolderID,
|
||||||
|
"primary_notice_id": t.NoticePublicationID,
|
||||||
|
"used_notice_id": usedNotice,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lookupSummaryForTender returns the overall_summary for a tender by trying its
|
||||||
|
// primary notice publication id first, then each related publication id in
|
||||||
|
// reverse (newest-first) order. The notice id that actually served the summary
|
||||||
|
// is returned alongside.
|
||||||
|
func (s *tenderService) lookupSummaryForTender(ctx context.Context, t *Tender) (string, string, error) {
|
||||||
|
if s.aiSummarizerStorage == nil {
|
||||||
|
return "", "", fmt.Errorf("AI summarizer storage is not configured")
|
||||||
|
}
|
||||||
|
candidates := orderedNoticeCandidates(t)
|
||||||
|
if len(candidates) == 0 {
|
||||||
|
return "", "", ai_summarizer.ErrObjectNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
var lastErr error
|
||||||
|
for _, noticeID := range candidates {
|
||||||
|
summary, err := s.aiSummarizerStorage.GetSummaryFromStorage(ctx, t.ContractFolderID, noticeID)
|
||||||
|
if err == nil {
|
||||||
|
return summary, noticeID, nil
|
||||||
|
}
|
||||||
|
lastErr = err
|
||||||
|
// Only "not found" / "not ready" justify trying the next candidate.
|
||||||
|
// Anything else (network/storage error) is propagated immediately.
|
||||||
|
if !errors.Is(err, ai_summarizer.ErrObjectNotFound) &&
|
||||||
|
!errors.Is(err, ai_summarizer.ErrSummaryNotReady) &&
|
||||||
|
!errors.Is(err, ai_summarizer.ErrNoOverallSummary) {
|
||||||
|
return "", noticeID, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if lastErr == nil {
|
||||||
|
lastErr = ai_summarizer.ErrObjectNotFound
|
||||||
|
}
|
||||||
|
return "", "", lastErr
|
||||||
|
}
|
||||||
|
|
||||||
|
// orderedNoticeCandidates returns the publication ids to try when resolving
|
||||||
|
// AI-side artefacts for a (possibly merged) tender, newest-first and deduped.
|
||||||
|
// The primary NoticePublicationID is tried first, then any older ids stored in
|
||||||
|
// RelatedNoticePublicationIDs (which we append oldest-first; reverse so newest
|
||||||
|
// wins).
|
||||||
|
func orderedNoticeCandidates(t *Tender) []string {
|
||||||
|
if t == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
seen := make(map[string]struct{}, len(t.RelatedNoticePublicationIDs)+1)
|
||||||
|
out := make([]string, 0, len(t.RelatedNoticePublicationIDs)+1)
|
||||||
|
add := func(id string) {
|
||||||
|
id = strings.TrimSpace(id)
|
||||||
|
if id == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, ok := seen[id]; ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
seen[id] = struct{}{}
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
add(t.NoticePublicationID)
|
||||||
|
for i := len(t.RelatedNoticePublicationIDs) - 1; i >= 0; i-- {
|
||||||
|
add(t.RelatedNoticePublicationIDs[i])
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// UpdateTender updates an existing tender
|
// UpdateTender updates an existing tender
|
||||||
@@ -282,14 +361,18 @@ func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDoc
|
|||||||
if tender == nil {
|
if tender == nil {
|
||||||
return nil, fmt.Errorf("tender not found")
|
return nil, fmt.Errorf("tender not found")
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(tender.ContractFolderID) == "" {
|
||||||
|
return nil, fmt.Errorf("tender has no contract folder id")
|
||||||
|
}
|
||||||
if strings.TrimSpace(tender.NoticePublicationID) == "" {
|
if strings.TrimSpace(tender.NoticePublicationID) == "" {
|
||||||
return nil, fmt.Errorf("tender notice publication ID is empty")
|
return nil, fmt.Errorf("tender notice publication ID is empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
storedDocuments, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
|
storedDocuments, err := s.listDocumentsForTender(ctx, tender)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to list tender documents from storage", map[string]interface{}{
|
s.logger.Error("Failed to list tender documents from storage", map[string]interface{}{
|
||||||
"tender_id": id,
|
"tender_id": id,
|
||||||
|
"contract_folder_id": tender.ContractFolderID,
|
||||||
"notice_id": tender.NoticePublicationID,
|
"notice_id": tender.NoticePublicationID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
@@ -308,6 +391,7 @@ func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDoc
|
|||||||
|
|
||||||
s.logger.Info("Retrieved scraped documents for tender", map[string]interface{}{
|
s.logger.Info("Retrieved scraped documents for tender", map[string]interface{}{
|
||||||
"tender_id": id,
|
"tender_id": id,
|
||||||
|
"contract_folder_id": tender.ContractFolderID,
|
||||||
"notice_id": tender.NoticePublicationID,
|
"notice_id": tender.NoticePublicationID,
|
||||||
"documents_count": len(documents),
|
"documents_count": len(documents),
|
||||||
})
|
})
|
||||||
@@ -318,6 +402,48 @@ func (s *tenderService) GetDocuments(ctx context.Context, id string) (*TenderDoc
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// listDocumentsForTender unions document listings across every notice tied to a
|
||||||
|
// tender (primary + RelatedNoticePublicationIDs), deduplicating by object name.
|
||||||
|
// This way, a merged tender (e.g. CN + Result for the same procedure) exposes
|
||||||
|
// the documents from every notice that contributed to it.
|
||||||
|
func (s *tenderService) listDocumentsForTender(ctx context.Context, t *Tender) ([]ai_summarizer.StoredDocument, error) {
|
||||||
|
if s.aiSummarizerStorage == nil {
|
||||||
|
return nil, ErrTenderDocumentStorageUnavailable
|
||||||
|
}
|
||||||
|
|
||||||
|
candidates := orderedNoticeCandidates(t)
|
||||||
|
if len(candidates) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
var aggregated []ai_summarizer.StoredDocument
|
||||||
|
var firstErr error
|
||||||
|
for _, noticeID := range candidates {
|
||||||
|
docs, err := s.aiSummarizerStorage.ListDocuments(ctx, t.ContractFolderID, noticeID)
|
||||||
|
if err != nil {
|
||||||
|
// Remember the first error, but keep trying other notices so a
|
||||||
|
// missing folder for an older notice does not hide newer docs.
|
||||||
|
if firstErr == nil {
|
||||||
|
firstErr = err
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, doc := range docs {
|
||||||
|
if _, ok := seen[doc.ObjectName]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[doc.ObjectName] = struct{}{}
|
||||||
|
aggregated = append(aggregated, doc)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(aggregated) == 0 && firstErr != nil {
|
||||||
|
return nil, firstErr
|
||||||
|
}
|
||||||
|
return aggregated, nil
|
||||||
|
}
|
||||||
|
|
||||||
// DownloadDocuments returns a ZIP stream containing all documents for a tender.
|
// DownloadDocuments returns a ZIP stream containing all documents for a tender.
|
||||||
func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error) {
|
func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*TenderDocumentDownload, error) {
|
||||||
if id == "" {
|
if id == "" {
|
||||||
@@ -338,14 +464,18 @@ func (s *tenderService) DownloadDocuments(ctx context.Context, id string) (*Tend
|
|||||||
if tender == nil {
|
if tender == nil {
|
||||||
return nil, fmt.Errorf("tender not found")
|
return nil, fmt.Errorf("tender not found")
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(tender.ContractFolderID) == "" {
|
||||||
|
return nil, ErrTenderDocumentNotFound
|
||||||
|
}
|
||||||
if strings.TrimSpace(tender.NoticePublicationID) == "" {
|
if strings.TrimSpace(tender.NoticePublicationID) == "" {
|
||||||
return nil, ErrTenderDocumentNotFound
|
return nil, ErrTenderDocumentNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
documents, err := s.aiSummarizerStorage.ListDocuments(ctx, tender.NoticePublicationID)
|
documents, err := s.listDocumentsForTender(ctx, tender)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.logger.Error("Failed to list tender documents for archive", map[string]interface{}{
|
s.logger.Error("Failed to list tender documents for archive", map[string]interface{}{
|
||||||
"tender_id": id,
|
"tender_id": id,
|
||||||
|
"contract_folder_id": tender.ContractFolderID,
|
||||||
"notice_id": tender.NoticePublicationID,
|
"notice_id": tender.NoticePublicationID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
@@ -722,39 +852,39 @@ func (s *tenderService) GetAISummary(ctx context.Context, id string) (*AISummary
|
|||||||
if noticeID == "" {
|
if noticeID == "" {
|
||||||
return nil, fmt.Errorf("tender has no notice publication ID")
|
return nil, fmt.Errorf("tender has no notice publication ID")
|
||||||
}
|
}
|
||||||
|
if t.ContractFolderID == "" {
|
||||||
|
return nil, fmt.Errorf("tender has no contract folder id")
|
||||||
|
}
|
||||||
|
|
||||||
// Try to get summary from MinIO storage (async pipeline)
|
if s.aiSummarizerStorage == nil {
|
||||||
if s.aiSummarizerStorage != nil {
|
return nil, fmt.Errorf("AI summarizer storage is not configured")
|
||||||
summary, err := s.aiSummarizerStorage.GetSummaryFromStorage(ctx, noticeID)
|
}
|
||||||
|
|
||||||
|
summary, usedNotice, err := s.lookupSummaryForTender(ctx, t)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return &AISummaryResponse{
|
return &AISummaryResponse{
|
||||||
NoticeID: noticeID,
|
NoticeID: usedNotice,
|
||||||
OverallSummary: summary,
|
OverallSummary: summary,
|
||||||
Source: "storage",
|
Source: "storage",
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Log but don't fail — the summary might just not be ready yet
|
|
||||||
s.logger.Info("AI summary not available from storage", map[string]interface{}{
|
s.logger.Info("AI summary not available from storage", map[string]interface{}{
|
||||||
"tender_id": id,
|
"tender_id": id,
|
||||||
|
"contract_folder_id": t.ContractFolderID,
|
||||||
"notice_id": noticeID,
|
"notice_id": noticeID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
|
|
||||||
// Return specific error if summary not ready
|
|
||||||
if errors.Is(err, ai_summarizer.ErrSummaryNotReady) {
|
if errors.Is(err, ai_summarizer.ErrSummaryNotReady) {
|
||||||
return nil, fmt.Errorf("AI summary is not ready yet, the pipeline is still processing")
|
return nil, fmt.Errorf("AI summary is not ready yet, the pipeline is still processing")
|
||||||
}
|
}
|
||||||
if errors.Is(err, ai_summarizer.ErrObjectNotFound) {
|
if errors.Is(err, ai_summarizer.ErrObjectNotFound) || errors.Is(err, ai_summarizer.ErrNoOverallSummary) {
|
||||||
return nil, fmt.Errorf("AI summary not found — this tender has not been processed by the AI pipeline yet")
|
return nil, fmt.Errorf("AI summary not found — this tender has not been processed by the AI pipeline yet")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("failed to get AI summary: %w", err)
|
return nil, fmt.Errorf("failed to get AI summary: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil, fmt.Errorf("AI summarizer storage is not configured")
|
|
||||||
}
|
|
||||||
|
|
||||||
// TriggerAISummarize triggers on-demand AI summarization for a tender.
|
// TriggerAISummarize triggers on-demand AI summarization for a tender.
|
||||||
// It calls the external AI service HTTP API to generate summaries.
|
// It calls the external AI service HTTP API to generate summaries.
|
||||||
func (s *tenderService) TriggerAISummarize(ctx context.Context, id string) (*AISummarizeResponse, error) {
|
func (s *tenderService) TriggerAISummarize(ctx context.Context, id string) (*AISummarizeResponse, error) {
|
||||||
|
|||||||
+121
-35
@@ -16,6 +16,12 @@ import (
|
|||||||
"tm/pkg/logger"
|
"tm/pkg/logger"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// procedurePrefix is prepended to every contract folder id in the MinIO key.
|
||||||
|
// The AI team's storage layout is:
|
||||||
|
//
|
||||||
|
// PROC_<contractFolderID>/<noticePublicationID>/{tender.json, *.pdf, *.docx, ...}
|
||||||
|
const procedurePrefix = "PROC_"
|
||||||
|
|
||||||
// StorageClient wraps a MinIO client configured to connect to the AI service's
|
// StorageClient wraps a MinIO client configured to connect to the AI service's
|
||||||
// MinIO bucket and retrieve tender.json files.
|
// MinIO bucket and retrieve tender.json files.
|
||||||
type StorageClient struct {
|
type StorageClient struct {
|
||||||
@@ -26,11 +32,49 @@ type StorageClient struct {
|
|||||||
|
|
||||||
// TenderSummaryItem represents a single tender's summary retrieved from storage.
|
// TenderSummaryItem represents a single tender's summary retrieved from storage.
|
||||||
type TenderSummaryItem struct {
|
type TenderSummaryItem struct {
|
||||||
|
ContractFolderID string `json:"contract_folder_id"`
|
||||||
NoticeID string `json:"notice_id"`
|
NoticeID string `json:"notice_id"`
|
||||||
OverallSummary string `json:"overall_summary"`
|
OverallSummary string `json:"overall_summary"`
|
||||||
Done bool `json:"done"`
|
Done bool `json:"done"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// noticeObjectPrefix returns the MinIO key prefix that contains all artefacts
|
||||||
|
// (tender.json + scraped documents) for a single notice within a procedure.
|
||||||
|
// The trailing slash is intentional so it can be passed directly to
|
||||||
|
// minio.ListObjectsOptions.Prefix and also concatenated with a filename.
|
||||||
|
func noticeObjectPrefix(contractFolderID, noticePublicationID string) string {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"%s%s/%s/",
|
||||||
|
procedurePrefix,
|
||||||
|
strings.TrimSpace(contractFolderID),
|
||||||
|
normalizeNoticeKey(noticePublicationID),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// tenderJSONKey returns the MinIO key for a notice's tender.json (overall summary + status).
|
||||||
|
func tenderJSONKey(contractFolderID, noticePublicationID string) string {
|
||||||
|
return noticeObjectPrefix(contractFolderID, noticePublicationID) + "tender.json"
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeNoticeKey trims the leading zeros that we add when persisting the
|
||||||
|
// notice publication id (e.g. "00322263-2026") so it matches the AI team's
|
||||||
|
// folder convention ("322263-2026"). Empty input is preserved.
|
||||||
|
func normalizeNoticeKey(noticePublicationID string) string {
|
||||||
|
id := strings.TrimSpace(noticePublicationID)
|
||||||
|
if id == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
// Strip leading zeros from the numeric portion only; keep the year suffix
|
||||||
|
// (and any non-numeric tail) intact.
|
||||||
|
out := strings.TrimLeft(id, "0")
|
||||||
|
if out == "" || strings.HasPrefix(out, "-") {
|
||||||
|
// All-zero numeric portion: keep a single "0" so the key is never empty
|
||||||
|
// and never starts with "-".
|
||||||
|
return "0" + out
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// NewStorageClient creates a new MinIO storage client for the AI service.
|
// NewStorageClient creates a new MinIO storage client for the AI service.
|
||||||
// It establishes a connection to the external MinIO instance and verifies
|
// It establishes a connection to the external MinIO instance and verifies
|
||||||
// that the configured bucket exists.
|
// that the configured bucket exists.
|
||||||
@@ -58,7 +102,8 @@ func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error)
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetSummaryFromStorage fetches the <noticeID>/tender.json file from the AI
|
// GetSummaryFromStorage fetches the
|
||||||
|
// PROC_<contractFolderID>/<noticePublicationID>/tender.json file from the AI
|
||||||
// service's MinIO bucket, parses it, and returns the overall_summary string.
|
// service's MinIO bucket, parses it, and returns the overall_summary string.
|
||||||
//
|
//
|
||||||
// It returns:
|
// It returns:
|
||||||
@@ -67,8 +112,14 @@ func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error)
|
|||||||
// - ("", ErrSummaryNotReady) if the file exists but summarize_status.done is false
|
// - ("", ErrSummaryNotReady) if the file exists but summarize_status.done is false
|
||||||
// - ("", ErrNoOverallSummary) if done is true but overall_summary is empty/null
|
// - ("", ErrNoOverallSummary) if done is true but overall_summary is empty/null
|
||||||
// - ("", error) for any other failure
|
// - ("", error) for any other failure
|
||||||
func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error) {
|
func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) (string, error) {
|
||||||
objectPath := fmt.Sprintf("%s/tender.json", noticeID)
|
if strings.TrimSpace(contractFolderID) == "" {
|
||||||
|
return "", fmt.Errorf("ai_summarizer: contractFolderID is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(noticePublicationID) == "" {
|
||||||
|
return "", fmt.Errorf("ai_summarizer: noticePublicationID is required")
|
||||||
|
}
|
||||||
|
objectPath := tenderJSONKey(contractFolderID, noticePublicationID)
|
||||||
|
|
||||||
s.logger.Debug("Fetching tender.json from MinIO", map[string]interface{}{
|
s.logger.Debug("Fetching tender.json from MinIO", map[string]interface{}{
|
||||||
"bucket": s.config.MinioBucket,
|
"bucket": s.config.MinioBucket,
|
||||||
@@ -118,7 +169,8 @@ func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID stri
|
|||||||
}
|
}
|
||||||
|
|
||||||
s.logger.Info("Retrieved overall_summary from storage", map[string]interface{}{
|
s.logger.Info("Retrieved overall_summary from storage", map[string]interface{}{
|
||||||
"notice_id": noticeID,
|
"contract_folder_id": contractFolderID,
|
||||||
|
"notice_id": noticePublicationID,
|
||||||
"summary_length": len(*tenderJSON.OverallSummary),
|
"summary_length": len(*tenderJSON.OverallSummary),
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -128,8 +180,11 @@ func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID stri
|
|||||||
// IsSummaryReady is a convenience method that checks whether the tender.json
|
// IsSummaryReady is a convenience method that checks whether the tender.json
|
||||||
// exists and the summarize_status.done flag is true, without returning the
|
// exists and the summarize_status.done flag is true, without returning the
|
||||||
// actual summary text.
|
// actual summary text.
|
||||||
func (s *StorageClient) IsSummaryReady(ctx context.Context, noticeID string) (bool, error) {
|
func (s *StorageClient) IsSummaryReady(ctx context.Context, contractFolderID, noticePublicationID string) (bool, error) {
|
||||||
objectPath := fmt.Sprintf("%s/tender.json", noticeID)
|
if strings.TrimSpace(contractFolderID) == "" || strings.TrimSpace(noticePublicationID) == "" {
|
||||||
|
return false, fmt.Errorf("ai_summarizer: contractFolderID and noticePublicationID are required")
|
||||||
|
}
|
||||||
|
objectPath := tenderJSONKey(contractFolderID, noticePublicationID)
|
||||||
|
|
||||||
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectPath, minio.GetObjectOptions{})
|
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectPath, minio.GetObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -154,14 +209,18 @@ func (s *StorageClient) IsSummaryReady(ctx context.Context, noticeID string) (bo
|
|||||||
return tenderJSON.SummarizeStatus.Done, nil
|
return tenderJSON.SummarizeStatus.Done, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListDocuments lists document files stored under <noticeID>/.
|
// ListDocuments lists document files stored under
|
||||||
func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]StoredDocument, error) {
|
// PROC_<contractFolderID>/<noticePublicationID>/. Non-document artefacts
|
||||||
noticeID = strings.TrimSpace(noticeID)
|
// (tender.json, AI cache files, etc.) are filtered out.
|
||||||
if noticeID == "" {
|
func (s *StorageClient) ListDocuments(ctx context.Context, contractFolderID, noticePublicationID string) ([]StoredDocument, error) {
|
||||||
return nil, fmt.Errorf("ai_summarizer: noticeID is required")
|
if strings.TrimSpace(contractFolderID) == "" {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: contractFolderID is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(noticePublicationID) == "" {
|
||||||
|
return nil, fmt.Errorf("ai_summarizer: noticePublicationID is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
prefix := fmt.Sprintf("%s/", noticeID)
|
prefix := noticeObjectPrefix(contractFolderID, noticePublicationID)
|
||||||
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
|
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
|
||||||
Prefix: prefix,
|
Prefix: prefix,
|
||||||
Recursive: true,
|
Recursive: true,
|
||||||
@@ -170,9 +229,9 @@ func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]S
|
|||||||
documents := make([]StoredDocument, 0)
|
documents := make([]StoredDocument, 0)
|
||||||
for object := range objectCh {
|
for object := range objectCh {
|
||||||
if object.Err != nil {
|
if object.Err != nil {
|
||||||
return nil, fmt.Errorf("ai_summarizer: failed to list documents for notice %s: %w", noticeID, object.Err)
|
return nil, fmt.Errorf("ai_summarizer: failed to list documents under %s: %w", prefix, object.Err)
|
||||||
}
|
}
|
||||||
if !isTenderDocumentObject(noticeID, object.Key) {
|
if !isTenderDocumentObject(prefix, object.Key) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -187,20 +246,26 @@ func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]S
|
|||||||
}
|
}
|
||||||
|
|
||||||
s.logger.Info("Listed tender documents from storage", map[string]interface{}{
|
s.logger.Info("Listed tender documents from storage", map[string]interface{}{
|
||||||
"notice_id": noticeID,
|
"contract_folder_id": contractFolderID,
|
||||||
|
"notice_id": noticePublicationID,
|
||||||
|
"prefix": prefix,
|
||||||
"documents_count": len(documents),
|
"documents_count": len(documents),
|
||||||
})
|
})
|
||||||
|
|
||||||
return documents, nil
|
return documents, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func isTenderDocumentObject(noticeID, objectName string) bool {
|
// isTenderDocumentObject reports whether an object key under the given prefix
|
||||||
|
// represents a downloadable tender document (PDF/DOCX/...) rather than an
|
||||||
|
// internal artefact such as tender.json or a cache file.
|
||||||
|
func isTenderDocumentObject(prefix, objectName string) bool {
|
||||||
if objectName == "" || strings.HasSuffix(objectName, "/") {
|
if objectName == "" || strings.HasSuffix(objectName, "/") {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
relativePath := strings.TrimPrefix(objectName, fmt.Sprintf("%s/", noticeID))
|
relativePath := strings.TrimPrefix(objectName, prefix)
|
||||||
if relativePath == "" {
|
if relativePath == "" || relativePath == objectName {
|
||||||
|
// objectName did not start with prefix; ignore.
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -237,18 +302,18 @@ func (s *StorageClient) DownloadDocument(ctx context.Context, objectName string)
|
|||||||
return object, nil
|
return object, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetAllSummaries lists all tender.json files in the MinIO bucket and returns
|
// GetAllSummaries walks the MinIO bucket for the AI team's procedure-scoped
|
||||||
// the overall_summary for each tender where summarize_status.done is true.
|
// layout (PROC_<contractFolderID>/<noticePublicationID>/tender.json) and
|
||||||
// This is useful for batch-retrieving all summaries that the AI pipeline has
|
// returns one TenderSummaryItem per notice. Tenders whose summarize_status.done
|
||||||
// already processed.
|
// is true are returned with their overall_summary; tenders still in progress
|
||||||
|
// are returned with Done=false and an empty summary.
|
||||||
func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryItem, error) {
|
func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryItem, error) {
|
||||||
s.logger.Debug("Listing all tender.json files from MinIO", map[string]interface{}{
|
s.logger.Debug("Listing all tender.json files from MinIO", map[string]interface{}{
|
||||||
"bucket": s.config.MinioBucket,
|
"bucket": s.config.MinioBucket,
|
||||||
})
|
})
|
||||||
|
|
||||||
// List all objects ending with /tender.json
|
|
||||||
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
|
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
|
||||||
Prefix: "",
|
Prefix: procedurePrefix,
|
||||||
Recursive: true,
|
Recursive: true,
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -262,30 +327,28 @@ func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryIte
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Only process tender.json files (path pattern: <notice_id>/tender.json)
|
// Expected key shape: PROC_<contractFolderID>/<noticePublicationID>/tender.json
|
||||||
if !strings.HasSuffix(object.Key, "/tender.json") {
|
if !strings.HasSuffix(object.Key, "/tender.json") {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Extract notice_id from the path
|
contractFolderID, noticePublicationID, ok := parseTenderJSONKey(object.Key)
|
||||||
noticeID := strings.TrimSuffix(object.Key, "/tender.json")
|
if !ok {
|
||||||
if noticeID == "" {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fetch and parse the tender.json
|
summary, err := s.GetSummaryFromStorage(ctx, contractFolderID, noticePublicationID)
|
||||||
summary, err := s.GetSummaryFromStorage(ctx, noticeID)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Log but skip tenders that aren't ready or have errors
|
|
||||||
s.logger.Debug("Skipping tender summary", map[string]interface{}{
|
s.logger.Debug("Skipping tender summary", map[string]interface{}{
|
||||||
"notice_id": noticeID,
|
"contract_folder_id": contractFolderID,
|
||||||
|
"notice_id": noticePublicationID,
|
||||||
"error": err.Error(),
|
"error": err.Error(),
|
||||||
})
|
})
|
||||||
|
|
||||||
// Still include items that aren't done so the caller knows they exist
|
|
||||||
if errors.Is(err, ErrSummaryNotReady) {
|
if errors.Is(err, ErrSummaryNotReady) {
|
||||||
results = append(results, TenderSummaryItem{
|
results = append(results, TenderSummaryItem{
|
||||||
NoticeID: noticeID,
|
ContractFolderID: contractFolderID,
|
||||||
|
NoticeID: noticePublicationID,
|
||||||
Done: false,
|
Done: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -293,7 +356,8 @@ func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryIte
|
|||||||
}
|
}
|
||||||
|
|
||||||
results = append(results, TenderSummaryItem{
|
results = append(results, TenderSummaryItem{
|
||||||
NoticeID: noticeID,
|
ContractFolderID: contractFolderID,
|
||||||
|
NoticeID: noticePublicationID,
|
||||||
OverallSummary: summary,
|
OverallSummary: summary,
|
||||||
Done: true,
|
Done: true,
|
||||||
})
|
})
|
||||||
@@ -307,6 +371,28 @@ func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryIte
|
|||||||
return results, nil
|
return results, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// parseTenderJSONKey splits a key of the form
|
||||||
|
// PROC_<contractFolderID>/<noticePublicationID>/tender.json into its components.
|
||||||
|
// The boolean is false if the key does not match the expected layout.
|
||||||
|
func parseTenderJSONKey(key string) (contractFolderID, noticePublicationID string, ok bool) {
|
||||||
|
if !strings.HasPrefix(key, procedurePrefix) {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
trimmed := strings.TrimSuffix(key, "/tender.json")
|
||||||
|
if trimmed == key {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
parts := strings.SplitN(trimmed, "/", 3)
|
||||||
|
if len(parts) != 2 {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
cfid := strings.TrimPrefix(parts[0], procedurePrefix)
|
||||||
|
if cfid == "" || parts[1] == "" {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
return cfid, parts[1], true
|
||||||
|
}
|
||||||
|
|
||||||
// countDone counts how many items have Done=true
|
// countDone counts how many items have Done=true
|
||||||
func countDone(items []TenderSummaryItem) int {
|
func countDone(items []TenderSummaryItem) int {
|
||||||
count := 0
|
count := 0
|
||||||
|
|||||||
Reference in New Issue
Block a user