mazyar/tm_back
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

document scraper logic and API refactor

Browse Source
This commit is contained in:
Mazyar
2026-05-11 15:52:32 +03:30
parent b4efb97f47
commit 7c42b343ce
4 changed files with 318 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/ai_summarizer/storage.go
+130 -44
View File
@@ -16,6 +16,12 @@ import (
"tm/pkg/logger"
)
// procedurePrefix is prepended to every contract folder id in the MinIO key.
// The AI team's storage layout is:
//
// PROC_<contractFolderID>/<noticePublicationID>/{tender.json, *.pdf, *.docx, ...}
const procedurePrefix = "PROC_"
// StorageClient wraps a MinIO client configured to connect to the AI service's
// MinIO bucket and retrieve tender.json files.
type StorageClient struct {
@@ -26,9 +32,47 @@ type StorageClient struct {
// TenderSummaryItem represents a single tender's summary retrieved from storage.
type TenderSummaryItem struct {
NoticeID string `json:"notice_id"`
OverallSummary string `json:"overall_summary"`
Done bool `json:"done"`
ContractFolderID string `json:"contract_folder_id"`
NoticeID string `json:"notice_id"`
OverallSummary string `json:"overall_summary"`
Done bool `json:"done"`
}
// noticeObjectPrefix returns the MinIO key prefix that contains all artefacts
// (tender.json + scraped documents) for a single notice within a procedure.
// The trailing slash is intentional so it can be passed directly to
// minio.ListObjectsOptions.Prefix and also concatenated with a filename.
func noticeObjectPrefix(contractFolderID, noticePublicationID string) string {
return fmt.Sprintf(
"%s%s/%s/",
procedurePrefix,
strings.TrimSpace(contractFolderID),
normalizeNoticeKey(noticePublicationID),
)
}
// tenderJSONKey returns the MinIO key for a notice's tender.json (overall summary + status).
func tenderJSONKey(contractFolderID, noticePublicationID string) string {
return noticeObjectPrefix(contractFolderID, noticePublicationID) + "tender.json"
}
// normalizeNoticeKey trims the leading zeros that we add when persisting the
// notice publication id (e.g. "00322263-2026") so it matches the AI team's
// folder convention ("322263-2026"). Empty input is preserved.
func normalizeNoticeKey(noticePublicationID string) string {
id := strings.TrimSpace(noticePublicationID)
if id == "" {
return ""
}
// Strip leading zeros from the numeric portion only; keep the year suffix
// (and any non-numeric tail) intact.
out := strings.TrimLeft(id, "0")
if out == "" || strings.HasPrefix(out, "-") {
// All-zero numeric portion: keep a single "0" so the key is never empty
// and never starts with "-".
return "0" + out
}
return out
}
// NewStorageClient creates a new MinIO storage client for the AI service.
@@ -58,7 +102,8 @@ func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error)
}, nil
}
// GetSummaryFromStorage fetches the <noticeID>/tender.json file from the AI
// GetSummaryFromStorage fetches the
// PROC_<contractFolderID>/<noticePublicationID>/tender.json file from the AI
// service's MinIO bucket, parses it, and returns the overall_summary string.
//
// It returns:
@@ -67,8 +112,14 @@ func NewStorageClient(config *Config, log logger.Logger) (*StorageClient, error)
// - ("", ErrSummaryNotReady) if the file exists but summarize_status.done is false
// - ("", ErrNoOverallSummary) if done is true but overall_summary is empty/null
// - ("", error) for any other failure
func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID string) (string, error) {
objectPath := fmt.Sprintf("%s/tender.json", noticeID)
func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) (string, error) {
if strings.TrimSpace(contractFolderID) == "" {
return "", fmt.Errorf("ai_summarizer: contractFolderID is required")
}
if strings.TrimSpace(noticePublicationID) == "" {
return "", fmt.Errorf("ai_summarizer: noticePublicationID is required")
}
objectPath := tenderJSONKey(contractFolderID, noticePublicationID)
s.logger.Debug("Fetching tender.json from MinIO", map[string]interface{}{
"bucket": s.config.MinioBucket,
@@ -118,8 +169,9 @@ func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID stri
}
s.logger.Info("Retrieved overall_summary from storage", map[string]interface{}{
"notice_id": noticeID,
"summary_length": len(*tenderJSON.OverallSummary),
"contract_folder_id": contractFolderID,
"notice_id": noticePublicationID,
"summary_length": len(*tenderJSON.OverallSummary),
})
return *tenderJSON.OverallSummary, nil
@@ -128,8 +180,11 @@ func (s *StorageClient) GetSummaryFromStorage(ctx context.Context, noticeID stri
// IsSummaryReady is a convenience method that checks whether the tender.json
// exists and the summarize_status.done flag is true, without returning the
// actual summary text.
func (s *StorageClient) IsSummaryReady(ctx context.Context, noticeID string) (bool, error) {
objectPath := fmt.Sprintf("%s/tender.json", noticeID)
func (s *StorageClient) IsSummaryReady(ctx context.Context, contractFolderID, noticePublicationID string) (bool, error) {
if strings.TrimSpace(contractFolderID) == "" || strings.TrimSpace(noticePublicationID) == "" {
return false, fmt.Errorf("ai_summarizer: contractFolderID and noticePublicationID are required")
}
objectPath := tenderJSONKey(contractFolderID, noticePublicationID)
object, err := s.client.GetObject(ctx, s.config.MinioBucket, objectPath, minio.GetObjectOptions{})
if err != nil {
@@ -154,14 +209,18 @@ func (s *StorageClient) IsSummaryReady(ctx context.Context, noticeID string) (bo
return tenderJSON.SummarizeStatus.Done, nil
}
// ListDocuments lists document files stored under <noticeID>/.
func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]StoredDocument, error) {
noticeID = strings.TrimSpace(noticeID)
if noticeID == "" {
return nil, fmt.Errorf("ai_summarizer: noticeID is required")
// ListDocuments lists document files stored under
// PROC_<contractFolderID>/<noticePublicationID>/. Non-document artefacts
// (tender.json, AI cache files, etc.) are filtered out.
func (s *StorageClient) ListDocuments(ctx context.Context, contractFolderID, noticePublicationID string) ([]StoredDocument, error) {
if strings.TrimSpace(contractFolderID) == "" {
return nil, fmt.Errorf("ai_summarizer: contractFolderID is required")
}
if strings.TrimSpace(noticePublicationID) == "" {
return nil, fmt.Errorf("ai_summarizer: noticePublicationID is required")
}
prefix := fmt.Sprintf("%s/", noticeID)
prefix := noticeObjectPrefix(contractFolderID, noticePublicationID)
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
Prefix: prefix,
Recursive: true,
@@ -170,9 +229,9 @@ func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]S
documents := make([]StoredDocument, 0)
for object := range objectCh {
if object.Err != nil {
return nil, fmt.Errorf("ai_summarizer: failed to list documents for notice %s: %w", noticeID, object.Err)
return nil, fmt.Errorf("ai_summarizer: failed to list documents under %s: %w", prefix, object.Err)
}
if !isTenderDocumentObject(noticeID, object.Key) {
if !isTenderDocumentObject(prefix, object.Key) {
continue
}
@@ -187,20 +246,26 @@ func (s *StorageClient) ListDocuments(ctx context.Context, noticeID string) ([]S
}
s.logger.Info("Listed tender documents from storage", map[string]interface{}{
"notice_id": noticeID,
"documents_count": len(documents),
"contract_folder_id": contractFolderID,
"notice_id": noticePublicationID,
"prefix": prefix,
"documents_count": len(documents),
})
return documents, nil
}
func isTenderDocumentObject(noticeID, objectName string) bool {
// isTenderDocumentObject reports whether an object key under the given prefix
// represents a downloadable tender document (PDF/DOCX/...) rather than an
// internal artefact such as tender.json or a cache file.
func isTenderDocumentObject(prefix, objectName string) bool {
if objectName == "" || strings.HasSuffix(objectName, "/") {
return false
}
relativePath := strings.TrimPrefix(objectName, fmt.Sprintf("%s/", noticeID))
if relativePath == "" {
relativePath := strings.TrimPrefix(objectName, prefix)
if relativePath == "" || relativePath == objectName {
// objectName did not start with prefix; ignore.
return false
}
@@ -237,18 +302,18 @@ func (s *StorageClient) DownloadDocument(ctx context.Context, objectName string)
return object, nil
}
// GetAllSummaries lists all tender.json files in the MinIO bucket and returns
// the overall_summary for each tender where summarize_status.done is true.
// This is useful for batch-retrieving all summaries that the AI pipeline has
// already processed.
// GetAllSummaries walks the MinIO bucket for the AI team's procedure-scoped
// layout (PROC_<contractFolderID>/<noticePublicationID>/tender.json) and
// returns one TenderSummaryItem per notice. Tenders whose summarize_status.done
// is true are returned with their overall_summary; tenders still in progress
// are returned with Done=false and an empty summary.
func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryItem, error) {
s.logger.Debug("Listing all tender.json files from MinIO", map[string]interface{}{
"bucket": s.config.MinioBucket,
})
// List all objects ending with /tender.json
objectCh := s.client.ListObjects(ctx, s.config.MinioBucket, minio.ListObjectsOptions{
Prefix: "",
Prefix: procedurePrefix,
Recursive: true,
})
@@ -262,40 +327,39 @@ func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryIte
continue
}
// Only process tender.json files (path pattern: <notice_id>/tender.json)
// Expected key shape: PROC_<contractFolderID>/<noticePublicationID>/tender.json
if !strings.HasSuffix(object.Key, "/tender.json") {
continue
}
// Extract notice_id from the path
noticeID := strings.TrimSuffix(object.Key, "/tender.json")
if noticeID == "" {
contractFolderID, noticePublicationID, ok := parseTenderJSONKey(object.Key)
if !ok {
continue
}
// Fetch and parse the tender.json
summary, err := s.GetSummaryFromStorage(ctx, noticeID)
summary, err := s.GetSummaryFromStorage(ctx, contractFolderID, noticePublicationID)
if err != nil {
// Log but skip tenders that aren't ready or have errors
s.logger.Debug("Skipping tender summary", map[string]interface{}{
"notice_id": noticeID,
"error": err.Error(),
"contract_folder_id": contractFolderID,
"notice_id": noticePublicationID,
"error": err.Error(),
})
// Still include items that aren't done so the caller knows they exist
if errors.Is(err, ErrSummaryNotReady) {
results = append(results, TenderSummaryItem{
NoticeID: noticeID,
Done: false,
ContractFolderID: contractFolderID,
NoticeID: noticePublicationID,
Done: false,
})
}
continue
}
results = append(results, TenderSummaryItem{
NoticeID: noticeID,
OverallSummary: summary,
Done: true,
ContractFolderID: contractFolderID,
NoticeID: noticePublicationID,
OverallSummary: summary,
Done: true,
})
}
@@ -307,6 +371,28 @@ func (s *StorageClient) GetAllSummaries(ctx context.Context) ([]TenderSummaryIte
return results, nil
}
// parseTenderJSONKey splits a key of the form
// PROC_<contractFolderID>/<noticePublicationID>/tender.json into its components.
// The boolean is false if the key does not match the expected layout.
func parseTenderJSONKey(key string) (contractFolderID, noticePublicationID string, ok bool) {
if !strings.HasPrefix(key, procedurePrefix) {
return "", "", false
}
trimmed := strings.TrimSuffix(key, "/tender.json")
if trimmed == key {
return "", "", false
}
parts := strings.SplitN(trimmed, "/", 3)
if len(parts) != 2 {
return "", "", false
}
cfid := strings.TrimPrefix(parts[0], procedurePrefix)
if cfid == "" || parts[1] == "" {
return "", "", false
}
return cfid, parts[1], true
}
// countDone counts how many items have Done=true
func countDone(items []TenderSummaryItem) int {
count := 0