68b170126d
- Updated `InitAISummarizerClient` to accept `mongoManager` for tracking translation success. - Introduced new `Statistics` endpoint in the dashboard to fetch scraping and translation statistics. - Enhanced `TranslationWorker` to utilize the new success counter for tracking successful translations. - Added necessary data structures and query forms for statistics reporting. This refactor improves the tracking of AI translation success and provides new insights through the dashboard statistics.
248 lines
8.5 KiB
Go
248 lines
8.5 KiB
Go
package ai_summarizer
|
|
|
|
import (
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// SummarizeRequest represents the request payload sent to the AI service
|
|
// POST /ai/summarize endpoint for on-demand summarization.
|
|
type SummarizeRequest struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
DocumentURL string `json:"document_url,omitempty"`
|
|
Title string `json:"title,omitempty"`
|
|
Description string `json:"description,omitempty"`
|
|
Country string `json:"country,omitempty"`
|
|
EstimatedValue *float64 `json:"estimated_value,omitempty"`
|
|
Currency string `json:"currency,omitempty"`
|
|
SubmissionDeadline string `json:"submission_deadline,omitempty"`
|
|
AuthorityName string `json:"authority_name,omitempty"`
|
|
}
|
|
|
|
// SummarizeRequestInput collects fields used to build SummarizeRequest.
|
|
type SummarizeRequestInput struct {
|
|
ContractFolderID string
|
|
NoticePublicationID string
|
|
DocumentURL string
|
|
Title string
|
|
Description string
|
|
Country string
|
|
Currency string
|
|
SubmissionDeadline int64 // Unix seconds; omitted from request when zero
|
|
EstimatedValue float64
|
|
AuthorityName string
|
|
}
|
|
|
|
// NewSummarizeRequest builds a SummarizeRequest for POST /ai/summarize.
|
|
func NewSummarizeRequest(in SummarizeRequestInput) SummarizeRequest {
|
|
req := SummarizeRequest{
|
|
ContractFolderID: strings.TrimSpace(in.ContractFolderID),
|
|
NoticePublicationID: strings.TrimSpace(in.NoticePublicationID),
|
|
DocumentURL: strings.TrimSpace(in.DocumentURL),
|
|
Title: in.Title,
|
|
Description: in.Description,
|
|
Country: in.Country,
|
|
Currency: in.Currency,
|
|
AuthorityName: strings.TrimSpace(in.AuthorityName),
|
|
}
|
|
if in.EstimatedValue > 0 {
|
|
v := in.EstimatedValue
|
|
req.EstimatedValue = &v
|
|
}
|
|
if in.SubmissionDeadline > 0 {
|
|
req.SubmissionDeadline = time.Unix(in.SubmissionDeadline, 0).UTC().Format(time.RFC3339)
|
|
}
|
|
return req
|
|
}
|
|
|
|
// SummarizeResponse represents the response from the AI summarization API.
|
|
type SummarizeResponse struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
NoticeID string `json:"notice_id"` // legacy field name
|
|
Summaries []DocumentSummary `json:"summaries"`
|
|
OverallSummary *string `json:"overall_summary"`
|
|
}
|
|
|
|
// ResolvedNoticePublicationID returns the notice publication ID from the response.
|
|
func (r *SummarizeResponse) ResolvedNoticePublicationID() string {
|
|
if r == nil {
|
|
return ""
|
|
}
|
|
if id := strings.TrimSpace(r.NoticePublicationID); id != "" {
|
|
return id
|
|
}
|
|
return strings.TrimSpace(r.NoticeID)
|
|
}
|
|
|
|
// DocumentSummary represents a summary for a single document within the AI response.
|
|
type DocumentSummary struct {
|
|
DocumentName string `json:"document_name"`
|
|
DocumentType string `json:"document_type"`
|
|
Summary string `json:"summary"`
|
|
SummaryModel string `json:"summary_model"`
|
|
Error string `json:"error"` // empty string when no error
|
|
}
|
|
|
|
// ProcedureDocumentsSummary reports scraped documents stored under one procedure folder in MinIO.
|
|
type ProcedureDocumentsSummary struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
DocumentCount int `json:"document_count"`
|
|
LatestModified int64 `json:"latest_modified"` // Unix seconds
|
|
}
|
|
|
|
// StoredDocument represents a document object stored for a tender in MinIO.
|
|
type StoredDocument struct {
|
|
ObjectName string `json:"-"`
|
|
DocumentName string `json:"document_name"`
|
|
Size int64 `json:"size"`
|
|
LastModified int64 `json:"last_modified"`
|
|
DocumentType string `json:"document_type,omitempty"`
|
|
}
|
|
|
|
// TranslateRequest represents the request payload sent to the AI service
|
|
// POST /ai/translate endpoint.
|
|
type TranslateRequest struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
Title string `json:"title"`
|
|
Description string `json:"description"`
|
|
Language string `json:"language"`
|
|
RequestSource string `json:"-"` // internal: daily_job | manual_trigger
|
|
}
|
|
|
|
const (
|
|
TranslationRequestSourceDailyJob = "daily_job"
|
|
TranslationRequestSourceManualTrigger = "manual_trigger"
|
|
)
|
|
|
|
// TranslateResponse represents the translation payload from POST /ai/translate.
|
|
type TranslateResponse struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
Language string `json:"language"`
|
|
TranslatedTitle string `json:"translated_title"`
|
|
TranslatedDescription string `json:"translated_description"`
|
|
}
|
|
|
|
// PipelineTranslateRequest represents the request payload for POST /pipeline/translate.
|
|
type PipelineTranslateRequest struct {
|
|
Languages []string `json:"languages"`
|
|
}
|
|
|
|
// PipelineTranslateResponse represents the response from POST /pipeline/translate.
|
|
type PipelineTranslateResponse struct {
|
|
Status string `json:"status"`
|
|
Languages []string `json:"languages"`
|
|
}
|
|
|
|
// StoredTranslation is one language entry inside tender.json translations map.
|
|
type StoredTranslation struct {
|
|
Title string `json:"title"`
|
|
Description string `json:"description"`
|
|
}
|
|
|
|
// TranslationStatus tracks which languages the async pipeline has finished.
|
|
type TranslationStatus struct {
|
|
Done []string `json:"done"`
|
|
DoneUpper []string `json:"Done"`
|
|
}
|
|
|
|
func (s TranslationStatus) languagesDone() []string {
|
|
if len(s.Done) > 0 {
|
|
return s.Done
|
|
}
|
|
return s.DoneUpper
|
|
}
|
|
|
|
func languageInDone(done []string, language string) bool {
|
|
lang := strings.ToLower(strings.TrimSpace(language))
|
|
for _, item := range done {
|
|
if strings.ToLower(strings.TrimSpace(item)) == lang {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// TenderJSON represents the structure of the <notice_id>/tender.json file
|
|
// stored in the AI service's MinIO bucket.
|
|
type TenderJSON struct {
|
|
// Summary is the current pipeline field for the tender-level summary text.
|
|
Summary string `json:"summary"`
|
|
OverallSummary *string `json:"overall_summary"`
|
|
OverallSummaryCamel *string `json:"overallSummary"`
|
|
SummarizeStatus SummarizeStatus `json:"summarize_status"`
|
|
SummarizeStatusCamel SummarizeStatus `json:"summarizeStatus"`
|
|
Translations map[string]StoredTranslation `json:"translations"`
|
|
TranslationStatus TranslationStatus `json:"translation_status"`
|
|
TranslationStatusCamel TranslationStatus `json:"translationStatus"`
|
|
}
|
|
|
|
// summaryText returns the best-effort tender-level summary (may be empty).
|
|
func (t *TenderJSON) summaryText() string {
|
|
if t == nil {
|
|
return ""
|
|
}
|
|
for _, candidate := range []*string{t.OverallSummary, t.OverallSummaryCamel} {
|
|
if candidate == nil {
|
|
continue
|
|
}
|
|
if text := strings.TrimSpace(*candidate); text != "" {
|
|
return text
|
|
}
|
|
}
|
|
return strings.TrimSpace(t.Summary)
|
|
}
|
|
|
|
// resolved returns whether summarization is marked done and the summary text
|
|
// (may be empty when the pipeline finished without text yet).
|
|
func (t *TenderJSON) resolved() (done bool, overall string) {
|
|
if t == nil {
|
|
return false, ""
|
|
}
|
|
done = t.SummarizeStatus.isDone() || t.SummarizeStatusCamel.isDone()
|
|
return done, t.summaryText()
|
|
}
|
|
|
|
// SummarizeStatus represents the summarization status inside tender.json.
|
|
type SummarizeStatus struct {
|
|
Done bool `json:"done"` // true when the pipeline has finished
|
|
DoneUpper bool `json:"Done"` // some writers emit PascalCase
|
|
}
|
|
|
|
func (s SummarizeStatus) isDone() bool {
|
|
return s.Done || s.DoneUpper
|
|
}
|
|
|
|
// translationsDone returns languages marked complete by the translation pipeline.
|
|
func (t *TenderJSON) translationsDone() []string {
|
|
done := t.TranslationStatus.languagesDone()
|
|
if len(done) > 0 {
|
|
return done
|
|
}
|
|
return t.TranslationStatusCamel.languagesDone()
|
|
}
|
|
|
|
// translationForLanguage returns stored translation text when the pipeline marked
|
|
// the language done and non-empty title exists.
|
|
func (t *TenderJSON) translationForLanguage(language string) (StoredTranslation, bool) {
|
|
if t == nil || !languageInDone(t.translationsDone(), language) {
|
|
return StoredTranslation{}, false
|
|
}
|
|
lang := strings.ToLower(strings.TrimSpace(language))
|
|
if t.Translations != nil {
|
|
for key, entry := range t.Translations {
|
|
if strings.ToLower(strings.TrimSpace(key)) != lang {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(entry.Title) == "" {
|
|
continue
|
|
}
|
|
return entry, true
|
|
}
|
|
}
|
|
return StoredTranslation{}, false
|
|
}
|