Mazyar
2b6e25f979
continuous-integration/drone/push Build is passing
- Changed the `Rank` field type from `string` to `int` in the `RecommendedTenderResponse` struct within the `company` domain for better data representation. - Updated the `Rank` field type from `string` to `int` in the `TenderResponse` struct within the `tender` domain to ensure consistency in ranking data. - Modified the `Rank` field type from `string` to `int` in the `RecommendedTender` struct within the AI summarizer package to align with the updated data structure. This update enhances the data integrity and consistency across the tender management system by standardizing the rank representation as an integer.
391 lines
14 KiB
Go
391 lines
14 KiB
Go
package ai_summarizer
|
|
|
|
import (
|
|
"strings"
|
|
)
|
|
|
|
// TenderRef identifies a tender for batch and pipeline requests.
|
|
type TenderRef struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
}
|
|
|
|
// NewTenderRef builds a TenderRef from contract folder and notice publication ids.
|
|
func NewTenderRef(contractFolderID, noticePublicationID string) TenderRef {
|
|
return TenderRef{
|
|
ContractFolderID: strings.TrimSpace(contractFolderID),
|
|
NoticePublicationID: strings.TrimSpace(noticePublicationID),
|
|
}
|
|
}
|
|
|
|
// SummarizeRequest is the payload for POST /ai/summarize.
|
|
type SummarizeRequest struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
}
|
|
|
|
// NewSummarizeRequest builds a SummarizeRequest for POST /ai/summarize.
|
|
func NewSummarizeRequest(contractFolderID, noticePublicationID string) SummarizeRequest {
|
|
return SummarizeRequest{
|
|
ContractFolderID: strings.TrimSpace(contractFolderID),
|
|
NoticePublicationID: strings.TrimSpace(noticePublicationID),
|
|
}
|
|
}
|
|
|
|
// SummarizeResponse represents the response from the AI summarization API.
|
|
type SummarizeResponse struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
NoticeID string `json:"notice_id"` // legacy field name
|
|
Summaries []DocumentSummary `json:"summaries"`
|
|
OverallSummary *string `json:"overall_summary"`
|
|
RollupS float64 `json:"rollup_s"`
|
|
TotalS float64 `json:"total_s"`
|
|
}
|
|
|
|
// ResolvedNoticePublicationID returns the notice publication ID from the response.
|
|
func (r *SummarizeResponse) ResolvedNoticePublicationID() string {
|
|
if r == nil {
|
|
return ""
|
|
}
|
|
if id := strings.TrimSpace(r.NoticePublicationID); id != "" {
|
|
return id
|
|
}
|
|
return strings.TrimSpace(r.NoticeID)
|
|
}
|
|
|
|
// DocumentSummary represents a summary for a single document within the AI response.
|
|
type DocumentSummary struct {
|
|
DocumentName string `json:"document_name"`
|
|
DocumentType string `json:"document_type"`
|
|
Summary string `json:"summary"`
|
|
SummaryModel string `json:"summary_model"`
|
|
Error string `json:"error"` // empty when no error
|
|
}
|
|
|
|
// ProcedureDocumentsSummary reports scraped documents stored under one procedure folder in MinIO.
|
|
type ProcedureDocumentsSummary struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
DocumentCount int `json:"document_count"`
|
|
LatestModified int64 `json:"latest_modified"` // Unix seconds
|
|
}
|
|
|
|
// StoredDocument represents a document object stored for a tender in MinIO.
|
|
type StoredDocument struct {
|
|
ObjectName string `json:"-"`
|
|
DocumentName string `json:"document_name"`
|
|
Size int64 `json:"size"`
|
|
LastModified int64 `json:"last_modified"`
|
|
DocumentType string `json:"document_type,omitempty"`
|
|
}
|
|
|
|
// TranslateRequest is the payload for POST /ai/translate.
|
|
type TranslateRequest struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
Language string `json:"language"`
|
|
Title string `json:"title,omitempty"`
|
|
Description string `json:"description,omitempty"`
|
|
RequestSource string `json:"-"` // internal: daily_job | manual_trigger
|
|
}
|
|
|
|
// NewTranslateRequest builds a TranslateRequest for POST /ai/translate.
|
|
func NewTranslateRequest(contractFolderID, noticePublicationID, language, title, description string) TranslateRequest {
|
|
return TranslateRequest{
|
|
ContractFolderID: strings.TrimSpace(contractFolderID),
|
|
NoticePublicationID: strings.TrimSpace(noticePublicationID),
|
|
Language: strings.ToLower(strings.TrimSpace(language)),
|
|
Title: strings.TrimSpace(title),
|
|
Description: strings.TrimSpace(description),
|
|
}
|
|
}
|
|
|
|
const (
|
|
TranslationRequestSourceDailyJob = "daily_job"
|
|
TranslationRequestSourceManualTrigger = "manual_trigger"
|
|
)
|
|
|
|
// TranslateResponse represents the translation payload from POST /ai/translate.
|
|
type TranslateResponse struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
Language string `json:"language"`
|
|
TranslatedTitle string `json:"translated_title"`
|
|
TranslatedDescription string `json:"translated_description"`
|
|
}
|
|
|
|
// AnalyzeRequest is the payload for POST /ai/analyze.
|
|
type AnalyzeRequest struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
}
|
|
|
|
// NewAnalyzeRequest builds an AnalyzeRequest for POST /ai/analyze.
|
|
func NewAnalyzeRequest(contractFolderID, noticePublicationID string) AnalyzeRequest {
|
|
return AnalyzeRequest{
|
|
ContractFolderID: strings.TrimSpace(contractFolderID),
|
|
NoticePublicationID: strings.TrimSpace(noticePublicationID),
|
|
}
|
|
}
|
|
|
|
// AnalyzeDocumentStats describes one document touched during agentic analysis.
|
|
type AnalyzeDocumentStats struct {
|
|
Filename string `json:"filename"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
TimesRead int `json:"times_read"`
|
|
TimesQueried int `json:"times_queried"`
|
|
Description string `json:"description"`
|
|
}
|
|
|
|
// AnalyzeResponse is the payload from POST /ai/analyze.
|
|
type AnalyzeResponse struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
Summary string `json:"summary"`
|
|
Documents []AnalyzeDocumentStats `json:"documents"`
|
|
Scratchpad map[string]interface{} `json:"scratchpad"`
|
|
ToolCallsLog []string `json:"tool_calls_log"`
|
|
Iterations int `json:"iterations"`
|
|
Model string `json:"model"`
|
|
}
|
|
|
|
// ScrapeDocumentsRequest is the payload for POST /scrape/documents.
|
|
type ScrapeDocumentsRequest struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
}
|
|
|
|
// ScrapeDocumentsFile describes one downloaded document.
|
|
type ScrapeDocumentsFile struct {
|
|
Filename string `json:"filename"`
|
|
Size int64 `json:"size"`
|
|
}
|
|
|
|
// ScrapeDocumentsResponse is returned by POST /scrape/documents.
|
|
type ScrapeDocumentsResponse struct {
|
|
Success bool `json:"success"`
|
|
FilesDownloaded int `json:"files_downloaded"`
|
|
Files []ScrapeDocumentsFile `json:"files"`
|
|
}
|
|
|
|
// ScrapeDocumentsBatchRequest is the payload for POST /scrape/documents/batch.
|
|
type ScrapeDocumentsBatchRequest struct {
|
|
Tenders []TenderRef `json:"tenders"`
|
|
}
|
|
|
|
// ScrapePortalsResponse is returned by GET /scrape/portals — portal identifiers.
|
|
type ScrapePortalsResponse []string
|
|
|
|
// SummarizeBatchRequest is the payload for POST /ai/summarize/batch.
|
|
type SummarizeBatchRequest struct {
|
|
Tenders []TenderRef `json:"tenders"`
|
|
}
|
|
|
|
// TranslateBatchRequest is the payload for POST /ai/translate/batch.
|
|
type TranslateBatchRequest struct {
|
|
Tenders []TenderRef `json:"tenders"`
|
|
Languages []string `json:"languages"`
|
|
}
|
|
|
|
// BatchAcceptedResponse is returned by background batch endpoints (HTTP 202).
|
|
type BatchAcceptedResponse struct {
|
|
Status string `json:"status"`
|
|
Count int `json:"count"`
|
|
}
|
|
|
|
// PipelineSyncResponse is returned by POST /pipeline/sync.
|
|
type PipelineSyncResponse struct {
|
|
Stored int `json:"stored"`
|
|
}
|
|
|
|
// PipelineRunRequest is the payload for POST /pipeline/run.
|
|
type PipelineRunRequest struct {
|
|
ContractFolderID string `json:"contract_folder_id"`
|
|
NoticePublicationID string `json:"notice_publication_id"`
|
|
Languages []string `json:"languages"`
|
|
}
|
|
|
|
// PipelineRunBatchRequest is the payload for POST /pipeline/run/batch.
|
|
type PipelineRunBatchRequest struct {
|
|
Tenders []TenderRef `json:"tenders"`
|
|
Languages []string `json:"languages"`
|
|
}
|
|
|
|
// PipelineStartedResponse is returned by POST /pipeline/daily-run and POST /pipeline/auto.
|
|
type PipelineStartedResponse struct {
|
|
Status string `json:"status"`
|
|
Report string `json:"report"`
|
|
}
|
|
|
|
// PipelineActionResponse is returned by fire-and-forget pipeline endpoints.
|
|
type PipelineActionResponse struct {
|
|
Status string `json:"status"`
|
|
}
|
|
|
|
// PipelineReportQuery holds optional query params for GET /pipeline/report and GET /pipeline/stats.
|
|
type PipelineReportQuery struct {
|
|
Window string `json:"window,omitempty"`
|
|
}
|
|
|
|
// PipelineReportResponse is returned by GET /pipeline/report.
|
|
type PipelineReportResponse struct {
|
|
Window string `json:"window"`
|
|
From string `json:"from"`
|
|
To string `json:"to"`
|
|
Summary map[string]interface{} `json:"summary,omitempty"`
|
|
Days map[string]interface{} `json:"days,omitempty"`
|
|
Status string `json:"status,omitempty"`
|
|
}
|
|
|
|
// PipelineStatsResponse is returned by GET /pipeline/stats.
|
|
type PipelineStatsResponse struct {
|
|
GeneratedAt string `json:"generated_at"`
|
|
EventLog map[string]interface{} `json:"event_log"`
|
|
Minio map[string]interface{} `json:"minio"`
|
|
}
|
|
|
|
// PipelineMinioStatsResponse is returned by GET /pipeline/minio-stats.
|
|
type PipelineMinioStatsResponse map[string]interface{}
|
|
|
|
// OnboardingRequest is the payload for POST /onboarding.
|
|
type OnboardingRequest struct {
|
|
CompanyID string `json:"company_id"`
|
|
Documents string `json:"documents"`
|
|
WebsiteURL string `json:"website_url"`
|
|
}
|
|
|
|
// OnboardingResponse is returned by POST /onboarding.
|
|
type OnboardingResponse struct {
|
|
Status string `json:"status"`
|
|
}
|
|
|
|
// RecommendRequest is the payload for POST /recommend.
|
|
type RecommendRequest struct {
|
|
CompanyID string `json:"company_id"`
|
|
}
|
|
|
|
// RecommendedTender is one ranked tender from POST /recommend.
|
|
type RecommendedTender struct {
|
|
Rank int `json:"rank"`
|
|
TenderID string `json:"tender_id"`
|
|
Analysis string `json:"analysis"`
|
|
}
|
|
|
|
// StoredTranslation is one language entry inside tender.json translations map.
|
|
type StoredTranslation struct {
|
|
Title string `json:"title"`
|
|
Description string `json:"description"`
|
|
}
|
|
|
|
// TranslationStatus tracks which languages the async pipeline has finished.
|
|
type TranslationStatus struct {
|
|
Done []string `json:"done"`
|
|
DoneUpper []string `json:"Done"`
|
|
}
|
|
|
|
func (s TranslationStatus) languagesDone() []string {
|
|
if len(s.Done) > 0 {
|
|
return s.Done
|
|
}
|
|
return s.DoneUpper
|
|
}
|
|
|
|
func languageInDone(done []string, language string) bool {
|
|
lang := strings.ToLower(strings.TrimSpace(language))
|
|
for _, item := range done {
|
|
if strings.ToLower(strings.TrimSpace(item)) == lang {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// TenderJSON represents the structure of the <notice_id>/tender.json file
|
|
// stored in the AI service's MinIO bucket.
|
|
type TenderJSON struct {
|
|
// Summary is the current pipeline field for the tender-level summary text.
|
|
Summary string `json:"summary"`
|
|
OverallSummary *string `json:"overall_summary"`
|
|
OverallSummaryCamel *string `json:"overallSummary"`
|
|
Summaries []DocumentSummary `json:"summaries"`
|
|
SummarizeStatus SummarizeStatus `json:"summarize_status"`
|
|
SummarizeStatusCamel SummarizeStatus `json:"summarizeStatus"`
|
|
Translations map[string]StoredTranslation `json:"translations"`
|
|
TranslationStatus TranslationStatus `json:"translation_status"`
|
|
TranslationStatusCamel TranslationStatus `json:"translationStatus"`
|
|
}
|
|
|
|
// summaryText returns the best-effort tender-level summary (may be empty).
|
|
func (t *TenderJSON) summaryText() string {
|
|
if t == nil {
|
|
return ""
|
|
}
|
|
for _, candidate := range []*string{t.OverallSummary, t.OverallSummaryCamel} {
|
|
if candidate == nil {
|
|
continue
|
|
}
|
|
if text := strings.TrimSpace(*candidate); text != "" {
|
|
return text
|
|
}
|
|
}
|
|
return strings.TrimSpace(t.Summary)
|
|
}
|
|
|
|
// resolved returns whether summarization is marked done and the summary text
|
|
// (may be empty when the pipeline finished without text yet).
|
|
func (t *TenderJSON) resolved() (done bool, overall string) {
|
|
if t == nil {
|
|
return false, ""
|
|
}
|
|
done = t.SummarizeStatus.isDone() || t.SummarizeStatusCamel.isDone()
|
|
return done, t.summaryText()
|
|
}
|
|
|
|
// SummarizeStatus represents the summarization status inside tender.json.
|
|
type SummarizeStatus struct {
|
|
Done bool `json:"done"` // true when the pipeline has finished
|
|
DoneUpper bool `json:"Done"` // some writers emit PascalCase
|
|
}
|
|
|
|
func (s SummarizeStatus) isDone() bool {
|
|
return s.Done || s.DoneUpper
|
|
}
|
|
|
|
// translationsDone returns languages marked complete by the translation pipeline.
|
|
func (t *TenderJSON) translationsDone() []string {
|
|
done := t.TranslationStatus.languagesDone()
|
|
if len(done) > 0 {
|
|
return done
|
|
}
|
|
return t.TranslationStatusCamel.languagesDone()
|
|
}
|
|
|
|
// storedTranslationText returns translation text from the translations map when
|
|
// title is non-empty. It does not require translation_status.done — on-demand
|
|
// writes may persist translations before updating status.
|
|
func (t *TenderJSON) storedTranslationText(language string) (StoredTranslation, bool) {
|
|
if t == nil || t.Translations == nil {
|
|
return StoredTranslation{}, false
|
|
}
|
|
lang := strings.ToLower(strings.TrimSpace(language))
|
|
for key, entry := range t.Translations {
|
|
if strings.ToLower(strings.TrimSpace(key)) != lang {
|
|
continue
|
|
}
|
|
if strings.TrimSpace(entry.Title) == "" {
|
|
continue
|
|
}
|
|
return entry, true
|
|
}
|
|
return StoredTranslation{}, false
|
|
}
|
|
|
|
// translationForLanguage returns stored translation when the pipeline marked the
|
|
// language done and non-empty title exists.
|
|
func (t *TenderJSON) translationForLanguage(language string) (StoredTranslation, bool) {
|
|
if t == nil || !languageInDone(t.translationsDone(), language) {
|
|
return StoredTranslation{}, false
|
|
}
|
|
return t.storedTranslationText(language)
|
|
}
|