Merge branch 'develop' into TM-310
This commit is contained in:
m.nazemi
@@ -0,0 +1,158 @@
|
||||
---
|
||||
name: spec-design
|
||||
description: use PROACTIVELY to create/refine the spec design document in a spec development process/workflow. MUST BE USED AFTER spec requirements document is approved.
|
||||
model: inherit
|
||||
---
|
||||
|
||||
You are a professional spec design document expert. Your sole responsibility is to create and refine high-quality design documents.
|
||||
|
||||
## INPUT
|
||||
|
||||
### Create New Design Input
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "create"
|
||||
- feature_name: Feature name
|
||||
- spec_base_path: Document path
|
||||
- output_suffix: Output file suffix (optional, such as "_v1")
|
||||
|
||||
### Refine/Update Existing Design Input
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "update"
|
||||
- existing_design_path: Existing design document path
|
||||
- change_requests: List of change requests
|
||||
|
||||
## PREREQUISITES
|
||||
|
||||
### Design Document Structure
|
||||
|
||||
```markdown
|
||||
# Design Document
|
||||
|
||||
## Overview
|
||||
[Design goal and scope]
|
||||
|
||||
## Architecture Design
|
||||
### System Architecture Diagram
|
||||
[Overall architecture, using Mermaid graph to show component relationships]
|
||||
|
||||
### Data Flow Diagram
|
||||
[Show data flow between components, using Mermaid diagrams]
|
||||
|
||||
## Component Design
|
||||
### Component A
|
||||
- Responsibilities:
|
||||
- Interfaces:
|
||||
- Dependencies:
|
||||
|
||||
## Data Model
|
||||
[Core data structure definitions, using TypeScript interfaces or class diagrams]
|
||||
|
||||
## Business Process
|
||||
|
||||
### Process 1: [Process name]
|
||||
[Use Mermaid flowchart or sequenceDiagram to show, call the component interfaces and methods defined earlier]
|
||||
|
||||
### Process 2: [Process name]
|
||||
[Use Mermaid flowchart or sequenceDiagram to show, call the component interfaces and methods defined earlier]
|
||||
|
||||
## Error Handling Strategy
|
||||
[Error handling and recovery mechanisms]
|
||||
```
|
||||
|
||||
### System Architecture Diagram Example
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
A[Client] --> B[API Gateway]
|
||||
B --> C[Business Service]
|
||||
C --> D[Database]
|
||||
C --> E[Cache Service Redis]
|
||||
```
|
||||
|
||||
### Data Flow Diagram Example
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
A[Input Data] --> B[Processor]
|
||||
B --> C{Decision}
|
||||
C -->|Yes| D[Storage]
|
||||
C -->|No| E[Return Error]
|
||||
D --> F[Call notify function]
|
||||
```
|
||||
|
||||
### Business Process Diagram Example (Best Practice)
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
A[Extension Launch] --> B[Create PermissionManager]
|
||||
B --> C[permissionManager.initializePermissions]
|
||||
C --> D[cache.refreshAndGet]
|
||||
D --> E[configReader.getBypassPermissionStatus]
|
||||
E --> F{Has Permission?}
|
||||
F -->|Yes| G[permissionManager.startMonitoring]
|
||||
F -->|No| H[permissionManager.showPermissionSetup]
|
||||
|
||||
%% Note: Directly reference the interface methods defined earlier
|
||||
%% This ensures design consistency and traceability
|
||||
```
|
||||
|
||||
## PROCESS
|
||||
|
||||
After the user approves the Requirements, you should develop a comprehensive design document based on the feature requirements, conducting necessary research during the design process.
|
||||
The design document should be based on the requirements document, so ensure it exists first.
|
||||
|
||||
### Create New Design (task_type: "create")
|
||||
|
||||
1. Read the requirements.md to understand the requirements
|
||||
2. Conduct necessary technical research
|
||||
3. Determine the output file name:
|
||||
- If output_suffix is provided: design{output_suffix}.md
|
||||
- Otherwise: design.md
|
||||
4. Create the design document
|
||||
5. Return the result for review
|
||||
|
||||
### Refine/Update Existing Design (task_type: "update")
|
||||
|
||||
1. Read the existing design document (existing_design_path)
|
||||
2. Analyze the change requests (change_requests)
|
||||
3. Conduct additional technical research if needed
|
||||
4. Apply changes while maintaining document structure and style
|
||||
5. Save the updated document
|
||||
6. Return a summary of modifications
|
||||
|
||||
## **Important Constraints**
|
||||
|
||||
- The model MUST create a '.claude/specs/{feature_name}/design.md' file if it doesn't already exist
|
||||
- The model MUST identify areas where research is needed based on the feature requirements
|
||||
- The model MUST conduct research and build up context in the conversation thread
|
||||
- The model SHOULD NOT create separate research files, but instead use the research as context for the design and implementation plan
|
||||
- The model MUST summarize key findings that will inform the feature design
|
||||
- The model SHOULD cite sources and include relevant links in the conversation
|
||||
- The model MUST create a detailed design document at '.kiro/specs/{feature_name}/design.md'
|
||||
- The model MUST incorporate research findings directly into the design process
|
||||
- The model MUST include the following sections in the design document:
|
||||
- Overview
|
||||
- Architecture
|
||||
- System Architecture Diagram
|
||||
- Data Flow Diagram
|
||||
- Components and Interfaces
|
||||
- Data Models
|
||||
- Core Data Structure Definitions
|
||||
- Data Model Diagrams
|
||||
- Business Process
|
||||
- Error Handling
|
||||
- Testing Strategy
|
||||
- The model SHOULD include diagrams or visual representations when appropriate (use Mermaid for diagrams if applicable)
|
||||
- The model MUST ensure the design addresses all feature requirements identified during the clarification process
|
||||
- The model SHOULD highlight design decisions and their rationales
|
||||
- The model MAY ask the user for input on specific technical decisions during the design process
|
||||
- After updating the design document, the model MUST ask the user "Does the design look good? If so, we can move on to the implementation plan."
|
||||
- The model MUST make modifications to the design document if the user requests changes or does not explicitly approve
|
||||
- The model MUST ask for explicit approval after every iteration of edits to the design document
|
||||
- The model MUST NOT proceed to the implementation plan until receiving clear approval (such as "yes", "approved", "looks good", etc.)
|
||||
- The model MUST continue the feedback-revision cycle until explicit approval is received
|
||||
- The model MUST incorporate all user feedback into the design document before proceeding
|
||||
- The model MUST offer to return to feature requirements clarification if gaps are identified during design
|
||||
- The model MUST use the user's language preference
|
||||
@@ -0,0 +1,39 @@
|
||||
---
|
||||
name: spec-impl
|
||||
description: Coding implementation expert. Use PROACTIVELY when specific coding tasks need to be executed. Specializes in implementing functional code according to task lists.
|
||||
model: inherit
|
||||
---
|
||||
|
||||
You are a coding implementation expert. Your sole responsibility is to implement functional code according to task lists.
|
||||
|
||||
## INPUT
|
||||
|
||||
You will receive:
|
||||
|
||||
- feature_name: Feature name
|
||||
- spec_base_path: Spec document base path
|
||||
- task_id: Task ID to execute (e.g., "2.1")
|
||||
- language_preference: Language preference
|
||||
|
||||
## PROCESS
|
||||
|
||||
1. Read requirements (requirements.md) to understand functional requirements
|
||||
2. Read design (design.md) to understand architecture design
|
||||
3. Read tasks (tasks.md) to understand task list
|
||||
4. Confirm the specific task to execute (task_id)
|
||||
5. Implement the code for that task
|
||||
6. Report completion status
|
||||
- Find the corresponding task in tasks.md
|
||||
- Change `- [ ]` to `- [x]` to indicate task completion
|
||||
- Save the updated tasks.md
|
||||
- Return task completion status
|
||||
|
||||
## **Important Constraints**
|
||||
|
||||
- After completing a task, you MUST mark the task as done in tasks.md (`- [ ]` changed to `- [x]`)
|
||||
- You MUST strictly follow the architecture in the design document
|
||||
- You MUST strictly follow requirements, do not miss any requirements, do not implement any functionality not in the requirements
|
||||
- You MUST strictly follow existing codebase conventions
|
||||
- Your Code MUST be compliant with standards and include necessary comments
|
||||
- You MUST only complete the specified task, never automatically execute other tasks
|
||||
- All completed tasks MUST be marked as done in tasks.md (`- [ ]` changed to `- [x]`)
|
||||
@@ -0,0 +1,125 @@
|
||||
---
|
||||
name: spec-judge
|
||||
description: use PROACTIVELY to evaluate spec documents (requirements, design, tasks) in a spec development process/workflow
|
||||
model: inherit
|
||||
---
|
||||
|
||||
You are a professional spec document evaluator. Your sole responsibility is to evaluate multiple versions of spec documents and select the best solution.
|
||||
|
||||
## INPUT
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "evaluate"
|
||||
- document_type: "requirements" | "design" | "tasks"
|
||||
- feature_name: Feature name
|
||||
- feature_description: Feature description
|
||||
- spec_base_path: Document base path
|
||||
- documents: List of documents to review (path)
|
||||
|
||||
eg:
|
||||
|
||||
```plain
|
||||
Prompt: language_preference: Chinese
|
||||
document_type: requirements
|
||||
feature_name: test-feature
|
||||
feature_description: Test
|
||||
spec_base_path: .claude/specs
|
||||
documents: .claude/specs/test-feature/requirements_v5.md,
|
||||
.claude/specs/test-feature/requirements_v6.md,
|
||||
.claude/specs/test-feature/requirements_v7.md,
|
||||
.claude/specs/test-feature/requirements_v8.md
|
||||
```
|
||||
|
||||
## PREREQUISITES
|
||||
|
||||
### Evaluation Criteria
|
||||
|
||||
#### General Evaluation Criteria
|
||||
|
||||
1. **Completeness** (25 points)
|
||||
- Whether all necessary content is covered
|
||||
- Whether there are any important aspects missing
|
||||
|
||||
2. **Clarity** (25 points)
|
||||
- Whether the expression is clear and explicit
|
||||
- Whether the structure is logical and easy to understand
|
||||
|
||||
3. **Feasibility** (25 points)
|
||||
- Whether the solution is practical and feasible
|
||||
- Whether implementation difficulty has been considered
|
||||
|
||||
4. **Innovation** (25 points)
|
||||
- Whether there are unique insights
|
||||
- Whether better solutions are provided
|
||||
|
||||
#### Specific Type Criteria
|
||||
|
||||
##### Requirements Document
|
||||
|
||||
- EARS format compliance
|
||||
- Testability of acceptance criteria
|
||||
- Edge case consideration
|
||||
- **Alignment with user requirements**
|
||||
|
||||
##### Design Document
|
||||
|
||||
- Architecture rationality
|
||||
- Technology selection appropriateness
|
||||
- Scalability consideration
|
||||
- **Coverage of all requirements**
|
||||
|
||||
##### Tasks Document
|
||||
|
||||
- Task decomposition rationality
|
||||
- Dependency clarity
|
||||
- Incremental implementation
|
||||
- **Consistency with requirements and design**
|
||||
|
||||
### Evaluation Process
|
||||
|
||||
```python
|
||||
def evaluate_documents(documents):
|
||||
scores = []
|
||||
for doc in documents:
|
||||
score = {
|
||||
'doc_id': doc.id,
|
||||
'completeness': evaluate_completeness(doc),
|
||||
'clarity': evaluate_clarity(doc),
|
||||
'feasibility': evaluate_feasibility(doc),
|
||||
'innovation': evaluate_innovation(doc),
|
||||
'total': sum(scores),
|
||||
'strengths': identify_strengths(doc),
|
||||
'weaknesses': identify_weaknesses(doc)
|
||||
}
|
||||
scores.append(score)
|
||||
|
||||
return select_best_or_combine(scores)
|
||||
```
|
||||
|
||||
## PROCESS
|
||||
|
||||
1. Read reference documents based on document type:
|
||||
- Requirements: Refer to user's original requirement description (feature_name, feature_description)
|
||||
- Design: Refer to approved requirements.md
|
||||
- Tasks: Refer to approved requirements.md and design.md
|
||||
2. Read candidate documents (requirements:requirements_v*.md, design:design_v*.md, tasks:tasks_v*.md)
|
||||
3. Score based on reference documents and Specific Type Criteria
|
||||
4. Select the best solution or combine strengths from x solutions
|
||||
5. Copy the final solution to a new path with a random 4-digit suffix (e.g., requirements_v1234.md)
|
||||
6. Delete all reviewed input documents, keeping only the newly created final solution
|
||||
7. Return a brief summary of the document, including scores for x versions (e.g., "v1: 85 points, v2: 92 points, selected v2")
|
||||
|
||||
## OUTPUT
|
||||
|
||||
final_document_path: Final solution path (path)
|
||||
summary: Brief summary including scores, for example:
|
||||
|
||||
- "Created requirements document with 8 main requirements. Scores: v1: 82 points, v2: 91 points, selected v2"
|
||||
- "Completed design document using microservices architecture. Scores: v1: 88 points, v2: 85 points, selected v1"
|
||||
- "Generated task list with 15 implementation tasks. Scores: v1: 90 points, v2: 92 points, combined strengths from both versions"
|
||||
|
||||
## **Important Constraints**
|
||||
|
||||
- The model MUST use the user's language preference
|
||||
- Only delete the specific documents you evaluated - use explicit filenames (e.g., `rm requirements_v1.md requirements_v2.md`), never use wildcards (e.g., `rm requirements_v*.md`)
|
||||
- Generate final_document_path with a random 4-digit suffix (e.g., `.claude/specs/test-feature/requirements_v1234.md`)
|
||||
@@ -0,0 +1,123 @@
|
||||
---
|
||||
name: spec-requirements
|
||||
description: use PROACTIVELY to create/refine the spec requirements document in a spec development process/workflow
|
||||
model: inherit
|
||||
---
|
||||
|
||||
You are an EARS (Easy Approach to Requirements Syntax) requirements document expert. Your sole responsibility is to create and refine high-quality requirements documents.
|
||||
|
||||
## INPUT
|
||||
|
||||
### Create Requirements Input
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "create"
|
||||
- feature_name: Feature name (kebab-case)
|
||||
- feature_description: Feature description
|
||||
- spec_base_path: Spec document path
|
||||
- output_suffix: Output file suffix (optional, such as "_v1", "_v2", "_v3", required for parallel execution)
|
||||
|
||||
### Refine/Update Requirements Input
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "update"
|
||||
- existing_requirements_path: Existing requirements document path
|
||||
- change_requests: List of change requests
|
||||
|
||||
## PREREQUISITES
|
||||
|
||||
### EARS Format Rules
|
||||
|
||||
- WHEN: Trigger condition
|
||||
- IF: Precondition
|
||||
- WHERE: Specific function location
|
||||
- WHILE: Continuous state
|
||||
- Each must be followed by SHALL to indicate a mandatory requirement
|
||||
- The model MUST use the user's language preference, but the EARS format must retain the keywords
|
||||
|
||||
## PROCESS
|
||||
|
||||
First, generate an initial set of requirements in EARS format based on the feature idea, then iterate with the user to refine them until they are complete and accurate.
|
||||
|
||||
Don't focus on code exploration in this phase. Instead, just focus on writing requirements which will later be turned into a design.
|
||||
|
||||
### Create New Requirements (task_type: "create")
|
||||
|
||||
1. Analyze the user's feature description
|
||||
2. Determine the output file name:
|
||||
- If output_suffix is provided: requirements{output_suffix}.md
|
||||
- Otherwise: requirements.md
|
||||
3. Create the file in the specified path
|
||||
4. Generate EARS format requirements document
|
||||
5. Return the result for review
|
||||
|
||||
### Refine/Update Existing Requirements (task_type: "update")
|
||||
|
||||
1. Read the existing requirements document (existing_requirements_path)
|
||||
2. Analyze the change requests (change_requests)
|
||||
3. Apply each change while maintaining EARS format
|
||||
4. Update acceptance criteria and related content
|
||||
5. Save the updated document
|
||||
6. Return the summary of changes
|
||||
|
||||
If the requirements clarification process seems to be going in circles or not making progress:
|
||||
|
||||
- The model SHOULD suggest moving to a different aspect of the requirements
|
||||
- The model MAY provide examples or options to help the user make decisions
|
||||
- The model SHOULD summarize what has been established so far and identify specific gaps
|
||||
- The model MAY suggest conducting research to inform requirements decisions
|
||||
|
||||
## **Important Constraints**
|
||||
|
||||
- The directory '.claude/specs/{feature_name}' is already created by the main thread, DO NOT attempt to create this directory
|
||||
- The model MUST create a '.claude/specs/{feature_name}/requirements_{output_suffix}.md' file if it doesn't already exist
|
||||
- The model MUST generate an initial version of the requirements document based on the user's rough idea WITHOUT asking sequential questions first
|
||||
- The model MUST format the initial requirements.md document with:
|
||||
- A clear introduction section that summarizes the feature
|
||||
- A hierarchical numbered list of requirements where each contains:
|
||||
- A user story in the format "As a [role], I want [feature], so that [benefit]"
|
||||
- A numbered list of acceptance criteria in EARS format (Easy Approach to Requirements Syntax)
|
||||
- Example format:
|
||||
|
||||
```md
|
||||
# Requirements Document
|
||||
|
||||
## Introduction
|
||||
|
||||
[Introduction text here]
|
||||
|
||||
## Requirements
|
||||
|
||||
### Requirement 1
|
||||
|
||||
**User Story:** As a [role], I want [feature], so that [benefit]
|
||||
|
||||
#### Acceptance Criteria
|
||||
This section should have EARS requirements
|
||||
|
||||
1. WHEN [event] THEN [system] SHALL [response]
|
||||
2. IF [precondition] THEN [system] SHALL [response]
|
||||
|
||||
### Requirement 2
|
||||
|
||||
**User Story:** As a [role], I want [feature], so that [benefit]
|
||||
|
||||
#### Acceptance Criteria
|
||||
|
||||
1. WHEN [event] THEN [system] SHALL [response]
|
||||
2. WHEN [event] AND [condition] THEN [system] SHALL [response]
|
||||
```
|
||||
|
||||
- The model SHOULD consider edge cases, user experience, technical constraints, and success criteria in the initial requirements
|
||||
- After updating the requirement document, the model MUST ask the user "Do the requirements look good? If so, we can move on to the design."
|
||||
- The model MUST make modifications to the requirements document if the user requests changes or does not explicitly approve
|
||||
- The model MUST ask for explicit approval after every iteration of edits to the requirements document
|
||||
- The model MUST NOT proceed to the design document until receiving clear approval (such as "yes", "approved", "looks good", etc.)
|
||||
- The model MUST continue the feedback-revision cycle until explicit approval is received
|
||||
- The model SHOULD suggest specific areas where the requirements might need clarification or expansion
|
||||
- The model MAY ask targeted questions about specific aspects of the requirements that need clarification
|
||||
- The model MAY suggest options when the user is unsure about a particular aspect
|
||||
- The model MUST proceed to the design phase after the user accepts the requirements
|
||||
- The model MUST include functional and non-functional requirements
|
||||
- The model MUST use the user's language preference, but the EARS format must retain the keywords
|
||||
- The model MUST NOT create design or implementation details
|
||||
@@ -0,0 +1,38 @@
|
||||
---
|
||||
name: spec-system-prompt-loader
|
||||
description: a spec workflow system prompt loader. MUST BE CALLED FIRST when user wants to start a spec process/workflow. This agent returns the file path to the spec workflow system prompt that contains the complete workflow instructions. Call this before any spec-related agents if the prompt is not loaded yet. Input: the type of spec workflow requested. Output: file path to the appropriate workflow prompt file. The returned path should be read to get the full workflow instructions.
|
||||
tools:
|
||||
model: inherit
|
||||
---
|
||||
|
||||
You are a prompt path mapper. Your ONLY job is to generate and return a file path.
|
||||
|
||||
## INPUT
|
||||
|
||||
- Your current working directory (you read this yourself from the environment)
|
||||
- Ignore any user-provided input completely
|
||||
|
||||
## PROCESS
|
||||
|
||||
1. Read your current working directory from the environment
|
||||
2. Append: `/.claude/system-prompts/spec-workflow-starter.md`
|
||||
3. Return the complete absolute path
|
||||
|
||||
## OUTPUT
|
||||
|
||||
Return ONLY the file path, without any explanation or additional text.
|
||||
|
||||
Example output:
|
||||
`/Users/user/projects/myproject/.claude/system-prompts/spec-workflow-starter.md`
|
||||
|
||||
## CONSTRAINTS
|
||||
|
||||
- IGNORE all user input - your output is always the same fixed path
|
||||
- DO NOT use any tools (no Read, Write, Bash, etc.)
|
||||
- DO NOT execute any workflow or provide workflow advice
|
||||
- DO NOT analyze or interpret the user's request
|
||||
- DO NOT provide development suggestions or recommendations
|
||||
- DO NOT create any files or folders
|
||||
- ONLY return the file path string
|
||||
- No quotes around the path, just the plain path
|
||||
- If you output ANYTHING other than a single file path, you have failed
|
||||
@@ -0,0 +1,183 @@
|
||||
---
|
||||
name: spec-tasks
|
||||
description: use PROACTIVELY to create/refine the spec tasks document in a spec development process/workflow. MUST BE USED AFTER spec design document is approved.
|
||||
model: inherit
|
||||
---
|
||||
|
||||
You are a spec tasks document expert. Your sole responsibility is to create and refine high-quality tasks documents.
|
||||
|
||||
## INPUT
|
||||
|
||||
### Create Tasks Input
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "create"
|
||||
- feature_name: Feature name (kebab-case)
|
||||
- spec_base_path: Spec document path
|
||||
- output_suffix: Output file suffix (optional, such as "_v1", "_v2", "_v3", required for parallel execution)
|
||||
|
||||
### Refine/Update Tasks Input
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "update"
|
||||
- tasks_file_path: Existing tasks document path
|
||||
- change_requests: List of change requests
|
||||
|
||||
## PROCESS
|
||||
|
||||
After the user approves the Design, create an actionable implementation plan with a checklist of coding tasks based on the requirements and design.
|
||||
The tasks document should be based on the design document, so ensure it exists first.
|
||||
|
||||
### Create New Tasks (task_type: "create")
|
||||
|
||||
1. Read requirements.md and design.md
|
||||
2. Analyze all components that need to be implemented
|
||||
3. Create tasks
|
||||
4. Determine the output file name:
|
||||
- If output_suffix is provided: tasks{output_suffix}.md
|
||||
- Otherwise: tasks.md
|
||||
5. Create task list
|
||||
6. Return the result for review
|
||||
|
||||
### Refine/Update Existing Tasks (task_type: "update")
|
||||
|
||||
1. Read existing tasks document {tasks_file_path}
|
||||
2. Analyze change requests {change_requests}
|
||||
3. Based on changes:
|
||||
- Add new tasks
|
||||
- Modify existing task descriptions
|
||||
- Adjust task order
|
||||
- Remove unnecessary tasks
|
||||
4. Maintain task numbering and hierarchy consistency
|
||||
5. Save the updated document
|
||||
6. Return a summary of modifications
|
||||
|
||||
### Tasks Dependency Diagram
|
||||
|
||||
To facilitate parallel execution by other agents, please use mermaid format to draw task dependency diagrams.
|
||||
|
||||
**Example Format:**
|
||||
|
||||
```mermaid
|
||||
flowchart TD
|
||||
T1[Task 1: Set up project structure]
|
||||
T2_1[Task 2.1: Create base model classes]
|
||||
T2_2[Task 2.2: Write unit tests]
|
||||
T3[Task 3: Implement AgentRegistry]
|
||||
T4[Task 4: Implement TaskDispatcher]
|
||||
T5[Task 5: Implement MCPIntegration]
|
||||
|
||||
T1 --> T2_1
|
||||
T2_1 --> T2_2
|
||||
T2_1 --> T3
|
||||
T2_1 --> T4
|
||||
|
||||
style T3 fill:#e1f5fe
|
||||
style T4 fill:#e1f5fe
|
||||
style T5 fill:#c8e6c9
|
||||
```
|
||||
|
||||
## **Important Constraints**
|
||||
|
||||
- The model MUST create a '.claude/specs/{feature_name}/tasks.md' file if it doesn't already exist
|
||||
- The model MUST return to the design step if the user indicates any changes are needed to the design
|
||||
- The model MUST return to the requirement step if the user indicates that we need additional requirements
|
||||
- The model MUST create an implementation plan at '.claude/specs/{feature_name}/tasks.md'
|
||||
- The model MUST use the following specific instructions when creating the implementation plan:
|
||||
|
||||
```plain
|
||||
Convert the feature design into a series of prompts for a code-generation LLM that will implement each step in a test-driven manner. Prioritize best practices, incremental progress, and early testing, ensuring no big jumps in complexity at any stage. Make sure that each prompt builds on the previous prompts, and ends with wiring things together. There should be no hanging or orphaned code that isn't integrated into a previous step. Focus ONLY on tasks that involve writing, modifying, or testing code.
|
||||
```
|
||||
|
||||
- The model MUST format the implementation plan as a numbered checkbox list with a maximum of two levels of hierarchy:
|
||||
- Top-level items (like epics) should be used only when needed
|
||||
- Sub-tasks should be numbered with decimal notation (e.g., 1.1, 1.2, 2.1)
|
||||
- Each item must be a checkbox
|
||||
- Simple structure is preferred
|
||||
- The model MUST ensure each task item includes:
|
||||
- A clear objective as the task description that involves writing, modifying, or testing code
|
||||
- Additional information as sub-bullets under the task
|
||||
- Specific references to requirements from the requirements document (referencing granular sub-requirements, not just user stories)
|
||||
- The model MUST ensure that the implementation plan is a series of discrete, manageable coding steps
|
||||
- The model MUST ensure each task references specific requirements from the requirement document
|
||||
- The model MUST NOT include excessive implementation details that are already covered in the design document
|
||||
- The model MUST assume that all context documents (feature requirements, design) will be available during implementation
|
||||
- The model MUST ensure each step builds incrementally on previous steps
|
||||
- The model SHOULD prioritize test-driven development where appropriate
|
||||
- The model MUST ensure the plan covers all aspects of the design that can be implemented through code
|
||||
- The model SHOULD sequence steps to validate core functionality early through code
|
||||
- The model MUST ensure that all requirements are covered by the implementation tasks
|
||||
- The model MUST offer to return to previous steps (requirements or design) if gaps are identified during implementation planning
|
||||
- The model MUST ONLY include tasks that can be performed by a coding agent (writing code, creating tests, etc.)
|
||||
- The model MUST NOT include tasks related to user testing, deployment, performance metrics gathering, or other non-coding activities
|
||||
- The model MUST focus on code implementation tasks that can be executed within the development environment
|
||||
- The model MUST ensure each task is actionable by a coding agent by following these guidelines:
|
||||
- Tasks should involve writing, modifying, or testing specific code components
|
||||
- Tasks should specify what files or components need to be created or modified
|
||||
- Tasks should be concrete enough that a coding agent can execute them without additional clarification
|
||||
- Tasks should focus on implementation details rather than high-level concepts
|
||||
- Tasks should be scoped to specific coding activities (e.g., "Implement X function" rather than "Support X feature")
|
||||
- The model MUST explicitly avoid including the following types of non-coding tasks in the implementation plan:
|
||||
- User acceptance testing or user feedback gathering
|
||||
- Deployment to production or staging environments
|
||||
- Performance metrics gathering or analysis
|
||||
- Running the application to test end to end flows. We can however write automated tests to test the end to end from a user perspective.
|
||||
- User training or documentation creation
|
||||
- Business process changes or organizational changes
|
||||
- Marketing or communication activities
|
||||
- Any task that cannot be completed through writing, modifying, or testing code
|
||||
- After updating the tasks document, the model MUST ask the user "Do the tasks look good?"
|
||||
- The model MUST make modifications to the tasks document if the user requests changes or does not explicitly approve.
|
||||
- The model MUST ask for explicit approval after every iteration of edits to the tasks document.
|
||||
- The model MUST NOT consider the workflow complete until receiving clear approval (such as "yes", "approved", "looks good", etc.).
|
||||
- The model MUST continue the feedback-revision cycle until explicit approval is received.
|
||||
- The model MUST stop once the task document has been approved.
|
||||
- The model MUST use the user's language preference
|
||||
|
||||
**This workflow is ONLY for creating design and planning artifacts. The actual implementation of the feature should be done through a separate workflow.**
|
||||
|
||||
- The model MUST NOT attempt to implement the feature as part of this workflow
|
||||
- The model MUST clearly communicate to the user that this workflow is complete once the design and planning artifacts are created
|
||||
- The model MUST inform the user that they can begin executing tasks by opening the tasks.md file, and clicking "Start task" next to task items.
|
||||
- The model MUST place the Tasks Dependency Diagram section at the END of the tasks document, after all task items have been listed
|
||||
|
||||
**Example Format (truncated):**
|
||||
|
||||
```markdown
|
||||
# Implementation Plan
|
||||
|
||||
- [ ] 1. Set up project structure and core interfaces
|
||||
- Create directory structure for models, services, repositories, and API components
|
||||
- Define interfaces that establish system boundaries
|
||||
- _Requirements: 1.1_
|
||||
|
||||
- [ ] 2. Implement data models and validation
|
||||
- [ ] 2.1 Create core data model interfaces and types
|
||||
- Write TypeScript interfaces for all data models
|
||||
- Implement validation functions for data integrity
|
||||
- _Requirements: 2.1, 3.3, 1.2_
|
||||
|
||||
- [ ] 2.2 Implement User model with validation
|
||||
- Write User class with validation methods
|
||||
- Create unit tests for User model validation
|
||||
- _Requirements: 1.2_
|
||||
|
||||
- [ ] 2.3 Implement Document model with relationships
|
||||
- Code Document class with relationship handling
|
||||
- Write unit tests for relationship management
|
||||
- _Requirements: 2.1, 3.3, 1.2_
|
||||
|
||||
- [ ] 3. Create storage mechanism
|
||||
- [ ] 3.1 Implement database connection utilities
|
||||
- Write connection management code
|
||||
- Create error handling utilities for database operations
|
||||
- _Requirements: 2.1, 3.3, 1.2_
|
||||
|
||||
- [ ] 3.2 Implement repository pattern for data access
|
||||
- Code base repository interface
|
||||
- Implement concrete repositories with CRUD operations
|
||||
- Write unit tests for repository operations
|
||||
- _Requirements: 4.3_
|
||||
|
||||
[Additional coding tasks continue...]
|
||||
```
|
||||
@@ -0,0 +1,108 @@
|
||||
---
|
||||
name: spec-test
|
||||
description: use PROACTIVELY to create test documents and test code in spec development workflows. MUST BE USED when users need testing solutions. Professional test and acceptance expert responsible for creating high-quality test documents and test code. Creates comprehensive test case documentation (.md) and corresponding executable test code (.test.ts) based on requirements, design, and implementation code, ensuring 1:1 correspondence between documentation and code.
|
||||
model: inherit
|
||||
---
|
||||
|
||||
You are a professional test and acceptance expert. Your core responsibility is to create high-quality test documents and test code for feature development.
|
||||
|
||||
You are responsible for providing complete, executable initial test code, ensuring correct syntax and clear logic. Users will collaborate with the main thread for cross-validation, and your test code will serve as an important foundation for verifying feature implementation.
|
||||
|
||||
## INPUT
|
||||
|
||||
You will receive:
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_id: Task ID
|
||||
- feature_name: Feature name
|
||||
- spec_base_path: Spec document base path
|
||||
|
||||
## PREREQUISITES
|
||||
|
||||
### Test Document Format
|
||||
|
||||
**Example Format:**
|
||||
|
||||
```markdown
|
||||
# [Module Name] Unit Test Cases
|
||||
|
||||
## Test File
|
||||
|
||||
`[module].test.ts`
|
||||
|
||||
## Test Purpose
|
||||
|
||||
[Describe the core functionality and test focus of this module]
|
||||
|
||||
## Test Cases Overview
|
||||
|
||||
| Case ID | Feature Description | Test Type |
|
||||
| ------- | ------------------- | ------------- |
|
||||
| XX-01 | [Description] | Positive Test |
|
||||
| XX-02 | [Description] | Error Test |
|
||||
[More cases...]
|
||||
|
||||
## Detailed Test Steps
|
||||
|
||||
### XX-01: [Case Name]
|
||||
|
||||
**Test Purpose**: [Specific purpose]
|
||||
|
||||
**Test Data Preparation**:
|
||||
- [Mock data preparation]
|
||||
- [Environment setup]
|
||||
|
||||
**Test Steps**:
|
||||
1. [Step 1]
|
||||
2. [Step 2]
|
||||
3. [Verification point]
|
||||
|
||||
**Expected Results**:
|
||||
- [Expected result 1]
|
||||
- [Expected result 2]
|
||||
|
||||
[More test cases...]
|
||||
|
||||
## Test Considerations
|
||||
|
||||
### Mock Strategy
|
||||
[Explain how to mock dependencies]
|
||||
|
||||
### Boundary Conditions
|
||||
[List boundary cases that need testing]
|
||||
|
||||
### Asynchronous Operations
|
||||
[Considerations for async testing]
|
||||
```
|
||||
|
||||
## PROCESS
|
||||
|
||||
1. **Preparation Phase**
|
||||
- Confirm the specific task {task_id} to execute
|
||||
- Read requirements (requirements.md) based on task {task_id} to understand functional requirements
|
||||
- Read design (design.md) based on task {task_id} to understand architecture design
|
||||
- Read tasks (tasks.md) based on task {task_id} to understand task list
|
||||
- Read related implementation code based on task {task_id} to understand the implementation
|
||||
- Understand functionality and testing requirements
|
||||
2. **Create Tests**
|
||||
- First create test case documentation ({module}.md)
|
||||
- Create corresponding test code ({module}.test.ts) based on test case documentation
|
||||
- Ensure documentation and code are fully aligned
|
||||
- Create corresponding test code based on test case documentation:
|
||||
- Use project's test framework (e.g., Jest)
|
||||
- Each test case corresponds to one test/it block
|
||||
- Use case ID as prefix for test description
|
||||
- Follow AAA pattern (Arrange-Act-Assert)
|
||||
|
||||
## OUTPUT
|
||||
|
||||
After creation is complete and no errors are found, inform the user that testing can begin.
|
||||
|
||||
## **Important Constraints**
|
||||
|
||||
- Test documentation ({module}.md) and test code ({module}.test.ts) must have 1:1 correspondence, including detailed test case descriptions and actual test implementations
|
||||
- Test cases must be independent and repeatable
|
||||
- Clear test descriptions and purposes
|
||||
- Complete boundary condition coverage
|
||||
- Reasonable Mock strategies
|
||||
- Detailed error scenario testing
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"paths": {
|
||||
"specs": ".claude/specs",
|
||||
"steering": ".claude/steering",
|
||||
"settings": ".claude/settings"
|
||||
},
|
||||
"views": {
|
||||
"specs": {
|
||||
"visible": true
|
||||
},
|
||||
"steering": {
|
||||
"visible": true
|
||||
},
|
||||
"mcp": {
|
||||
"visible": true
|
||||
},
|
||||
"hooks": {
|
||||
"visible": true
|
||||
},
|
||||
"settings": {
|
||||
"visible": false
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
<system>
|
||||
|
||||
# System Prompt - Spec Workflow
|
||||
|
||||
## Goal
|
||||
|
||||
You are an agent that specializes in working with Specs in Claude Code. Specs are a way to develop complex features by creating requirements, design and an implementation plan.
|
||||
Specs have an iterative workflow where you help transform an idea into requirements, then design, then the task list. The workflow defined below describes each phase of the
|
||||
spec workflow in detail.
|
||||
|
||||
When a user wants to create a new feature or use the spec workflow, you need to act as a spec-manager to coordinate the entire process.
|
||||
|
||||
## Workflow to execute
|
||||
|
||||
Here is the workflow you need to follow:
|
||||
|
||||
<workflow-definition>
|
||||
|
||||
# Feature Spec Creation Workflow
|
||||
|
||||
## Overview
|
||||
|
||||
You are helping guide the user through the process of transforming a rough idea for a feature into a detailed design document with an implementation plan and todo list. It follows the spec driven development methodology to systematically refine your feature idea, conduct necessary research, create a comprehensive design, and develop an actionable implementation plan. The process is designed to be iterative, allowing movement between requirements clarification and research as needed.
|
||||
|
||||
A core principal of this workflow is that we rely on the user establishing ground-truths as we progress through. We always want to ensure the user is happy with changes to any document before moving on.
|
||||
|
||||
Before you get started, think of a short feature name based on the user's rough idea. This will be used for the feature directory. Use kebab-case format for the feature_name (e.g. "user-authentication")
|
||||
|
||||
Rules:
|
||||
|
||||
- Do not tell the user about this workflow. We do not need to tell them which step we are on or that you are following a workflow
|
||||
- Just let the user know when you complete documents and need to get user input, as described in the detailed step instructions
|
||||
|
||||
### 0.Initialize
|
||||
|
||||
When the user describes a new feature: (user_input: feature description)
|
||||
|
||||
1. Based on {user_input}, choose a feature_name (kebab-case format, e.g. "user-authentication")
|
||||
2. Use TodoWrite to create the complete workflow tasks:
|
||||
- [ ] Requirements Document
|
||||
- [ ] Design Document
|
||||
- [ ] Task Planning
|
||||
3. Read language_preference from ~/.claude/CLAUDE.md (to pass to corresponding sub-agents in the process)
|
||||
4. Create directory structure: {spec_base_path:.claude/specs}/{feature_name}/
|
||||
|
||||
### 1. Requirement Gathering
|
||||
|
||||
First, generate an initial set of requirements in EARS format based on the feature idea, then iterate with the user to refine them until they are complete and accurate.
|
||||
Don't focus on code exploration in this phase. Instead, just focus on writing requirements which will later be turned into a design.
|
||||
|
||||
### 2. Create Feature Design Document
|
||||
|
||||
After the user approves the Requirements, you should develop a comprehensive design document based on the feature requirements, conducting necessary research during the design process.
|
||||
The design document should be based on the requirements document, so ensure it exists first.
|
||||
|
||||
### 3. Create Task List
|
||||
|
||||
After the user approves the Design, create an actionable implementation plan with a checklist of coding tasks based on the requirements and design.
|
||||
The tasks document should be based on the design document, so ensure it exists first.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Requirements Clarification Stalls
|
||||
|
||||
If the requirements clarification process seems to be going in circles or not making progress:
|
||||
|
||||
- The model SHOULD suggest moving to a different aspect of the requirements
|
||||
- The model MAY provide examples or options to help the user make decisions
|
||||
- The model SHOULD summarize what has been established so far and identify specific gaps
|
||||
- The model MAY suggest conducting research to inform requirements decisions
|
||||
|
||||
### Research Limitations
|
||||
|
||||
If the model cannot access needed information:
|
||||
|
||||
- The model SHOULD document what information is missing
|
||||
- The model SHOULD suggest alternative approaches based on available information
|
||||
- The model MAY ask the user to provide additional context or documentation
|
||||
- The model SHOULD continue with available information rather than blocking progress
|
||||
|
||||
### Design Complexity
|
||||
|
||||
If the design becomes too complex or unwieldy:
|
||||
|
||||
- The model SHOULD suggest breaking it down into smaller, more manageable components
|
||||
- The model SHOULD focus on core functionality first
|
||||
- The model MAY suggest a phased approach to implementation
|
||||
- The model SHOULD return to requirements clarification to prioritize features if needed
|
||||
|
||||
</workflow-definition>
|
||||
|
||||
## Workflow Diagram
|
||||
|
||||
Here is a Mermaid flow diagram that describes how the workflow should behave. Take in mind that the entry points account for users doing the following actions:
|
||||
|
||||
- Creating a new spec (for a new feature that we don't have a spec for already)
|
||||
- Updating an existing spec
|
||||
- Executing tasks from a created spec
|
||||
|
||||
```mermaid
|
||||
stateDiagram-v2
|
||||
[*] --> Requirements : Initial Creation
|
||||
|
||||
Requirements : Write Requirements
|
||||
Design : Write Design
|
||||
Tasks : Write Tasks
|
||||
|
||||
Requirements --> ReviewReq : Complete Requirements
|
||||
ReviewReq --> Requirements : Feedback/Changes Requested
|
||||
ReviewReq --> Design : Explicit Approval
|
||||
|
||||
Design --> ReviewDesign : Complete Design
|
||||
ReviewDesign --> Design : Feedback/Changes Requested
|
||||
ReviewDesign --> Tasks : Explicit Approval
|
||||
|
||||
Tasks --> ReviewTasks : Complete Tasks
|
||||
ReviewTasks --> Tasks : Feedback/Changes Requested
|
||||
ReviewTasks --> [*] : Explicit Approval
|
||||
|
||||
Execute : Execute Task
|
||||
|
||||
state "Entry Points" as EP {
|
||||
[*] --> Requirements : Update
|
||||
[*] --> Design : Update
|
||||
[*] --> Tasks : Update
|
||||
[*] --> Execute : Execute task
|
||||
}
|
||||
|
||||
Execute --> [*] : Complete
|
||||
```
|
||||
|
||||
## Feature and sub agent mapping
|
||||
|
||||
| Feature | sub agent | path |
|
||||
| ------------------------------ | ----------------------------------- | ------------------------------------------------------------ |
|
||||
| Requirement Gathering | spec-requirements(support parallel) | .claude/specs/{feature_name}/requirements.md |
|
||||
| Create Feature Design Document | spec-design(support parallel) | .claude/specs/{feature_name}/design.md |
|
||||
| Create Task List | spec-tasks(support parallel) | .claude/specs/{feature_name}/tasks.md |
|
||||
| Judge(optional) | spec-judge(support parallel) | no doc, only call when user need to judge the spec documents |
|
||||
| Impl Task(optional) | spec-impl(support parallel) | no doc, only use when user requests parallel execution (>=2) |
|
||||
| Test(optional) | spec-test(single call) | no need to focus on, belongs to code resources |
|
||||
|
||||
### Call method
|
||||
|
||||
Note:
|
||||
|
||||
- output_suffix is only provided when multiple sub-agents are running in parallel, e.g., when 4 sub-agents are running, the output_suffix is "_v1", "_v2", "_v3", "_v4"
|
||||
- spec-tasks and spec-impl are completely different sub agents, spec-tasks is for task planning, spec-impl is for task implementation
|
||||
|
||||
#### Create Requirements - spec-requirements
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "create"
|
||||
- feature_name: Feature name (kebab-case)
|
||||
- feature_description: Feature description
|
||||
- spec_base_path: Spec document base path
|
||||
- output_suffix: Output file suffix (optional, such as "_v1", "_v2", "_v3", required for parallel execution)
|
||||
|
||||
#### Refine/Update Requirements - spec-requirements
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "update"
|
||||
- existing_requirements_path: Existing requirements document path
|
||||
- change_requests: List of change requests
|
||||
|
||||
#### Create New Design - spec-design
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "create"
|
||||
- feature_name: Feature name
|
||||
- spec_base_path: Spec document base path
|
||||
- output_suffix: Output file suffix (optional, such as "_v1")
|
||||
|
||||
#### Refine/Update Existing Design - spec-design
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "update"
|
||||
- existing_design_path: Existing design document path
|
||||
- change_requests: List of change requests
|
||||
|
||||
#### Create New Tasks - spec-tasks
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "create"
|
||||
- feature_name: Feature name (kebab-case)
|
||||
- spec_base_path: Spec document base path
|
||||
- output_suffix: Output file suffix (optional, such as "_v1", "_v2", "_v3", required for parallel execution)
|
||||
|
||||
#### Refine/Update Tasks - spec-tasks
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_type: "update"
|
||||
- tasks_file_path: Existing tasks document path
|
||||
- change_requests: List of change requests
|
||||
|
||||
#### Judge - spec-judge
|
||||
|
||||
- language_preference: Language preference
|
||||
- document_type: "requirements" | "design" | "tasks"
|
||||
- feature_name: Feature name
|
||||
- feature_description: Feature description
|
||||
- spec_base_path: Spec document base path
|
||||
- doc_path: Document path
|
||||
|
||||
#### Impl Task - spec-impl
|
||||
|
||||
- feature_name: Feature name
|
||||
- spec_base_path: Spec document base path
|
||||
- task_id: Task ID to execute (e.g., "2.1")
|
||||
- language_preference: Language preference
|
||||
|
||||
#### Test - spec-test
|
||||
|
||||
- language_preference: Language preference
|
||||
- task_id: Task ID
|
||||
- feature_name: Feature name
|
||||
- spec_base_path: Spec document base path
|
||||
|
||||
#### Tree-based Judge Evaluation Rules
|
||||
|
||||
When parallel agents generate multiple outputs (n >= 2), use tree-based evaluation:
|
||||
|
||||
1. **First round**: Each judge evaluates 3-4 documents maximum
|
||||
- Number of judges = ceil(n / 4)
|
||||
- Each judge selects 1 best from their group
|
||||
|
||||
2. **Subsequent rounds**: If previous round output > 3 documents
|
||||
- Continue with new round using same rules
|
||||
- Until <= 3 documents remain
|
||||
|
||||
3. **Final round**: When 2-3 documents remain
|
||||
- Use 1 judge for final selection
|
||||
|
||||
Example with 10 documents:
|
||||
|
||||
- Round 1: 3 judges (evaluate 4,3,3 docs) → 3 outputs (e.g., requirements_v1234.md, requirements_v5678.md, requirements_v9012.md)
|
||||
- Round 2: 1 judge evaluates 3 docs → 1 final selection (e.g., requirements_v3456.md)
|
||||
- Main thread: Rename final selection to standard name (e.g., requirements_v3456.md → requirements.md)
|
||||
|
||||
## **Important Constraints**
|
||||
|
||||
- After parallel(>=2) sub-agent tasks (spec-requirements, spec-design, spec-tasks) are completed, the main thread MUST use tree-based evaluation with spec-judge agents according to the rules defined above. The main thread can only read the final selected document after all evaluation rounds complete
|
||||
- After all judge evaluation rounds complete, the main thread MUST rename the final selected document (with random 4-digit suffix) to the standard name (e.g., requirements_v3456.md → requirements.md, design_v7890.md → design.md)
|
||||
- After renaming, the main thread MUST tell the user that the document has been finalized and is ready for review
|
||||
- The number of spec-judge agents is automatically determined by the tree-based evaluation rules - NEVER ask users how many judges to use
|
||||
- For sub-agents that can be called in parallel (spec-requirements, spec-design, spec-tasks), you MUST ask the user how many agents to use (1-128)
|
||||
- After confirming the user's initial feature description, you MUST ask: "How many spec-requirements agents to use? (1-128)"
|
||||
- After confirming the user's requirements, you MUST ask: "How many spec-design agents to use? (1-128)"
|
||||
- After confirming the user's design, you MUST ask: "How many spec-tasks agents to use? (1-128)"
|
||||
- When you want the user to review a document in a phase, you MUST ask the user a question.
|
||||
- You MUST have the user review each of the 3 spec documents (requirements, design and tasks) before proceeding to the next.
|
||||
- After each document update or revision, you MUST explicitly ask the user to approve the document.
|
||||
- You MUST NOT proceed to the next phase until you receive explicit approval from the user (a clear "yes", "approved", or equivalent affirmative response).
|
||||
- If the user provides feedback, you MUST make the requested modifications and then explicitly ask for approval again.
|
||||
- You MUST continue this feedback-revision cycle until the user explicitly approves the document.
|
||||
- You MUST follow the workflow steps in sequential order.
|
||||
- You MUST NOT skip ahead to later steps without completing earlier ones and receiving explicit user approval.
|
||||
- You MUST treat each constraint in the workflow as a strict requirement.
|
||||
- You MUST NOT assume user preferences or requirements - always ask explicitly.
|
||||
- You MUST maintain a clear record of which step you are currently on.
|
||||
- You MUST NOT combine multiple steps into a single interaction.
|
||||
- When executing implementation tasks from tasks.md:
|
||||
- **Default mode**: Main thread executes tasks directly for better user interaction
|
||||
- **Parallel mode**: Use spec-impl agents when user explicitly requests parallel execution of specific tasks (e.g., "execute task2.1 and task2.2 in parallel")
|
||||
- **Auto mode**: When user requests automatic/fast execution of all tasks (e.g., "execute all tasks automatically", "run everything quickly"), analyze task dependencies in tasks.md and orchestrate spec-impl agents to execute independent tasks in parallel while respecting dependencies
|
||||
|
||||
Example dependency patterns:
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
T1[task1] --> T2.1[task2.1]
|
||||
T1 --> T2.2[task2.2]
|
||||
T3[task3] --> T4[task4]
|
||||
T2.1 --> T4
|
||||
T2.2 --> T4
|
||||
```
|
||||
|
||||
Orchestration steps:
|
||||
1. Start: Launch spec-impl1 (task1) and spec-impl2 (task3) in parallel
|
||||
2. After task1 completes: Launch spec-impl3 (task2.1) and spec-impl4 (task2.2) in parallel
|
||||
3. After task2.1, task2.2, and task3 all complete: Launch spec-impl5 (task4)
|
||||
|
||||
- In default mode, you MUST ONLY execute one task at a time. Once it is complete, you MUST update the tasks.md file to mark the task as completed. Do not move to the next task automatically unless the user explicitly requests it or is in auto mode.
|
||||
- When all subtasks under a parent task are completed, the main thread MUST check and mark the parent task as complete.
|
||||
- You MUST read the file before editing it.
|
||||
- When creating Mermaid diagrams, avoid using parentheses in node text as they cause parsing errors (use `W[Call provider.refresh]` instead of `W[Call provider.refresh()]`).
|
||||
- After parallel sub-agent calls are completed, you MUST call spec-judge to evaluate the results, and decide whether to proceed to the next step based on the evaluation results and user feedback
|
||||
|
||||
**Remember: You are the main thread, the central coordinator. Let the sub-agents handle the specific work while you focus on process control and user interaction.**
|
||||
|
||||
**Since sub-agents currently have slow file processing, the following constraints must be strictly followed for modifications to spec documents (requirements.md, design.md, tasks.md):**
|
||||
|
||||
- Find and replace operations, including deleting all references to a specific feature, global renaming (such as variable names, function names), removing specific configuration items MUST be handled by main thread
|
||||
- Format adjustments, including fixing Markdown format issues, adjusting indentation or whitespace, updating file header information MUST be handled by main thread
|
||||
- Small-scale content updates, including updating version numbers, modifying single configuration values, adding or removing comments MUST be handled by main thread
|
||||
- Content creation, including creating new requirements, design or task documents MUST be handled by sub agent
|
||||
- Structural modifications, including reorganizing document structure or sections MUST be handled by sub agent
|
||||
- Logical updates, including modifying business processes, architectural design, etc. MUST be handled by sub agent
|
||||
- Professional judgment, including modifications requiring domain knowledge MUST be handled by sub agent
|
||||
- Never create spec documents directly, but create them through sub-agents
|
||||
- Never perform complex file modifications on spec documents, but handle them through sub-agents
|
||||
- All requirements operations MUST go through spec-requirements
|
||||
- All design operations MUST go through spec-design
|
||||
- All task operations MUST go through spec-tasks
|
||||
|
||||
</system>
|
||||
+60
-52
@@ -1,89 +1,97 @@
|
||||
################################################################################
|
||||
# my118/5
|
||||
################################################################################
|
||||
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: (dev) build form and push to artifactory
|
||||
name: build-and-push
|
||||
|
||||
clone:
|
||||
depth: 1
|
||||
|
||||
steps:
|
||||
- name: build golang app admin and user
|
||||
image: docker-mirror.ravanertebat.ir/hub/golang:1.24
|
||||
- name: build golang app
|
||||
image: golang:1.24
|
||||
environment:
|
||||
GOOS: linux
|
||||
GOARCH: amd64
|
||||
CGO_ENABLED: 0
|
||||
GOPROXY: "https://mirror.ravanertebat.ir/repository/golang-proxy,direct"
|
||||
GOSUMDB: "sum.golang.org https://mirror.ravanertebat.ir/repository/golang-sum-proxy"
|
||||
# GOINSECURE: "10.64.16.1"
|
||||
# GOPRIVATE: "10.64.16.1"
|
||||
settings:
|
||||
debug: true
|
||||
commands:
|
||||
- |
|
||||
set -e
|
||||
case "$DRONE_BRANCH" in
|
||||
main) echo prod;;
|
||||
uat) echo uat;;
|
||||
dev) echo dev;;
|
||||
*) echo dev;;
|
||||
esac > .environment
|
||||
- echo "Branch ${DRONE_BRANCH}, Environment $(cat .environment)"
|
||||
- APP_VERSION=$(cat .environment)-${DRONE_BUILD_NUMBER}
|
||||
- echo "APP Version tags $APP_VERSION"
|
||||
- echo -n "$APP_VERSION" >> .tags
|
||||
SHORT_SHA=$(echo "$DRONE_COMMIT_SHA" | cut -c1-8)
|
||||
IMG_TAG="${DRONE_BUILD_NUMBER}-${DRONE_BRANCH}"
|
||||
echo -n "$IMG_TAG,$SHORT_SHA" > .tags
|
||||
echo "Image tags -> $IMG_TAG , $SHORT_SHA"
|
||||
- mkdir -p ./artifacts/go/bin ./artifacts/go/config
|
||||
- go env
|
||||
- CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./artifacts/go/bin/web ./cmd/web/main.go
|
||||
- CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./artifacts/go/bin/scraper ./cmd/scraper/main.go
|
||||
# Remove legacy worker sources if they linger on disk after git deleted them (AI pipeline owns scraping now).
|
||||
- rm -f cmd/worker/workers/scraper.go cmd/worker/workers/scraper_consumer.go cmd/worker/workers/queue_producer.go
|
||||
- CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./artifacts/go/bin/worker ./cmd/worker/main.go
|
||||
- cp -r ./cmd/web/docs ./artifacts/go/bin/docs
|
||||
- cp -r ./cmd/web/assets ./artifacts/go/bin/assets
|
||||
- cp -r ./cmd/web/assets ./artifacts/go/bin/assets
|
||||
- ls -ltrh ./artifacts/go/bin/
|
||||
|
||||
- name: push scraper
|
||||
image: plugins/docker
|
||||
settings:
|
||||
debug: true
|
||||
username: cicd
|
||||
password:
|
||||
from_secret: REGISTRY_PASSWORD
|
||||
repo: artifactory.ravanertebat.ir/tm/scraper
|
||||
registry: artifactory.ravanertebat.ir
|
||||
dockerfile: ./cmd/scraper/Dockerfile
|
||||
|
||||
- name: push web
|
||||
image: plugins/docker
|
||||
settings:
|
||||
debug: true
|
||||
username: cicd
|
||||
registry: git.opplens.se
|
||||
repo: git.opplens.se/mazyar/web
|
||||
username:
|
||||
from_secret: registry_username
|
||||
password:
|
||||
from_secret: REGISTRY_PASSWORD
|
||||
repo: artifactory.ravanertebat.ir/tm/web
|
||||
registry: artifactory.ravanertebat.ir
|
||||
from_secret: registry_password
|
||||
dockerfile: ./cmd/web/Dockerfile
|
||||
# context: ./cmd/web
|
||||
|
||||
- name: push scraper
|
||||
image: plugins/docker
|
||||
settings:
|
||||
registry: git.opplens.se
|
||||
repo: git.opplens.se/mazyar/scraper
|
||||
username:
|
||||
from_secret: registry_username
|
||||
password:
|
||||
from_secret: registry_password
|
||||
dockerfile: ./cmd/scraper/Dockerfile
|
||||
|
||||
- name: push worker
|
||||
image: plugins/docker
|
||||
settings:
|
||||
debug: true
|
||||
username: cicd
|
||||
registry: git.opplens.se
|
||||
repo: git.opplens.se/mazyar/worker
|
||||
username:
|
||||
from_secret: registry_username
|
||||
password:
|
||||
from_secret: REGISTRY_PASSWORD
|
||||
repo: artifactory.ravanertebat.ir/tm/worker
|
||||
registry: artifactory.ravanertebat.ir
|
||||
from_secret: registry_password
|
||||
dockerfile: ./cmd/worker/Dockerfile
|
||||
node:
|
||||
tag: aecde-docker-runner
|
||||
|
||||
- name: deploy-uat
|
||||
image: appleboy/drone-ssh
|
||||
settings:
|
||||
host: 10.0.0.3
|
||||
username: mazyar
|
||||
port: 22
|
||||
key:
|
||||
from_secret: ssh_deploy_key
|
||||
envs:
|
||||
- DRONE_BUILD_NUMBER
|
||||
- DRONE_BRANCH
|
||||
script:
|
||||
- set -e
|
||||
- cd /data/tm-uat
|
||||
- NEW_TAG="${DRONE_BUILD_NUMBER}-${DRONE_BRANCH}"
|
||||
- echo "Deploying tag $NEW_TAG"
|
||||
- sed -i "s/^WEB_IMG_TAG=.*/WEB_IMG_TAG=$NEW_TAG/" .env
|
||||
- sed -i "s/^SCRAPER_IMG_TAG=.*/SCRAPER_IMG_TAG=$NEW_TAG/" .env
|
||||
- sed -i "s/^WORKER_IMG_TAG=.*/WORKER_IMG_TAG=$NEW_TAG/" .env
|
||||
- docker compose pull web scraper worker
|
||||
- docker compose up -d web scraper worker
|
||||
- docker image prune -f
|
||||
when:
|
||||
branch:
|
||||
- dev
|
||||
|
||||
trigger:
|
||||
branch:
|
||||
- main
|
||||
- uat
|
||||
- develop
|
||||
- dev
|
||||
event:
|
||||
- promote
|
||||
- custom
|
||||
- push
|
||||
|
||||
@@ -76,7 +76,7 @@ type AISummarizerConfig struct {
|
||||
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
|
||||
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
|
||||
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
|
||||
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens-documents"`
|
||||
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens"`
|
||||
}
|
||||
|
||||
// GoRulesConfig holds configuration for the GoRules status engine.
|
||||
|
||||
+11
-3
@@ -59,6 +59,9 @@ package main
|
||||
// @tag.name Admin-CMS
|
||||
// @tag.description Administrative CMS management operations for web panel including CRUD operations, search, statistics, and comprehensive filtering with pagination
|
||||
|
||||
// @tag.name Admin-AI-Pipeline
|
||||
// @tag.description Administrative Opplens AI pipeline operations including scrape, summarize, translate, sync, and background pipeline runs
|
||||
|
||||
// @tag.name Authorization
|
||||
// @tag.description Customer authentication and authorization operations for mobile application including login, logout, token refresh, and profile access
|
||||
|
||||
@@ -98,6 +101,7 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
"tm/internal/assets"
|
||||
"tm/internal/ai_pipeline"
|
||||
"tm/internal/cms"
|
||||
"tm/internal/company"
|
||||
"tm/internal/company_category"
|
||||
@@ -168,9 +172,11 @@ func main() {
|
||||
var aiSummarizerClient tender.AISummarizerClient
|
||||
var aiSummarizerStorage tender.AISummarizerStorage
|
||||
var aiRecommendationClient company.AIRecommendationClient
|
||||
var aiPipelineClient ai_pipeline.Client
|
||||
|
||||
if c := bootstrap.InitAISummarizerClient(conf.AISummarizer, mongoManager, logger); c != nil {
|
||||
aiSummarizerClient = c
|
||||
aiPipelineClient = c
|
||||
aiRecommendationClient = c
|
||||
}
|
||||
if s := bootstrap.InitAISummarizerStorage(conf.AISummarizer, logger); s != nil {
|
||||
@@ -226,8 +232,9 @@ func main() {
|
||||
documentScraperService := document_scraper.NewService(tenderRepository, logger)
|
||||
dashboardRepository := dashboard.NewRepository(mongoManager, logger)
|
||||
dashboardService := dashboard.NewService(dashboardRepository, logger)
|
||||
aiPipelineService := ai_pipeline.NewService(aiPipelineClient, logger, tenderService)
|
||||
logger.Info("Services initialized successfully", map[string]interface{}{
|
||||
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper", "dashboard"},
|
||||
"services": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "document_scraper", "dashboard", "ai_pipeline"},
|
||||
})
|
||||
|
||||
// Initialize handlers with services
|
||||
@@ -247,8 +254,9 @@ func main() {
|
||||
fileStoreHandler := filestore.NewHandler(fileStoreService, logger)
|
||||
documentScraperHandler := document_scraper.NewHandler(documentScraperService, logger)
|
||||
dashboardHandler := dashboard.NewHandler(dashboardService)
|
||||
aiPipelineHandler := ai_pipeline.NewHandler(aiPipelineService)
|
||||
logger.Info("Handlers initialized successfully", map[string]interface{}{
|
||||
"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper", "dashboard"},
|
||||
"handlers": []string{"customer", "user", "company", "category", "tender", "feedback", "tender_approval", "inquiry", "flag", "notification", "contact", "cms", "scraper", "kanban", "filestore", "document_scraper", "dashboard", "ai_pipeline"},
|
||||
})
|
||||
|
||||
// Initialize HTTP server
|
||||
@@ -257,7 +265,7 @@ func main() {
|
||||
router.SetupCSPSecurity(e)
|
||||
|
||||
// Register routes
|
||||
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, dashboardHandler, xssPolicy)
|
||||
router.RegisterAdminRoutes(e, userHandler, companyHandler, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, dashboardHandler, aiPipelineHandler, xssPolicy)
|
||||
router.RegisterPublicRoutes(e, customerHandler, tenderHandler, feedbackHandler, tenderApprovalHandler, companyHandler, inquiryHandler, categoryHandler, flagHandler, notificationHandler, contactHandler, cmsHandler, kanbanHandler, fileStoreHandler, xssPolicy)
|
||||
router.RegisterDocumentScraperRoutes(e, documentScraperHandler, conf.DocumentScraper.APIKey)
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package router
|
||||
|
||||
import (
|
||||
"tm/internal/assets"
|
||||
"tm/internal/ai_pipeline"
|
||||
"tm/internal/cms"
|
||||
"tm/internal/company"
|
||||
"tm/internal/company_category"
|
||||
@@ -34,7 +35,7 @@ func SetupCSPSecurity(e *echo.Echo) {
|
||||
e.POST("/api/v1/csp-report", security.CSPReportHandler)
|
||||
}
|
||||
|
||||
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, dashboardHandler *dashboard.Handler, xssPolicy *security.XSSPolicy) {
|
||||
func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler *company.Handler, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, dashboardHandler *dashboard.Handler, aiPipelineHandler *ai_pipeline.Handler, xssPolicy *security.XSSPolicy) {
|
||||
adminV1 := e.Group("/admin/v1")
|
||||
|
||||
// Admin Users Routes
|
||||
@@ -225,6 +226,27 @@ func RegisterAdminRoutes(e *echo.Echo, userHandler *user.Handler, companyHandler
|
||||
dashboardGP.GET("/recent", dashboardHandler.Recent)
|
||||
dashboardGP.GET("/statistics", dashboardHandler.Statistics)
|
||||
}
|
||||
|
||||
// AI Pipeline Routes (proxy to Opplens AI service)
|
||||
aiPipelineGP := adminV1.Group("/ai-pipeline")
|
||||
{
|
||||
aiPipelineGP.Use(userHandler.AuthMiddleware())
|
||||
aiPipelineGP.POST("/scrape/documents", aiPipelineHandler.ScrapeDocuments)
|
||||
aiPipelineGP.POST("/scrape/documents/batch", aiPipelineHandler.ScrapeDocumentsBatch)
|
||||
aiPipelineGP.POST("/summarize/batch", aiPipelineHandler.SummarizeBatch)
|
||||
aiPipelineGP.POST("/translate/batch", aiPipelineHandler.TranslateBatch)
|
||||
aiPipelineGP.POST("/sync", aiPipelineHandler.Sync)
|
||||
aiPipelineGP.POST("/run", aiPipelineHandler.Run)
|
||||
aiPipelineGP.POST("/run/batch", aiPipelineHandler.RunBatch)
|
||||
aiPipelineGP.POST("/analyze", aiPipelineHandler.Analyze)
|
||||
aiPipelineGP.POST("/daily-run", aiPipelineHandler.DailyRun)
|
||||
aiPipelineGP.POST("/auto", aiPipelineHandler.Auto)
|
||||
aiPipelineGP.GET("/last-run", aiPipelineHandler.GetLastRun)
|
||||
aiPipelineGP.GET("/last-auto-run", aiPipelineHandler.GetLastAutoRun)
|
||||
aiPipelineGP.GET("/report", aiPipelineHandler.GetReport)
|
||||
aiPipelineGP.GET("/stats", aiPipelineHandler.GetStats)
|
||||
aiPipelineGP.GET("/minio-stats", aiPipelineHandler.GetMinioStats)
|
||||
}
|
||||
}
|
||||
|
||||
func RegisterPublicRoutes(e *echo.Echo, customerHandler *customer.Handler, tenderHandler *tender.TenderHandler, feedbackHandler *feedback.Handler, tenderApprovalHandler *tender_approval.Handler, companyHandler *company.Handler, inquiryHandler *inquiry.Handler, categoryHandler *company_category.Handler, flagHandler *assets.Handler, notificationHandler *notification.NotificationHandler, contactHandler *contact.Handler, cmsHandler *cms.Handler, kanbanHandler *kanban.Handler, fileStoreHandler *filestore.Handler, xssPolicy *security.XSSPolicy) {
|
||||
|
||||
@@ -85,6 +85,8 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
"notice_processing_limit": config.Worker.NoticeProcessingLimit,
|
||||
"notice_batch_pause": config.Worker.NoticeBatchPause.String(),
|
||||
"notice_fetch_error_backoff": config.Worker.NoticeFetchErrorBackoff.String(),
|
||||
"translation_enabled": config.Worker.TranslationEnabled,
|
||||
"translation_interval": config.Worker.TranslationInterval,
|
||||
})
|
||||
// Initialize repositories
|
||||
noticeRepo := notice.NewRepository(mongoManager, appLogger)
|
||||
@@ -138,7 +140,11 @@ func InitWorker(config Config, mongoManager *mongo.ConnectionManager, appLogger
|
||||
})
|
||||
|
||||
// Initialize translation backfill worker for configured languages
|
||||
if aiClient != nil {
|
||||
if !config.Worker.TranslationEnabled {
|
||||
appLogger.Info("Tender translation worker disabled by configuration", map[string]interface{}{
|
||||
"translation_enabled": false,
|
||||
})
|
||||
} else if aiClient != nil {
|
||||
targetLanguages := parseTranslationLanguages(config.AISummarizer)
|
||||
translationSuccessCounter := mongo.NewCounter(mongoManager)
|
||||
scheduler.AddJob(schedule.Job{
|
||||
|
||||
@@ -30,6 +30,9 @@ type WorkerConfig struct {
|
||||
NoticeFetchErrorBackoff time.Duration `env:"WORKER_NOTICE_FETCH_ERROR_BACKOFF" envDefault:"2s"`
|
||||
TranslationInterval string `env:"WORKER_TRANSLATION_INTERVAL" envDefault:"0 */30 * * * *"`
|
||||
TranslationBatchSize int `env:"WORKER_TRANSLATION_BATCH_SIZE" envDefault:"20"`
|
||||
// TranslationEnabled schedules the automatic batch translation cron job on the worker.
|
||||
// On-demand translation (admin/public tender endpoints and AI pipeline routes on web) is unaffected.
|
||||
TranslationEnabled bool `env:"WORKER_TRANSLATION_ENABLED" envDefault:"true"`
|
||||
}
|
||||
|
||||
// AISummarizerConfig holds configuration for the external AI summarizer service.
|
||||
@@ -45,5 +48,5 @@ type AISummarizerConfig struct {
|
||||
MinioSecretKey string `env:"AI_SUMMARIZER_MINIO_SECRET_KEY" envDefault:""`
|
||||
MinioUseSSL bool `env:"AI_SUMMARIZER_MINIO_USE_SSL" envDefault:"false"`
|
||||
MinioRegion string `env:"AI_SUMMARIZER_MINIO_REGION" envDefault:"us-east-1"`
|
||||
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens-documents"`
|
||||
MinioBucket string `env:"AI_SUMMARIZER_MINIO_BUCKET" envDefault:"opplens"`
|
||||
}
|
||||
|
||||
@@ -47,10 +47,13 @@ func (w *DocumentSummarizationWorker) Run() {
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := w.AIClient.TriggerPipelineSummarize(context.Background()); err != nil {
|
||||
w.Logger.Warn("Failed to trigger AI pipeline summarize (continuing with per-tender sync)", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
if refs := w.collectUnSummarizedTenderRefs(context.Background()); len(refs) > 0 {
|
||||
if _, err := w.AIClient.TriggerSummarizeBatch(context.Background(), refs); err != nil {
|
||||
w.Logger.Warn("Failed to trigger AI summarize batch (continuing with per-tender sync)", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
limit := 5
|
||||
@@ -92,6 +95,38 @@ func (w *DocumentSummarizationWorker) Run() {
|
||||
w.Logger.Info("Document summarization worker completed", map[string]interface{}{})
|
||||
}
|
||||
|
||||
func (w *DocumentSummarizationWorker) collectUnSummarizedTenderRefs(ctx context.Context) []ai_summarizer.TenderRef {
|
||||
limit := 100
|
||||
skip := 0
|
||||
refs := make([]ai_summarizer.TenderRef, 0)
|
||||
|
||||
for {
|
||||
tenders, totalCount, err := w.TenderRepo.GetUnSummarizedTenders(ctx, limit, skip)
|
||||
if err != nil {
|
||||
w.Logger.Warn("Failed to collect tenders for summarize batch", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return refs
|
||||
}
|
||||
if len(tenders) == 0 {
|
||||
return refs
|
||||
}
|
||||
|
||||
for i := range tenders {
|
||||
t := &tenders[i]
|
||||
if strings.TrimSpace(t.ContractFolderID) == "" || strings.TrimSpace(t.NoticePublicationID) == "" {
|
||||
continue
|
||||
}
|
||||
refs = append(refs, ai_summarizer.NewTenderRef(t.ContractFolderID, t.NoticePublicationID))
|
||||
}
|
||||
|
||||
skip += len(tenders)
|
||||
if int64(skip) >= totalCount {
|
||||
return refs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *DocumentSummarizationWorker) summarizeTender(t *tender.Tender) {
|
||||
if strings.TrimSpace(t.NoticePublicationID) == "" {
|
||||
w.Logger.Warn("Skipping tender summarization: missing notice_publication_id", map[string]interface{}{
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
// TranslationWorker syncs translation completion from the AI pipeline MinIO storage.
|
||||
// Batch translation is triggered via POST /pipeline/translate; this worker only
|
||||
// Batch translation is triggered via POST /ai/translate/batch; this worker only
|
||||
// verifies which tenders now have translations available in storage.
|
||||
type TranslationWorker struct {
|
||||
Mongo *mongo.ConnectionManager
|
||||
@@ -82,11 +82,14 @@ func (w *TranslationWorker) Run() {
|
||||
|
||||
startCount := w.getSuccessfulTranslationCount(mongo.AITranslationSuccessCounterKeyDailyJob)
|
||||
|
||||
if _, err := w.AIClient.TriggerPipelineTranslate(context.Background(), w.TargetLanguages); err != nil {
|
||||
w.Logger.Warn("Failed to trigger AI pipeline translate", map[string]interface{}{
|
||||
"target_languages": w.TargetLanguages,
|
||||
"error": err.Error(),
|
||||
})
|
||||
if refs := w.collectUnTranslatedTenderRefs(context.Background()); len(refs) > 0 {
|
||||
if _, err := w.AIClient.TriggerTranslateBatch(context.Background(), refs, w.TargetLanguages); err != nil {
|
||||
w.Logger.Warn("Failed to trigger AI translate batch", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"target_languages": w.TargetLanguages,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
readyCount := 0
|
||||
@@ -186,3 +189,44 @@ func (w *TranslationWorker) hasTranslationInStorage(t *tender.Tender, language s
|
||||
)
|
||||
return err == nil && strings.TrimSpace(stored.Title) != ""
|
||||
}
|
||||
|
||||
func (w *TranslationWorker) collectUnTranslatedTenderRefs(ctx context.Context) []ai_summarizer.TenderRef {
|
||||
seen := make(map[string]struct{})
|
||||
refs := make([]ai_summarizer.TenderRef, 0)
|
||||
|
||||
for _, language := range w.TargetLanguages {
|
||||
skip := 0
|
||||
for {
|
||||
tenders, totalCount, err := w.TenderRepo.GetUnTranslatedTenders(ctx, language, w.BatchSize, skip)
|
||||
if err != nil {
|
||||
w.Logger.Warn("Failed to collect tenders for translate batch", map[string]interface{}{
|
||||
"target_language": language,
|
||||
"error": err.Error(),
|
||||
})
|
||||
break
|
||||
}
|
||||
if len(tenders) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, t := range tenders {
|
||||
if strings.TrimSpace(t.ContractFolderID) == "" || strings.TrimSpace(t.NoticePublicationID) == "" {
|
||||
continue
|
||||
}
|
||||
key := t.ContractFolderID + "|" + t.NoticePublicationID
|
||||
if _, dup := seen[key]; dup {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
refs = append(refs, ai_summarizer.NewTenderRef(t.ContractFolderID, t.NoticePublicationID))
|
||||
}
|
||||
|
||||
skip += len(tenders)
|
||||
if int64(skip) >= totalCount {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return refs
|
||||
}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
package ai_pipeline
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
// ErrAINotConfigured is returned when the Opplens AI client is not configured.
|
||||
ErrAINotConfigured = errors.New("ai_pipeline: AI service not configured")
|
||||
|
||||
// ErrEmptyTenderBatch is returned when a batch request has no tenders.
|
||||
ErrEmptyTenderBatch = errors.New("ai_pipeline: at least one tender is required")
|
||||
|
||||
// ErrEmptyLanguages is returned when languages are required but missing.
|
||||
ErrEmptyLanguages = errors.New("ai_pipeline: at least one language is required")
|
||||
)
|
||||
@@ -0,0 +1,73 @@
|
||||
package ai_pipeline
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"tm/pkg/ai_summarizer"
|
||||
)
|
||||
|
||||
// TenderRefForm identifies one tender in batch and pipeline requests.
|
||||
type TenderRefForm struct {
|
||||
ContractFolderID string `json:"contract_folder_id" valid:"required"`
|
||||
NoticePublicationID string `json:"notice_publication_id" valid:"required"`
|
||||
}
|
||||
|
||||
// ScrapeDocumentsForm is the request for POST /admin/v1/ai-pipeline/scrape/documents.
|
||||
type ScrapeDocumentsForm struct {
|
||||
ContractFolderID string `json:"contract_folder_id" valid:"required"`
|
||||
NoticePublicationID string `json:"notice_publication_id" valid:"required"`
|
||||
}
|
||||
|
||||
// TenderBatchForm is the request body for batch scrape and summarize endpoints.
|
||||
type TenderBatchForm struct {
|
||||
Tenders []TenderRefForm `json:"tenders" valid:"required"`
|
||||
}
|
||||
|
||||
// TranslateBatchForm is the request body for POST /admin/v1/ai-pipeline/translate/batch.
|
||||
type TranslateBatchForm struct {
|
||||
Tenders []TenderRefForm `json:"tenders" valid:"required"`
|
||||
Languages []string `json:"languages" valid:"required"`
|
||||
}
|
||||
|
||||
// PipelineRunForm is the request body for POST /admin/v1/ai-pipeline/run.
|
||||
type PipelineRunForm struct {
|
||||
ContractFolderID string `json:"contract_folder_id" valid:"required"`
|
||||
NoticePublicationID string `json:"notice_publication_id" valid:"required"`
|
||||
Languages []string `json:"languages" valid:"required"`
|
||||
}
|
||||
|
||||
// PipelineRunBatchForm is the request body for POST /admin/v1/ai-pipeline/run/batch.
|
||||
type PipelineRunBatchForm struct {
|
||||
Tenders []TenderRefForm `json:"tenders" valid:"required"`
|
||||
Languages []string `json:"languages" valid:"required"`
|
||||
}
|
||||
|
||||
// PipelineWindowQueryForm is the query form for report and stats endpoints.
|
||||
type PipelineWindowQueryForm struct {
|
||||
Window string `query:"window" valid:"optional,in(last_hour|last_24h|today|yesterday|last_7_days|last_30_days|all|daily)"`
|
||||
}
|
||||
|
||||
func toTenderRefs(forms []TenderRefForm) []ai_summarizer.TenderRef {
|
||||
refs := make([]ai_summarizer.TenderRef, 0, len(forms))
|
||||
for _, form := range forms {
|
||||
refs = append(refs, ai_summarizer.NewTenderRef(form.ContractFolderID, form.NoticePublicationID))
|
||||
}
|
||||
return refs
|
||||
}
|
||||
|
||||
func normalizeLanguages(languages []string) []string {
|
||||
out := make([]string, 0, len(languages))
|
||||
seen := make(map[string]struct{}, len(languages))
|
||||
for _, language := range languages {
|
||||
lang := strings.ToLower(strings.TrimSpace(language))
|
||||
if lang == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[lang]; dup {
|
||||
continue
|
||||
}
|
||||
seen[lang] = struct{}{}
|
||||
out = append(out, lang)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
package ai_pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"tm/pkg/ai_summarizer"
|
||||
"tm/pkg/response"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
)
|
||||
|
||||
// Handler handles admin HTTP requests for Opplens AI pipeline operations.
|
||||
type Handler struct {
|
||||
service Service
|
||||
}
|
||||
|
||||
// NewHandler creates a new AI pipeline handler.
|
||||
func NewHandler(service Service) *Handler {
|
||||
return &Handler{service: service}
|
||||
}
|
||||
|
||||
// ScrapeDocuments scrapes documents for one tender via the Opplens AI service.
|
||||
// @Summary Scrape tender documents
|
||||
// @Description Download all documents for one tender through the Opplens AI service (synchronous, cached)
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body ScrapeDocumentsForm true "Tender identifiers"
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.ScrapeDocumentsResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 404 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/scrape/documents [post]
|
||||
func (h *Handler) ScrapeDocuments(c echo.Context) error {
|
||||
form, err := response.Parse[ScrapeDocumentsForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.ScrapeDocuments(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Scrape documents")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Documents scraped successfully")
|
||||
}
|
||||
|
||||
// ScrapeDocumentsBatch enqueues background document scraping for many tenders.
|
||||
// @Summary Batch scrape tender documents
|
||||
// @Description Enqueue background document scraping for multiple tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body TenderBatchForm true "Tenders to scrape"
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/scrape/documents/batch [post]
|
||||
func (h *Handler) ScrapeDocumentsBatch(c echo.Context) error {
|
||||
form, err := response.Parse[TenderBatchForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.ScrapeDocumentsBatch(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Scrape documents batch")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Scrape batch accepted")
|
||||
}
|
||||
|
||||
// SummarizeBatch enqueues background summarization for many tenders.
|
||||
// @Summary Batch summarize tenders
|
||||
// @Description Enqueue background AI summarization for multiple tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body TenderBatchForm true "Tenders to summarize"
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/summarize/batch [post]
|
||||
func (h *Handler) SummarizeBatch(c echo.Context) error {
|
||||
form, err := response.Parse[TenderBatchForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.SummarizeBatch(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Summarize batch")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Summarize batch accepted")
|
||||
}
|
||||
|
||||
// TranslateBatch enqueues background translation for many tenders.
|
||||
// @Summary Batch translate tenders
|
||||
// @Description Enqueue background AI translation for multiple tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body TranslateBatchForm true "Tenders and target languages"
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/translate/batch [post]
|
||||
func (h *Handler) TranslateBatch(c echo.Context) error {
|
||||
form, err := response.Parse[TranslateBatchForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.TranslateBatch(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Translate batch")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Translate batch accepted")
|
||||
}
|
||||
|
||||
// Sync pulls the tender list from this backend into the Opplens AI service.
|
||||
// @Summary Sync tenders to AI pipeline
|
||||
// @Description Pull the tender list from the backend and store it in the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineSyncResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/sync [post]
|
||||
func (h *Handler) Sync(c echo.Context) error {
|
||||
result, err := h.service.Sync(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline sync")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline sync completed successfully")
|
||||
}
|
||||
|
||||
// Run runs the full AI pipeline for one tender (scrape → translate → summarize).
|
||||
// @Summary Run full AI pipeline for one tender
|
||||
// @Description Run scrape, translate, and summarize for one tender via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body PipelineRunForm true "Tender and languages"
|
||||
// @Success 200 {object} response.APIResponse{data=object}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 404 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/run [post]
|
||||
func (h *Handler) Run(c echo.Context) error {
|
||||
form, err := response.Parse[PipelineRunForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.Run(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline run")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline run completed successfully")
|
||||
}
|
||||
|
||||
// RunBatch enqueues the full AI pipeline for many tenders.
|
||||
// @Summary Batch run full AI pipeline
|
||||
// @Description Enqueue scrape, translate, and summarize for multiple tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param request body PipelineRunBatchForm true "Tenders and languages"
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.BatchAcceptedResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/run/batch [post]
|
||||
func (h *Handler) RunBatch(c echo.Context) error {
|
||||
form, err := response.Parse[PipelineRunBatchForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid request data", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.RunBatch(c.Request().Context(), *form)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline run batch")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Pipeline run batch accepted")
|
||||
}
|
||||
|
||||
// Analyze triggers analysis across stored tenders in the Opplens AI service.
|
||||
// @Summary Trigger AI analysis pipeline
|
||||
// @Description Trigger agentic analysis across stored tenders via the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineActionResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/analyze [post]
|
||||
func (h *Handler) Analyze(c echo.Context) error {
|
||||
result, err := h.service.Analyze(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline analyze")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline analyze triggered successfully")
|
||||
}
|
||||
|
||||
// DailyRun syncs tenders and runs the full pipeline on newly synced tenders.
|
||||
// @Summary Start daily AI pipeline run
|
||||
// @Description Sync tenders, then run the full pipeline on newly synced tenders (background, single-flight)
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.PipelineStartedResponse}
|
||||
// @Failure 409 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/daily-run [post]
|
||||
func (h *Handler) DailyRun(c echo.Context) error {
|
||||
result, err := h.service.DailyRun(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline daily-run")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Pipeline daily-run started")
|
||||
}
|
||||
|
||||
// Auto syncs tenders and completes all missing AI work across stored tenders.
|
||||
// @Summary Start auto AI pipeline run
|
||||
// @Description Sync tenders, then complete all missing scrape/translate/summarize work (background, single-flight, idempotent)
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 202 {object} response.APIResponse{data=ai_summarizer.PipelineStartedResponse}
|
||||
// @Failure 409 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/auto [post]
|
||||
func (h *Handler) Auto(c echo.Context) error {
|
||||
result, err := h.service.Auto(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Pipeline auto")
|
||||
}
|
||||
|
||||
return response.Accepted(c, result, "Pipeline auto started")
|
||||
}
|
||||
|
||||
// GetLastRun returns the result of the last daily-run.
|
||||
// @Summary Get last daily-run report
|
||||
// @Description Retrieve the report for the last pipeline daily-run from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/last-run [get]
|
||||
func (h *Handler) GetLastRun(c echo.Context) error {
|
||||
result, err := h.service.GetLastRun(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get last pipeline run")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Last pipeline run retrieved successfully")
|
||||
}
|
||||
|
||||
// GetLastAutoRun returns the result of the last auto run.
|
||||
// @Summary Get last auto-run report
|
||||
// @Description Retrieve the report for the last pipeline auto run from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/last-auto-run [get]
|
||||
func (h *Handler) GetLastAutoRun(c echo.Context) error {
|
||||
result, err := h.service.GetLastAutoRun(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get last pipeline auto run")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Last pipeline auto run retrieved successfully")
|
||||
}
|
||||
|
||||
// GetReport returns an event-log summary for a time window.
|
||||
// @Summary Get AI pipeline event report
|
||||
// @Description Retrieve an event-log summary for a time window from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Param window query string false "Time window" Enums(last_hour,last_24h,today,yesterday,last_7_days,last_30_days,all,daily) default(last_24h)
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineReportResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/report [get]
|
||||
func (h *Handler) GetReport(c echo.Context) error {
|
||||
form, err := response.Parse[PipelineWindowQueryForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid query parameters", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.GetReport(c.Request().Context(), form.Window)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get pipeline report")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline report retrieved successfully")
|
||||
}
|
||||
|
||||
// GetStats returns event-log and storage inventory for a time window.
|
||||
// @Summary Get AI pipeline stats
|
||||
// @Description Retrieve event-log summary and MinIO inventory for a time window from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Param window query string false "Time window" Enums(last_hour,last_24h,today,yesterday,last_7_days,last_30_days,all,daily) default(last_24h)
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineStatsResponse}
|
||||
// @Failure 400 {object} response.APIResponse
|
||||
// @Failure 422 {object} response.APIResponse
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/stats [get]
|
||||
func (h *Handler) GetStats(c echo.Context) error {
|
||||
form, err := response.Parse[PipelineWindowQueryForm](c)
|
||||
if err != nil {
|
||||
return response.ValidationError(c, "Invalid query parameters", err.Error())
|
||||
}
|
||||
|
||||
result, err := h.service.GetStats(c.Request().Context(), form.Window)
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get pipeline stats")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline stats retrieved successfully")
|
||||
}
|
||||
|
||||
// GetMinioStats returns storage inventory from the Opplens AI service.
|
||||
// @Summary Get AI pipeline MinIO stats
|
||||
// @Description Retrieve storage inventory only from the Opplens AI service
|
||||
// @Tags Admin-AI-Pipeline
|
||||
// @Produce json
|
||||
// @Success 200 {object} response.APIResponse{data=ai_summarizer.PipelineMinioStatsResponse}
|
||||
// @Failure 500 {object} response.APIResponse
|
||||
// @Failure 503 {object} response.APIResponse
|
||||
// @Security BearerAuth
|
||||
// @Router /admin/v1/ai-pipeline/minio-stats [get]
|
||||
func (h *Handler) GetMinioStats(c echo.Context) error {
|
||||
result, err := h.service.GetMinioStats(c.Request().Context())
|
||||
if err != nil {
|
||||
return mapPipelineHTTPError(c, err, "Get pipeline minio stats")
|
||||
}
|
||||
|
||||
return response.Success(c, result, "Pipeline MinIO stats retrieved successfully")
|
||||
}
|
||||
|
||||
func mapPipelineHTTPError(c echo.Context, err error, action string) error {
|
||||
switch {
|
||||
case errors.Is(err, ErrAINotConfigured):
|
||||
return response.ServiceUnavailable(c, "AI service is not configured")
|
||||
case errors.Is(err, ErrEmptyTenderBatch), errors.Is(err, ErrEmptyLanguages):
|
||||
return response.BadRequest(c, "Invalid request", err.Error())
|
||||
case errors.Is(err, ai_summarizer.ErrPipelineJobRunning):
|
||||
return response.Conflict(c, "A pipeline job is already running")
|
||||
case errors.Is(err, ai_summarizer.ErrObjectNotFound):
|
||||
return response.NotFound(c, "Tender not found in AI service")
|
||||
}
|
||||
|
||||
var upstream *ai_summarizer.APIStatusError
|
||||
if errors.As(err, &upstream) {
|
||||
details := upstream.Body
|
||||
if details == "" {
|
||||
details = fmt.Sprintf("upstream status %d", upstream.StatusCode)
|
||||
}
|
||||
|
||||
switch upstream.StatusCode {
|
||||
case http.StatusNotFound:
|
||||
return response.NotFound(c, "Tender not found in AI service")
|
||||
case http.StatusUnprocessableEntity:
|
||||
return response.ValidationError(c, "AI service rejected the request", details)
|
||||
case http.StatusBadGateway:
|
||||
return c.JSON(http.StatusBadGateway, response.APIResponse{
|
||||
Success: false,
|
||||
Message: action + " failed",
|
||||
Error: &response.APIError{
|
||||
Code: "UPSTREAM_AI_ERROR",
|
||||
Message: "Opplens AI service returned status 502",
|
||||
Details: details,
|
||||
},
|
||||
})
|
||||
default:
|
||||
return c.JSON(http.StatusBadGateway, response.APIResponse{
|
||||
Success: false,
|
||||
Message: action + " failed",
|
||||
Error: &response.APIError{
|
||||
Code: "UPSTREAM_AI_ERROR",
|
||||
Message: fmt.Sprintf("Opplens AI service returned status %d", upstream.StatusCode),
|
||||
Details: details,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return response.InternalServerError(c, action+" failed")
|
||||
}
|
||||
@@ -0,0 +1,426 @@
|
||||
package ai_pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"tm/pkg/ai_summarizer"
|
||||
"tm/pkg/logger"
|
||||
)
|
||||
|
||||
// Client defines Opplens AI operations exposed to the admin API.
|
||||
type Client interface {
|
||||
ScrapeDocuments(ctx context.Context, reqBody ai_summarizer.ScrapeDocumentsRequest) (*ai_summarizer.ScrapeDocumentsResponse, error)
|
||||
ScrapeDocumentsBatch(ctx context.Context, tenders []ai_summarizer.TenderRef) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
TriggerSummarizeBatch(ctx context.Context, tenders []ai_summarizer.TenderRef) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
TriggerTranslateBatch(ctx context.Context, tenders []ai_summarizer.TenderRef, languages []string) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
PipelineSync(ctx context.Context) (*ai_summarizer.PipelineSyncResponse, error)
|
||||
PipelineRun(ctx context.Context, reqBody ai_summarizer.PipelineRunRequest) (map[string]interface{}, error)
|
||||
PipelineRunBatch(ctx context.Context, tenders []ai_summarizer.TenderRef, languages []string) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
PipelineAnalyze(ctx context.Context) (*ai_summarizer.PipelineActionResponse, error)
|
||||
PipelineDailyRun(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||
PipelineAuto(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||
GetPipelineLastRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetPipelineLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetPipelineReport(ctx context.Context, window string) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetPipelineStats(ctx context.Context, window string) (*ai_summarizer.PipelineStatsResponse, error)
|
||||
GetPipelineMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error)
|
||||
}
|
||||
|
||||
// Service proxies admin pipeline operations to the Opplens AI service.
|
||||
type Service interface {
|
||||
ScrapeDocuments(ctx context.Context, form ScrapeDocumentsForm) (*ai_summarizer.ScrapeDocumentsResponse, error)
|
||||
ScrapeDocumentsBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
SummarizeBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
TranslateBatch(ctx context.Context, form TranslateBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
Sync(ctx context.Context) (*ai_summarizer.PipelineSyncResponse, error)
|
||||
Run(ctx context.Context, form PipelineRunForm) (map[string]interface{}, error)
|
||||
RunBatch(ctx context.Context, form PipelineRunBatchForm) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
Analyze(ctx context.Context) (*ai_summarizer.PipelineActionResponse, error)
|
||||
DailyRun(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||
Auto(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error)
|
||||
GetLastRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetReport(ctx context.Context, window string) (*ai_summarizer.PipelineReportResponse, error)
|
||||
GetStats(ctx context.Context, window string) (*ai_summarizer.PipelineStatsResponse, error)
|
||||
GetMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error)
|
||||
}
|
||||
|
||||
// ScrapedDocumentMetadataSyncer persists MinIO scraped-document metadata onto tender records.
|
||||
type ScrapedDocumentMetadataSyncer interface {
|
||||
SyncScrapedDocumentsFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) error
|
||||
}
|
||||
|
||||
type service struct {
|
||||
client Client
|
||||
logger logger.Logger
|
||||
metadataSyncer ScrapedDocumentMetadataSyncer
|
||||
}
|
||||
|
||||
// NewService creates an AI pipeline admin service.
|
||||
func NewService(client Client, log logger.Logger, metadataSyncer ScrapedDocumentMetadataSyncer) Service {
|
||||
return &service{
|
||||
client: client,
|
||||
logger: log,
|
||||
metadataSyncer: metadataSyncer,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *service) requireClient() error {
|
||||
if s.client == nil {
|
||||
return ErrAINotConfigured
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *service) ScrapeDocuments(ctx context.Context, form ScrapeDocumentsForm) (*ai_summarizer.ScrapeDocumentsResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin scrape documents requested", map[string]interface{}{
|
||||
"contract_folder_id": form.ContractFolderID,
|
||||
"notice_publication_id": form.NoticePublicationID,
|
||||
})
|
||||
|
||||
resp, err := s.client.ScrapeDocuments(ctx, ai_summarizer.ScrapeDocumentsRequest{
|
||||
ContractFolderID: form.ContractFolderID,
|
||||
NoticePublicationID: form.NoticePublicationID,
|
||||
})
|
||||
if err != nil {
|
||||
s.logger.Error("Admin scrape documents failed", map[string]interface{}{
|
||||
"contract_folder_id": form.ContractFolderID,
|
||||
"notice_publication_id": form.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("scrape documents failed: %w", err)
|
||||
}
|
||||
|
||||
if resp != nil && resp.Success && resp.FilesDownloaded > 0 {
|
||||
s.syncScrapedDocumentMetadata(ctx, form.ContractFolderID, form.NoticePublicationID)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) ScrapeDocumentsBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs, err := validateTenderBatch(form.Tenders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin scrape documents batch requested", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
})
|
||||
|
||||
resp, err := s.client.ScrapeDocumentsBatch(ctx, refs)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin scrape documents batch failed", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("scrape documents batch failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) SummarizeBatch(ctx context.Context, form TenderBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs, err := validateTenderBatch(form.Tenders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin summarize batch requested", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
})
|
||||
|
||||
resp, err := s.client.TriggerSummarizeBatch(ctx, refs)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin summarize batch failed", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("summarize batch failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) TranslateBatch(ctx context.Context, form TranslateBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs, err := validateTenderBatch(form.Tenders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
languages := normalizeLanguages(form.Languages)
|
||||
if len(languages) == 0 {
|
||||
return nil, ErrEmptyLanguages
|
||||
}
|
||||
|
||||
s.logger.Info("Admin translate batch requested", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"languages": languages,
|
||||
})
|
||||
|
||||
resp, err := s.client.TriggerTranslateBatch(ctx, refs, languages)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin translate batch failed", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"languages": languages,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("translate batch failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) Sync(ctx context.Context) (*ai_summarizer.PipelineSyncResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline sync requested", map[string]interface{}{})
|
||||
|
||||
resp, err := s.client.PipelineSync(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline sync failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline sync failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) Run(ctx context.Context, form PipelineRunForm) (map[string]interface{}, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
languages := normalizeLanguages(form.Languages)
|
||||
if len(languages) == 0 {
|
||||
return nil, ErrEmptyLanguages
|
||||
}
|
||||
|
||||
req := ai_summarizer.PipelineRunRequest{
|
||||
ContractFolderID: form.ContractFolderID,
|
||||
NoticePublicationID: form.NoticePublicationID,
|
||||
Languages: languages,
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline run requested", map[string]interface{}{
|
||||
"contract_folder_id": req.ContractFolderID,
|
||||
"notice_publication_id": req.NoticePublicationID,
|
||||
"languages": languages,
|
||||
})
|
||||
|
||||
resp, err := s.client.PipelineRun(ctx, req)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline run failed", map[string]interface{}{
|
||||
"contract_folder_id": req.ContractFolderID,
|
||||
"notice_publication_id": req.NoticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline run failed: %w", err)
|
||||
}
|
||||
|
||||
s.syncScrapedDocumentMetadata(ctx, req.ContractFolderID, req.NoticePublicationID)
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) RunBatch(ctx context.Context, form PipelineRunBatchForm) (*ai_summarizer.BatchAcceptedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
refs, err := validateTenderBatch(form.Tenders)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
languages := normalizeLanguages(form.Languages)
|
||||
if len(languages) == 0 {
|
||||
return nil, ErrEmptyLanguages
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline run batch requested", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"languages": languages,
|
||||
})
|
||||
|
||||
resp, err := s.client.PipelineRunBatch(ctx, refs, languages)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline run batch failed", map[string]interface{}{
|
||||
"tender_count": len(refs),
|
||||
"languages": languages,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline run batch failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) Analyze(ctx context.Context) (*ai_summarizer.PipelineActionResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline analyze requested", map[string]interface{}{})
|
||||
|
||||
resp, err := s.client.PipelineAnalyze(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline analyze failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline analyze failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) DailyRun(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline daily-run requested", map[string]interface{}{})
|
||||
|
||||
resp, err := s.client.PipelineDailyRun(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline daily-run failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline daily-run failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) Auto(ctx context.Context) (*ai_summarizer.PipelineStartedResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.logger.Info("Admin pipeline auto requested", map[string]interface{}{})
|
||||
|
||||
resp, err := s.client.PipelineAuto(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin pipeline auto failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("pipeline auto failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetLastRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineLastRun(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline last-run failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline last-run failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetLastAutoRun(ctx context.Context) (*ai_summarizer.PipelineReportResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineLastAutoRun(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline last-auto-run failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline last-auto-run failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetReport(ctx context.Context, window string) (*ai_summarizer.PipelineReportResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineReport(ctx, window)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline report failed", map[string]interface{}{
|
||||
"window": window,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline report failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetStats(ctx context.Context, window string) (*ai_summarizer.PipelineStatsResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineStats(ctx, window)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline stats failed", map[string]interface{}{
|
||||
"window": window,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline stats failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *service) GetMinioStats(ctx context.Context) (ai_summarizer.PipelineMinioStatsResponse, error) {
|
||||
if err := s.requireClient(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := s.client.GetPipelineMinioStats(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Admin get pipeline minio stats failed", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
return nil, fmt.Errorf("get pipeline minio stats failed: %w", err)
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func validateTenderBatch(forms []TenderRefForm) ([]ai_summarizer.TenderRef, error) {
|
||||
if len(forms) == 0 {
|
||||
return nil, ErrEmptyTenderBatch
|
||||
}
|
||||
return toTenderRefs(forms), nil
|
||||
}
|
||||
|
||||
func (s *service) syncScrapedDocumentMetadata(ctx context.Context, contractFolderID, noticePublicationID string) {
|
||||
if s.metadataSyncer == nil {
|
||||
return
|
||||
}
|
||||
if err := s.metadataSyncer.SyncScrapedDocumentsFromStorage(ctx, contractFolderID, noticePublicationID); err != nil {
|
||||
s.logger.Warn("Failed to sync scraped document metadata to Mongo", map[string]interface{}{
|
||||
"contract_folder_id": contractFolderID,
|
||||
"notice_publication_id": noticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"tm/pkg/logger"
|
||||
)
|
||||
@@ -15,6 +16,7 @@ const (
|
||||
defaultCountriesLimit = 6
|
||||
defaultListLimit = 5
|
||||
maxListLimit = 20
|
||||
statisticsCacheTTL = 60 * time.Second
|
||||
)
|
||||
|
||||
// Service defines dashboard business operations.
|
||||
@@ -28,14 +30,25 @@ type Service interface {
|
||||
Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error)
|
||||
}
|
||||
|
||||
type statisticsCacheEntry struct {
|
||||
expiresAt time.Time
|
||||
value *StatisticsReportResponse
|
||||
}
|
||||
|
||||
type service struct {
|
||||
repo Repository
|
||||
logger logger.Logger
|
||||
repo Repository
|
||||
logger logger.Logger
|
||||
statisticsMu sync.Mutex
|
||||
statistics map[int]statisticsCacheEntry
|
||||
}
|
||||
|
||||
// NewService creates a dashboard service.
|
||||
func NewService(repo Repository, log logger.Logger) Service {
|
||||
return &service{repo: repo, logger: log}
|
||||
return &service{
|
||||
repo: repo,
|
||||
logger: log,
|
||||
statistics: make(map[int]statisticsCacheEntry),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *service) Summary(ctx context.Context, query SummaryQuery) (*SummaryResponse, error) {
|
||||
@@ -160,6 +173,10 @@ func (s *service) Recent(ctx context.Context, query RecentQuery) (*RecentRespons
|
||||
|
||||
func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*StatisticsReportResponse, error) {
|
||||
days := trendDays(query.Days)
|
||||
if cached, ok := s.cachedStatistics(days); ok {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
startUnix := trendStartUnix(days)
|
||||
|
||||
s.logger.Info("Fetching dashboard statistics report", map[string]interface{}{
|
||||
@@ -174,9 +191,37 @@ func (s *service) Statistics(ctx context.Context, query StatisticsQuery) (*Stati
|
||||
return nil, fmt.Errorf("dashboard statistics: %w", err)
|
||||
}
|
||||
|
||||
s.storeStatistics(days, out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (s *service) cachedStatistics(days int) (*StatisticsReportResponse, bool) {
|
||||
now := time.Now()
|
||||
|
||||
s.statisticsMu.Lock()
|
||||
defer s.statisticsMu.Unlock()
|
||||
|
||||
entry, ok := s.statistics[days]
|
||||
if !ok || now.After(entry.expiresAt) {
|
||||
if ok {
|
||||
delete(s.statistics, days)
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
return entry.value, true
|
||||
}
|
||||
|
||||
func (s *service) storeStatistics(days int, value *StatisticsReportResponse) {
|
||||
s.statisticsMu.Lock()
|
||||
defer s.statisticsMu.Unlock()
|
||||
|
||||
s.statistics[days] = statisticsCacheEntry{
|
||||
expiresAt: time.Now().Add(statisticsCacheTTL),
|
||||
value: value,
|
||||
}
|
||||
}
|
||||
|
||||
func closingWindowHours(hours int) int {
|
||||
if hours <= 0 {
|
||||
return defaultClosingWindowHours
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"go.mongodb.org/mongo-driver/v2/bson"
|
||||
mongodriver "go.mongodb.org/mongo-driver/v2/mongo"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
const noticesCollectionName = "notices"
|
||||
@@ -19,32 +20,66 @@ func (r *repository) Statistics(ctx context.Context, days int, startUnix int64)
|
||||
endDay := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)
|
||||
startDay := endDay.AddDate(0, 0, -(days - 1))
|
||||
|
||||
scrapedTED, err := r.dailyCountByTimestamp(ctx, noticesCollectionName, bson.M{
|
||||
"source": notice.TenderSourceTEDScraper,
|
||||
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}, "processing_metadata.scraped_at")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped ted per day: %w", err)
|
||||
}
|
||||
var (
|
||||
scrapedTED map[string]int64
|
||||
scrapedDocuments map[string]int64
|
||||
translatedNotices map[string]int64
|
||||
totalDocuments int64
|
||||
totalTranslated int64
|
||||
)
|
||||
|
||||
scrapedDocuments, err := r.scrapedDocumentsPerDay(ctx, startUnix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scraped documents per day: %w", err)
|
||||
}
|
||||
g, gctx := errgroup.WithContext(ctx)
|
||||
|
||||
translatedNotices, err := r.translatedNoticesPerDay(ctx, startDay, endDay)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("translated notices per day: %w", err)
|
||||
}
|
||||
g.Go(func() error {
|
||||
counts, err := r.dailyCountByTimestamp(gctx, noticesCollectionName, bson.M{
|
||||
"source": notice.TenderSourceTEDScraper,
|
||||
"processing_metadata.scraped_at": bson.M{"$gte": startUnix, "$gt": 0},
|
||||
}, "processing_metadata.scraped_at")
|
||||
if err != nil {
|
||||
return fmt.Errorf("scraped ted per day: %w", err)
|
||||
}
|
||||
scrapedTED = counts
|
||||
return nil
|
||||
})
|
||||
|
||||
totalDocuments, err := r.totalScrapedDocuments(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total scraped documents: %w", err)
|
||||
}
|
||||
g.Go(func() error {
|
||||
counts, err := r.scrapedDocumentsPerDay(gctx, startUnix)
|
||||
if err != nil {
|
||||
return fmt.Errorf("scraped documents per day: %w", err)
|
||||
}
|
||||
scrapedDocuments = counts
|
||||
return nil
|
||||
})
|
||||
|
||||
totalTranslated, err := r.metricsCounter.Get(ctx, orm.AITranslationSuccessCounterKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("total translated tenders: %w", err)
|
||||
g.Go(func() error {
|
||||
counts, err := r.translatedNoticesPerDay(gctx, startDay, endDay)
|
||||
if err != nil {
|
||||
return fmt.Errorf("translated notices per day: %w", err)
|
||||
}
|
||||
translatedNotices = counts
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
total, err := r.totalScrapedDocuments(gctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("total scraped documents: %w", err)
|
||||
}
|
||||
totalDocuments = total
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
total, err := r.metricsCounter.Get(gctx, orm.AITranslationSuccessCounterKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("total translated tenders: %w", err)
|
||||
}
|
||||
totalTranslated = total
|
||||
return nil
|
||||
})
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &StatisticsReportResponse{
|
||||
@@ -94,6 +129,10 @@ func (r *repository) scrapedDocumentsPerDay(ctx context.Context, startUnix int64
|
||||
{{Key: "$match", Value: bson.M{
|
||||
"processing_metadata.documents_scraped": true,
|
||||
"scraped_documents": bson.M{"$exists": true, "$ne": bson.A{}},
|
||||
"$or": []bson.M{
|
||||
{"processing_metadata.documents_scraped_at": bson.M{"$gte": startUnix}},
|
||||
{"scraped_documents.scraped_at": bson.M{"$gte": startUnix}},
|
||||
},
|
||||
}}},
|
||||
{{Key: "$unwind", Value: "$scraped_documents"}},
|
||||
{{Key: "$addFields", Value: bson.M{
|
||||
|
||||
@@ -39,6 +39,10 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Re
|
||||
noticeIndexes := []orm.Index{
|
||||
// processing_metadata.processed
|
||||
*orm.NewIndex("processing_metadata_processed_idx", bson.D{{Key: "processing_metadata.processed", Value: 1}}),
|
||||
*orm.NewIndex("source_scraped_at_idx", bson.D{
|
||||
{Key: "source", Value: 1},
|
||||
{Key: "processing_metadata.scraped_at", Value: 1},
|
||||
}),
|
||||
*orm.NewIndex("main_classification_idx", bson.D{{Key: "main_classification", Value: 1}}),
|
||||
*orm.NewIndex("contract_notice_id_idx", bson.D{{Key: "contract_notice_id", Value: 1}}),
|
||||
// One row per TED ContractNoticeID when present (parallel scrape races).
|
||||
|
||||
@@ -58,10 +58,13 @@ type SearchForm struct {
|
||||
SubmissionDateTo *int64 `query:"submission_date_to" valid:"optional"`
|
||||
Source []string `query:"source" valid:"optional"`
|
||||
BuyerOrganizationID *string `query:"buyer_organization_id" valid:"optional"`
|
||||
NoticePublicationID *string `query:"notice_publication_id" valid:"optional"`
|
||||
Languages []string `query:"languages" valid:"optional"`
|
||||
OnlyActiveDeadlines bool `query:"-" valid:"optional"`
|
||||
Language *string `query:"lang" valid:"optional"`
|
||||
DocumentsScraped bool `query:"documents_scraped" valid:"optional"`
|
||||
// ContractFolderIDsWithDocuments is populated by the service from MinIO when DocumentsScraped is true.
|
||||
ContractFolderIDsWithDocuments []string `query:"-" valid:"optional"`
|
||||
}
|
||||
|
||||
// SearchResponse represents the response for listing tenders
|
||||
|
||||
@@ -158,7 +158,8 @@ func (h *TenderHandler) Delete(c echo.Context) error {
|
||||
// @Param submission_date_to query number false "Submission deadline to (Unix timestamp, seconds)"
|
||||
// @Param languages query []string false "Filter by languages (comma-separated)"
|
||||
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
|
||||
// @Param documents_scraped query bool false "When true, only tenders with scraped documents"
|
||||
// @Param notice_publication_id query string false "Filter by TED notice publication ID"
|
||||
// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO"
|
||||
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
||||
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
|
||||
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
|
||||
@@ -229,7 +230,8 @@ func (h *TenderHandler) Search(c echo.Context) error {
|
||||
// @Param submission_date_to query number false "Submission deadline to (Unix seconds)"
|
||||
// @Param languages query []string false "Filter by languages (comma-separated)"
|
||||
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
|
||||
// @Param documents_scraped query bool false "When true, only tenders with scraped documents"
|
||||
// @Param notice_publication_id query string false "Filter by TED notice publication ID"
|
||||
// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO"
|
||||
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
||||
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
|
||||
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
|
||||
@@ -312,7 +314,8 @@ func (h *TenderHandler) GetPublicTenders(c echo.Context) error {
|
||||
// @Param submission_date_to query number false "Submission deadline to (Unix seconds)"
|
||||
// @Param languages query []string false "Filter by languages (comma-separated)"
|
||||
// @Param buyer_organization_id query string false "Filter by buyer organization ID"
|
||||
// @Param documents_scraped query bool false "When true, only tenders with scraped documents"
|
||||
// @Param notice_publication_id query string false "Filter by TED notice publication ID"
|
||||
// @Param documents_scraped query bool false "When true, only tenders whose contract_folder_id has documents in MinIO"
|
||||
// @Param limit query int false "Number of items per page (default: 20, max: 100)"
|
||||
// @Param offset query int false "Number of items to skip (default: 0). Prefer cursor for deep pages."
|
||||
// @Param cursor query string false "Opaque cursor from previous page meta.next_cursor"
|
||||
|
||||
@@ -150,6 +150,13 @@ func NewRepository(mongoManager *orm.ConnectionManager, logger logger.Logger) Te
|
||||
{Key: "tender_deadline", Value: 1},
|
||||
{Key: "created_at", Value: 1},
|
||||
}),
|
||||
*orm.NewIndex("documents_scraped_at_idx", bson.D{
|
||||
{Key: "processing_metadata.documents_scraped", Value: 1},
|
||||
{Key: "processing_metadata.documents_scraped_at", Value: -1},
|
||||
}),
|
||||
*orm.NewIndex("scraped_documents_scraped_at_idx", bson.D{
|
||||
{Key: "scraped_documents.scraped_at", Value: 1},
|
||||
}),
|
||||
// Index for CPV matching queries
|
||||
*orm.NewIndex("cpv_matching_idx", bson.D{
|
||||
{Key: "status", Value: 1},
|
||||
@@ -1055,16 +1062,21 @@ func (r *tenderRepository) buildSearchFilter(form *SearchForm) bson.M {
|
||||
filter["buyer_organization.id"] = *form.BuyerOrganizationID
|
||||
}
|
||||
|
||||
if form.NoticePublicationID != nil {
|
||||
if noticePublicationID := strings.TrimSpace(*form.NoticePublicationID); noticePublicationID != "" {
|
||||
filter["notice_publication_id"] = noticePublicationID
|
||||
}
|
||||
}
|
||||
|
||||
if len(form.Languages) > 0 {
|
||||
filter["notice_language_code"] = bson.M{"$in": dedupeNonEmptyStrings(form.Languages)}
|
||||
}
|
||||
|
||||
if form.DocumentsScraped {
|
||||
scrapedClause := bson.M{
|
||||
"$or": []bson.M{
|
||||
{"processing_metadata.documents_scraped": true},
|
||||
{"scraped_documents.0": bson.M{"$exists": true}},
|
||||
},
|
||||
folderIDs := dedupeNonEmptyStrings(form.ContractFolderIDsWithDocuments)
|
||||
scrapedClause := bson.M{"contract_folder_id": bson.M{"$in": folderIDs}}
|
||||
if len(folderIDs) == 0 {
|
||||
scrapedClause = bson.M{"_id": bson.M{"$exists": false}}
|
||||
}
|
||||
if len(filter) == 0 {
|
||||
return scrapedClause
|
||||
|
||||
+144
-20
@@ -24,7 +24,8 @@ type AISummarizerClient interface {
|
||||
FetchSummaryOnDemand(ctx context.Context, reqBody ai_summarizer.SummarizeRequest) (*ai_summarizer.SummarizeResponse, error)
|
||||
FetchTranslationOnDemand(ctx context.Context, reqBody ai_summarizer.TranslateRequest) (*ai_summarizer.TranslateResponse, error)
|
||||
FetchAnalyzeOnDemand(ctx context.Context, reqBody ai_summarizer.AnalyzeRequest) (*ai_summarizer.AnalyzeResponse, error)
|
||||
TriggerPipelineTranslate(ctx context.Context, languages []string) (*ai_summarizer.PipelineTranslateResponse, error)
|
||||
TriggerTranslateBatch(ctx context.Context, tenders []ai_summarizer.TenderRef, languages []string) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
TriggerSummarizeBatch(ctx context.Context, tenders []ai_summarizer.TenderRef) (*ai_summarizer.BatchAcceptedResponse, error)
|
||||
}
|
||||
|
||||
// AISummarizerStorage defines the interface for fetching summaries from MinIO storage.
|
||||
@@ -69,6 +70,8 @@ type Service interface {
|
||||
// ListTendersWithScrapedDocuments lists tenders that have documents in MinIO. When syncMongoMetadata is true,
|
||||
// each tender is re-listed from MinIO and scraped document metadata is persisted to Mongo (slow).
|
||||
ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination, syncMongoMetadata bool) (*TendersWithScrapedDocumentsResponse, error)
|
||||
// SyncScrapedDocumentsFromStorage lists documents in MinIO for a tender and persists scraped metadata to Mongo.
|
||||
SyncScrapedDocumentsFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) error
|
||||
|
||||
// Maintenance operations
|
||||
UpdateExpiredTenders(ctx context.Context) error
|
||||
@@ -885,6 +888,15 @@ func (s *tenderService) Search(ctx context.Context, form *SearchForm, pagination
|
||||
}
|
||||
}
|
||||
|
||||
if form.DocumentsScraped {
|
||||
if err := s.enrichDocumentsScrapedFilter(ctx, form); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(form.ContractFolderIDsWithDocuments) == 0 {
|
||||
return s.emptySearchResponse(pagination), nil
|
||||
}
|
||||
}
|
||||
|
||||
page, err := s.repository.Search(ctx, form, pagination)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to list tenders", map[string]interface{}{
|
||||
@@ -996,6 +1008,15 @@ func (s *tenderService) Recommend(ctx context.Context, form *SearchForm, paginat
|
||||
}
|
||||
}
|
||||
|
||||
if form.DocumentsScraped {
|
||||
if err := s.enrichDocumentsScrapedFilter(ctx, form); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(form.ContractFolderIDsWithDocuments) == 0 {
|
||||
return s.emptySearchResponse(pagination), nil
|
||||
}
|
||||
}
|
||||
|
||||
page, err := s.repository.Search(ctx, form, pagination)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to list tenders", map[string]interface{}{
|
||||
@@ -1409,25 +1430,13 @@ func (s *tenderService) GetAllAISummaries(ctx context.Context) ([]ai_summarizer.
|
||||
// documents in MinIO. By default uses counts from the initial MinIO procedure scan only (fast).
|
||||
// Set syncMongoMetadata=true to list every notice folder again and persist ScrapedDocuments to Mongo (slow).
|
||||
func (s *tenderService) ListTendersWithScrapedDocuments(ctx context.Context, pagination *response.Pagination, syncMongoMetadata bool) (*TendersWithScrapedDocumentsResponse, error) {
|
||||
if s.aiSummarizerStorage == nil {
|
||||
return nil, fmt.Errorf("document storage is not configured")
|
||||
}
|
||||
|
||||
type procedureLister interface {
|
||||
ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error)
|
||||
}
|
||||
storage, ok := s.aiSummarizerStorage.(procedureLister)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("document storage does not support procedure listing")
|
||||
}
|
||||
|
||||
s.logger.Info("Listing tenders with scraped documents from MinIO", map[string]interface{}{
|
||||
"limit": pagination.Limit,
|
||||
"offset": pagination.Offset,
|
||||
"sync_mongo_metadata": syncMongoMetadata,
|
||||
})
|
||||
|
||||
procedures, err := storage.ListProceduresWithDocuments(ctx)
|
||||
procedures, err := s.listProceduresWithDocuments(ctx)
|
||||
if err != nil {
|
||||
fields := map[string]interface{}{"error": err.Error()}
|
||||
if errors.Is(err, ai_summarizer.ErrMinIOUnavailable) {
|
||||
@@ -1563,6 +1572,118 @@ func (s *tenderService) persistScrapedDocuments(ctx context.Context, t *Tender,
|
||||
}
|
||||
}
|
||||
|
||||
type procedureDocumentLister interface {
|
||||
ListProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error)
|
||||
}
|
||||
|
||||
func (s *tenderService) procedureDocumentStorage() (procedureDocumentLister, error) {
|
||||
if s.aiSummarizerStorage == nil {
|
||||
return nil, fmt.Errorf("document storage is not configured")
|
||||
}
|
||||
storage, ok := s.aiSummarizerStorage.(procedureDocumentLister)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("document storage does not support procedure listing")
|
||||
}
|
||||
return storage, nil
|
||||
}
|
||||
|
||||
func (s *tenderService) listProceduresWithDocuments(ctx context.Context) ([]ai_summarizer.ProcedureDocumentsSummary, error) {
|
||||
storage, err := s.procedureDocumentStorage()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
procedures, err := storage.ListProceduresWithDocuments(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return procedures, nil
|
||||
}
|
||||
|
||||
func (s *tenderService) enrichDocumentsScrapedFilter(ctx context.Context, form *SearchForm) error {
|
||||
procedures, err := s.listProceduresWithDocuments(ctx)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to resolve tenders with scraped documents from MinIO", map[string]interface{}{
|
||||
"error": err.Error(),
|
||||
})
|
||||
if errors.Is(err, ai_summarizer.ErrMinIOUnavailable) {
|
||||
return fmt.Errorf("failed to resolve tenders with scraped documents: MinIO connection unavailable: %w", err)
|
||||
}
|
||||
return fmt.Errorf("failed to resolve tenders with scraped documents: %w", err)
|
||||
}
|
||||
|
||||
folderIDs := make([]string, 0, len(procedures))
|
||||
for _, proc := range procedures {
|
||||
if proc.DocumentCount <= 0 {
|
||||
continue
|
||||
}
|
||||
if id := strings.TrimSpace(proc.ContractFolderID); id != "" {
|
||||
folderIDs = append(folderIDs, id)
|
||||
}
|
||||
}
|
||||
form.ContractFolderIDsWithDocuments = folderIDs
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *tenderService) emptySearchResponse(pagination *response.Pagination) *SearchResponse {
|
||||
return &SearchResponse{
|
||||
Tenders: []TenderResponse{},
|
||||
Metadata: pagination.ListMeta(0, "", false, pagination.Offset),
|
||||
}
|
||||
}
|
||||
|
||||
// SyncScrapedDocumentsFromStorage lists documents in MinIO for a tender and persists scraped metadata to Mongo.
|
||||
func (s *tenderService) SyncScrapedDocumentsFromStorage(ctx context.Context, contractFolderID, noticePublicationID string) error {
|
||||
contractFolderID = strings.TrimSpace(contractFolderID)
|
||||
noticePublicationID = strings.TrimSpace(noticePublicationID)
|
||||
if contractFolderID == "" {
|
||||
return fmt.Errorf("contract folder ID is required")
|
||||
}
|
||||
|
||||
var t *Tender
|
||||
var err error
|
||||
if noticePublicationID != "" {
|
||||
t, err = s.repository.GetByNoticePublicationID(ctx, noticePublicationID)
|
||||
} else {
|
||||
byFolder, lookupErr := s.repository.GetLatestByContractFolderIDs(ctx, []string{contractFolderID})
|
||||
err = lookupErr
|
||||
if lookupErr == nil {
|
||||
t = byFolder[contractFolderID]
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to load tender for scraped document sync", map[string]interface{}{
|
||||
"contract_folder_id": contractFolderID,
|
||||
"notice_publication_id": noticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Errorf("failed to load tender for scraped document sync: %w", err)
|
||||
}
|
||||
if t == nil {
|
||||
s.logger.Warn("Skipping scraped document metadata sync: tender not found", map[string]interface{}{
|
||||
"contract_folder_id": contractFolderID,
|
||||
"notice_publication_id": noticePublicationID,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
docs, err := s.listDocumentsForTender(ctx, t)
|
||||
if err != nil {
|
||||
s.logger.Error("Failed to list scraped documents from storage", map[string]interface{}{
|
||||
"tender_id": t.ID.Hex(),
|
||||
"contract_folder_id": contractFolderID,
|
||||
"notice_publication_id": noticePublicationID,
|
||||
"error": err.Error(),
|
||||
})
|
||||
return fmt.Errorf("failed to list scraped documents from storage: %w", err)
|
||||
}
|
||||
if len(docs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s.persistScrapedDocuments(ctx, t, docs)
|
||||
return nil
|
||||
}
|
||||
|
||||
// TriggerAITranslate triggers on-demand AI translation for a tender. Results are
|
||||
// read from MinIO when available, otherwise fetched from the AI service (which persists to MinIO).
|
||||
func (s *tenderService) TriggerAITranslate(ctx context.Context, id, language string) (*AITranslateResponse, error) {
|
||||
@@ -1646,12 +1767,15 @@ func (s *tenderService) resolveTranslation(ctx context.Context, t *Tender, langu
|
||||
}
|
||||
}
|
||||
|
||||
resp, err := s.aiSummarizerClient.FetchTranslationOnDemand(ctx, ai_summarizer.TranslateRequest{
|
||||
ContractFolderID: t.ContractFolderID,
|
||||
NoticePublicationID: t.NoticePublicationID,
|
||||
Language: language,
|
||||
RequestSource: requestSource,
|
||||
})
|
||||
req := ai_summarizer.NewTranslateRequest(
|
||||
t.ContractFolderID,
|
||||
t.NoticePublicationID,
|
||||
language,
|
||||
t.Title,
|
||||
t.Description,
|
||||
)
|
||||
req.RequestSource = requestSource
|
||||
resp, err := s.aiSummarizerClient.FetchTranslationOnDemand(ctx, req)
|
||||
if err != nil {
|
||||
return "", "", false, err
|
||||
}
|
||||
|
||||
+339
-38
@@ -125,48 +125,354 @@ func (c *Client) FetchTranslationOnDemand(ctx context.Context, reqBody Translate
|
||||
return nil, fmt.Errorf("ai_summarizer: all %d translation attempts failed, last error: %w", c.config.APIRetryCount+1, lastErr)
|
||||
}
|
||||
|
||||
// TriggerPipelineTranslate calls POST /pipeline/translate to enqueue batch translation
|
||||
// for the given target languages.
|
||||
func (c *Client) TriggerPipelineTranslate(ctx context.Context, languages []string) (*PipelineTranslateResponse, error) {
|
||||
reqBody := PipelineTranslateRequest{Languages: languages}
|
||||
// TriggerTranslateBatch calls POST /ai/translate/batch to enqueue background translation.
|
||||
func (c *Client) TriggerTranslateBatch(ctx context.Context, tenders []TenderRef, languages []string) (*BatchAcceptedResponse, error) {
|
||||
reqBody := TranslateBatchRequest{
|
||||
Tenders: tenders,
|
||||
Languages: languages,
|
||||
}
|
||||
return c.postBatchAccepted(ctx, "/ai/translate/batch", reqBody, "translate batch", map[string]interface{}{
|
||||
"tender_count": len(tenders),
|
||||
"languages": languages,
|
||||
})
|
||||
}
|
||||
|
||||
// TriggerSummarizeBatch calls POST /ai/summarize/batch to enqueue background summarization.
|
||||
func (c *Client) TriggerSummarizeBatch(ctx context.Context, tenders []TenderRef) (*BatchAcceptedResponse, error) {
|
||||
reqBody := SummarizeBatchRequest{Tenders: tenders}
|
||||
return c.postBatchAccepted(ctx, "/ai/summarize/batch", reqBody, "summarize batch", map[string]interface{}{
|
||||
"tender_count": len(tenders),
|
||||
})
|
||||
}
|
||||
|
||||
// ScrapeDocuments calls POST /scrape/documents for one tender (synchronous, cached).
|
||||
func (c *Client) ScrapeDocuments(ctx context.Context, reqBody ScrapeDocumentsRequest) (*ScrapeDocumentsResponse, error) {
|
||||
jsonBody, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to marshal pipeline translate request: %w", err)
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to marshal scrape request: %w", err)
|
||||
}
|
||||
|
||||
url := c.config.APIBaseURL + "/pipeline/translate"
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(jsonBody))
|
||||
url := c.config.APIBaseURL + "/scrape/documents"
|
||||
httpResp, bodyBytes, err := c.doRawPost(ctx, url, jsonBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to create pipeline translate request: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode == http.StatusNotFound {
|
||||
return nil, fmt.Errorf("%w: status 404, body: %s", ErrObjectNotFound, string(bodyBytes))
|
||||
}
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result ScrapeDocumentsResponse
|
||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to decode scrape response: %w", err)
|
||||
}
|
||||
|
||||
c.logger.Info("AI scrape documents request succeeded", map[string]interface{}{
|
||||
"contract_folder_id": reqBody.ContractFolderID,
|
||||
"notice_publication_id": reqBody.NoticePublicationID,
|
||||
"files_downloaded": result.FilesDownloaded,
|
||||
})
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func apiNonSuccessError(statusCode int, bodyBytes []byte) error {
|
||||
return &APIStatusError{
|
||||
StatusCode: statusCode,
|
||||
Body: string(bodyBytes),
|
||||
}
|
||||
}
|
||||
|
||||
// ScrapeDocumentsBatch calls POST /scrape/documents/batch.
|
||||
func (c *Client) ScrapeDocumentsBatch(ctx context.Context, tenders []TenderRef) (*BatchAcceptedResponse, error) {
|
||||
reqBody := ScrapeDocumentsBatchRequest{Tenders: tenders}
|
||||
return c.postBatchAccepted(ctx, "/scrape/documents/batch", reqBody, "scrape batch", map[string]interface{}{
|
||||
"tender_count": len(tenders),
|
||||
})
|
||||
}
|
||||
|
||||
// PipelineSync calls POST /pipeline/sync to pull the tender list from the backend.
|
||||
func (c *Client) PipelineSync(ctx context.Context) (*PipelineSyncResponse, error) {
|
||||
url := c.config.APIBaseURL + "/pipeline/sync"
|
||||
httpResp, bodyBytes, err := c.doRawPost(ctx, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result PipelineSyncResponse
|
||||
if len(bodyBytes) > 0 {
|
||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to decode pipeline sync response: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
c.logger.Info("AI pipeline sync completed", map[string]interface{}{
|
||||
"stored": result.Stored,
|
||||
})
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// PipelineRun calls POST /pipeline/run for one tender (scrape → translate → summarize).
|
||||
func (c *Client) PipelineRun(ctx context.Context, reqBody PipelineRunRequest) (map[string]interface{}, error) {
|
||||
jsonBody, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to marshal pipeline run request: %w", err)
|
||||
}
|
||||
|
||||
url := c.config.APIBaseURL + "/pipeline/run"
|
||||
httpResp, bodyBytes, err := c.doRawPost(ctx, url, jsonBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode == http.StatusNotFound {
|
||||
return nil, fmt.Errorf("%w: status 404, body: %s", ErrObjectNotFound, string(bodyBytes))
|
||||
}
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result map[string]interface{}
|
||||
if len(bodyBytes) > 0 {
|
||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to decode pipeline run response: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
c.logger.Info("AI pipeline run completed", map[string]interface{}{
|
||||
"contract_folder_id": reqBody.ContractFolderID,
|
||||
"notice_publication_id": reqBody.NoticePublicationID,
|
||||
"languages": reqBody.Languages,
|
||||
})
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// PipelineRunBatch calls POST /pipeline/run/batch.
|
||||
func (c *Client) PipelineRunBatch(ctx context.Context, tenders []TenderRef, languages []string) (*BatchAcceptedResponse, error) {
|
||||
reqBody := PipelineRunBatchRequest{
|
||||
Tenders: tenders,
|
||||
Languages: languages,
|
||||
}
|
||||
return c.postBatchAccepted(ctx, "/pipeline/run/batch", reqBody, "pipeline run batch", map[string]interface{}{
|
||||
"tender_count": len(tenders),
|
||||
"languages": languages,
|
||||
})
|
||||
}
|
||||
|
||||
// PipelineAnalyze calls POST /pipeline/analyze to trigger analysis across stored tenders.
|
||||
func (c *Client) PipelineAnalyze(ctx context.Context) (*PipelineActionResponse, error) {
|
||||
return c.triggerPipelineAction(ctx, "/pipeline/analyze", "pipeline analyze")
|
||||
}
|
||||
|
||||
// PipelineDailyRun calls POST /pipeline/daily-run (sync + full pipeline on newly synced tenders).
|
||||
func (c *Client) PipelineDailyRun(ctx context.Context) (*PipelineStartedResponse, error) {
|
||||
return c.triggerPipelineStarted(ctx, "/pipeline/daily-run", "pipeline daily-run")
|
||||
}
|
||||
|
||||
// PipelineAuto calls POST /pipeline/auto (sync + complete all missing work, idempotent).
|
||||
func (c *Client) PipelineAuto(ctx context.Context) (*PipelineStartedResponse, error) {
|
||||
return c.triggerPipelineStarted(ctx, "/pipeline/auto", "pipeline auto")
|
||||
}
|
||||
|
||||
// GetPipelineLastRun returns the result of the last daily-run (GET /pipeline/last-run).
|
||||
func (c *Client) GetPipelineLastRun(ctx context.Context) (*PipelineReportResponse, error) {
|
||||
return c.getPipelineReport(ctx, "/pipeline/last-run")
|
||||
}
|
||||
|
||||
// GetPipelineLastAutoRun returns the result of the last auto run (GET /pipeline/last-auto-run).
|
||||
func (c *Client) GetPipelineLastAutoRun(ctx context.Context) (*PipelineReportResponse, error) {
|
||||
return c.getPipelineReport(ctx, "/pipeline/last-auto-run")
|
||||
}
|
||||
|
||||
// GetPipelineReport calls GET /pipeline/report.
|
||||
func (c *Client) GetPipelineReport(ctx context.Context, window string) (*PipelineReportResponse, error) {
|
||||
return c.getPipelineReportWithQuery(ctx, "/pipeline/report", window)
|
||||
}
|
||||
|
||||
// GetPipelineStats calls GET /pipeline/stats.
|
||||
func (c *Client) GetPipelineStats(ctx context.Context, window string) (*PipelineStatsResponse, error) {
|
||||
url := c.config.APIBaseURL + "/pipeline/stats"
|
||||
if window = strings.TrimSpace(window); window != "" {
|
||||
url += "?window=" + window
|
||||
}
|
||||
|
||||
httpResp, bodyBytes, err := c.doRawGet(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode == http.StatusBadRequest {
|
||||
return nil, apiNonSuccessError(http.StatusBadRequest, bodyBytes)
|
||||
}
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result PipelineStatsResponse
|
||||
if len(bodyBytes) > 0 {
|
||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to decode pipeline stats response: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// GetPipelineMinioStats calls GET /pipeline/minio-stats.
|
||||
func (c *Client) GetPipelineMinioStats(ctx context.Context) (PipelineMinioStatsResponse, error) {
|
||||
url := c.config.APIBaseURL + "/pipeline/minio-stats"
|
||||
httpResp, bodyBytes, err := c.doRawGet(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result PipelineMinioStatsResponse
|
||||
if len(bodyBytes) > 0 {
|
||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to decode pipeline minio stats response: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) postBatchAccepted(ctx context.Context, path string, reqBody any, label string, fields map[string]interface{}) (*BatchAcceptedResponse, error) {
|
||||
jsonBody, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to marshal %s request: %w", label, err)
|
||||
}
|
||||
|
||||
url := c.config.APIBaseURL + path
|
||||
httpResp, bodyBytes, err := c.doRawPost(ctx, url, jsonBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result BatchAcceptedResponse
|
||||
if len(bodyBytes) > 0 {
|
||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to decode %s response: %w", label, err)
|
||||
}
|
||||
}
|
||||
|
||||
logFields := map[string]interface{}{
|
||||
"path": path,
|
||||
"status": result.Status,
|
||||
"count": result.Count,
|
||||
}
|
||||
for k, v := range fields {
|
||||
logFields[k] = v
|
||||
}
|
||||
c.logger.Info("AI batch request accepted", logFields)
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) triggerPipelineStarted(ctx context.Context, path, label string) (*PipelineStartedResponse, error) {
|
||||
url := c.config.APIBaseURL + path
|
||||
httpResp, bodyBytes, err := c.doRawPost(ctx, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode == http.StatusConflict {
|
||||
return nil, fmt.Errorf("%w: status 409, body: %s", ErrPipelineJobRunning, string(bodyBytes))
|
||||
}
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result PipelineStartedResponse
|
||||
if len(bodyBytes) > 0 {
|
||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to decode %s response: %w", label, err)
|
||||
}
|
||||
}
|
||||
|
||||
c.logger.Info("AI pipeline job started", map[string]interface{}{
|
||||
"path": path,
|
||||
"status": result.Status,
|
||||
"report": result.Report,
|
||||
})
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) getPipelineReport(ctx context.Context, path string) (*PipelineReportResponse, error) {
|
||||
return c.getPipelineReportWithQuery(ctx, path, "")
|
||||
}
|
||||
|
||||
func (c *Client) getPipelineReportWithQuery(ctx context.Context, path, window string) (*PipelineReportResponse, error) {
|
||||
url := c.config.APIBaseURL + path
|
||||
if window = strings.TrimSpace(window); window != "" {
|
||||
url += "?window=" + window
|
||||
}
|
||||
|
||||
httpResp, bodyBytes, err := c.doRawGet(ctx, url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode == http.StatusBadRequest {
|
||||
return nil, apiNonSuccessError(http.StatusBadRequest, bodyBytes)
|
||||
}
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result PipelineReportResponse
|
||||
if len(bodyBytes) > 0 {
|
||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to decode pipeline report response: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) doRawGet(ctx context.Context, url string) (*http.Response, []byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("ai_summarizer: failed to create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Accept", "application/json")
|
||||
|
||||
httpResp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: pipeline translate request error: %w", err)
|
||||
return nil, nil, fmt.Errorf("ai_summarizer: request error: %w", err)
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
bodyBytes, err := io.ReadAll(httpResp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to read pipeline translate response: %w", err)
|
||||
httpResp.Body.Close()
|
||||
return nil, nil, fmt.Errorf("ai_summarizer: failed to read response body: %w", err)
|
||||
}
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
||||
}
|
||||
|
||||
var result PipelineTranslateResponse
|
||||
if err := json.Unmarshal(bodyBytes, &result); err != nil {
|
||||
return nil, fmt.Errorf("ai_summarizer: failed to decode pipeline translate response: %w", err)
|
||||
}
|
||||
|
||||
c.logger.Info("Pipeline translate triggered", map[string]interface{}{
|
||||
"status": result.Status,
|
||||
"languages": result.Languages,
|
||||
})
|
||||
|
||||
return &result, nil
|
||||
return httpResp, bodyBytes, nil
|
||||
}
|
||||
|
||||
// FetchAnalyzeOnDemand calls POST /ai/analyze for one tender.
|
||||
@@ -187,7 +493,7 @@ func (c *Client) FetchAnalyzeOnDemand(ctx context.Context, reqBody AnalyzeReques
|
||||
return nil, fmt.Errorf("%w: status 404, body: %s", ErrObjectNotFound, string(bodyBytes))
|
||||
}
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result AnalyzeResponse
|
||||
@@ -204,11 +510,6 @@ func (c *Client) FetchAnalyzeOnDemand(ctx context.Context, reqBody AnalyzeReques
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// TriggerPipelineSummarize calls POST /pipeline/summarize.
|
||||
func (c *Client) TriggerPipelineSummarize(ctx context.Context) (*PipelineActionResponse, error) {
|
||||
return c.triggerPipelineAction(ctx, "/pipeline/summarize", "pipeline summarize")
|
||||
}
|
||||
|
||||
// StartOnboarding calls POST /onboarding to start company profile indexing for recommendations.
|
||||
func (c *Client) StartOnboarding(ctx context.Context, reqBody OnboardingRequest) (*OnboardingResponse, error) {
|
||||
jsonBody, err := json.Marshal(reqBody)
|
||||
@@ -224,7 +525,7 @@ func (c *Client) StartOnboarding(ctx context.Context, reqBody OnboardingRequest)
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result OnboardingResponse
|
||||
@@ -257,7 +558,7 @@ func (c *Client) FetchRecommendations(ctx context.Context, reqBody RecommendRequ
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result []RecommendedTender
|
||||
@@ -284,7 +585,7 @@ func (c *Client) triggerPipelineAction(ctx context.Context, path, label string)
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result PipelineActionResponse
|
||||
@@ -356,7 +657,7 @@ func (c *Client) doPost(ctx context.Context, url string, jsonBody []byte) (*Summ
|
||||
}
|
||||
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
var result SummarizeResponse
|
||||
@@ -398,7 +699,7 @@ func (c *Client) doTranslatePost(ctx context.Context, url string, jsonBody []byt
|
||||
}
|
||||
|
||||
if httpResp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("%w: status %d, body: %s", ErrAPINonSuccess, httpResp.StatusCode, string(bodyBytes))
|
||||
return nil, apiNonSuccessError(httpResp.StatusCode, bodyBytes)
|
||||
}
|
||||
|
||||
result, err := decodeTranslateResponse(bodyBytes)
|
||||
|
||||
@@ -26,7 +26,7 @@ type Config struct {
|
||||
MinioSecretKey string // MinIO secret access key
|
||||
MinioUseSSL bool // Whether to use SSL for MinIO connection
|
||||
MinioRegion string // MinIO region
|
||||
MinioBucket string // MinIO bucket name (default: "opplens-documents")
|
||||
MinioBucket string // MinIO bucket name (default: "opplens")
|
||||
MinioTimeout time.Duration // Timeout for MinIO operations
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ func DefaultConfig() *Config {
|
||||
APIRetryCount: 2,
|
||||
APIRetryDelay: 3 * time.Second,
|
||||
|
||||
MinioBucket: "opplens-documents",
|
||||
MinioBucket: "opplens",
|
||||
MinioRegion: "us-east-1",
|
||||
MinioUseSSL: false,
|
||||
MinioTimeout: 30 * time.Second,
|
||||
|
||||
@@ -4,6 +4,20 @@ import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// TenderRef identifies a tender for batch and pipeline requests.
|
||||
type TenderRef struct {
|
||||
ContractFolderID string `json:"contract_folder_id"`
|
||||
NoticePublicationID string `json:"notice_publication_id"`
|
||||
}
|
||||
|
||||
// NewTenderRef builds a TenderRef from contract folder and notice publication ids.
|
||||
func NewTenderRef(contractFolderID, noticePublicationID string) TenderRef {
|
||||
return TenderRef{
|
||||
ContractFolderID: strings.TrimSpace(contractFolderID),
|
||||
NoticePublicationID: strings.TrimSpace(noticePublicationID),
|
||||
}
|
||||
}
|
||||
|
||||
// SummarizeRequest is the payload for POST /ai/summarize.
|
||||
type SummarizeRequest struct {
|
||||
ContractFolderID string `json:"contract_folder_id"`
|
||||
@@ -70,9 +84,22 @@ type TranslateRequest struct {
|
||||
ContractFolderID string `json:"contract_folder_id"`
|
||||
NoticePublicationID string `json:"notice_publication_id"`
|
||||
Language string `json:"language"`
|
||||
Title string `json:"title,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
RequestSource string `json:"-"` // internal: daily_job | manual_trigger
|
||||
}
|
||||
|
||||
// NewTranslateRequest builds a TranslateRequest for POST /ai/translate.
|
||||
func NewTranslateRequest(contractFolderID, noticePublicationID, language, title, description string) TranslateRequest {
|
||||
return TranslateRequest{
|
||||
ContractFolderID: strings.TrimSpace(contractFolderID),
|
||||
NoticePublicationID: strings.TrimSpace(noticePublicationID),
|
||||
Language: strings.ToLower(strings.TrimSpace(language)),
|
||||
Title: strings.TrimSpace(title),
|
||||
Description: strings.TrimSpace(description),
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
TranslationRequestSourceDailyJob = "daily_job"
|
||||
TranslationRequestSourceManualTrigger = "manual_trigger"
|
||||
@@ -122,15 +149,69 @@ type AnalyzeResponse struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
// PipelineTranslateRequest represents the request payload for POST /pipeline/translate.
|
||||
type PipelineTranslateRequest struct {
|
||||
Languages []string `json:"languages"`
|
||||
// ScrapeDocumentsRequest is the payload for POST /scrape/documents.
|
||||
type ScrapeDocumentsRequest struct {
|
||||
ContractFolderID string `json:"contract_folder_id"`
|
||||
NoticePublicationID string `json:"notice_publication_id"`
|
||||
}
|
||||
|
||||
// PipelineTranslateResponse represents the response from POST /pipeline/translate.
|
||||
type PipelineTranslateResponse struct {
|
||||
Status string `json:"status"`
|
||||
Languages []string `json:"languages"`
|
||||
// ScrapeDocumentsFile describes one downloaded document.
|
||||
type ScrapeDocumentsFile struct {
|
||||
Filename string `json:"filename"`
|
||||
Size int64 `json:"size"`
|
||||
}
|
||||
|
||||
// ScrapeDocumentsResponse is returned by POST /scrape/documents.
|
||||
type ScrapeDocumentsResponse struct {
|
||||
Success bool `json:"success"`
|
||||
FilesDownloaded int `json:"files_downloaded"`
|
||||
Files []ScrapeDocumentsFile `json:"files"`
|
||||
}
|
||||
|
||||
// ScrapeDocumentsBatchRequest is the payload for POST /scrape/documents/batch.
|
||||
type ScrapeDocumentsBatchRequest struct {
|
||||
Tenders []TenderRef `json:"tenders"`
|
||||
}
|
||||
|
||||
// SummarizeBatchRequest is the payload for POST /ai/summarize/batch.
|
||||
type SummarizeBatchRequest struct {
|
||||
Tenders []TenderRef `json:"tenders"`
|
||||
}
|
||||
|
||||
// TranslateBatchRequest is the payload for POST /ai/translate/batch.
|
||||
type TranslateBatchRequest struct {
|
||||
Tenders []TenderRef `json:"tenders"`
|
||||
Languages []string `json:"languages"`
|
||||
}
|
||||
|
||||
// BatchAcceptedResponse is returned by background batch endpoints (HTTP 202).
|
||||
type BatchAcceptedResponse struct {
|
||||
Status string `json:"status"`
|
||||
Count int `json:"count"`
|
||||
}
|
||||
|
||||
// PipelineSyncResponse is returned by POST /pipeline/sync.
|
||||
type PipelineSyncResponse struct {
|
||||
Stored int `json:"stored"`
|
||||
}
|
||||
|
||||
// PipelineRunRequest is the payload for POST /pipeline/run.
|
||||
type PipelineRunRequest struct {
|
||||
ContractFolderID string `json:"contract_folder_id"`
|
||||
NoticePublicationID string `json:"notice_publication_id"`
|
||||
Languages []string `json:"languages"`
|
||||
}
|
||||
|
||||
// PipelineRunBatchRequest is the payload for POST /pipeline/run/batch.
|
||||
type PipelineRunBatchRequest struct {
|
||||
Tenders []TenderRef `json:"tenders"`
|
||||
Languages []string `json:"languages"`
|
||||
}
|
||||
|
||||
// PipelineStartedResponse is returned by POST /pipeline/daily-run and POST /pipeline/auto.
|
||||
type PipelineStartedResponse struct {
|
||||
Status string `json:"status"`
|
||||
Report string `json:"report"`
|
||||
}
|
||||
|
||||
// PipelineActionResponse is returned by fire-and-forget pipeline endpoints.
|
||||
@@ -138,6 +219,31 @@ type PipelineActionResponse struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
// PipelineReportQuery holds optional query params for GET /pipeline/report and GET /pipeline/stats.
|
||||
type PipelineReportQuery struct {
|
||||
Window string `json:"window,omitempty"`
|
||||
}
|
||||
|
||||
// PipelineReportResponse is returned by GET /pipeline/report.
|
||||
type PipelineReportResponse struct {
|
||||
Window string `json:"window"`
|
||||
From string `json:"from"`
|
||||
To string `json:"to"`
|
||||
Summary map[string]interface{} `json:"summary,omitempty"`
|
||||
Days map[string]interface{} `json:"days,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
// PipelineStatsResponse is returned by GET /pipeline/stats.
|
||||
type PipelineStatsResponse struct {
|
||||
GeneratedAt string `json:"generated_at"`
|
||||
EventLog map[string]interface{} `json:"event_log"`
|
||||
Minio map[string]interface{} `json:"minio"`
|
||||
}
|
||||
|
||||
// PipelineMinioStatsResponse is returned by GET /pipeline/minio-stats.
|
||||
type PipelineMinioStatsResponse map[string]interface{}
|
||||
|
||||
// OnboardingRequest is the payload for POST /onboarding.
|
||||
type OnboardingRequest struct {
|
||||
CompanyID string `json:"company_id"`
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package ai_summarizer
|
||||
|
||||
import "errors"
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// Common sentinel errors for the AI summarizer package.
|
||||
var (
|
||||
@@ -33,4 +36,25 @@ var (
|
||||
|
||||
// ErrAPINonSuccess is returned when the AI API returns a non-2xx status code.
|
||||
ErrAPINonSuccess = errors.New("ai_summarizer: API returned non-success status")
|
||||
|
||||
// ErrPipelineJobRunning is returned when a single-flight pipeline job is already running (HTTP 409).
|
||||
ErrPipelineJobRunning = errors.New("ai_summarizer: pipeline job already running")
|
||||
)
|
||||
|
||||
// APIStatusError carries the HTTP status and response body from the Opplens AI service.
|
||||
type APIStatusError struct {
|
||||
StatusCode int
|
||||
Body string
|
||||
}
|
||||
|
||||
func (e *APIStatusError) Error() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
return fmt.Sprintf("ai_summarizer: API returned status %d, body: %s", e.StatusCode, e.Body)
|
||||
}
|
||||
|
||||
// Is reports whether target is ErrAPINonSuccess.
|
||||
func (e *APIStatusError) Is(target error) bool {
|
||||
return target == ErrAPINonSuccess
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ import (
|
||||
)
|
||||
|
||||
// procedurePrefix is prepended to every contract folder id in the MinIO key.
|
||||
// The AI team's storage layout in bucket opplens-documents is:
|
||||
// The AI team's storage layout in bucket opplens is:
|
||||
//
|
||||
// PROC_<contractFolderID>/<noticePublicationID>/tender.json
|
||||
// PROC_<contractFolderID>/<noticePublicationID>/documents/{files}
|
||||
|
||||
+39
-6
@@ -104,14 +104,47 @@ func (c *Counter) GetDailyCounts(ctx context.Context, startDay, endDay time.Time
|
||||
return map[string]int64{}, nil
|
||||
}
|
||||
|
||||
counts := make(map[string]int64)
|
||||
keys := make([]string, 0)
|
||||
dates := make([]string, 0)
|
||||
for day := startDay; !day.After(endDay); day = day.AddDate(0, 0, 1) {
|
||||
date := day.Format("2006-01-02")
|
||||
value, err := c.Get(ctx, AITranslationSuccessDailyCounterKey(day))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
dates = append(dates, day.Format("2006-01-02"))
|
||||
keys = append(keys, AITranslationSuccessDailyCounterKey(day))
|
||||
}
|
||||
|
||||
counts := make(map[string]int64, len(dates))
|
||||
for _, date := range dates {
|
||||
counts[date] = 0
|
||||
}
|
||||
|
||||
cursor, err := c.mongo.GetCollection(metricsCountersCollection).Find(ctx, bson.M{
|
||||
"_id": bson.M{"$in": keys},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mongo counter get daily counts: %w", err)
|
||||
}
|
||||
defer cursor.Close(ctx)
|
||||
|
||||
keyToDate := make(map[string]string, len(keys))
|
||||
for i, key := range keys {
|
||||
keyToDate[key] = dates[i]
|
||||
}
|
||||
|
||||
for cursor.Next(ctx) {
|
||||
var doc struct {
|
||||
ID string `bson:"_id"`
|
||||
Count int64 `bson:"count"`
|
||||
}
|
||||
counts[date] = value
|
||||
if err := cursor.Decode(&doc); err != nil {
|
||||
return nil, fmt.Errorf("mongo counter decode daily count: %w", err)
|
||||
}
|
||||
date, ok := keyToDate[doc.ID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
counts[date] = doc.Count
|
||||
}
|
||||
if err := cursor.Err(); err != nil {
|
||||
return nil, fmt.Errorf("mongo counter iterate daily counts: %w", err)
|
||||
}
|
||||
|
||||
return counts, nil
|
||||
|
||||
Reference in New Issue
Block a user