diff --git a/agent/robot/DESIGN.md b/agent/robot/DESIGN.md index 86c558a6..c0ce644c 100644 --- a/agent/robot/DESIGN.md +++ b/agent/robot/DESIGN.md @@ -372,7 +372,7 @@ type Task struct { ``` ┌─────────────────────────────────────────────────────────────┐ │ run.go (P3 Entry) │ -│ - RunConfig: retries, threshold, continue-on-failure │ +│ - RunConfig: threshold, continue-on-failure, max-turns │ │ - RunExecution: main execution loop │ └─────────────────────┬───────────────────────────────────────┘ │ @@ -383,6 +383,7 @@ type Task struct { │ - Runner │ │ - Validator │ │ - Task exec │ │ - Two-layer │ │ - Multi-turn │ │ - Rule+Semantic│ +│ conversation │ │ - NeedReply │ └────────┬────────┘ └────────┬────────┘ │ │ │ ▼ @@ -395,8 +396,8 @@ type Task struct { ┌─────────────────────────────────────────┐ │ Executor Types │ │ - assistant: AI Agent (multi-turn) │ -│ - mcp: MCP Tool (clientID.toolName) │ -│ - process: Yao Process │ +│ - mcp: MCP Tool (single-call) │ +│ - process: Yao Process (single-call) │ └─────────────────────────────────────────┘ ``` @@ -404,10 +405,15 @@ type Task struct { For each task: -1. **Execute** via appropriate executor (Assistant/MCP/Process) -2. **Validate** using two-layer validation -3. **Retry** if validation fails (with feedback to expert agent) -4. **Update** task status and store result +1. **Build Context**: Include previous task results as context +2. **Execute**: Call appropriate executor (Assistant/MCP/Process) +3. **Validate**: Use two-layer validation (rule-based + semantic) +4. **Continue or Complete**: + - For Assistant tasks: If `NeedReply`, continue conversation with `ReplyContent` + - For MCP/Process tasks: Single-call execution, no multi-turn +5. **Update**: Set task status and store result + +**Task Dependency**: Previous task results are automatically passed as context to subsequent tasks via `Runner.BuildTaskContext()` and formatted using `FormatPreviousResultsAsContext()`. **Two-Layer Validation:** @@ -424,27 +430,42 @@ For each task: | `mcp` | `clientID.toolName` | `filesystem.read_file` | | `process` | Process name | `models.user.Find` | -**Retry Mechanism:** +**Multi-Turn Conversation Flow:** -- Retries only on validation failure (not execution error) -- Validation feedback sent to expert agent on retry -- Configurable: `MaxRetries`, `RetryOnValidationFailure` +For assistant tasks, P3 uses a multi-turn conversation approach: +1. **Call**: Call assistant and get result +2. **Validate**: Validate result (determines: passed, complete, needReply, replyContent) +3. **Reply**: If needReply, continue conversation with replyContent +4. **Repeat**: Until complete or max turns exceeded + +The `Validator.ValidateWithContext()` method determines: +- `Complete`: Whether the expected result is obtained +- `NeedReply`: Whether to continue conversation +- `ReplyContent`: What to send in the next turn (validation feedback, clarification request, etc.) + +This replaces the traditional retry mechanism with intelligent conversation continuation. ```go +// RunConfig configures P3 execution behavior type RunConfig struct { - MaxRetries int // default: 3 - RetryOnValidationFailure bool // default: true - ContinueOnFailure bool // default: false - ValidationThreshold float64 // default: 0.6 - MaxTurnsPerTask int // default: 10 + ContinueOnFailure bool // continue to next task even if current fails (default: false) + ValidationThreshold float64 // minimum score to pass validation (default: 0.6) + MaxTurnsPerTask int // max conversation turns per task (default: 10) } +// ValidationResult with multi-turn conversation support type ValidationResult struct { + // Basic validation result Passed bool // overall validation passed Score float64 // 0-1 confidence score Issues []string // what failed Suggestions []string // how to improve Details string // detailed report (markdown) + + // Execution state (for multi-turn conversation control) + Complete bool // whether expected result is obtained + NeedReply bool // whether to continue conversation + ReplyContent string // content for next turn (if NeedReply) } ``` diff --git a/agent/robot/TECHNICAL.md b/agent/robot/TECHNICAL.md index ca84f332..55b82ee4 100644 --- a/agent/robot/TECHNICAL.md +++ b/agent/robot/TECHNICAL.md @@ -1384,13 +1384,19 @@ type TaskResult struct { Validation *ValidationResult `json:"validation,omitempty"` // P3 validation result } -// ValidationResult - P3 semantic validation result +// ValidationResult - P3 validation result with multi-turn conversation support type ValidationResult struct { + // Basic validation result Passed bool `json:"passed"` // overall validation passed Score float64 `json:"score,omitempty"` // 0-1 confidence score Issues []string `json:"issues,omitempty"` // what failed Suggestions []string `json:"suggestions,omitempty"` // how to improve - Details string `json:"details,omitempty"` // detailed validation report + Details string `json:"details,omitempty"` // detailed validation report (markdown) + + // Execution state (for multi-turn conversation control) + Complete bool `json:"complete"` // whether expected result is obtained + NeedReply bool `json:"need_reply,omitempty"` // whether to continue conversation + ReplyContent string `json:"reply_content,omitempty"` // content for next turn (if NeedReply) } // DeliveryResult - P4 delivery output diff --git a/agent/robot/TODO.md b/agent/robot/TODO.md index 7253b5c7..263a5e5b 100644 --- a/agent/robot/TODO.md +++ b/agent/robot/TODO.md @@ -777,22 +777,30 @@ Each phase test uses different expert combinations: ### 9.1 Implementation ✅ - [x] `executor/run.go` - `RunExecution(ctx, exec, data)` - real implementation - - [x] `RunConfig` - configuration for retries, validation threshold, etc. + - [x] `RunConfig` - configuration (ContinueOnFailure, ValidationThreshold, MaxTurnsPerTask) - [x] Sequential task execution with progress tracking - [x] Task status updates (Running → Completed/Failed/Skipped) - [x] `ContinueOnFailure` option for graceful failure handling + - [x] Previous task results passed as context to subsequent tasks - [x] `executor/runner.go` - `Runner` struct for task execution - - [x] `ExecuteWithRetry()` - retry mechanism for validation failures - - [x] `ExecuteTask()` - dispatch to correct executor type - - [x] `ExecuteAssistantTask()` - AI assistant execution with multi-turn support + - [x] `ExecuteWithRetry()` - multi-turn conversation flow for assistant tasks + - [x] `executeNonAssistantTask()` - single-call execution for MCP/Process + - [x] `executeAssistantWithMultiTurn()` - AI assistant with conversation support - [x] `ExecuteMCPTask()` - MCP tool execution (format: `clientID.toolName`) - [x] `ExecuteProcessTask()` - Yao process execution - [x] `BuildTaskContext()` - context with previous results - - [x] `GenerateAutoReply()` - auto-reply for multi-turn conversations - - [x] `FormatValidationFeedback()` - feedback for retry attempts + - [x] `BuildAssistantMessages()` - build messages for assistant + - [x] `FormatPreviousResultsAsContext()` - format previous results as context + - [x] `extractOutput()` - extract output from CallResult + - [x] `generateDefaultReply()` - fallback reply generation - [x] `executor/validator.go` - Two-layer validation system - [x] Layer 1: Rule-based validation using `yao/assert` - [x] Layer 2: Semantic validation using Validation Agent + - [x] `ValidateWithContext()` - validation with multi-turn support + - [x] `isComplete()` - determine if expected result is obtained + - [x] `checkNeedReply()` - determine if conversation should continue + - [x] `generateFeedbackReply()` - generate validation feedback for next turn + - [x] `detectNeedMoreInfo()` - detect if assistant needs clarification - [x] `convertStringRule()` - natural language rules to assertions - [x] `parseRules()` - JSON and string rule parsing - [x] `mergeResults()` - combine rule and semantic results @@ -829,14 +837,16 @@ Created new `yao/assert` package for universal assertion/validation: - [ ] Test: ContinueOnFailure option - [ ] Test: remaining tasks marked as skipped on failure - [ ] `executor/standard/runner_test.go` - Runner tests - - [ ] Test: ExecuteWithRetry with validation failures - - [ ] Test: ExecuteAssistantTask with multi-turn conversation + - [ ] Test: ExecuteWithRetry with multi-turn conversation flow + - [ ] Test: executeAssistantWithMultiTurn conversation continuation - [ ] Test: ExecuteMCPTask with correct ID parsing - [ ] Test: ExecuteProcessTask with Yao process - [ ] Test: BuildTaskContext with previous results - - [ ] Test: GenerateAutoReply for tool results + - [ ] Test: FormatPreviousResultsAsContext formatting - [ ] `executor/standard/validator_test.go` - Validator tests - - [ ] Test: two-layer validation (rules + semantic) + - [ ] Test: ValidateWithContext with multi-turn state + - [ ] Test: isComplete determination logic + - [ ] Test: checkNeedReply scenarios (clarification, feedback, incomplete) - [ ] Test: convertStringRule for natural language rules - [ ] Test: parseRules for JSON assertions - [ ] Test: validateSemantic with Validation Agent @@ -846,9 +856,10 @@ Created new `yao/assert` package for universal assertion/validation: ``` ┌─────────────────────────────────────────────────────────────┐ -│ run.go (P3 入口) │ -│ - RunConfig 配置 │ -│ - RunExecution 主循环 │ +│ run.go (P3 Entry) │ +│ - RunConfig: ContinueOnFailure, ValidationThreshold, │ +│ MaxTurnsPerTask │ +│ - RunExecution: main loop with task dependency passing │ └─────────────────────┬───────────────────────────────────────┘ │ ┌────────────┴────────────┐ @@ -856,34 +867,66 @@ Created new `yao/assert` package for universal assertion/validation: ┌─────────────────┐ ┌─────────────────┐ │ runner.go │ │ validator.go │ │ - Runner │ │ - Validator │ -│ - 任务执行 │ │ - 两层验证 │ -│ - 多轮对话 │ │ - 规则 + 语义 │ +│ - Multi-turn │ │ - Two-layer │ +│ conversation │ │ - Rule+Semantic│ +│ - Task context │ │ - NeedReply │ +│ building │ │ - ReplyContent │ └────────┬────────┘ └────────┬────────┘ │ │ │ ▼ │ ┌─────────────────┐ │ │ yao/assert │ │ │ - Asserter │ - │ │ - 8种断言类型 │ - │ │ - 可扩展接口 │ + │ │ - 8 types │ + │ │ - Extensible │ │ └─────────────────┘ ▼ ┌─────────────────────────────────────────┐ -│ 执行器类型 │ -│ - ExecutorAssistant → AI 助手 │ -│ - ExecutorMCP → MCP 工具 │ -│ - ExecutorProcess → Yao 进程 │ +│ Executor Types │ +│ - ExecutorAssistant → Multi-turn AI │ +│ - ExecutorMCP → Single-call MCP tool │ +│ - ExecutorProcess → Single-call Process │ └─────────────────────────────────────────┘ ``` +**Multi-Turn Conversation Flow (Assistant Tasks):** + +``` +┌──────────────────────────────────────────────────────────────┐ +│ executeAssistantWithMultiTurn │ +├──────────────────────────────────────────────────────────────┤ +│ 1. Create Conversation (single instance for entire task) │ +│ 2. Build initial messages with task context │ +│ │ +│ ┌─────────────────── Turn Loop ───────────────────────────┐ │ +│ │ Phase 1: Call assistant via conv.Turn() │ │ +│ │ Phase 2: ValidateWithContext() determines: │ │ +│ │ - Complete: task done? │ │ +│ │ - NeedReply: continue conversation? │ │ +│ │ - ReplyContent: what to send next? │ │ +│ │ Phase 3: If NeedReply, use ReplyContent as next input │ │ +│ │ Break if: Complete && Passed, or !NeedReply │ │ +│ └──────────────────────────────────────────────────────────┘ │ +│ │ +│ 3. Return output, validation, error │ +└──────────────────────────────────────────────────────────────┘ +``` + ### 9.5 Notes - Validation rules support two formats: 1. Natural language: `"output must be valid JSON"`, `"must contain 'field'"` 2. Structured JSON: `{"type": "type", "path": "field", "value": "array"}` -- Retry mechanism only triggers on validation failures, not execution errors -- Multi-turn conversation uses auto-reply generation for tool results +- Multi-turn conversation is validator-driven: + - `ValidateWithContext()` returns `NeedReply` and `ReplyContent` + - Conversation continues until `Complete && Passed` or `!NeedReply` + - Max turns controlled by `RunConfig.MaxTurnsPerTask` +- Task dependencies handled automatically: + - `BuildTaskContext()` collects previous task results + - `FormatPreviousResultsAsContext()` formats them for assistant +- MCP and Process tasks use single-call execution (no multi-turn) - `yao/assert` is a standalone package, can be used by other modules +- Agent context is properly released via `defer agentCtx.Release()` in `AgentCaller.Call()` --- diff --git a/agent/robot/executor/standard/agent.go b/agent/robot/executor/standard/agent.go index f09d42a5..e7577e72 100644 --- a/agent/robot/executor/standard/agent.go +++ b/agent/robot/executor/standard/agent.go @@ -178,6 +178,7 @@ func (c *AgentCaller) Call(ctx *robottypes.Context, assistantID string, messages // Convert robot context to agent context agentCtx := c.buildAgentContext(ctx) + defer agentCtx.Release() // IMPORTANT: Release agent context to prevent resource leaks // Call assistant with streaming response, err := ast.Stream(agentCtx, messages, opts) diff --git a/agent/robot/executor/standard/run.go b/agent/robot/executor/standard/run.go index 94f6902e..68f7d4de 100644 --- a/agent/robot/executor/standard/run.go +++ b/agent/robot/executor/standard/run.go @@ -9,36 +9,30 @@ import ( // RunConfig configures P3 execution behavior type RunConfig struct { - // MaxRetries is the maximum number of retry attempts per task (default: 3) - MaxRetries int - - // RetryOnValidationFailure enables retry when validation fails (default: true) - RetryOnValidationFailure bool - // ContinueOnFailure continues to next task even if current task fails (default: false) ContinueOnFailure bool // ValidationThreshold is the minimum score to pass validation (default: 0.6) ValidationThreshold float64 - // MaxTurnsPerTask is the maximum conversation turns for multi-turn agents (default: 10) + // MaxTurnsPerTask is the maximum conversation turns for multi-turn tasks (default: 10) + // This controls how many times the assistant can be called for a single task + // (including retries with validation feedback) MaxTurnsPerTask int } // DefaultRunConfig returns the default P3 configuration func DefaultRunConfig() *RunConfig { return &RunConfig{ - MaxRetries: 3, - RetryOnValidationFailure: true, - ContinueOnFailure: false, - ValidationThreshold: 0.6, - MaxTurnsPerTask: 10, + ContinueOnFailure: false, + ValidationThreshold: 0.6, + MaxTurnsPerTask: 10, } } // RunExecution executes P3: Run phase // Executes each task using the appropriate executor (Assistant, MCP, Process) -// with validation and retry mechanism +// with multi-turn conversation and validation // // Input: // - Tasks (from P2) @@ -46,12 +40,12 @@ func DefaultRunConfig() *RunConfig { // Output: // - TaskResult for each task with output and validation // -// Features: -// 1. Sequential task execution with progress tracking -// 2. Validation after each task using Validation Agent -// 3. Retry mechanism with feedback loop to expert agent -// 4. Multi-turn conversation support for complex tasks -// 5. Previous task results passed as context to next task +// Execution Flow (per task): +// 1. Call assistant/MCP/process and get result +// 2. Validate result using two-layer validation (rule-based + semantic) +// 3. If validation.NeedReply, continue conversation with validation.ReplyContent +// 4. Repeat until validation.Complete or max turns exceeded +// 5. Pass previous task results as context to next task func (e *Executor) RunExecution(ctx *robottypes.Context, exec *robottypes.Execution, _ interface{}) error { robot := exec.GetRobot() if robot == nil { @@ -90,13 +84,16 @@ func (e *Executor) RunExecution(ctx *robottypes.Context, exec *robottypes.Execut // Build task context with previous results taskCtx := runner.BuildTaskContext(exec, i) - // Execute task with retry + // Execute task with multi-turn conversation support result := runner.ExecuteWithRetry(task, taskCtx) // Update task status based on result endTime := time.Now() task.EndTime = &endTime - if result.Success && (result.Validation == nil || result.Validation.Passed) { + + // Determine task status from result + // Note: result.Success is already set to (validation.Complete && validation.Passed) in runner + if result.Success { task.Status = robottypes.TaskCompleted } else { task.Status = robottypes.TaskFailed diff --git a/agent/robot/executor/standard/runner.go b/agent/robot/executor/standard/runner.go index c3ee2229..a38e97b6 100644 --- a/agent/robot/executor/standard/runner.go +++ b/agent/robot/executor/standard/runner.go @@ -61,7 +61,11 @@ func (r *Runner) BuildTaskContext(exec *robottypes.Execution, taskIndex int) *Ru return ctx } -// ExecuteWithRetry executes a task with retry mechanism +// ExecuteWithRetry executes a task with the new multi-turn conversation flow: +// 1. Call assistant and get result +// 2. Validate result (determines: passed, complete, needReply, replyContent) +// 3. If needReply, continue conversation with replyContent +// 4. Repeat until complete or max turns exceeded func (r *Runner) ExecuteWithRetry(task *robottypes.Task, taskCtx *RunnerContext) *robottypes.TaskResult { startTime := time.Now() @@ -69,86 +73,74 @@ func (r *Runner) ExecuteWithRetry(task *robottypes.Task, taskCtx *RunnerContext) TaskID: task.ID, } - var lastOutput interface{} - var lastValidation *robottypes.ValidationResult - var allErrors []string // Collect all errors for debugging - - for attempt := 0; attempt <= r.config.MaxRetries; attempt++ { - // Execute the task - output, err := r.ExecuteTask(task, taskCtx, lastValidation) + // For non-assistant tasks (MCP, Process), use simple single-call execution + if task.ExecutorType != robottypes.ExecutorAssistant { + output, err := r.executeNonAssistantTask(task, taskCtx) if err != nil { - // Execution error - don't retry, return immediately - // (Retries are only for validation failures, not execution errors) - errMsg := fmt.Sprintf("execution failed on attempt %d: %s", attempt+1, err.Error()) - allErrors = append(allErrors, errMsg) result.Success = false - result.Error = strings.Join(allErrors, "; ") + result.Error = fmt.Sprintf("execution failed: %s", err.Error()) result.Duration = time.Since(startTime).Milliseconds() return result } - lastOutput = output result.Output = output - - // Validate the result (reuse validator instance) - validation := r.validator.Validate(task, output) - lastValidation = validation + validation := r.validator.ValidateWithContext(task, output, nil) result.Validation = validation + // For non-assistant tasks (MCP, Process): + // - No multi-turn conversation, so Complete is determined by validation alone + // - Success if passed OR score meets threshold (for partial success scenarios) + result.Success = validation.Complete || (validation.Passed && validation.Score >= r.config.ValidationThreshold) + result.Duration = time.Since(startTime).Milliseconds() - // Check if validation passed (unified logic) - validationPassed := validation.Passed || validation.Score >= r.config.ValidationThreshold - if validationPassed { - result.Success = true - result.Duration = time.Since(startTime).Milliseconds() - return result + if !result.Success && validation != nil { + result.Error = fmt.Sprintf("validation failed: %v", validation.Issues) } - - // Validation failed - check if we should retry - if !r.config.RetryOnValidationFailure || attempt >= r.config.MaxRetries { - break - } - - // Prepare for retry with validation feedback - // The next iteration will include validation issues in the context + return result } - // All retries exhausted - result.Success = false - result.Output = lastOutput - result.Validation = lastValidation + // For assistant tasks, use multi-turn conversation flow + output, validation, err := r.executeAssistantWithMultiTurn(task, taskCtx) + if err != nil { + result.Success = false + result.Error = err.Error() + result.Output = output // Preserve partial output for debugging + result.Validation = validation // Preserve validation result for debugging + result.Duration = time.Since(startTime).Milliseconds() + return result + } + + result.Output = output + result.Validation = validation + result.Success = validation.Complete && validation.Passed result.Duration = time.Since(startTime).Milliseconds() - if len(allErrors) > 0 { - result.Error = strings.Join(allErrors, "; ") - } else if lastValidation != nil { - result.Error = fmt.Sprintf("validation failed after %d attempts: %v", - r.config.MaxRetries+1, lastValidation.Issues) + if !result.Success && validation != nil { + result.Error = fmt.Sprintf("task incomplete: %v", validation.Issues) } return result } -// ExecuteTask executes a single task based on its executor type -func (r *Runner) ExecuteTask(task *robottypes.Task, taskCtx *RunnerContext, prevValidation *robottypes.ValidationResult) (interface{}, error) { +// executeNonAssistantTask executes MCP or Process tasks (single-call, no multi-turn) +func (r *Runner) executeNonAssistantTask(task *robottypes.Task, taskCtx *RunnerContext) (interface{}, error) { switch task.ExecutorType { - case robottypes.ExecutorAssistant: - return r.ExecuteAssistantTask(task, taskCtx, prevValidation) case robottypes.ExecutorMCP: return r.ExecuteMCPTask(task, taskCtx) case robottypes.ExecutorProcess: return r.ExecuteProcessTask(task, taskCtx) default: - return nil, fmt.Errorf("unknown executor type: %s", task.ExecutorType) + return nil, fmt.Errorf("unsupported executor type: %s (expected mcp or process)", task.ExecutorType) } } -// ExecuteAssistantTask executes a task using an AI assistant -// Supports multi-turn conversation for complex tasks -func (r *Runner) ExecuteAssistantTask(task *robottypes.Task, taskCtx *RunnerContext, prevValidation *robottypes.ValidationResult) (interface{}, error) { - // Build messages for the assistant - messages := r.BuildAssistantMessages(task, taskCtx, prevValidation) - - // Create conversation for multi-turn support +// executeAssistantWithMultiTurn executes an assistant task with multi-turn conversation support +// This is the main execution flow for assistant tasks: +// 1. Call assistant and get result +// 2. Validate result (determines: passed, complete, needReply, replyContent) +// 3. If needReply, continue conversation with replyContent +// 4. Repeat until complete or max turns exceeded +func (r *Runner) executeAssistantWithMultiTurn(task *robottypes.Task, taskCtx *RunnerContext) (interface{}, *robottypes.ValidationResult, error) { + // Create conversation for the entire task execution (shared across all turns) chatID := fmt.Sprintf("robot-%s-task-%s", r.robot.MemberID, task.ID) conv := NewConversation(task.ExecutorID, chatID, r.config.MaxTurnsPerTask) @@ -157,55 +149,106 @@ func (r *Runner) ExecuteAssistantTask(task *robottypes.Task, taskCtx *RunnerCont conv.WithSystemPrompt(taskCtx.SystemPrompt) } - // First turn: send the task - firstInput := r.FormatMessagesAsText(messages) - turnResult, err := conv.Turn(r.ctx, firstInput) - if err != nil { - return nil, fmt.Errorf("assistant call failed: %w", err) + // Build initial messages + messages := r.BuildAssistantMessages(task, taskCtx) + input := r.FormatMessagesAsText(messages) + + // Ensure we have valid input for the first turn + if strings.TrimSpace(input) == "" { + return nil, nil, fmt.Errorf("no valid input messages for task %s", task.ID) } - // Check if the assistant needs more information (multi-turn) - // We detect this by checking if the response indicates incompleteness - // or if there are tool calls that need results - response := turnResult.Result + var lastOutput interface{} + var lastValidation *robottypes.ValidationResult + var lastCallResult *CallResult - // For simple tasks, return the result directly - if response.Response == nil || len(response.Response.Tools) == 0 { - // Try to extract structured output - if data, err := response.GetJSON(); err == nil { - return data, nil - } - // Return text content - return response.GetText(), nil - } - - // Handle multi-turn conversation with auto-reply simulation - // Similar to the test framework's dynamic runner - for turn := 2; turn <= r.config.MaxTurnsPerTask; turn++ { - // Check if we have a complete response - if r.IsResponseComplete(response) { - break - } - - // Generate auto-reply based on tool results or context - autoReply := r.GenerateAutoReply(response, task) - if autoReply == "" { - break // No more input needed - } - - // Continue conversation - turnResult, err = conv.Turn(r.ctx, autoReply) + for turn := 1; turn <= r.config.MaxTurnsPerTask; turn++ { + // Phase 1: Call assistant + turnResult, err := conv.Turn(r.ctx, input) if err != nil { - return nil, fmt.Errorf("assistant turn %d failed: %w", turn, err) + return lastOutput, lastValidation, fmt.Errorf("turn %d failed: %w", turn, err) + } + + lastCallResult = turnResult.Result + lastOutput = r.extractOutput(lastCallResult) + + // Phase 2: Validate result + lastValidation = r.validator.ValidateWithContext(task, lastOutput, lastCallResult) + + // Check if complete + if lastValidation.Complete && lastValidation.Passed { + return lastOutput, lastValidation, nil // Success! + } + + // Phase 3: Check if we should continue conversation + if !lastValidation.NeedReply { + // No need to continue, but not complete either + // This could be a validation failure that can't be fixed by conversation + if lastValidation.Passed { + // Passed but not complete (e.g., empty output) + return lastOutput, lastValidation, nil + } + // Failed and can't retry + return lastOutput, lastValidation, fmt.Errorf("validation failed: %v", lastValidation.Issues) + } + + // Prepare next turn input + input = lastValidation.ReplyContent + if input == "" { + // Fallback: generate default reply + input = r.generateDefaultReply(lastValidation, task) } - response = turnResult.Result } - // Extract final output - if data, err := response.GetJSON(); err == nil { - return data, nil + // Max turns exceeded + if lastValidation == nil { + lastValidation = &robottypes.ValidationResult{ + Passed: false, + Complete: false, + Issues: []string{fmt.Sprintf("max turns (%d) exceeded without completion", r.config.MaxTurnsPerTask)}, + } + } else { + lastValidation.Issues = append(lastValidation.Issues, + fmt.Sprintf("max turns (%d) exceeded without completion", r.config.MaxTurnsPerTask)) } - return response.GetText(), nil + + return lastOutput, lastValidation, fmt.Errorf("max turns (%d) exceeded without completion", r.config.MaxTurnsPerTask) +} + +// extractOutput extracts the output from a CallResult +func (r *Runner) extractOutput(result *CallResult) interface{} { + if result == nil { + return nil + } + + // Try to extract structured JSON output + if data, err := result.GetJSON(); err == nil { + return data + } + + // Fall back to text content + return result.GetText() +} + +// generateDefaultReply generates a default reply when validation doesn't provide one +func (r *Runner) generateDefaultReply(validation *robottypes.ValidationResult, task *robottypes.Task) string { + var sb strings.Builder + + if len(validation.Issues) > 0 { + sb.WriteString("Please address the following issues:\n") + for _, issue := range validation.Issues { + sb.WriteString(fmt.Sprintf("- %s\n", issue)) + } + sb.WriteString("\n") + } + + if task.ExpectedOutput != "" { + sb.WriteString(fmt.Sprintf("Expected output: %s\n", task.ExpectedOutput)) + } + + sb.WriteString("\nPlease provide an improved response.") + + return sb.String() } // ExecuteMCPTask executes a task using an MCP tool @@ -269,7 +312,8 @@ func (r *Runner) ExecuteProcessTask(task *robottypes.Task, taskCtx *RunnerContex } // BuildAssistantMessages builds messages for an assistant task -func (r *Runner) BuildAssistantMessages(task *robottypes.Task, taskCtx *RunnerContext, prevValidation *robottypes.ValidationResult) []agentcontext.Message { +// Note: In the new multi-turn flow, validation feedback is handled via ValidateWithContext.ReplyContent +func (r *Runner) BuildAssistantMessages(task *robottypes.Task, taskCtx *RunnerContext) []agentcontext.Message { messages := make([]agentcontext.Message, 0) // Add context from previous tasks if available @@ -286,15 +330,6 @@ func (r *Runner) BuildAssistantMessages(task *robottypes.Task, taskCtx *RunnerCo // Add task messages messages = append(messages, task.Messages...) - // Add validation feedback if this is a retry - if prevValidation != nil && !prevValidation.Passed { - feedbackMsg := r.FormatValidationFeedback(prevValidation) - messages = append(messages, agentcontext.Message{ - Role: agentcontext.RoleUser, - Content: feedbackMsg, - }) - } - return messages } @@ -357,89 +392,3 @@ func (r *Runner) FormatPreviousResultsAsContext(results []robottypes.TaskResult) return sb.String() } - -// FormatValidationFeedback formats validation feedback for retry -func (r *Runner) FormatValidationFeedback(validation *robottypes.ValidationResult) string { - var sb strings.Builder - sb.WriteString("## Validation Feedback\n\n") - sb.WriteString("Your previous response did not pass validation. Please address the following issues:\n\n") - - if len(validation.Issues) > 0 { - sb.WriteString("### Issues\n") - for _, issue := range validation.Issues { - sb.WriteString(fmt.Sprintf("- %s\n", issue)) - } - sb.WriteString("\n") - } - - if len(validation.Suggestions) > 0 { - sb.WriteString("### Suggestions\n") - for _, suggestion := range validation.Suggestions { - sb.WriteString(fmt.Sprintf("- %s\n", suggestion)) - } - sb.WriteString("\n") - } - - sb.WriteString("Please provide an improved response that addresses these issues.\n") - - return sb.String() -} - -// IsResponseComplete checks if an assistant response is complete -// (no pending tool calls, has content) -func (r *Runner) IsResponseComplete(result *CallResult) bool { - if result == nil || result.Response == nil { - return true - } - - // If there are tool calls, check if all have results - if len(result.Response.Tools) > 0 { - for _, tool := range result.Response.Tools { - if tool.Result == nil { - return false // Still waiting for tool results - } - } - // All tools have results - response is complete - return true - } - - // No tools - check if there's content - return result.Content != "" || result.Next != nil -} - -// GenerateAutoReply generates an automatic reply for multi-turn conversation -// This simulates user responses when the assistant needs more information -func (r *Runner) GenerateAutoReply(result *CallResult, task *robottypes.Task) string { - if result == nil || result.Response == nil { - return "" - } - - // If there are tool results, format them as the reply - if len(result.Response.Tools) > 0 { - var replies []string - for _, tool := range result.Response.Tools { - if tool.Result != nil { - resultJSON, err := json.Marshal(tool.Result) - if err == nil { - replies = append(replies, fmt.Sprintf("Tool %s result: %s", tool.Tool, string(resultJSON))) - } - } - } - if len(replies) > 0 { - return fmt.Sprintf("Tool execution results:\n%s\n\nPlease continue with the task.", strings.Join(replies, "\n")) - } - } - - // If the response asks for clarification, provide generic guidance - // Use case-insensitive matching - content := strings.ToLower(result.GetText()) - clarificationKeywords := []string{"need more", "clarify", "please provide", "what", "which"} - for _, keyword := range clarificationKeywords { - if strings.Contains(content, keyword) { - return fmt.Sprintf("Please proceed with the task as best as you can based on the available information. "+ - "The expected output is: %s", task.ExpectedOutput) - } - } - - return "" -} diff --git a/agent/robot/executor/standard/validator.go b/agent/robot/executor/standard/validator.go index 659569be..bbfe331c 100644 --- a/agent/robot/executor/standard/validator.go +++ b/agent/robot/executor/standard/validator.go @@ -36,15 +36,30 @@ func NewValidator(ctx *robottypes.Context, robot *robottypes.Robot, config *RunC return v } -// Validate validates task output using two-layer validation: -// 1. First, run rule-based assertions (fast, deterministic) -// 2. Then, if ExpectedOutput is set, run semantic validation via Agent +// Validate validates task output using two-layer validation (without multi-turn context) +// Equivalent to ValidateWithContext(task, output, nil) +// Use ValidateWithContext when you have a CallResult for better multi-turn support func (v *Validator) Validate(task *robottypes.Task, output interface{}) *robottypes.ValidationResult { - // If no validation rules and no expected output, return passed + return v.ValidateWithContext(task, output, nil) +} + +// ValidateWithContext validates task output and determines execution state for multi-turn conversation. +// It extends basic validation with: +// - Complete: whether expected result is obtained +// - NeedReply: whether to continue conversation +// - ReplyContent: content for next turn +// +// Parameters: +// - task: the task being executed +// - output: the output from assistant/mcp/process +// - callResult: the full call result (for detecting assistant's need for more info) +func (v *Validator) ValidateWithContext(task *robottypes.Task, output interface{}, callResult *CallResult) *robottypes.ValidationResult { + // If no validation rules and no expected output, return passed and complete if task.ExpectedOutput == "" && len(task.ValidationRules) == 0 { return &robottypes.ValidationResult{ - Passed: true, - Score: 1.0, + Passed: true, + Score: 1.0, + Complete: v.hasValidOutput(output), } } @@ -57,6 +72,9 @@ func (v *Validator) Validate(task *robottypes.Task, output interface{}) *robotty if len(task.ValidationRules) > 0 { ruleResult := v.validateRules(task.ValidationRules, output) if !ruleResult.Passed { + // Rule validation failed - check if we should retry with feedback + ruleResult.Complete = false + ruleResult.NeedReply, ruleResult.ReplyContent = v.checkNeedReplyOnFailure(task, ruleResult) return ruleResult } // Merge rule validation results @@ -71,9 +89,186 @@ func (v *Validator) Validate(task *robottypes.Task, output interface{}) *robotty result = v.mergeResults(result, semanticResult) } + // Determine execution state + result.Complete = v.isComplete(task, output, result) + result.NeedReply, result.ReplyContent = v.checkNeedReply(task, output, callResult, result) + return result } +// hasValidOutput checks if output is non-empty and valid +func (v *Validator) hasValidOutput(output interface{}) bool { + if output == nil { + return false + } + switch o := output.(type) { + case string: + return strings.TrimSpace(o) != "" + case []interface{}: + return len(o) > 0 + case map[string]interface{}: + return len(o) > 0 + default: + return true + } +} + +// isComplete determines if the expected result has been obtained +func (v *Validator) isComplete(task *robottypes.Task, output interface{}, result *robottypes.ValidationResult) bool { + // If validation failed, not complete + if !result.Passed { + return false + } + + // Must have valid output + if !v.hasValidOutput(output) { + return false + } + + // If score is below threshold, consider incomplete + if result.Score < v.config.ValidationThreshold { + return false + } + + return true +} + +// checkNeedReply determines if conversation should continue and generates reply content +func (v *Validator) checkNeedReply(task *robottypes.Task, output interface{}, callResult *CallResult, result *robottypes.ValidationResult) (bool, string) { + // If already complete, no need to reply + if result.Complete { + return false, "" + } + + // Scenario 1: Assistant explicitly asks for more information + if callResult != nil { + text := callResult.GetText() + if v.detectNeedMoreInfo(text) { + return true, v.generateClarificationReply(task, text) + } + } + + // Scenario 2: Validation passed but output is incomplete/empty + if result.Passed && !v.hasValidOutput(output) { + return true, "Please continue and provide the complete result as specified in the task." + } + + // Scenario 3: Validation failed with suggestions - can retry with feedback + if !result.Passed && len(result.Suggestions) > 0 { + return true, v.generateFeedbackReply(result) + } + + // Scenario 4: Low confidence score - ask for improvement + if result.Passed && result.Score < v.config.ValidationThreshold { + return true, fmt.Sprintf("The result is partially correct (score: %.2f), but needs improvement. Please refine your response to better match the expected output: %s", result.Score, task.ExpectedOutput) + } + + // No need to continue + return false, "" +} + +// checkNeedReplyOnFailure handles the case when rule validation fails +func (v *Validator) checkNeedReplyOnFailure(task *robottypes.Task, result *robottypes.ValidationResult) (bool, string) { + // If there are suggestions, we can try to fix + if len(result.Suggestions) > 0 { + return true, v.generateFeedbackReply(result) + } + + // If there are issues, provide feedback + if len(result.Issues) > 0 { + var sb strings.Builder + sb.WriteString("Your response did not pass validation. Please fix the following issues:\n\n") + for _, issue := range result.Issues { + sb.WriteString(fmt.Sprintf("- %s\n", issue)) + } + sb.WriteString(fmt.Sprintf("\nExpected output: %s", task.ExpectedOutput)) + return true, sb.String() + } + + return false, "" +} + +// detectNeedMoreInfo checks if assistant's response indicates need for more information +func (v *Validator) detectNeedMoreInfo(text string) bool { + if text == "" { + return false + } + + textLower := strings.ToLower(text) + keywords := []string{ + "need more information", + "please clarify", + "could you provide", + "can you specify", + "what is the", + "which one", + "please provide", + "i need to know", + "could you tell me", + "what do you mean", + } + + for _, kw := range keywords { + if strings.Contains(textLower, kw) { + return true + } + } + + // Check for question marks at the end (likely asking for clarification) + // Note: We require 2+ question marks to avoid false positives from rhetorical questions + // or questions that are part of the output (e.g., "How can I help you?") + // Single questions are often just conversational and don't need clarification + trimmed := strings.TrimSpace(text) + if strings.HasSuffix(trimmed, "?") { + if strings.Count(text, "?") >= 2 { + return true + } + } + + return false +} + +// generateClarificationReply generates a reply when assistant asks for clarification +func (v *Validator) generateClarificationReply(task *robottypes.Task, assistantText string) string { + var sb strings.Builder + sb.WriteString("Please proceed with the task based on the available information.\n\n") + + if task.ExpectedOutput != "" { + sb.WriteString(fmt.Sprintf("**Expected Output**: %s\n\n", task.ExpectedOutput)) + } + + sb.WriteString("If you need to make assumptions, please state them clearly and proceed with the most reasonable interpretation.") + + return sb.String() +} + +// generateFeedbackReply generates a reply with validation feedback +func (v *Validator) generateFeedbackReply(result *robottypes.ValidationResult) string { + var sb strings.Builder + sb.WriteString("## Validation Feedback\n\n") + sb.WriteString("Your previous response needs improvement. Please address the following:\n\n") + + if len(result.Issues) > 0 { + sb.WriteString("### Issues\n") + for _, issue := range result.Issues { + sb.WriteString(fmt.Sprintf("- %s\n", issue)) + } + sb.WriteString("\n") + } + + if len(result.Suggestions) > 0 { + sb.WriteString("### Suggestions\n") + for _, suggestion := range result.Suggestions { + sb.WriteString(fmt.Sprintf("- %s\n", suggestion)) + } + sb.WriteString("\n") + } + + sb.WriteString("Please provide an improved response that addresses these points.") + + return sb.String() +} + // validateRules validates output against rule-based assertions func (v *Validator) validateRules(rules []string, output interface{}) *robottypes.ValidationResult { result := &robottypes.ValidationResult{ @@ -233,16 +428,21 @@ func (v *Validator) validateSemantic(task *robottypes.Task, output interface{}) } // BuildSemanticPrompt builds the prompt for semantic validation +// Format matches the Validation Agent's expected input structure: +// 1. Task: task definition with expected_output and validation_rules +// 2. Result: actual output from task execution +// 3. Success Criteria: overall criteria (optional) func (v *Validator) BuildSemanticPrompt(task *robottypes.Task, output interface{}) string { var sb strings.Builder - sb.WriteString("## Task Definition\n\n") + // Section 1: Task (matches Agent's expected "Task" input) + sb.WriteString("## Task\n\n") sb.WriteString(fmt.Sprintf("**Task ID**: %s\n", task.ID)) sb.WriteString(fmt.Sprintf("**Executor**: %s (%s)\n\n", task.ExecutorID, task.ExecutorType)) - // Task description + // Task description (instructions) if len(task.Messages) > 0 { - sb.WriteString("**Task Instructions**:\n") + sb.WriteString("**Instructions**:\n") for _, msg := range task.Messages { if content, ok := msg.Content.(string); ok { sb.WriteString(content + "\n") @@ -253,21 +453,21 @@ func (v *Validator) BuildSemanticPrompt(task *robottypes.Task, output interface{ // Expected output (primary criterion for semantic validation) if task.ExpectedOutput != "" { - sb.WriteString(fmt.Sprintf("**Expected Output**: %s\n\n", task.ExpectedOutput)) + sb.WriteString(fmt.Sprintf("**expected_output**: %s\n\n", task.ExpectedOutput)) } - // Semantic validation rules (rules that couldn't be converted to assertions) + // Validation rules semanticRules := v.getSemanticRules(task.ValidationRules) if len(semanticRules) > 0 { - sb.WriteString("**Validation Criteria**:\n") + sb.WriteString("**validation_rules**:\n") for _, rule := range semanticRules { sb.WriteString(fmt.Sprintf("- %s\n", rule)) } sb.WriteString("\n") } - // Actual output - sb.WriteString("## Actual Output\n\n") + // Section 2: Result (matches Agent's expected "Result" input) + sb.WriteString("## Result\n\n") if output != nil { outputJSON, err := json.MarshalIndent(output, "", " ") if err == nil { @@ -279,10 +479,14 @@ func (v *Validator) BuildSemanticPrompt(task *robottypes.Task, output interface{ sb.WriteString("(no output)\n") } - sb.WriteString("\n## Validation Request\n\n") - sb.WriteString("Please validate the actual output against the expected output and validation criteria. ") - sb.WriteString("Focus on semantic correctness and completeness. ") - sb.WriteString("Return a JSON object with: passed (bool), score (0-1), issues (array), suggestions (array), details (markdown report).\n") + // Section 3: Success Criteria (optional, from goals if available) + // Note: This could be extended to include criteria from exec.Goals if needed + sb.WriteString("\n## Success Criteria\n\n") + if task.ExpectedOutput != "" { + sb.WriteString(fmt.Sprintf("The task should produce: %s\n", task.ExpectedOutput)) + } else { + sb.WriteString("Complete the task successfully with valid output.\n") + } return sb.String() } diff --git a/agent/robot/types/robot.go b/agent/robot/types/robot.go index 05da4605..19c4b41d 100644 --- a/agent/robot/types/robot.go +++ b/agent/robot/types/robot.go @@ -249,11 +249,17 @@ type TaskResult struct { // ValidationResult - P3 semantic validation result type ValidationResult struct { + // Basic validation result Passed bool `json:"passed"` // overall validation passed Score float64 `json:"score,omitempty"` // 0-1 confidence score Issues []string `json:"issues,omitempty"` // what failed Suggestions []string `json:"suggestions,omitempty"` // how to improve Details string `json:"details,omitempty"` // detailed validation report (markdown) + + // Execution state (for multi-turn conversation control) + Complete bool `json:"complete"` // whether expected result is obtained + NeedReply bool `json:"need_reply,omitempty"` // whether to continue conversation + ReplyContent string `json:"reply_content,omitempty"` // content for next turn (if NeedReply) } // DeliveryResult - P4 delivery output diff --git a/agent/robot/types/robot_test.go b/agent/robot/types/robot_test.go index 97c4db88..71c84395 100644 --- a/agent/robot/types/robot_test.go +++ b/agent/robot/types/robot_test.go @@ -474,6 +474,65 @@ func TestValidationResultStructure(t *testing.T) { assert.Len(t, validation.Suggestions, 2) } +func TestValidationResultMultiTurnFields(t *testing.T) { + // Test new multi-turn conversation control fields + t.Run("complete and passed", func(t *testing.T) { + validation := &types.ValidationResult{ + Passed: true, + Score: 0.95, + Complete: true, + } + assert.True(t, validation.Passed) + assert.True(t, validation.Complete) + assert.False(t, validation.NeedReply) + assert.Empty(t, validation.ReplyContent) + }) + + t.Run("passed but not complete - needs reply", func(t *testing.T) { + validation := &types.ValidationResult{ + Passed: true, + Score: 0.7, + Complete: false, + NeedReply: true, + ReplyContent: "Please continue and provide the complete result.", + } + assert.True(t, validation.Passed) + assert.False(t, validation.Complete) + assert.True(t, validation.NeedReply) + assert.NotEmpty(t, validation.ReplyContent) + }) + + t.Run("failed with suggestions - needs reply", func(t *testing.T) { + validation := &types.ValidationResult{ + Passed: false, + Score: 0.3, + Complete: false, + Issues: []string{"Missing required field"}, + Suggestions: []string{"Add the field"}, + NeedReply: true, + ReplyContent: "## Validation Feedback\n\nPlease fix: Missing required field", + } + assert.False(t, validation.Passed) + assert.False(t, validation.Complete) + assert.True(t, validation.NeedReply) + assert.Contains(t, validation.ReplyContent, "Validation Feedback") + }) + + t.Run("failed without suggestions - no reply", func(t *testing.T) { + validation := &types.ValidationResult{ + Passed: false, + Score: 0.0, + Complete: false, + Issues: []string{"Critical error: invalid format"}, + NeedReply: false, + } + assert.False(t, validation.Passed) + assert.False(t, validation.Complete) + assert.False(t, validation.NeedReply) + assert.Empty(t, validation.ReplyContent) + }) +} + func TestDeliveryResultStructure(t *testing.T) { sentAt := time.Now() delivery := &types.DeliveryResult{