Update TODO.md and run_test.go for P3 Run Implementation Completion
- Marked the P3 Run Implementation as complete in TODO.md, reflecting the successful integration of task execution and validation. - Updated the status of tests related to the ContinueOnFailure option, indicating their completion with detailed test cases for various execution scenarios. - Enhanced run_test.go with new tests to validate the behavior of task execution under different ContinueOnFailure configurations, ensuring robust error handling and task management. - Revised the RunExecution method to accept configuration data, improving flexibility in execution parameters.
This commit is contained in:
Max
parent
9cc5be78ef
commit
603ed69a9e
3 changed files with 222 additions and 8 deletions
|
|
@ -766,13 +766,13 @@ Each phase test uses different expert combinations:
|
|||
|
||||
---
|
||||
|
||||
## Phase 9: P3 Run Implementation 🟡
|
||||
## Phase 9: P3 Run Implementation ✅
|
||||
|
||||
**Goal:** Implement P3 (Task Execution + Validation). P2 → P3 → stub P4-P5.
|
||||
|
||||
**Depends on:** Phase 8 (P2 Tasks + Validation Agent)
|
||||
|
||||
**Status:** Implementation complete, unit tests pending
|
||||
**Status:** Complete
|
||||
|
||||
### 9.1 Implementation ✅
|
||||
|
||||
|
|
@ -855,8 +855,11 @@ Created new `yao/assert` package for universal assertion/validation:
|
|||
- [x] Test: validateSemantic with Validation Agent
|
||||
- [x] Test: mergeResults logic (rule + semantic)
|
||||
|
||||
**TODO (Future):**
|
||||
- [ ] Test: ContinueOnFailure option (run_test.go)
|
||||
**Completed:**
|
||||
- [x] Test: ContinueOnFailure option (run_test.go) ✅
|
||||
- [x] `stops_on_first_failure_when_ContinueOnFailure_is_false`
|
||||
- [x] `continues_execution_when_ContinueOnFailure_is_true`
|
||||
- [x] `multiple_failures_with_ContinueOnFailure`
|
||||
|
||||
---
|
||||
|
||||
|
|
@ -1096,7 +1099,7 @@ func TestWithLLM(t *testing.T) {
|
|||
| 6. P0 Inspiration | ✅ | Inspiration Agent integration |
|
||||
| 7. P1 Goals | ✅ | Goal Generation Agent integration |
|
||||
| 8. P2 Tasks | ✅ | Task Planning Agent integration |
|
||||
| 9. P3 Run | 🟡 | Task execution + validation + yao/assert (tests pending) |
|
||||
| 9. P3 Run | ✅ | Task execution + validation + yao/assert + multi-turn conversation |
|
||||
| 10. P4 Delivery | ⬜ | Output delivery (email/file/webhook/notify) |
|
||||
| 11. P5 Learning | ⬜ | Learning Agent + KB save |
|
||||
| 12. API & Integration | ⬜ | Complete API, end-to-end tests |
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ func DefaultRunConfig() *RunConfig {
|
|||
// 3. If validation.NeedReply, continue conversation with validation.ReplyContent
|
||||
// 4. Repeat until validation.Complete or max turns exceeded
|
||||
// 5. Pass previous task results as context to next task
|
||||
func (e *Executor) RunExecution(ctx *robottypes.Context, exec *robottypes.Execution, _ interface{}) error {
|
||||
func (e *Executor) RunExecution(ctx *robottypes.Context, exec *robottypes.Execution, data interface{}) error {
|
||||
robot := exec.GetRobot()
|
||||
if robot == nil {
|
||||
return fmt.Errorf("robot not found in execution")
|
||||
|
|
@ -56,8 +56,13 @@ func (e *Executor) RunExecution(ctx *robottypes.Context, exec *robottypes.Execut
|
|||
return fmt.Errorf("no tasks to execute")
|
||||
}
|
||||
|
||||
// Get run configuration
|
||||
config := DefaultRunConfig()
|
||||
// Get run configuration from data or use default
|
||||
var config *RunConfig
|
||||
if cfg, ok := data.(*RunConfig); ok && cfg != nil {
|
||||
config = cfg
|
||||
} else {
|
||||
config = DefaultRunConfig()
|
||||
}
|
||||
|
||||
// Initialize results slice
|
||||
exec.Results = make([]robottypes.TaskResult, 0, len(exec.Tasks))
|
||||
|
|
|
|||
|
|
@ -321,6 +321,212 @@ func TestRunExecutionErrorHandling(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestRunExecutionContinueOnFailure(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
}
|
||||
|
||||
testutils.Prepare(t)
|
||||
defer testutils.Clean(t)
|
||||
|
||||
ctx := types.NewContext(context.Background(), testAuth())
|
||||
|
||||
t.Run("stops on first failure when ContinueOnFailure is false", func(t *testing.T) {
|
||||
robot := createRunTestRobot(t)
|
||||
exec := createRunTestExecution(robot)
|
||||
|
||||
// First task will fail (non-existent assistant), second should be skipped
|
||||
exec.Tasks = []types.Task{
|
||||
{
|
||||
ID: "task-001",
|
||||
ExecutorType: types.ExecutorAssistant,
|
||||
ExecutorID: "non.existent.assistant.xyz123",
|
||||
Messages: []agentcontext.Message{
|
||||
{Role: agentcontext.RoleUser, Content: "This will fail"},
|
||||
},
|
||||
Order: 0,
|
||||
Status: types.TaskPending,
|
||||
},
|
||||
{
|
||||
ID: "task-002",
|
||||
ExecutorType: types.ExecutorAssistant,
|
||||
ExecutorID: "experts.text-writer",
|
||||
Messages: []agentcontext.Message{
|
||||
{Role: agentcontext.RoleUser, Content: "Write a greeting"},
|
||||
},
|
||||
Order: 1,
|
||||
Status: types.TaskPending,
|
||||
},
|
||||
}
|
||||
|
||||
// Use default config (ContinueOnFailure = false)
|
||||
config := standard.DefaultRunConfig()
|
||||
assert.False(t, config.ContinueOnFailure)
|
||||
|
||||
e := standard.New()
|
||||
err := e.RunExecution(ctx, exec, config)
|
||||
|
||||
// Should return error
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "task-001")
|
||||
|
||||
// Only first task should have a result
|
||||
assert.Len(t, exec.Results, 1)
|
||||
|
||||
// First task failed
|
||||
assert.Equal(t, types.TaskFailed, exec.Tasks[0].Status)
|
||||
|
||||
// Second task should be skipped (not executed)
|
||||
assert.Equal(t, types.TaskSkipped, exec.Tasks[1].Status)
|
||||
})
|
||||
|
||||
t.Run("continues execution when ContinueOnFailure is true", func(t *testing.T) {
|
||||
robot := createRunTestRobot(t)
|
||||
exec := createRunTestExecution(robot)
|
||||
|
||||
// First task will fail, but second should still execute
|
||||
exec.Tasks = []types.Task{
|
||||
{
|
||||
ID: "task-001",
|
||||
ExecutorType: types.ExecutorAssistant,
|
||||
ExecutorID: "non.existent.assistant.xyz123",
|
||||
Messages: []agentcontext.Message{
|
||||
{Role: agentcontext.RoleUser, Content: "This will fail"},
|
||||
},
|
||||
Order: 0,
|
||||
Status: types.TaskPending,
|
||||
},
|
||||
{
|
||||
ID: "task-002",
|
||||
ExecutorType: types.ExecutorAssistant,
|
||||
ExecutorID: "experts.text-writer",
|
||||
Messages: []agentcontext.Message{
|
||||
{Role: agentcontext.RoleUser, Content: "Write a short greeting message"},
|
||||
},
|
||||
ExpectedOutput: "A greeting message",
|
||||
Order: 1,
|
||||
Status: types.TaskPending,
|
||||
},
|
||||
{
|
||||
ID: "task-003",
|
||||
ExecutorType: types.ExecutorAssistant,
|
||||
ExecutorID: "experts.text-writer",
|
||||
Messages: []agentcontext.Message{
|
||||
{Role: agentcontext.RoleUser, Content: "Write a farewell message"},
|
||||
},
|
||||
ExpectedOutput: "A farewell message",
|
||||
Order: 2,
|
||||
Status: types.TaskPending,
|
||||
},
|
||||
}
|
||||
|
||||
// Enable ContinueOnFailure
|
||||
config := standard.DefaultRunConfig()
|
||||
config.ContinueOnFailure = true
|
||||
|
||||
e := standard.New()
|
||||
err := e.RunExecution(ctx, exec, config)
|
||||
|
||||
// Should NOT return error when ContinueOnFailure is true
|
||||
assert.NoError(t, err)
|
||||
|
||||
// All tasks should have results
|
||||
assert.Len(t, exec.Results, 3)
|
||||
|
||||
// First task failed
|
||||
assert.Equal(t, types.TaskFailed, exec.Tasks[0].Status)
|
||||
assert.False(t, exec.Results[0].Success)
|
||||
|
||||
// Second and third tasks should have executed and completed
|
||||
assert.Equal(t, types.TaskCompleted, exec.Tasks[1].Status)
|
||||
assert.True(t, exec.Results[1].Success)
|
||||
|
||||
assert.Equal(t, types.TaskCompleted, exec.Tasks[2].Status)
|
||||
assert.True(t, exec.Results[2].Success)
|
||||
|
||||
t.Logf("Task 1 (failed): %v", exec.Results[0].Error)
|
||||
t.Logf("Task 2 (success): %v", exec.Results[1].Output)
|
||||
t.Logf("Task 3 (success): %v", exec.Results[2].Output)
|
||||
})
|
||||
|
||||
t.Run("multiple failures with ContinueOnFailure", func(t *testing.T) {
|
||||
robot := createRunTestRobot(t)
|
||||
exec := createRunTestExecution(robot)
|
||||
|
||||
// Mix of failing and succeeding tasks
|
||||
exec.Tasks = []types.Task{
|
||||
{
|
||||
ID: "task-001",
|
||||
ExecutorType: types.ExecutorAssistant,
|
||||
ExecutorID: "non.existent.assistant.1",
|
||||
Messages: []agentcontext.Message{
|
||||
{Role: agentcontext.RoleUser, Content: "Fail 1"},
|
||||
},
|
||||
Order: 0,
|
||||
Status: types.TaskPending,
|
||||
},
|
||||
{
|
||||
ID: "task-002",
|
||||
ExecutorType: types.ExecutorAssistant,
|
||||
ExecutorID: "experts.text-writer",
|
||||
Messages: []agentcontext.Message{
|
||||
{Role: agentcontext.RoleUser, Content: "Say hello"},
|
||||
},
|
||||
Order: 1,
|
||||
Status: types.TaskPending,
|
||||
},
|
||||
{
|
||||
ID: "task-003",
|
||||
ExecutorType: types.ExecutorAssistant,
|
||||
ExecutorID: "non.existent.assistant.2",
|
||||
Messages: []agentcontext.Message{
|
||||
{Role: agentcontext.RoleUser, Content: "Fail 2"},
|
||||
},
|
||||
Order: 2,
|
||||
Status: types.TaskPending,
|
||||
},
|
||||
{
|
||||
ID: "task-004",
|
||||
ExecutorType: types.ExecutorAssistant,
|
||||
ExecutorID: "experts.text-writer",
|
||||
Messages: []agentcontext.Message{
|
||||
{Role: agentcontext.RoleUser, Content: "Say goodbye"},
|
||||
},
|
||||
Order: 3,
|
||||
Status: types.TaskPending,
|
||||
},
|
||||
}
|
||||
|
||||
config := standard.DefaultRunConfig()
|
||||
config.ContinueOnFailure = true
|
||||
|
||||
e := standard.New()
|
||||
err := e.RunExecution(ctx, exec, config)
|
||||
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, exec.Results, 4)
|
||||
|
||||
// Check status pattern: fail, success, fail, success
|
||||
assert.Equal(t, types.TaskFailed, exec.Tasks[0].Status)
|
||||
assert.Equal(t, types.TaskCompleted, exec.Tasks[1].Status)
|
||||
assert.Equal(t, types.TaskFailed, exec.Tasks[2].Status)
|
||||
assert.Equal(t, types.TaskCompleted, exec.Tasks[3].Status)
|
||||
|
||||
// Count successes and failures
|
||||
successCount := 0
|
||||
failCount := 0
|
||||
for _, result := range exec.Results {
|
||||
if result.Success {
|
||||
successCount++
|
||||
} else {
|
||||
failCount++
|
||||
}
|
||||
}
|
||||
assert.Equal(t, 2, successCount)
|
||||
assert.Equal(t, 2, failCount)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunExecutionValidation(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping integration test")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue