Update TODO.md and run_test.go for P3 Run Implementation Completion
- Marked the P3 Run Implementation as complete in TODO.md, reflecting the successful integration of task execution and validation. - Updated the status of tests related to the ContinueOnFailure option, indicating their completion with detailed test cases for various execution scenarios. - Enhanced run_test.go with new tests to validate the behavior of task execution under different ContinueOnFailure configurations, ensuring robust error handling and task management. - Revised the RunExecution method to accept configuration data, improving flexibility in execution parameters.
This commit is contained in:
parent
9cc5be78ef
commit
603ed69a9e
3 changed files with 222 additions and 8 deletions
|
|
@ -766,13 +766,13 @@ Each phase test uses different expert combinations:
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Phase 9: P3 Run Implementation 🟡
|
## Phase 9: P3 Run Implementation ✅
|
||||||
|
|
||||||
**Goal:** Implement P3 (Task Execution + Validation). P2 → P3 → stub P4-P5.
|
**Goal:** Implement P3 (Task Execution + Validation). P2 → P3 → stub P4-P5.
|
||||||
|
|
||||||
**Depends on:** Phase 8 (P2 Tasks + Validation Agent)
|
**Depends on:** Phase 8 (P2 Tasks + Validation Agent)
|
||||||
|
|
||||||
**Status:** Implementation complete, unit tests pending
|
**Status:** Complete
|
||||||
|
|
||||||
### 9.1 Implementation ✅
|
### 9.1 Implementation ✅
|
||||||
|
|
||||||
|
|
@ -855,8 +855,11 @@ Created new `yao/assert` package for universal assertion/validation:
|
||||||
- [x] Test: validateSemantic with Validation Agent
|
- [x] Test: validateSemantic with Validation Agent
|
||||||
- [x] Test: mergeResults logic (rule + semantic)
|
- [x] Test: mergeResults logic (rule + semantic)
|
||||||
|
|
||||||
**TODO (Future):**
|
**Completed:**
|
||||||
- [ ] Test: ContinueOnFailure option (run_test.go)
|
- [x] Test: ContinueOnFailure option (run_test.go) ✅
|
||||||
|
- [x] `stops_on_first_failure_when_ContinueOnFailure_is_false`
|
||||||
|
- [x] `continues_execution_when_ContinueOnFailure_is_true`
|
||||||
|
- [x] `multiple_failures_with_ContinueOnFailure`
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
@ -1096,7 +1099,7 @@ func TestWithLLM(t *testing.T) {
|
||||||
| 6. P0 Inspiration | ✅ | Inspiration Agent integration |
|
| 6. P0 Inspiration | ✅ | Inspiration Agent integration |
|
||||||
| 7. P1 Goals | ✅ | Goal Generation Agent integration |
|
| 7. P1 Goals | ✅ | Goal Generation Agent integration |
|
||||||
| 8. P2 Tasks | ✅ | Task Planning Agent integration |
|
| 8. P2 Tasks | ✅ | Task Planning Agent integration |
|
||||||
| 9. P3 Run | 🟡 | Task execution + validation + yao/assert (tests pending) |
|
| 9. P3 Run | ✅ | Task execution + validation + yao/assert + multi-turn conversation |
|
||||||
| 10. P4 Delivery | ⬜ | Output delivery (email/file/webhook/notify) |
|
| 10. P4 Delivery | ⬜ | Output delivery (email/file/webhook/notify) |
|
||||||
| 11. P5 Learning | ⬜ | Learning Agent + KB save |
|
| 11. P5 Learning | ⬜ | Learning Agent + KB save |
|
||||||
| 12. API & Integration | ⬜ | Complete API, end-to-end tests |
|
| 12. API & Integration | ⬜ | Complete API, end-to-end tests |
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ func DefaultRunConfig() *RunConfig {
|
||||||
// 3. If validation.NeedReply, continue conversation with validation.ReplyContent
|
// 3. If validation.NeedReply, continue conversation with validation.ReplyContent
|
||||||
// 4. Repeat until validation.Complete or max turns exceeded
|
// 4. Repeat until validation.Complete or max turns exceeded
|
||||||
// 5. Pass previous task results as context to next task
|
// 5. Pass previous task results as context to next task
|
||||||
func (e *Executor) RunExecution(ctx *robottypes.Context, exec *robottypes.Execution, _ interface{}) error {
|
func (e *Executor) RunExecution(ctx *robottypes.Context, exec *robottypes.Execution, data interface{}) error {
|
||||||
robot := exec.GetRobot()
|
robot := exec.GetRobot()
|
||||||
if robot == nil {
|
if robot == nil {
|
||||||
return fmt.Errorf("robot not found in execution")
|
return fmt.Errorf("robot not found in execution")
|
||||||
|
|
@ -56,8 +56,13 @@ func (e *Executor) RunExecution(ctx *robottypes.Context, exec *robottypes.Execut
|
||||||
return fmt.Errorf("no tasks to execute")
|
return fmt.Errorf("no tasks to execute")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get run configuration
|
// Get run configuration from data or use default
|
||||||
config := DefaultRunConfig()
|
var config *RunConfig
|
||||||
|
if cfg, ok := data.(*RunConfig); ok && cfg != nil {
|
||||||
|
config = cfg
|
||||||
|
} else {
|
||||||
|
config = DefaultRunConfig()
|
||||||
|
}
|
||||||
|
|
||||||
// Initialize results slice
|
// Initialize results slice
|
||||||
exec.Results = make([]robottypes.TaskResult, 0, len(exec.Tasks))
|
exec.Results = make([]robottypes.TaskResult, 0, len(exec.Tasks))
|
||||||
|
|
|
||||||
|
|
@ -321,6 +321,212 @@ func TestRunExecutionErrorHandling(t *testing.T) {
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunExecutionContinueOnFailure(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration test")
|
||||||
|
}
|
||||||
|
|
||||||
|
testutils.Prepare(t)
|
||||||
|
defer testutils.Clean(t)
|
||||||
|
|
||||||
|
ctx := types.NewContext(context.Background(), testAuth())
|
||||||
|
|
||||||
|
t.Run("stops on first failure when ContinueOnFailure is false", func(t *testing.T) {
|
||||||
|
robot := createRunTestRobot(t)
|
||||||
|
exec := createRunTestExecution(robot)
|
||||||
|
|
||||||
|
// First task will fail (non-existent assistant), second should be skipped
|
||||||
|
exec.Tasks = []types.Task{
|
||||||
|
{
|
||||||
|
ID: "task-001",
|
||||||
|
ExecutorType: types.ExecutorAssistant,
|
||||||
|
ExecutorID: "non.existent.assistant.xyz123",
|
||||||
|
Messages: []agentcontext.Message{
|
||||||
|
{Role: agentcontext.RoleUser, Content: "This will fail"},
|
||||||
|
},
|
||||||
|
Order: 0,
|
||||||
|
Status: types.TaskPending,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "task-002",
|
||||||
|
ExecutorType: types.ExecutorAssistant,
|
||||||
|
ExecutorID: "experts.text-writer",
|
||||||
|
Messages: []agentcontext.Message{
|
||||||
|
{Role: agentcontext.RoleUser, Content: "Write a greeting"},
|
||||||
|
},
|
||||||
|
Order: 1,
|
||||||
|
Status: types.TaskPending,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use default config (ContinueOnFailure = false)
|
||||||
|
config := standard.DefaultRunConfig()
|
||||||
|
assert.False(t, config.ContinueOnFailure)
|
||||||
|
|
||||||
|
e := standard.New()
|
||||||
|
err := e.RunExecution(ctx, exec, config)
|
||||||
|
|
||||||
|
// Should return error
|
||||||
|
assert.Error(t, err)
|
||||||
|
assert.Contains(t, err.Error(), "task-001")
|
||||||
|
|
||||||
|
// Only first task should have a result
|
||||||
|
assert.Len(t, exec.Results, 1)
|
||||||
|
|
||||||
|
// First task failed
|
||||||
|
assert.Equal(t, types.TaskFailed, exec.Tasks[0].Status)
|
||||||
|
|
||||||
|
// Second task should be skipped (not executed)
|
||||||
|
assert.Equal(t, types.TaskSkipped, exec.Tasks[1].Status)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("continues execution when ContinueOnFailure is true", func(t *testing.T) {
|
||||||
|
robot := createRunTestRobot(t)
|
||||||
|
exec := createRunTestExecution(robot)
|
||||||
|
|
||||||
|
// First task will fail, but second should still execute
|
||||||
|
exec.Tasks = []types.Task{
|
||||||
|
{
|
||||||
|
ID: "task-001",
|
||||||
|
ExecutorType: types.ExecutorAssistant,
|
||||||
|
ExecutorID: "non.existent.assistant.xyz123",
|
||||||
|
Messages: []agentcontext.Message{
|
||||||
|
{Role: agentcontext.RoleUser, Content: "This will fail"},
|
||||||
|
},
|
||||||
|
Order: 0,
|
||||||
|
Status: types.TaskPending,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "task-002",
|
||||||
|
ExecutorType: types.ExecutorAssistant,
|
||||||
|
ExecutorID: "experts.text-writer",
|
||||||
|
Messages: []agentcontext.Message{
|
||||||
|
{Role: agentcontext.RoleUser, Content: "Write a short greeting message"},
|
||||||
|
},
|
||||||
|
ExpectedOutput: "A greeting message",
|
||||||
|
Order: 1,
|
||||||
|
Status: types.TaskPending,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "task-003",
|
||||||
|
ExecutorType: types.ExecutorAssistant,
|
||||||
|
ExecutorID: "experts.text-writer",
|
||||||
|
Messages: []agentcontext.Message{
|
||||||
|
{Role: agentcontext.RoleUser, Content: "Write a farewell message"},
|
||||||
|
},
|
||||||
|
ExpectedOutput: "A farewell message",
|
||||||
|
Order: 2,
|
||||||
|
Status: types.TaskPending,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Enable ContinueOnFailure
|
||||||
|
config := standard.DefaultRunConfig()
|
||||||
|
config.ContinueOnFailure = true
|
||||||
|
|
||||||
|
e := standard.New()
|
||||||
|
err := e.RunExecution(ctx, exec, config)
|
||||||
|
|
||||||
|
// Should NOT return error when ContinueOnFailure is true
|
||||||
|
assert.NoError(t, err)
|
||||||
|
|
||||||
|
// All tasks should have results
|
||||||
|
assert.Len(t, exec.Results, 3)
|
||||||
|
|
||||||
|
// First task failed
|
||||||
|
assert.Equal(t, types.TaskFailed, exec.Tasks[0].Status)
|
||||||
|
assert.False(t, exec.Results[0].Success)
|
||||||
|
|
||||||
|
// Second and third tasks should have executed and completed
|
||||||
|
assert.Equal(t, types.TaskCompleted, exec.Tasks[1].Status)
|
||||||
|
assert.True(t, exec.Results[1].Success)
|
||||||
|
|
||||||
|
assert.Equal(t, types.TaskCompleted, exec.Tasks[2].Status)
|
||||||
|
assert.True(t, exec.Results[2].Success)
|
||||||
|
|
||||||
|
t.Logf("Task 1 (failed): %v", exec.Results[0].Error)
|
||||||
|
t.Logf("Task 2 (success): %v", exec.Results[1].Output)
|
||||||
|
t.Logf("Task 3 (success): %v", exec.Results[2].Output)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("multiple failures with ContinueOnFailure", func(t *testing.T) {
|
||||||
|
robot := createRunTestRobot(t)
|
||||||
|
exec := createRunTestExecution(robot)
|
||||||
|
|
||||||
|
// Mix of failing and succeeding tasks
|
||||||
|
exec.Tasks = []types.Task{
|
||||||
|
{
|
||||||
|
ID: "task-001",
|
||||||
|
ExecutorType: types.ExecutorAssistant,
|
||||||
|
ExecutorID: "non.existent.assistant.1",
|
||||||
|
Messages: []agentcontext.Message{
|
||||||
|
{Role: agentcontext.RoleUser, Content: "Fail 1"},
|
||||||
|
},
|
||||||
|
Order: 0,
|
||||||
|
Status: types.TaskPending,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "task-002",
|
||||||
|
ExecutorType: types.ExecutorAssistant,
|
||||||
|
ExecutorID: "experts.text-writer",
|
||||||
|
Messages: []agentcontext.Message{
|
||||||
|
{Role: agentcontext.RoleUser, Content: "Say hello"},
|
||||||
|
},
|
||||||
|
Order: 1,
|
||||||
|
Status: types.TaskPending,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "task-003",
|
||||||
|
ExecutorType: types.ExecutorAssistant,
|
||||||
|
ExecutorID: "non.existent.assistant.2",
|
||||||
|
Messages: []agentcontext.Message{
|
||||||
|
{Role: agentcontext.RoleUser, Content: "Fail 2"},
|
||||||
|
},
|
||||||
|
Order: 2,
|
||||||
|
Status: types.TaskPending,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "task-004",
|
||||||
|
ExecutorType: types.ExecutorAssistant,
|
||||||
|
ExecutorID: "experts.text-writer",
|
||||||
|
Messages: []agentcontext.Message{
|
||||||
|
{Role: agentcontext.RoleUser, Content: "Say goodbye"},
|
||||||
|
},
|
||||||
|
Order: 3,
|
||||||
|
Status: types.TaskPending,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
config := standard.DefaultRunConfig()
|
||||||
|
config.ContinueOnFailure = true
|
||||||
|
|
||||||
|
e := standard.New()
|
||||||
|
err := e.RunExecution(ctx, exec, config)
|
||||||
|
|
||||||
|
assert.NoError(t, err)
|
||||||
|
assert.Len(t, exec.Results, 4)
|
||||||
|
|
||||||
|
// Check status pattern: fail, success, fail, success
|
||||||
|
assert.Equal(t, types.TaskFailed, exec.Tasks[0].Status)
|
||||||
|
assert.Equal(t, types.TaskCompleted, exec.Tasks[1].Status)
|
||||||
|
assert.Equal(t, types.TaskFailed, exec.Tasks[2].Status)
|
||||||
|
assert.Equal(t, types.TaskCompleted, exec.Tasks[3].Status)
|
||||||
|
|
||||||
|
// Count successes and failures
|
||||||
|
successCount := 0
|
||||||
|
failCount := 0
|
||||||
|
for _, result := range exec.Results {
|
||||||
|
if result.Success {
|
||||||
|
successCount++
|
||||||
|
} else {
|
||||||
|
failCount++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.Equal(t, 2, successCount)
|
||||||
|
assert.Equal(t, 2, failCount)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunExecutionValidation(t *testing.T) {
|
func TestRunExecutionValidation(t *testing.T) {
|
||||||
if testing.Short() {
|
if testing.Short() {
|
||||||
t.Skip("Skipping integration test")
|
t.Skip("Skipping integration test")
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue