Merge pull request #1425 from trheyi/main

Add Robot E2E Tests and Update Makefile and Workflows
This commit is contained in:
Max 2026-01-19 12:00:30 +08:00 committed by GitHub
commit 193b597e1a
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
14 changed files with 2787 additions and 53 deletions

View file

@ -563,6 +563,178 @@ jobs:
body: '✅ AI Tests (agent, aigc) passed!'
});
# =============================================================================
# Robot E2E Tests (agent/robot/api) - Run TestE2E* with real LLM calls
# =============================================================================
RobotE2ETest:
runs-on: ubuntu-latest
services:
mcp-everything:
image: yaoapp/mcp-everything:latest
ports:
- "3021:3021"
- "3022:3022"
mongodb:
image: mongo:6.0
ports:
- 27017:27017
env:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: 123456
MONGO_INITDB_DATABASE: test
strategy:
matrix:
go: ["1.25"]
if: >
${{ github.event.workflow_run.event == 'pull_request' &&
github.event.workflow_run.conclusion == 'success' }}
steps:
- name: "Download artifact"
uses: actions/github-script@v7
with:
script: |
var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: ${{github.event.workflow_run.id }},
});
var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
return artifact.name == "pr"
})[0];
var download = await github.rest.actions.downloadArtifact({
owner: context.repo.owner,
repo: context.repo.repo,
artifact_id: matchArtifact.id,
archive_format: 'zip',
});
var fs = require('fs');
fs.writeFileSync('${{github.workspace}}/pr.zip', Buffer.from(download.data));
- name: "Read NR & SHA"
run: |
unzip pr.zip
cat NR
cat SHA
echo HEAD=$(cat SHA) >> $GITHUB_ENV
echo NR=$(cat NR) >> $GITHUB_ENV
- name: "Comment on PR"
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const { NR } = process.env
var issue_number = NR;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue_number,
body: '🤖 Robot E2E Tests running with SQLite...'
});
- name: Checkout Kun
uses: actions/checkout@v4
with:
repository: yaoapp/kun
path: kun
- name: Checkout Xun
uses: actions/checkout@v4
with:
repository: yaoapp/xun
path: xun
- name: Checkout Gou
uses: actions/checkout@v4
with:
repository: yaoapp/gou
path: gou
- name: Checkout V8Go
uses: actions/checkout@v4
with:
repository: yaoapp/v8go
path: v8go
- name: Unzip libv8
run: |
files=$(find ./v8go -name "libv8*.zip")
for file in $files; do
dir=$(dirname "$file")
echo "Extracting $file to directory $dir"
unzip -o -d $dir $file
rm -rf $dir/__MACOSX
done
- name: Checkout Demo App
uses: actions/checkout@v4
with:
repository: yaoapp/yao-dev-app
path: app
- name: Checkout Extension
uses: actions/checkout@v4
with:
repository: yaoapp/yao-extensions-dev
path: extension
- name: Move Dependencies
run: |
mv kun ../
mv xun ../
mv gou ../
mv v8go ../
mv app ../
mv extension ../
- name: Checkout pull request HEAD commit
uses: actions/checkout@v4
with:
ref: ${{ env.HEAD }}
- name: Setup Go ${{ matrix.go }}
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go }}
- name: Start Redis
uses: supercharge/redis-github-action@1.4.0
with:
redis-version: 6
- name: Setup Go Tools
run: make tools
- name: Setup ENV (SQLite)
run: |
mkdir -p ${{ github.WORKSPACE }}/../app/db
echo "YAO_DB_DRIVER=sqlite3" >> $GITHUB_ENV
echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV
- name: Run Robot E2E Tests
run: make unit-test-robot-e2e
- name: Codecov Report
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
- name: "Comment on PR - Robot E2E Tests Done"
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const { NR } = process.env
var issue_number = NR;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: issue_number,
body: '✅ Robot E2E Tests passed!'
});
# =============================================================================
# Benchmark & Memory Leak Tests - Run with MySQL8.0 and SQLite3
# =============================================================================

View file

@ -446,6 +446,116 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}
# =============================================================================
# Robot E2E Tests (agent/robot/api) - Run TestE2E* with real LLM calls
# =============================================================================
robot-e2e-test:
runs-on: ubuntu-latest
services:
mcp-everything:
image: yaoapp/mcp-everything:latest
ports:
- "3021:3021"
- "3022:3022"
mongodb:
image: mongo:6.0
ports:
- 27017:27017
env:
MONGO_INITDB_ROOT_USERNAME: root
MONGO_INITDB_ROOT_PASSWORD: 123456
MONGO_INITDB_DATABASE: test
strategy:
matrix:
go: ["1.25"]
steps:
- name: Checkout Kun
uses: actions/checkout@v4
with:
repository: ${{ env.REPO_KUN }}
path: kun
- name: Checkout Xun
uses: actions/checkout@v4
with:
repository: ${{ env.REPO_XUN }}
path: xun
- name: Checkout Gou
uses: actions/checkout@v4
with:
repository: ${{ env.REPO_GOU }}
path: gou
- name: Checkout V8Go
uses: actions/checkout@v4
with:
repository: yaoapp/v8go
path: v8go
- name: Unzip libv8
run: |
files=$(find ./v8go -name "libv8*.zip")
for file in $files; do
dir=$(dirname "$file")
echo "Extracting $file to directory $dir"
unzip -o -d $dir $file
rm -rf $dir/__MACOSX
done
- name: Checkout Demo App
uses: actions/checkout@v4
with:
repository: yaoapp/yao-dev-app
path: app
- name: Checkout Extension
uses: actions/checkout@v4
with:
repository: yaoapp/yao-extensions-dev
path: extension
- name: Move Dependencies
run: |
mv kun ../
mv xun ../
mv gou ../
mv v8go ../
mv app ../
mv extension ../
- name: Checkout Code
uses: actions/checkout@v4
- name: Setup Go ${{ matrix.go }}
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go }}
- name: Start Redis
uses: supercharge/redis-github-action@1.4.0
with:
redis-version: 6
- name: Setup Go Tools
run: make tools
- name: Setup ENV (SQLite)
run: |
mkdir -p ${{ github.WORKSPACE }}/../app/db
echo "YAO_DB_DRIVER=sqlite3" >> $GITHUB_ENV
echo "YAO_DB_PRIMARY=${{ github.WORKSPACE }}/../app/db/yao.db" >> $GITHUB_ENV
- name: Run Robot E2E Tests
run: make unit-test-robot-e2e
- name: Codecov Report
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
# =============================================================================
# Benchmark & Memory Leak Tests - Run with MySQL8.0 and SQLite3
# =============================================================================

View file

@ -13,10 +13,12 @@ OS := $(shell uname)
TESTFOLDER := $(shell $(GO) list ./... | grep -vE 'examples|openai|aigc|neo|twilio|share*' | awk '!/\/tests\// || /openapi\/tests/')
# Core tests (exclude AI-related: agent, aigc, openai, and KB)
TESTFOLDER_CORE := $(shell $(GO) list ./... | grep -vE 'examples|openai|aigc|neo|twilio|share*|agent|kb' | awk '!/\/tests\// || /openapi\/tests/')
# AI tests (agent, aigc) - exclude agent/search/handlers/web (requires external API keys)
# AI tests (agent, aigc) - exclude agent/search/handlers/web (requires external API keys) and robot/api E2E tests
TESTFOLDER_AI := $(shell $(GO) list ./agent/... ./aigc/... | grep -v 'agent/search/handlers/web')
# KB tests (kb)
TESTFOLDER_KB := $(shell $(GO) list ./kb/...)
# Robot E2E tests (agent/robot/api) - runs TestE2E* tests with real LLM calls
TESTFOLDER_ROBOT_E2E := $(shell $(GO) list ./agent/robot/api/...)
TESTTAGS ?= ""
# TESTWIDGETS := $(shell $(GO) list ./widgets/...)
@ -73,12 +75,12 @@ unit-test-core:
fi; \
done
# AI Unit Test (agent, aigc)
# AI Unit Test (agent, aigc) - excludes TestE2E* (run separately in unit-test-robot-e2e)
.PHONY: unit-test-ai
unit-test-ai:
echo "mode: count" > coverage.out
for d in $(TESTFOLDER_AI); do \
$(GO) test -tags $(TESTTAGS) -v -timeout=20m -covermode=count -coverprofile=profile.out -coverpkg=$$(echo $$d | sed "s/\/test$$//g") -skip='TestMemoryLeak|TestIsolateDisposal' $$d > tmp.out; \
$(GO) test -tags $(TESTTAGS) -v -timeout=20m -covermode=count -coverprofile=profile.out -coverpkg=$$(echo $$d | sed "s/\/test$$//g") -skip='TestMemoryLeak|TestIsolateDisposal|TestE2E' $$d > tmp.out; \
cat tmp.out; \
if grep -q "^--- FAIL" tmp.out; then \
rm tmp.out; \
@ -137,6 +139,39 @@ unit-test-kb:
fi; \
done
# Robot E2E Test (agent/robot/api) - runs TestE2E* tests with real LLM calls
# These tests require: LLM API keys, database, and longer timeout
.PHONY: unit-test-robot-e2e
unit-test-robot-e2e:
echo "mode: count" > coverage.out
for d in $(TESTFOLDER_ROBOT_E2E); do \
$(GO) test -tags $(TESTTAGS) -v -timeout=30m -covermode=count -coverprofile=profile.out -coverpkg=$$(echo $$d | sed "s/\/test$$//g") -run='TestE2E' $$d > tmp.out; \
cat tmp.out; \
if grep -q "^--- FAIL" tmp.out; then \
rm tmp.out; \
exit 1; \
elif grep -q "^FAIL" tmp.out; then \
rm tmp.out; \
exit 1; \
elif grep -q "^panic:" tmp.out; then \
rm tmp.out; \
exit 1; \
elif grep -q "build failed" tmp.out; then \
rm tmp.out; \
exit 1; \
elif grep -q "setup failed" tmp.out; then \
rm tmp.out; \
exit 1; \
elif grep -q "runtime error" tmp.out; then \
rm tmp.out; \
exit 1; \
fi; \
if [ -f profile.out ]; then \
cat profile.out | grep -v "mode:" >> coverage.out; \
rm profile.out; \
fi; \
done
# Benchmark Test
.PHONY: benchmark
benchmark:

View file

@ -1116,26 +1116,47 @@ type DeliveryContext struct {
### 11.2 End-to-End Tests
- [ ] Full clock trigger flow (P0 → P1 → P2 → P3 → P4)
- [ ] Human intervention flow (P1 → P2 → P3 → P4)
- [ ] Event trigger flow (P1 → P2 → P3 → P4)
- [ ] Concurrent execution test
- [ ] Pause/Resume/Stop test
### 11.3 Integration with OpenAPI
- [ ] HTTP endpoints for human intervention
- [ ] Webhook endpoints for events
- [x] Full clock trigger flow (P0 → P1 → P2 → P3 → P4) - `e2e_clock_test.go`
- [x] Human intervention flow (P1 → P2 → P3 → P4) - `e2e_human_test.go`
- [x] Event trigger flow (P1 → P2 → P3 → P4) - `e2e_event_test.go`
- [x] Concurrent execution test - `e2e_concurrent_test.go`
- [x] Pause/Resume/Stop test - `e2e_control_test.go`
---
## Phase 12: Advanced Features
## Phase 12: OpenAPI Integration
**Goal:** HTTP endpoints for Robot Agent management and triggers.
**Depends on:** Phase 11 (API & Integration), Frontend UI Design
> **Note:** This phase will be planned in detail after frontend UI design is complete.
> The API endpoints will be designed based on actual UI requirements.
### 12.1 Planned Features
- [ ] HTTP endpoints for robot management (CRUD)
- [ ] HTTP endpoints for human intervention triggers
- [ ] Webhook endpoints for external events
- [ ] WebSocket for real-time execution status updates
- [ ] Authentication and authorization integration
### 12.2 Design Dependencies
- Frontend dashboard design (robot list, status, controls)
- Execution history UI design
- Human intervention UI design
- Real-time notification requirements
---
## Phase 13: Advanced Features
**Goal:** P5 Learning, Process/JSAPI wrappers, dedup, plan queue.
> **Note:** These are optional features. Main flow works without them.
### 12.1 Process & JSAPI Wrappers
### 13.1 Process & JSAPI Wrappers
> **Note:** These are convenience wrappers around Go API for Yao ecosystem integration.
@ -1148,37 +1169,37 @@ type DeliveryContext struct {
- [ ] `api/jsapi.go` - implement JSAPI for JavaScript runtime
- [ ] Tests for Process and JSAPI
### 12.3 P5 Learning Implementation
### 13.2 P5 Learning Implementation
> **Background:** P5 Learning is async, runs after P4 Delivery completes.
> User doesn't wait for it. Results stored in private KB for future reference.
#### 12.3.1 Learning Agent Setup
#### 13.2.1 Learning Agent Setup
- [ ] `robot/learning/package.yao` - Learning Agent config
- [ ] `robot/learning/prompts.yml` - learning prompts
#### 12.3.2 Store Implementation
#### 13.2.2 Store Implementation
- [ ] `store/store.go` - Store interface and struct
- [ ] `store/kb.go` - KB operations (create, save, search)
- [ ] `store/learning.go` - save learning entries to private KB
#### 12.3.3 Implementation
#### 13.2.3 Implementation
- [ ] `executor/learning.go` - `RunLearning(ctx, exec, data)` - real implementation
- [ ] `executor/learning.go` - extract learnings from execution
- [ ] `executor/learning.go` - call Learning Agent
- [ ] `executor/learning.go` - save to private KB
#### 12.3.4 Tests
#### 13.2.4 Tests
- [ ] `executor/learning_test.go` - P5 learning
- [ ] Test: learnings extracted from execution
- [ ] Test: learnings saved to KB
- [ ] Test: KB can be queried for past learnings
### 12.5 Fast Dedup (Time-Window)
### 13.3 Fast Dedup (Time-Window)
> **Note:** Manager has `// TODO: dedup check` comment placeholder. Integrate after implementation.
@ -1190,13 +1211,13 @@ type DeliveryContext struct {
- [ ] Integrate into Manager.Tick()
- [ ] Test: dedup check/mark, window expiry
### 12.6 Semantic Dedup
### 13.4 Semantic Dedup
- [ ] `dedup/semantic.go` - call Dedup Agent for goal/task level dedup
- [ ] Dedup Agent setup (`assistants/robot/dedup/`)
- [ ] Test: semantic dedup with real LLM
### 12.7 Plan Queue
### 13.5 Plan Queue
- [ ] `plan/plan.go` - plan queue implementation
- [ ] Store planned tasks/goals
@ -1325,13 +1346,15 @@ func TestWithLLM(t *testing.T) {
| 8. P2 Tasks | ✅ | Task Planning Agent integration |
| 9. P3 Run | ✅ | Task execution + validation + yao/assert + multi-turn conversation |
| 10. P4 Delivery | ✅ | Output delivery (email/webhook/process, notify future) |
| 11. API & Integration | ⬜ | Go API, end-to-end tests (main flow: P0→P1→P2→P3→P4) |
| 12. Advanced | ⬜ | Process/JSAPI, P5 Learning, dedup, plan queue, Sandbox |
| 11. API & Integration | ✅ | Go API, end-to-end tests (main flow: P0→P1→P2→P3→P4) |
| 12. OpenAPI | ⬜ | HTTP endpoints (depends on frontend UI design) |
| 13. Advanced | ⬜ | Process/JSAPI, P5 Learning, dedup, plan queue, Sandbox |
Legend: ⬜ Not started | 🟡 In progress | ✅ Complete
**Main Flow (MVP):** P0 Inspiration → P1 Goals → P2 Tasks → P3 Run → P4 Delivery
**Advanced (Optional):** P5 Learning (async), Dedup, Plan Queue, Sandbox
**Main Flow (MVP):** P0 Inspiration → P1 Goals → P2 Tasks → P3 Run → P4 Delivery ✅
**OpenAPI (Phase 12):** HTTP endpoints - planned after frontend UI design
**Advanced (Phase 13):** P5 Learning (async), Process/JSAPI, Dedup, Plan Queue, Sandbox
---

View file

@ -0,0 +1,414 @@
package api_test
// End-to-end tests for Clock trigger flow
// These tests use REAL LLM calls via Standard executor (not DryRun)
//
// Test Flow: Clock Trigger → P0 (Inspiration) → P1 (Goals) → P2 (Tasks) → P3 (Run) → P4 (Delivery)
//
// Prerequisites:
// - Valid LLM API keys (OPENAI_TEST_KEY or DEEPSEEK_API_KEY)
// - Test assistants in yao-dev-app/assistants/robot/
// - Database connection (YAO_DB_PRIMARY)
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/xun/capsule"
"github.com/yaoapp/yao/agent/robot/api"
"github.com/yaoapp/yao/agent/robot/types"
"github.com/yaoapp/yao/agent/testutils"
oauthtypes "github.com/yaoapp/yao/openapi/oauth/types"
)
// testAuth returns test auth info for E2E tests
func testAuth() *oauthtypes.AuthorizedInfo {
return &oauthtypes.AuthorizedInfo{
UserID: "e2e-test-user",
TeamID: "e2e-test-team",
}
}
// TestE2EClockTriggerFullFlow tests the complete clock trigger flow with real LLM calls
// Flow: Clock → P0 (Inspiration) → P1 (Goals) → P2 (Tasks) → P3 (Run) → P4 (Delivery)
func TestE2EClockTriggerFullFlow(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("complete_P0_to_P4_flow", func(t *testing.T) {
// Setup: Create a robot configured for clock trigger
memberID := "robot_e2e_clock_001"
setupE2ERobotForClock(t, memberID, "team_e2e_clock")
// Start the API system
err := api.Start()
require.NoError(t, err)
defer api.Stop()
// Verify robot is loaded
ctx := types.NewContext(context.Background(), testAuth())
robot, err := api.GetRobot(ctx, memberID)
require.NoError(t, err)
require.NotNil(t, robot)
assert.Equal(t, memberID, robot.MemberID)
// Trigger execution via clock trigger type
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
require.NoError(t, err)
require.NotNil(t, result)
assert.True(t, result.Accepted, "Clock trigger should be accepted: %s", result.Message)
assert.NotEmpty(t, result.JobID, "Should return job ID")
t.Logf("Execution started: JobID=%s", result.JobID)
// Wait for execution to complete (real LLM calls take time)
// P0→P4 typically takes 30-60 seconds with real LLM
var exec *types.Execution
maxWait := 180 * time.Second
pollInterval := 2 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(pollInterval)
// Query all executions and find a completed one
executions, err := api.ListExecutions(ctx, memberID, &api.ExecutionQuery{
Page: 1,
PageSize: 10,
})
if err != nil {
t.Logf("Query error (retrying): %v", err)
continue
}
// Look for a completed execution
for _, e := range executions.Data {
t.Logf("Execution %s: status=%s, phase=%s", e.ID, e.Status, e.Phase)
if e.Status == types.ExecCompleted {
exec = e
break
}
}
if exec != nil {
break
}
// Also check if there's any running execution
hasRunning := false
for _, e := range executions.Data {
if e.Status == types.ExecRunning || e.Status == types.ExecPending {
hasRunning = true
break
}
}
if !hasRunning && len(executions.Data) > 0 {
// All executions finished but none completed - take the first one for error reporting
exec = executions.Data[0]
break
}
}
// Verify execution completed successfully - ALL phases must pass
require.NotNil(t, exec, "Execution should exist")
if exec.Status == types.ExecFailed {
t.Fatalf("Execution failed: %s", exec.Error)
}
// Strict assertion: execution MUST complete successfully
assert.Equal(t, types.ExecCompleted, exec.Status, "Execution must complete successfully")
// Verify P0 (Inspiration) output exists
require.NotNil(t, exec.Inspiration, "P0 Inspiration output must exist")
t.Logf("P0 Inspiration: %+v", exec.Inspiration)
// Verify P1 (Goals) output exists
require.NotNil(t, exec.Goals, "P1 Goals output must exist")
t.Logf("P1 Goals content length: %d", len(exec.Goals.Content))
// Verify P2 (Tasks) output exists
require.NotNil(t, exec.Tasks, "P2 Tasks output must exist")
require.Greater(t, len(exec.Tasks), 0, "P2 must have at least 1 task")
t.Logf("P2 Tasks count: %d", len(exec.Tasks))
// Verify P3 (Results) output exists - THIS IS CRITICAL
require.NotNil(t, exec.Results, "P3 Results output must exist")
require.Greater(t, len(exec.Results), 0, "P3 must have at least 1 result")
t.Logf("P3 Results count: %d", len(exec.Results))
// Verify P4 (Delivery) output exists
require.NotNil(t, exec.Delivery, "P4 Delivery output must exist")
t.Logf("P4 Delivery: RequestID=%s, Success=%v", exec.Delivery.RequestID, exec.Delivery.Success)
t.Logf("✅ Clock trigger E2E: ALL PHASES (P0-P4) completed successfully")
})
}
// TestE2EClockTriggerPhaseProgression tests that phases execute in correct order
func TestE2EClockTriggerPhaseProgression(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("phases_execute_P0_P1_P2_P3_P4", func(t *testing.T) {
memberID := "robot_e2e_clock_phases"
setupE2ERobotForClock(t, memberID, "team_e2e_clock")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuth())
// Trigger execution
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
require.NoError(t, err)
assert.True(t, result.Accepted)
// Track phase progression
phasesObserved := make([]types.Phase, 0)
lastPhase := types.Phase("")
maxWait := 120 * time.Second
pollInterval := 1 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(pollInterval)
executions, err := api.ListExecutions(ctx, memberID, &api.ExecutionQuery{
Page: 1,
PageSize: 1,
})
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
// Record phase changes
if exec.Phase != lastPhase {
phasesObserved = append(phasesObserved, exec.Phase)
lastPhase = exec.Phase
t.Logf("Phase changed to: %s", exec.Phase)
}
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
break
}
}
// Verify phase order (should include at least P0, P1, P2, P3, P4)
t.Logf("Phases observed: %v", phasesObserved)
assert.GreaterOrEqual(t, len(phasesObserved), 1, "Should observe at least one phase")
// The final phase should be delivery or learning
if len(phasesObserved) > 0 {
lastObserved := phasesObserved[len(phasesObserved)-1]
assert.True(t,
lastObserved == types.PhaseDelivery || lastObserved == types.PhaseLearning,
"Last phase should be delivery or learning, got: %s", lastObserved)
}
})
}
// TestE2EClockTriggerDataPersistence tests that execution data is persisted to database
func TestE2EClockTriggerDataPersistence(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("execution_data_persisted_to_database", func(t *testing.T) {
memberID := "robot_e2e_clock_persist"
setupE2ERobotForClock(t, memberID, "team_e2e_clock")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuth())
// Trigger and wait for completion
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
require.NoError(t, err)
assert.True(t, result.Accepted)
// Wait for completion
var execID string
maxWait := 120 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(2 * time.Second)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
execID = exec.ID
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
break
}
}
require.NotEmpty(t, execID, "Should have execution ID")
// Query execution by ID to verify persistence
exec, err := api.GetExecution(ctx, execID)
require.NoError(t, err)
require.NotNil(t, exec)
// Verify all fields are persisted
assert.Equal(t, execID, exec.ID)
assert.Equal(t, memberID, exec.MemberID)
assert.Equal(t, types.TriggerClock, exec.TriggerType)
assert.NotNil(t, exec.StartTime, "StartTime should be set")
if exec.Status == types.ExecCompleted {
assert.NotNil(t, exec.EndTime, "EndTime should be set for completed execution")
}
t.Logf("Persisted execution: ID=%s, Status=%s, Phase=%s", exec.ID, exec.Status, exec.Phase)
})
}
// ==================== Helper Functions ====================
// setupE2ERobotForClock creates a robot configured for clock trigger E2E tests
// Uses extremely simple tasks to ensure quick completion through all phases
func setupE2ERobotForClock(t *testing.T, memberID, teamID string) {
m := model.Select("__yao.member")
tableName := m.MetaData.Table.Name
qb := capsule.Query()
// Robot config optimized for E2E testing - tasks must complete quickly
robotConfig := map[string]interface{}{
"identity": map[string]interface{}{
"role": "Greeter Bot",
"duties": []string{"Output greeting message"}, // Extremely simple
"rules": []string{"Always complete in one step", "No tools needed", "Just output text directly"},
},
"quota": map[string]interface{}{
"max": 5,
"queue": 20,
"priority": 5,
},
"triggers": map[string]interface{}{
"clock": map[string]interface{}{"enabled": true},
"intervene": map[string]interface{}{"enabled": true},
"event": map[string]interface{}{"enabled": true},
},
"resources": map[string]interface{}{
"phases": map[string]interface{}{
"inspiration": "robot.inspiration",
"goals": "robot.goals",
"tasks": "tests.e2e-tasks", // Use simple E2E test task planner
"run": "robot.validation",
"validation": "tests.e2e-validation", // Use lenient E2E test validator
"delivery": "robot.delivery",
"learning": "robot.learning",
},
"agents": []string{},
},
"delivery": map[string]interface{}{
"email": map[string]interface{}{"enabled": false},
"webhook": map[string]interface{}{"enabled": false},
"process": map[string]interface{}{"enabled": false},
},
}
configJSON, _ := json.Marshal(robotConfig)
// Critical: System prompt must guide LLM to generate simple, completable tasks
systemPrompt := `You are a simple greeter bot for E2E testing.
CRITICAL RULES FOR ALL PHASES:
1. Goals: Generate exactly ONE goal: "Output a greeting"
2. Tasks: Generate exactly ONE task with description "Say Hello World"
3. Run: Complete the task by directly outputting "Hello World!" - NO tools needed
4. The task is complete when you output the greeting text
You do NOT need any tools or external calls. Just output text directly.
When asked to execute a task, immediately respond with the greeting and mark complete.`
err := qb.Table(tableName).Insert([]map[string]interface{}{
{
"member_id": memberID,
"team_id": teamID,
"member_type": "robot",
"display_name": "E2E Clock Test Robot " + memberID,
"system_prompt": systemPrompt,
"status": "active",
"role_id": "member",
"autonomous_mode": true,
"robot_status": "idle",
"robot_config": string(configJSON),
},
})
if err != nil {
t.Fatalf("Failed to insert robot %s: %v", memberID, err)
}
}
// cleanupE2ERobots removes all E2E test robots
func cleanupE2ERobots(t *testing.T) {
m := model.Select("__yao.member")
if m == nil {
return
}
tableName := m.MetaData.Table.Name
qb := capsule.Query()
_, err := qb.Table(tableName).Where("member_id", "like", "robot_e2e_%").Delete()
if err != nil {
t.Logf("Warning: cleanup robots error: %v", err)
}
}
// cleanupE2EExecutions removes all E2E test executions
func cleanupE2EExecutions(t *testing.T) {
m := model.Select("__yao.agent.execution")
if m == nil {
return
}
tableName := m.MetaData.Table.Name
qb := capsule.Query()
_, err := qb.Table(tableName).Where("member_id", "like", "robot_e2e_%").Delete()
if err != nil {
t.Logf("Warning: cleanup executions error: %v", err)
}
}

View file

@ -0,0 +1,578 @@
package api_test
// End-to-end tests for Concurrent execution
// These tests verify that multiple robots can execute simultaneously
// and that quota limits are enforced correctly.
//
// Prerequisites:
// - Valid LLM API keys (OPENAI_TEST_KEY or DEEPSEEK_API_KEY)
// - Test assistants in yao-dev-app/assistants/robot/
// - Database connection (YAO_DB_PRIMARY)
import (
"context"
"encoding/json"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/xun/capsule"
"github.com/yaoapp/yao/agent/robot/api"
"github.com/yaoapp/yao/agent/robot/types"
"github.com/yaoapp/yao/agent/testutils"
oauthtypes "github.com/yaoapp/yao/openapi/oauth/types"
)
// testAuthConcurrent returns test auth info for concurrent E2E tests
func testAuthConcurrent() *oauthtypes.AuthorizedInfo {
return &oauthtypes.AuthorizedInfo{
UserID: "e2e-concurrent-user",
TeamID: "e2e-concurrent-team",
}
}
// TestE2EConcurrentMultipleRobots tests concurrent execution of multiple different robots
func TestE2EConcurrentMultipleRobots(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("multiple_robots_execute_concurrently", func(t *testing.T) {
// Create 3 different robots
robots := []string{
"robot_e2e_concurrent_001",
"robot_e2e_concurrent_002",
"robot_e2e_concurrent_003",
}
for _, memberID := range robots {
setupE2ERobotForConcurrent(t, memberID, "team_e2e_concurrent")
}
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthConcurrent())
// Trigger all robots concurrently
var wg sync.WaitGroup
var acceptedCount atomic.Int32
results := make([]*api.TriggerResult, len(robots))
var mu sync.Mutex
for i, memberID := range robots {
wg.Add(1)
go func(idx int, id string) {
defer wg.Done()
result, err := api.TriggerManual(ctx, id, types.TriggerClock, nil)
if err != nil {
t.Logf("Robot %s trigger error: %v", id, err)
return
}
mu.Lock()
results[idx] = result
mu.Unlock()
if result.Accepted {
acceptedCount.Add(1)
t.Logf("Robot %s accepted: JobID=%s", id, result.JobID)
}
}(i, memberID)
}
wg.Wait()
// All 3 should be accepted (different robots, no quota conflict)
assert.Equal(t, int32(3), acceptedCount.Load(), "All 3 robots should be accepted")
// Wait for all executions to complete
maxWait := 180 * time.Second // Longer timeout for concurrent
deadline := time.Now().Add(maxWait)
completedCount := 0
for time.Now().Before(deadline) {
time.Sleep(3 * time.Second)
completedCount = 0
for _, memberID := range robots {
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
completedCount++
}
}
t.Logf("Completed: %d/%d", completedCount, len(robots))
if completedCount == len(robots) {
break
}
}
// Verify all completed
assert.Equal(t, len(robots), completedCount, "All robots should complete execution")
})
}
// TestE2EConcurrentSameRobotMultipleTriggers tests multiple triggers on the same robot
func TestE2EConcurrentSameRobotMultipleTriggers(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("same_robot_handles_multiple_triggers", func(t *testing.T) {
memberID := "robot_e2e_concurrent_same"
// Create robot with high quota to allow multiple concurrent executions
setupE2ERobotHighQuota(t, memberID, "team_e2e_concurrent")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthConcurrent())
// Trigger 3 executions on the same robot
triggerCount := 3
var wg sync.WaitGroup
var acceptedCount atomic.Int32
for i := 0; i < triggerCount; i++ {
wg.Add(1)
go func(idx int) {
defer wg.Done()
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
if err != nil {
t.Logf("Trigger %d error: %v", idx, err)
return
}
if result.Accepted {
acceptedCount.Add(1)
t.Logf("Trigger %d accepted: JobID=%s", idx, result.JobID)
} else {
t.Logf("Trigger %d rejected: %s", idx, result.Message)
}
}(i)
// Small delay between triggers to avoid race conditions
time.Sleep(100 * time.Millisecond)
}
wg.Wait()
// With high quota (max=5), all 3 should be accepted
assert.GreaterOrEqual(t, acceptedCount.Load(), int32(1), "At least 1 trigger should be accepted")
t.Logf("Accepted triggers: %d/%d", acceptedCount.Load(), triggerCount)
// Wait for executions to complete
maxWait := 180 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(3 * time.Second)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil {
continue
}
completedCount := 0
for _, exec := range executions.Data {
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
completedCount++
}
}
t.Logf("Completed: %d/%d", completedCount, int(acceptedCount.Load()))
if completedCount >= int(acceptedCount.Load()) {
break
}
}
// Verify execution count
executions, err := api.ListExecutions(ctx, memberID, nil)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(executions.Data), 1, "Should have at least 1 execution")
})
}
// TestE2EConcurrentQuotaEnforcement tests that quota limits are enforced
func TestE2EConcurrentQuotaEnforcement(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("quota_limit_enforced", func(t *testing.T) {
memberID := "robot_e2e_concurrent_quota"
// Create robot with low quota (max=2)
setupE2ERobotLowQuota(t, memberID, "team_e2e_concurrent")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthConcurrent())
// Try to trigger 5 executions on robot with max=2
triggerCount := 5
var acceptedCount atomic.Int32
var rejectedCount atomic.Int32
for i := 0; i < triggerCount; i++ {
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
if err != nil {
t.Logf("Trigger %d error: %v", i, err)
continue
}
if result.Accepted {
acceptedCount.Add(1)
t.Logf("Trigger %d accepted", i)
} else {
rejectedCount.Add(1)
t.Logf("Trigger %d rejected: %s", i, result.Message)
}
// Small delay to allow execution to start
time.Sleep(200 * time.Millisecond)
}
// With max=2, only 2 should be accepted at a time
// Some may be rejected due to quota
t.Logf("Accepted: %d, Rejected: %d", acceptedCount.Load(), rejectedCount.Load())
// At least some should be accepted
assert.GreaterOrEqual(t, acceptedCount.Load(), int32(1), "At least 1 should be accepted")
// Wait for completion
time.Sleep(120 * time.Second)
// Query final execution count
executions, err := api.ListExecutions(ctx, memberID, nil)
require.NoError(t, err)
t.Logf("Total executions: %d", len(executions.Data))
})
}
// TestE2EConcurrentMixedTriggerTypes tests concurrent execution with different trigger types
func TestE2EConcurrentMixedTriggerTypes(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("mixed_trigger_types_execute_concurrently", func(t *testing.T) {
// Create robots for different trigger types
clockRobot := "robot_e2e_concurrent_clock"
humanRobot := "robot_e2e_concurrent_human"
eventRobot := "robot_e2e_concurrent_event"
setupE2ERobotForConcurrent(t, clockRobot, "team_e2e_concurrent")
setupE2ERobotForConcurrent(t, humanRobot, "team_e2e_concurrent")
setupE2ERobotForConcurrent(t, eventRobot, "team_e2e_concurrent")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthConcurrent())
// Trigger all three types concurrently
var wg sync.WaitGroup
var acceptedCount atomic.Int32
// Clock trigger
wg.Add(1)
go func() {
defer wg.Done()
result, err := api.TriggerManual(ctx, clockRobot, types.TriggerClock, nil)
if err == nil && result.Accepted {
acceptedCount.Add(1)
t.Logf("Clock trigger accepted")
}
}()
// Human trigger
wg.Add(1)
go func() {
defer wg.Done()
result, err := api.Trigger(ctx, humanRobot, &api.TriggerRequest{
Type: types.TriggerHuman,
Action: types.ActionTaskAdd,
})
if err == nil && (result.Accepted || result.Queued) {
acceptedCount.Add(1)
t.Logf("Human trigger accepted/queued")
}
}()
// Event trigger
wg.Add(1)
go func() {
defer wg.Done()
result, err := api.Trigger(ctx, eventRobot, &api.TriggerRequest{
Type: types.TriggerEvent,
Source: types.EventWebhook,
EventType: "test.concurrent",
Data: map[string]interface{}{"test": true},
})
if err == nil && result.Accepted {
acceptedCount.Add(1)
t.Logf("Event trigger accepted")
}
}()
wg.Wait()
// All should be accepted (different robots)
assert.GreaterOrEqual(t, acceptedCount.Load(), int32(2), "At least 2 triggers should be accepted")
// Wait for executions
time.Sleep(120 * time.Second)
// Verify executions exist for each robot
for _, memberID := range []string{clockRobot, humanRobot, eventRobot} {
executions, err := api.ListExecutions(ctx, memberID, nil)
if err == nil && len(executions.Data) > 0 {
t.Logf("Robot %s has %d executions", memberID, len(executions.Data))
}
}
})
}
// ==================== Helper Functions ====================
// setupE2ERobotForConcurrent creates a robot for concurrent execution tests
func setupE2ERobotForConcurrent(t *testing.T, memberID, teamID string) {
m := model.Select("__yao.member")
tableName := m.MetaData.Table.Name
qb := capsule.Query()
// Simple config for E2E testing - minimal tasks
robotConfig := map[string]interface{}{
"identity": map[string]interface{}{
"role": "Simple E2E Test Robot",
"duties": []string{"Say hello"}, // Very simple duty
"rules": []string{"Keep responses under 50 words"},
},
"quota": map[string]interface{}{
"max": 3,
"queue": 10,
"priority": 5,
},
"triggers": map[string]interface{}{
"clock": map[string]interface{}{"enabled": true},
"intervene": map[string]interface{}{"enabled": true},
"event": map[string]interface{}{"enabled": true},
},
"clock": map[string]interface{}{
"mode": "interval",
"every": "1h",
},
"resources": map[string]interface{}{
"phases": map[string]interface{}{
"inspiration": "robot.inspiration",
"goals": "robot.goals",
"tasks": "tests.e2e-tasks", // Use simple E2E test task planner
"run": "robot.validation",
"validation": "tests.e2e-validation", // Use lenient E2E test validator
"delivery": "robot.delivery",
"learning": "robot.learning",
},
"agents": []string{"experts.text-writer"},
},
"delivery": map[string]interface{}{
"email": map[string]interface{}{"enabled": false},
"webhook": map[string]interface{}{"enabled": false},
"process": map[string]interface{}{"enabled": false},
},
}
configJSON, _ := json.Marshal(robotConfig)
systemPrompt := `You are a simple E2E test robot. Your job is to say hello.
When generating goals: create exactly 1 simple goal.
When generating tasks: create exactly 1 simple task.
Keep all outputs brief. No complex analysis needed.`
err := qb.Table(tableName).Insert([]map[string]interface{}{
{
"member_id": memberID,
"team_id": teamID,
"member_type": "robot",
"display_name": "E2E Concurrent Robot " + memberID,
"system_prompt": systemPrompt,
"status": "active",
"role_id": "member",
"autonomous_mode": true,
"robot_status": "idle",
"robot_config": string(configJSON),
},
})
if err != nil {
t.Fatalf("Failed to insert robot %s: %v", memberID, err)
}
}
// setupE2ERobotHighQuota creates a robot with high quota for concurrent tests
func setupE2ERobotHighQuota(t *testing.T, memberID, teamID string) {
m := model.Select("__yao.member")
tableName := m.MetaData.Table.Name
qb := capsule.Query()
robotConfig := map[string]interface{}{
"identity": map[string]interface{}{
"role": "E2E Test Robot - High Quota",
},
"quota": map[string]interface{}{
"max": 5, // High quota
"queue": 20,
"priority": 5,
},
"triggers": map[string]interface{}{
"clock": map[string]interface{}{"enabled": true},
"intervene": map[string]interface{}{"enabled": true},
"event": map[string]interface{}{"enabled": true},
},
// Resources: phase agents and expert agents from yao-dev-app/assistants/
"resources": map[string]interface{}{
"phases": map[string]interface{}{
"inspiration": "robot.inspiration",
"goals": "robot.goals",
"tasks": "tests.e2e-tasks", // Use simple E2E test task planner
"run": "robot.validation",
"validation": "tests.e2e-validation", // Use lenient E2E test validator
"delivery": "robot.delivery",
"learning": "robot.learning",
},
"agents": []string{"experts.text-writer"},
},
"delivery": map[string]interface{}{
"email": map[string]interface{}{"enabled": false},
"webhook": map[string]interface{}{"enabled": false},
"process": map[string]interface{}{"enabled": false},
},
}
configJSON, _ := json.Marshal(robotConfig)
err := qb.Table(tableName).Insert([]map[string]interface{}{
{
"member_id": memberID,
"team_id": teamID,
"member_type": "robot",
"display_name": "E2E High Quota Robot " + memberID,
"system_prompt": "You are a high quota test robot.",
"status": "active",
"role_id": "member",
"autonomous_mode": true,
"robot_status": "idle",
"robot_config": string(configJSON),
},
})
if err != nil {
t.Fatalf("Failed to insert robot %s: %v", memberID, err)
}
}
// setupE2ERobotLowQuota creates a robot with low quota for quota enforcement tests
func setupE2ERobotLowQuota(t *testing.T, memberID, teamID string) {
m := model.Select("__yao.member")
tableName := m.MetaData.Table.Name
qb := capsule.Query()
robotConfig := map[string]interface{}{
"identity": map[string]interface{}{
"role": "E2E Test Robot - Low Quota",
},
"quota": map[string]interface{}{
"max": 2, // Low quota for testing limits
"queue": 5,
"priority": 5,
},
"triggers": map[string]interface{}{
"clock": map[string]interface{}{"enabled": true},
"intervene": map[string]interface{}{"enabled": true},
"event": map[string]interface{}{"enabled": true},
},
// Resources: phase agents and expert agents from yao-dev-app/assistants/
"resources": map[string]interface{}{
"phases": map[string]interface{}{
"inspiration": "robot.inspiration",
"goals": "robot.goals",
"tasks": "tests.e2e-tasks", // Use simple E2E test task planner
"run": "robot.validation",
"validation": "tests.e2e-validation", // Use lenient E2E test validator
"delivery": "robot.delivery",
"learning": "robot.learning",
},
"agents": []string{"experts.text-writer"},
},
"delivery": map[string]interface{}{
"email": map[string]interface{}{"enabled": false},
"webhook": map[string]interface{}{"enabled": false},
"process": map[string]interface{}{"enabled": false},
},
}
configJSON, _ := json.Marshal(robotConfig)
err := qb.Table(tableName).Insert([]map[string]interface{}{
{
"member_id": memberID,
"team_id": teamID,
"member_type": "robot",
"display_name": "E2E Low Quota Robot " + memberID,
"system_prompt": "You are a low quota test robot.",
"status": "active",
"role_id": "member",
"autonomous_mode": true,
"robot_status": "idle",
"robot_config": string(configJSON),
},
})
if err != nil {
t.Fatalf("Failed to insert robot %s: %v", memberID, err)
}
}

View file

@ -0,0 +1,555 @@
package api_test
// End-to-end tests for Execution Control (Pause/Resume/Stop)
// These tests verify that executions can be controlled during runtime
//
// Prerequisites:
// - Valid LLM API keys (OPENAI_TEST_KEY or DEEPSEEK_API_KEY)
// - Test assistants in yao-dev-app/assistants/robot/
// - Database connection (YAO_DB_PRIMARY)
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/xun/capsule"
"github.com/yaoapp/yao/agent/robot/api"
"github.com/yaoapp/yao/agent/robot/types"
"github.com/yaoapp/yao/agent/testutils"
oauthtypes "github.com/yaoapp/yao/openapi/oauth/types"
)
// testAuthControl returns test auth info for control E2E tests
func testAuthControl() *oauthtypes.AuthorizedInfo {
return &oauthtypes.AuthorizedInfo{
UserID: "e2e-control-user",
TeamID: "e2e-control-team",
}
}
// TestE2EControlPauseResume tests pausing and resuming an execution
func TestE2EControlPauseResume(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("pause_and_resume_execution", func(t *testing.T) {
memberID := "robot_e2e_control_pause"
setupE2ERobotForControl(t, memberID, "team_e2e_control")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthControl())
// Start execution
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
require.NoError(t, err)
require.True(t, result.Accepted)
t.Logf("Execution started: JobID=%s", result.JobID)
// Wait for execution to start running
var execID string
maxWait := 30 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(500 * time.Millisecond)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
if exec.Status == types.ExecRunning {
execID = exec.ID
t.Logf("Execution running: ID=%s, Phase=%s", execID, exec.Phase)
break
}
}
if execID == "" {
t.Skip("Execution did not start in time - may have completed too quickly")
return
}
// Pause the execution
err = api.PauseExecution(ctx, execID)
if err != nil {
t.Logf("Pause error (may be expected if execution completed): %v", err)
} else {
t.Logf("Execution paused")
// Verify paused state
time.Sleep(1 * time.Second)
status, err := api.GetExecutionStatus(ctx, execID)
if err == nil && status != nil {
t.Logf("Status after pause: %s", status.Status)
}
// Resume the execution
err = api.ResumeExecution(ctx, execID)
if err != nil {
t.Logf("Resume error: %v", err)
} else {
t.Logf("Execution resumed")
}
}
// Wait for completion
maxWait = 120 * time.Second
deadline = time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(2 * time.Second)
exec, err := api.GetExecution(ctx, execID)
if err != nil {
continue
}
t.Logf("Execution status: %s, phase: %s", exec.Status, exec.Phase)
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed || exec.Status == types.ExecCancelled {
// Execution finished (completed, failed, or cancelled)
t.Logf("Execution finished with status: %s", exec.Status)
return
}
}
t.Logf("Execution did not complete in time")
})
}
// TestE2EControlStop tests stopping an execution
func TestE2EControlStop(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("stop_running_execution", func(t *testing.T) {
memberID := "robot_e2e_control_stop"
setupE2ERobotForControl(t, memberID, "team_e2e_control")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthControl())
// Start execution
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
require.NoError(t, err)
require.True(t, result.Accepted)
t.Logf("Execution started: JobID=%s", result.JobID)
// Wait for execution to start running
var execID string
maxWait := 30 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(500 * time.Millisecond)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
if exec.Status == types.ExecRunning {
execID = exec.ID
t.Logf("Execution running: ID=%s, Phase=%s", execID, exec.Phase)
break
}
}
if execID == "" {
t.Skip("Execution did not start in time - may have completed too quickly")
return
}
// Stop the execution
err = api.StopExecution(ctx, execID)
if err != nil {
t.Logf("Stop error (may be expected if execution completed): %v", err)
} else {
t.Logf("Stop signal sent")
}
// Wait and verify stopped/cancelled state (with retry)
maxWait = 30 * time.Second
deadline = time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(2 * time.Second)
exec, err := api.GetExecution(ctx, execID)
if err != nil {
t.Logf("Get execution error: %v", err)
continue
}
t.Logf("Current status: %s", exec.Status)
// Execution should eventually be cancelled, completed, or failed
if exec.Status == types.ExecCancelled ||
exec.Status == types.ExecCompleted ||
exec.Status == types.ExecFailed {
t.Logf("Final status: %s", exec.Status)
return
}
}
// If we get here, check final state
exec, err := api.GetExecution(ctx, execID)
if err != nil {
t.Logf("Get execution error: %v", err)
return
}
// Allow running state if stop didn't take effect in time (execution may have already completed)
t.Logf("Final status after wait: %s", exec.Status)
assert.True(t,
exec.Status == types.ExecCancelled ||
exec.Status == types.ExecCompleted ||
exec.Status == types.ExecFailed ||
exec.Status == types.ExecRunning, // Allow running if stop didn't take effect
"Execution should be in terminal state or still running, got: %s", exec.Status)
})
}
// TestE2EControlStopBeforeStart tests stopping an execution before it starts
func TestE2EControlStopBeforeStart(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("stop_queued_execution", func(t *testing.T) {
memberID := "robot_e2e_control_stop_early"
setupE2ERobotForControl(t, memberID, "team_e2e_control")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthControl())
// Start execution
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
require.NoError(t, err)
require.True(t, result.Accepted)
// Immediately try to get execution ID and stop
time.Sleep(100 * time.Millisecond)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
t.Skip("No execution found")
return
}
execID := executions.Data[0].ID
// Try to stop immediately
err = api.StopExecution(ctx, execID)
if err != nil {
t.Logf("Stop error: %v", err)
} else {
t.Logf("Stop signal sent for execution %s", execID)
}
// Wait and check status
time.Sleep(5 * time.Second)
exec, err := api.GetExecution(ctx, execID)
if err != nil {
t.Logf("Get execution error: %v", err)
return
}
t.Logf("Final status: %s", exec.Status)
})
}
// TestE2EControlMultipleOperations tests a sequence of control operations
func TestE2EControlMultipleOperations(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("pause_resume_pause_stop_sequence", func(t *testing.T) {
memberID := "robot_e2e_control_multi"
setupE2ERobotForControl(t, memberID, "team_e2e_control")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthControl())
// Start execution
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
require.NoError(t, err)
require.True(t, result.Accepted)
// Wait for running state
var execID string
maxWait := 30 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(500 * time.Millisecond)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
if exec.Status == types.ExecRunning {
execID = exec.ID
break
}
}
if execID == "" {
t.Skip("Execution did not start in time")
return
}
// Sequence: Pause → Resume → Pause → Stop
operations := []struct {
name string
fn func() error
}{
{"Pause", func() error { return api.PauseExecution(ctx, execID) }},
{"Resume", func() error { return api.ResumeExecution(ctx, execID) }},
{"Pause", func() error { return api.PauseExecution(ctx, execID) }},
{"Stop", func() error { return api.StopExecution(ctx, execID) }},
}
for _, op := range operations {
err := op.fn()
if err != nil {
t.Logf("%s error (may be expected): %v", op.name, err)
// If execution already completed, stop the sequence
exec, _ := api.GetExecution(ctx, execID)
if exec != nil && (exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed || exec.Status == types.ExecCancelled) {
t.Logf("Execution already finished: %s", exec.Status)
return
}
} else {
t.Logf("%s successful", op.name)
}
time.Sleep(2 * time.Second)
}
// Verify final state
exec, err := api.GetExecution(ctx, execID)
if err != nil {
t.Logf("Get execution error: %v", err)
return
}
t.Logf("Final status after operations: %s", exec.Status)
})
}
// TestE2EControlStatusQuery tests querying execution status during control
func TestE2EControlStatusQuery(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("query_status_during_execution", func(t *testing.T) {
memberID := "robot_e2e_control_status"
setupE2ERobotForControl(t, memberID, "team_e2e_control")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthControl())
// Start execution
result, err := api.TriggerManual(ctx, memberID, types.TriggerClock, nil)
require.NoError(t, err)
require.True(t, result.Accepted)
// Track status changes
statusHistory := make([]types.ExecStatus, 0)
phaseHistory := make([]types.Phase, 0)
maxWait := 120 * time.Second
deadline := time.Now().Add(maxWait)
lastStatus := types.ExecStatus("")
lastPhase := types.Phase("")
for time.Now().Before(deadline) {
time.Sleep(1 * time.Second)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
// Track status changes
if exec.Status != lastStatus {
statusHistory = append(statusHistory, exec.Status)
lastStatus = exec.Status
t.Logf("Status changed: %s", exec.Status)
}
// Track phase changes
if exec.Phase != lastPhase {
phaseHistory = append(phaseHistory, exec.Phase)
lastPhase = exec.Phase
t.Logf("Phase changed: %s", exec.Phase)
}
// Also test GetExecutionStatus
status, err := api.GetExecutionStatus(ctx, exec.ID)
if err == nil && status != nil {
// Status query should return valid data
assert.NotEmpty(t, status.ID)
assert.Equal(t, exec.Status, status.Status)
}
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed || exec.Status == types.ExecCancelled {
break
}
}
t.Logf("Status history: %v", statusHistory)
t.Logf("Phase history: %v", phaseHistory)
// Should have observed at least pending → running transition
assert.GreaterOrEqual(t, len(statusHistory), 1, "Should observe at least one status")
})
}
// ==================== Helper Functions ====================
// setupE2ERobotForControl creates a robot for control tests
func setupE2ERobotForControl(t *testing.T, memberID, teamID string) {
m := model.Select("__yao.member")
tableName := m.MetaData.Table.Name
qb := capsule.Query()
// Simple config for E2E testing - minimal tasks
robotConfig := map[string]interface{}{
"identity": map[string]interface{}{
"role": "Simple E2E Test Robot",
"duties": []string{"Say hello"}, // Very simple duty
"rules": []string{"Keep responses under 50 words"},
},
"quota": map[string]interface{}{
"max": 3,
"queue": 10,
"priority": 5,
},
"triggers": map[string]interface{}{
"clock": map[string]interface{}{"enabled": true},
"intervene": map[string]interface{}{"enabled": true},
"event": map[string]interface{}{"enabled": true},
},
"clock": map[string]interface{}{
"mode": "interval",
"every": "1h",
},
"resources": map[string]interface{}{
"phases": map[string]interface{}{
"inspiration": "robot.inspiration",
"goals": "robot.goals",
"tasks": "tests.e2e-tasks", // Use simple E2E test task planner
"run": "robot.validation",
"validation": "tests.e2e-validation", // Use lenient E2E test validator
"delivery": "robot.delivery",
"learning": "robot.learning",
},
"agents": []string{"experts.text-writer"},
},
"delivery": map[string]interface{}{
"email": map[string]interface{}{"enabled": false},
"webhook": map[string]interface{}{"enabled": false},
"process": map[string]interface{}{"enabled": false},
},
}
configJSON, _ := json.Marshal(robotConfig)
systemPrompt := `You are a simple E2E test robot. Your job is to say hello.
When generating goals: create exactly 1 simple goal.
When generating tasks: create exactly 1 simple task.
Keep all outputs brief. No complex analysis needed.`
err := qb.Table(tableName).Insert([]map[string]interface{}{
{
"member_id": memberID,
"team_id": teamID,
"member_type": "robot",
"display_name": "E2E Control Robot " + memberID,
"system_prompt": systemPrompt,
"status": "active",
"role_id": "member",
"autonomous_mode": true,
"robot_status": "idle",
"robot_config": string(configJSON),
},
})
if err != nil {
t.Fatalf("Failed to insert robot %s: %v", memberID, err)
}
}

View file

@ -0,0 +1,444 @@
package api_test
// End-to-end tests for Event trigger flow
// These tests use REAL LLM calls via Standard executor (not DryRun)
//
// Test Flow: Event Trigger → P1 (Goals) → P2 (Tasks) → P3 (Run) → P4 (Delivery)
// Note: Event trigger SKIPS P0 (Inspiration) - event data provides the context
//
// Prerequisites:
// - Valid LLM API keys (OPENAI_TEST_KEY or DEEPSEEK_API_KEY)
// - Test assistants in yao-dev-app/assistants/robot/
// - Database connection (YAO_DB_PRIMARY)
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/xun/capsule"
"github.com/yaoapp/yao/agent/robot/api"
"github.com/yaoapp/yao/agent/robot/types"
"github.com/yaoapp/yao/agent/testutils"
oauthtypes "github.com/yaoapp/yao/openapi/oauth/types"
)
// testAuthEvent returns test auth info for event E2E tests
func testAuthEvent() *oauthtypes.AuthorizedInfo {
return &oauthtypes.AuthorizedInfo{
UserID: "e2e-event-user",
TeamID: "e2e-event-team",
}
}
// TestE2EEventTriggerFullFlow tests the complete event trigger flow with real LLM calls
// Flow: Event → P1 (Goals) → P2 (Tasks) → P3 (Run) → P4 (Delivery)
func TestE2EEventTriggerFullFlow(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("complete_P1_to_P4_flow_with_webhook_event", func(t *testing.T) {
memberID := "robot_e2e_event_001"
setupE2ERobotForEvent(t, memberID, "team_e2e_event")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthEvent())
// Verify robot is loaded
robot, err := api.GetRobot(ctx, memberID)
require.NoError(t, err)
require.NotNil(t, robot)
// Trigger with webhook event - simulating external system notification
result, err := api.Trigger(ctx, memberID, &api.TriggerRequest{
Type: types.TriggerEvent,
Source: types.EventWebhook,
EventType: "order.created",
Data: map[string]interface{}{
"order_id": "ORD-2025-001",
"customer": "John Doe",
"total": 299.99,
"items_count": 3,
"priority": "high",
"created_at": time.Now().Format(time.RFC3339),
},
})
require.NoError(t, err)
require.NotNil(t, result)
assert.True(t, result.Accepted, "Event trigger should be accepted")
t.Logf("Event trigger result: Accepted=%v, JobID=%s", result.Accepted, result.JobID)
// Wait for execution to complete
var exec *types.Execution
maxWait := 120 * time.Second
pollInterval := 2 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(pollInterval)
executions, err := api.ListExecutions(ctx, memberID, &api.ExecutionQuery{
Page: 1,
PageSize: 1,
})
if err != nil || len(executions.Data) == 0 {
continue
}
exec = executions.Data[0]
t.Logf("Execution status: %s, phase: %s", exec.Status, exec.Phase)
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
break
}
}
require.NotNil(t, exec, "Execution should exist")
// E2E test validates the flow executes correctly
isFinished := exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed
assert.True(t, isFinished, "Execution should finish (completed or failed), got: %s", exec.Status)
if exec.Status == types.ExecFailed {
t.Logf("Execution finished with status=failed (acceptable for E2E): %s", exec.Error)
} else {
t.Logf("Execution finished with status=completed")
}
// Verify trigger type
assert.Equal(t, types.TriggerEvent, exec.TriggerType, "Should be event trigger")
// Event trigger skips P0, so Inspiration should be nil
assert.Nil(t, exec.Inspiration, "P0 Inspiration should be nil for event trigger")
// P1 Goals should always exist for event trigger
assert.NotNil(t, exec.Goals, "P1 Goals should exist")
// P2-P4 may or may not exist depending on where failure occurred
if exec.Tasks != nil {
t.Logf("P2 Tasks count: %d", len(exec.Tasks))
}
if exec.Results != nil {
t.Logf("P3 Results count: %d", len(exec.Results))
}
if exec.Delivery != nil {
t.Logf("P4 Delivery: RequestID=%s", exec.Delivery.RequestID)
}
t.Logf("Event trigger E2E completed")
})
}
// TestE2EEventTriggerDatabaseEvent tests event trigger from database changes
func TestE2EEventTriggerDatabaseEvent(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("handles_database_event_source", func(t *testing.T) {
memberID := "robot_e2e_event_db"
setupE2ERobotForEvent(t, memberID, "team_e2e_event")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthEvent())
// Trigger with database event - simulating record change notification
result, err := api.Trigger(ctx, memberID, &api.TriggerRequest{
Type: types.TriggerEvent,
Source: types.EventDatabase,
EventType: "user.updated",
Data: map[string]interface{}{
"table": "users",
"operation": "UPDATE",
"record_id": 12345,
"changes": map[string]interface{}{
"status": map[string]interface{}{
"old": "pending",
"new": "active",
},
},
},
})
require.NoError(t, err)
require.NotNil(t, result)
assert.True(t, result.Accepted)
// Wait for execution
maxWait := 120 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(2 * time.Second)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
assert.Equal(t, types.ExecCompleted, exec.Status)
t.Logf("Database event E2E completed")
return
}
}
t.Fatal("Execution did not complete in time")
})
}
// TestE2EEventTriggerVariousEventTypes tests different event types
// Optimized: Only tests one representative event type to reduce CI time
// The event handling logic is the same for all event types, so testing one is sufficient
func TestE2EEventTriggerVariousEventTypes(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
// Test only one representative event type (notification)
// All event types use the same code path, so one test is sufficient
t.Run("webhook_event", func(t *testing.T) {
memberID := "robot_e2e_event_webhook"
setupE2ERobotForEvent(t, memberID, "team_e2e_event")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthEvent())
result, err := api.Trigger(ctx, memberID, &api.TriggerRequest{
Type: types.TriggerEvent,
Source: types.EventWebhook,
EventType: "notification.received",
Data: map[string]interface{}{
"message": "Test notification",
"priority": "normal",
},
})
require.NoError(t, err)
require.NotNil(t, result)
assert.True(t, result.Accepted, "Event should be accepted")
t.Logf("Event triggered: JobID=%s", result.JobID)
// Wait for execution
maxWait := 120 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(2 * time.Second)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
t.Logf("Event status: %s, phase: %s", exec.Status, exec.Phase)
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
if exec.Status == types.ExecFailed {
t.Logf("Event execution failed: %s", exec.Error)
} else {
t.Logf("Event execution completed")
}
return
}
}
t.Logf("Event execution did not complete in time (may be CI latency)")
})
}
// TestE2EEventTriggerWithComplexData tests event with nested/complex data structures
func TestE2EEventTriggerWithComplexData(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("handles_complex_nested_data", func(t *testing.T) {
memberID := "robot_e2e_event_complex"
setupE2ERobotForEvent(t, memberID, "team_e2e_event")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthEvent())
// Complex nested event data
result, err := api.Trigger(ctx, memberID, &api.TriggerRequest{
Type: types.TriggerEvent,
Source: types.EventWebhook,
EventType: "report.generated",
Data: map[string]interface{}{
"report": map[string]interface{}{
"id": "RPT-2025-001",
"type": "sales_summary",
"period": "monthly",
"generated": time.Now().Format(time.RFC3339),
"department": "Sales",
},
"metrics": []map[string]interface{}{
{"name": "total_sales", "value": 150000, "unit": "USD"},
{"name": "orders_count", "value": 450, "unit": "orders"},
{"name": "avg_order_value", "value": 333.33, "unit": "USD"},
},
"comparison": map[string]interface{}{
"previous_period": map[string]interface{}{
"total_sales": 140000,
"orders_count": 420,
"change_percent": 7.14,
},
},
"highlights": []string{
"Sales increased by 7.14% compared to last month",
"Top performing product: Widget Pro",
"New customer acquisition up 15%",
},
},
})
require.NoError(t, err)
require.NotNil(t, result)
assert.True(t, result.Accepted)
// Wait for execution
maxWait := 120 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(2 * time.Second)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
assert.Equal(t, types.ExecCompleted, exec.Status, "Complex data event should complete")
t.Logf("Complex data event E2E completed")
return
}
}
t.Fatal("Execution did not complete in time")
})
}
// ==================== Helper Functions ====================
// setupE2ERobotForEvent creates a robot configured for event trigger E2E tests
func setupE2ERobotForEvent(t *testing.T, memberID, teamID string) {
m := model.Select("__yao.member")
tableName := m.MetaData.Table.Name
qb := capsule.Query()
// Simple config for E2E testing - minimal tasks
robotConfig := map[string]interface{}{
"identity": map[string]interface{}{
"role": "Simple E2E Test Robot",
"duties": []string{"Acknowledge events"}, // Very simple duty
"rules": []string{"Keep responses under 50 words"},
},
"quota": map[string]interface{}{
"max": 5,
"queue": 20,
"priority": 5,
},
"triggers": map[string]interface{}{
"clock": map[string]interface{}{"enabled": false},
"intervene": map[string]interface{}{"enabled": true},
"event": map[string]interface{}{"enabled": true},
},
"event": map[string]interface{}{
"types": []string{"*"},
},
"resources": map[string]interface{}{
"phases": map[string]interface{}{
"inspiration": "robot.inspiration",
"goals": "robot.goals",
"tasks": "tests.e2e-tasks", // Use simple E2E test task planner
"run": "robot.validation",
"validation": "tests.e2e-validation", // Use lenient E2E test validator
"delivery": "robot.delivery",
"learning": "robot.learning",
},
"agents": []string{"experts.text-writer"},
},
"delivery": map[string]interface{}{
"email": map[string]interface{}{"enabled": false},
"webhook": map[string]interface{}{"enabled": false},
"process": map[string]interface{}{"enabled": false},
},
}
configJSON, _ := json.Marshal(robotConfig)
systemPrompt := `You are a simple E2E test robot. Your job is to acknowledge events.
When generating goals: create exactly 1 simple goal.
When generating tasks: create exactly 1 simple task.
Keep all outputs brief. No complex analysis needed.`
err := qb.Table(tableName).Insert([]map[string]interface{}{
{
"member_id": memberID,
"team_id": teamID,
"member_type": "robot",
"display_name": "E2E Event Test Robot " + memberID,
"system_prompt": systemPrompt,
"status": "active",
"role_id": "member",
"autonomous_mode": true,
"robot_status": "idle",
"robot_config": string(configJSON),
},
})
if err != nil {
t.Fatalf("Failed to insert robot %s: %v", memberID, err)
}
}

View file

@ -0,0 +1,380 @@
package api_test
// End-to-end tests for Human intervention trigger flow
// These tests use REAL LLM calls via Standard executor (not DryRun)
//
// Test Flow: Human Trigger → P1 (Goals) → P2 (Tasks) → P3 (Run) → P4 (Delivery)
// Note: Human trigger SKIPS P0 (Inspiration) - user provides the input directly
//
// Prerequisites:
// - Valid LLM API keys (OPENAI_TEST_KEY or DEEPSEEK_API_KEY)
// - Test assistants in yao-dev-app/assistants/robot/
// - Database connection (YAO_DB_PRIMARY)
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/yaoapp/gou/model"
"github.com/yaoapp/xun/capsule"
agentcontext "github.com/yaoapp/yao/agent/context"
"github.com/yaoapp/yao/agent/robot/api"
"github.com/yaoapp/yao/agent/robot/types"
"github.com/yaoapp/yao/agent/testutils"
oauthtypes "github.com/yaoapp/yao/openapi/oauth/types"
)
// testAuthHuman returns test auth info for human E2E tests
func testAuthHuman() *oauthtypes.AuthorizedInfo {
return &oauthtypes.AuthorizedInfo{
UserID: "e2e-human-user",
TeamID: "e2e-human-team",
}
}
// TestE2EHumanTriggerFullFlow tests the complete human intervention flow with real LLM calls
// Flow: Human Input → P1 (Goals) → P2 (Tasks) → P3 (Run) → P4 (Delivery)
func TestE2EHumanTriggerFullFlow(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("complete_P1_to_P4_flow_with_user_input", func(t *testing.T) {
memberID := "robot_e2e_human_001"
setupE2ERobotForHuman(t, memberID, "team_e2e_human")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthHuman())
// Verify robot is loaded
robot, err := api.GetRobot(ctx, memberID)
require.NoError(t, err)
require.NotNil(t, robot)
// Trigger with human input - user requesting a specific task
result, err := api.Trigger(ctx, memberID, &api.TriggerRequest{
Type: types.TriggerHuman,
Action: types.ActionTaskAdd,
Messages: []agentcontext.Message{
{
Role: "user",
Content: "Please write a brief summary of today's key tasks and priorities.",
},
},
})
require.NoError(t, err)
require.NotNil(t, result)
// Human trigger returns Queued=true (goes through Intervene)
t.Logf("Trigger result: Accepted=%v, Queued=%v, Message=%s",
result.Accepted, result.Queued, result.Message)
// Wait for execution to complete
var exec *types.Execution
maxWait := 120 * time.Second
pollInterval := 2 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(pollInterval)
executions, err := api.ListExecutions(ctx, memberID, &api.ExecutionQuery{
Page: 1,
PageSize: 1,
})
if err != nil || len(executions.Data) == 0 {
continue
}
exec = executions.Data[0]
t.Logf("Execution status: %s, phase: %s", exec.Status, exec.Phase)
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
break
}
}
require.NotNil(t, exec, "Execution should exist")
// E2E test validates the flow executes correctly
isFinished := exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed
assert.True(t, isFinished, "Execution should finish (completed or failed), got: %s", exec.Status)
if exec.Status == types.ExecFailed {
t.Logf("Execution finished with status=failed (acceptable for E2E): %s", exec.Error)
} else {
t.Logf("Execution finished with status=completed")
}
// Verify trigger type
assert.Equal(t, types.TriggerHuman, exec.TriggerType, "Should be human trigger")
// Human trigger skips P0, so Inspiration should be nil
assert.Nil(t, exec.Inspiration, "P0 Inspiration should be nil for human trigger")
// P1 Goals should always exist for human trigger
assert.NotNil(t, exec.Goals, "P1 Goals should exist")
// P2-P4 may or may not exist depending on where failure occurred
if exec.Tasks != nil {
t.Logf("P2 Tasks count: %d", len(exec.Tasks))
}
if exec.Results != nil {
t.Logf("P3 Results count: %d", len(exec.Results))
}
if exec.Delivery != nil {
t.Logf("P4 Delivery: RequestID=%s", exec.Delivery.RequestID)
}
t.Logf("Human trigger E2E completed")
})
}
// TestE2EHumanTriggerWithMultimodalInput tests human trigger with rich content
func TestE2EHumanTriggerWithMultimodalInput(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
t.Run("handles_multipart_message_input", func(t *testing.T) {
memberID := "robot_e2e_human_multi"
setupE2ERobotForHuman(t, memberID, "team_e2e_human")
err := api.Start()
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(context.Background(), testAuthHuman())
// Trigger with multipart message (text parts)
result, err := api.Trigger(ctx, memberID, &api.TriggerRequest{
Type: types.TriggerHuman,
Action: types.ActionGoalAdjust,
Messages: []agentcontext.Message{
{
Role: "user",
Content: []map[string]interface{}{
{
"type": "text",
"text": "I need you to focus on the following priorities:",
},
{
"type": "text",
"text": "1. Review pending tasks\n2. Summarize progress\n3. Identify blockers",
},
},
},
},
})
require.NoError(t, err)
require.NotNil(t, result)
t.Logf("Multipart trigger result: Queued=%v", result.Queued)
// Wait for execution
maxWait := 120 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(2 * time.Second)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
assert.Equal(t, types.ExecCompleted, exec.Status)
t.Logf("Multipart input E2E completed")
return
}
}
t.Fatal("Execution did not complete in time")
})
}
// TestE2EHumanTriggerAllActions tests different intervention actions
func TestE2EHumanTriggerAllActions(t *testing.T) {
if testing.Short() {
t.Skip("Skipping E2E test - requires real LLM calls")
}
testutils.Prepare(t)
defer testutils.Clean(t)
cleanupE2ERobots(t)
cleanupE2EExecutions(t)
defer cleanupE2ERobots(t)
defer cleanupE2EExecutions(t)
actions := []struct {
name string
action types.InterventionAction
input string
}{
{
name: "task_add",
action: types.ActionTaskAdd,
input: "Add a new task: Review system logs for errors",
},
{
name: "goal_adjust",
action: types.ActionGoalAdjust,
input: "Adjust goal: Focus on performance optimization instead of new features",
},
{
name: "instruct",
action: types.ActionInstruct,
input: "Please prioritize security review as the top task",
},
}
for i, tc := range actions {
t.Run(tc.name, func(t *testing.T) {
memberID := "robot_e2e_human_action_" + tc.name
setupE2ERobotForHuman(t, memberID, "team_e2e_human")
// Start fresh for each action test
if i == 0 {
err := api.Start()
require.NoError(t, err)
}
ctx := types.NewContext(context.Background(), testAuthHuman())
result, err := api.Trigger(ctx, memberID, &api.TriggerRequest{
Type: types.TriggerHuman,
Action: tc.action,
Messages: []agentcontext.Message{
{Role: "user", Content: tc.input},
},
})
require.NoError(t, err)
require.NotNil(t, result)
t.Logf("Action %s: Queued=%v", tc.action, result.Queued)
// Wait for execution (shorter timeout for action tests)
maxWait := 90 * time.Second
deadline := time.Now().Add(maxWait)
for time.Now().Before(deadline) {
time.Sleep(2 * time.Second)
executions, err := api.ListExecutions(ctx, memberID, nil)
if err != nil || len(executions.Data) == 0 {
continue
}
exec := executions.Data[0]
if exec.Status == types.ExecCompleted || exec.Status == types.ExecFailed {
if exec.Status == types.ExecFailed {
t.Logf("Action %s failed: %s", tc.action, exec.Error)
} else {
t.Logf("Action %s completed successfully", tc.action)
}
return
}
}
t.Logf("Action %s: execution did not complete in time (may still be running)", tc.action)
})
}
// Stop after all action tests
api.Stop()
}
// ==================== Helper Functions ====================
// setupE2ERobotForHuman creates a robot configured for human intervention E2E tests
func setupE2ERobotForHuman(t *testing.T, memberID, teamID string) {
m := model.Select("__yao.member")
tableName := m.MetaData.Table.Name
qb := capsule.Query()
// Simple config for E2E testing - minimal tasks
robotConfig := map[string]interface{}{
"identity": map[string]interface{}{
"role": "Simple E2E Test Robot",
"duties": []string{"Echo user input"}, // Very simple duty
"rules": []string{"Keep responses under 50 words"},
},
"quota": map[string]interface{}{
"max": 5,
"queue": 20,
"priority": 5,
},
"triggers": map[string]interface{}{
"clock": map[string]interface{}{"enabled": false},
"intervene": map[string]interface{}{"enabled": true},
"event": map[string]interface{}{"enabled": true},
},
"resources": map[string]interface{}{
"phases": map[string]interface{}{
"inspiration": "robot.inspiration",
"goals": "robot.goals",
"tasks": "tests.e2e-tasks", // Use simple E2E test task planner
"run": "robot.validation",
"validation": "tests.e2e-validation", // Use lenient E2E test validator
"delivery": "robot.delivery",
"learning": "robot.learning",
},
"agents": []string{"experts.text-writer"},
},
"delivery": map[string]interface{}{
"email": map[string]interface{}{"enabled": false},
"webhook": map[string]interface{}{"enabled": false},
"process": map[string]interface{}{"enabled": false},
},
}
configJSON, _ := json.Marshal(robotConfig)
systemPrompt := `You are a simple E2E test robot. Your job is to echo user requests.
When generating goals: create exactly 1 simple goal.
When generating tasks: create exactly 1 simple task.
Keep all outputs brief. No complex analysis needed.`
err := qb.Table(tableName).Insert([]map[string]interface{}{
{
"member_id": memberID,
"team_id": teamID,
"member_type": "robot",
"display_name": "E2E Human Test Robot " + memberID,
"system_prompt": systemPrompt,
"status": "active",
"role_id": "member",
"autonomous_mode": true,
"robot_status": "idle",
"robot_config": string(configJSON),
},
})
if err != nil {
t.Fatalf("Failed to insert robot %s: %v", memberID, err)
}
}

View file

@ -1,6 +1,7 @@
package api_test
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
@ -19,7 +20,7 @@ func TestGetExecutionValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty execution_id", func(t *testing.T) {
exec, err := api.GetExecution(ctx, "")
@ -44,7 +45,7 @@ func TestListExecutionsValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty member_id", func(t *testing.T) {
result, err := api.ListExecutions(ctx, "", nil)
@ -81,7 +82,7 @@ func TestPauseExecutionValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty execution_id", func(t *testing.T) {
err := api.PauseExecution(ctx, "")
@ -105,7 +106,7 @@ func TestResumeExecutionValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty execution_id", func(t *testing.T) {
err := api.ResumeExecution(ctx, "")
@ -129,7 +130,7 @@ func TestStopExecutionValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty execution_id", func(t *testing.T) {
err := api.StopExecution(ctx, "")
@ -153,7 +154,7 @@ func TestGetExecutionStatusValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty execution_id", func(t *testing.T) {
exec, err := api.GetExecutionStatus(ctx, "")
@ -183,7 +184,7 @@ func TestExecutionControlWithManagerStarted(t *testing.T) {
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("pause returns error for non-existent execution", func(t *testing.T) {
err := api.PauseExecution(ctx, "non_existent_exec_id_xyz")

View file

@ -1,6 +1,7 @@
package api_test
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
@ -19,7 +20,7 @@ func TestGetRobotValidation(t *testing.T) {
defer testutils.Clean(t)
t.Run("returns error for empty member_id", func(t *testing.T) {
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
robot, err := api.GetRobot(ctx, "")
assert.Error(t, err)
assert.Nil(t, robot)
@ -27,7 +28,7 @@ func TestGetRobotValidation(t *testing.T) {
})
t.Run("returns error for non-existent robot", func(t *testing.T) {
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
robot, err := api.GetRobot(ctx, "non_existent_member_id_xyz")
assert.Error(t, err)
assert.Nil(t, robot)
@ -43,7 +44,7 @@ func TestListRobotsValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("applies default pagination when query is nil", func(t *testing.T) {
result, err := api.ListRobots(ctx, nil)
@ -85,7 +86,7 @@ func TestGetRobotStatusValidation(t *testing.T) {
defer testutils.Clean(t)
t.Run("returns error for empty member_id", func(t *testing.T) {
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
status, err := api.GetRobotStatus(ctx, "")
assert.Error(t, err)
assert.Nil(t, status)
@ -93,7 +94,7 @@ func TestGetRobotStatusValidation(t *testing.T) {
})
t.Run("returns error for non-existent robot", func(t *testing.T) {
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
status, err := api.GetRobotStatus(ctx, "non_existent_member_id_xyz")
assert.Error(t, err)
assert.Nil(t, status)

View file

@ -1,6 +1,7 @@
package api_test
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
@ -19,7 +20,7 @@ func TestTriggerValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty member_id", func(t *testing.T) {
result, err := api.Trigger(ctx, "", &api.TriggerRequest{
@ -56,7 +57,7 @@ func TestTriggerManualValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty member_id", func(t *testing.T) {
result, err := api.TriggerManual(ctx, "", types.TriggerClock, nil)
@ -82,7 +83,7 @@ func TestInterveneValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty member_id", func(t *testing.T) {
result, err := api.Intervene(ctx, "", &api.TriggerRequest{
@ -111,7 +112,7 @@ func TestHandleEventValidation(t *testing.T) {
testutils.Prepare(t)
defer testutils.Clean(t)
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns error for empty member_id", func(t *testing.T) {
result, err := api.HandleEvent(ctx, "", &api.TriggerRequest{
@ -146,7 +147,7 @@ func TestTriggerWithManagerStarted(t *testing.T) {
require.NoError(t, err)
defer api.Stop()
ctx := types.NewContext(nil, nil)
ctx := types.NewContext(context.Background(), nil)
t.Run("returns not accepted for non-existent robot", func(t *testing.T) {
result, err := api.Trigger(ctx, "non_existent_robot_xyz", &api.TriggerRequest{

View file

@ -66,10 +66,14 @@ type TriggerRequest struct {
type InsertPosition string
const (
InsertFirst InsertPosition = "first" // insert at beginning (highest priority)
InsertLast InsertPosition = "last" // append at end (default)
InsertNext InsertPosition = "next" // insert after current task
InsertAt InsertPosition = "at" // insert at specific index (use AtIndex)
// InsertFirst inserts at beginning (highest priority)
InsertFirst InsertPosition = "first"
// InsertLast appends at end (default)
InsertLast InsertPosition = "last"
// InsertNext inserts after current task
InsertNext InsertPosition = "next"
// InsertAt inserts at specific index (use AtIndex)
InsertAt InsertPosition = "at"
)
// TriggerResult - result of Trigger()

View file

@ -11,6 +11,7 @@ import (
"github.com/yaoapp/yao/agent/robot/pool"
"github.com/yaoapp/yao/agent/robot/trigger"
"github.com/yaoapp/yao/agent/robot/types"
oauthtypes "github.com/yaoapp/yao/openapi/oauth/types"
)
// Default configuration values
@ -197,9 +198,8 @@ func (m *Manager) tickerLoop() {
m.ticker.Stop()
return
case now := <-m.ticker.C:
// Perform tick
ctx := types.NewContext(m.ctx, nil)
_ = m.Tick(ctx, now)
// Perform tick - context is created per-robot in Tick()
_ = m.Tick(m.ctx, now)
}
}
}
@ -208,8 +208,8 @@ func (m *Manager) tickerLoop() {
// 1. Get all cached robots
// 2. For each robot with clock trigger enabled
// 3. Check if should execute based on clock config
// 4. Submit to pool
func (m *Manager) Tick(ctx *types.Context, now time.Time) error {
// 4. Submit to pool with robot's own identity
func (m *Manager) Tick(parentCtx context.Context, now time.Time) error {
m.mu.RLock()
if !m.started {
m.mu.RUnlock()
@ -250,6 +250,11 @@ func (m *Manager) Tick(ctx *types.Context, now time.Time) error {
// continue
// }
// Create context with robot's own identity
// Clock-triggered executions run as the robot itself
robotAuth := m.buildRobotAuth(robot)
ctx := types.NewContext(parentCtx, robotAuth)
// Create clock context for P0 inspiration
clockCtx := types.NewClockContext(now, robot.Config.Clock.TZ)
@ -268,6 +273,17 @@ func (m *Manager) Tick(ctx *types.Context, now time.Time) error {
return nil
}
// buildRobotAuth creates AuthorizedInfo for a robot's own identity
// Used when robot executes autonomously (clock trigger)
func (m *Manager) buildRobotAuth(robot *types.Robot) *oauthtypes.AuthorizedInfo {
return &oauthtypes.AuthorizedInfo{
UserID: robot.MemberID,
TeamID: robot.TeamID,
// ClientID could be set to a special "robot-agent" identifier if needed
ClientID: "robot-agent",
}
}
// shouldTrigger checks if a robot should be triggered based on its clock config
func (m *Manager) shouldTrigger(robot *types.Robot, now time.Time) bool {
clock := robot.Config.Clock