ce19e9bdb7
- Added GetManager function to retrieve the global manager instance, returning nil if not started. - Introduced ExecRecovered event to notify about recovered non-terminal executions during manager startup. - Updated execution store to support querying by multiple statuses with ListByStatuses method. - Enhanced integration tests to accommodate longer tick intervals and added sleep delays for stability. - Improved cleanup logic in integration tests to prevent interference from previous execution records.
301 lines
8.5 KiB
Go
301 lines
8.5 KiB
Go
package manager_test
|
|
|
|
import (
|
|
"encoding/json"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/yaoapp/gou/model"
|
|
"github.com/yaoapp/xun/capsule"
|
|
"github.com/yaoapp/yao/agent/robot/manager"
|
|
"github.com/yaoapp/yao/agent/testutils"
|
|
)
|
|
|
|
const recoveryTestPrefix = "_test_recovery_"
|
|
|
|
func TestRecoveryOnRestart(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping integration test")
|
|
}
|
|
|
|
testutils.Prepare(t)
|
|
defer testutils.Clean(t)
|
|
|
|
t.Run("marks_running_as_failed_on_restart", func(t *testing.T) {
|
|
cleanupRecoveryData(t)
|
|
defer cleanupRecoveryData(t)
|
|
|
|
insertRecoveryExec(t, recoveryTestPrefix+"run_001", recoveryTestPrefix+"member_001", "team_r", "running")
|
|
insertRecoveryRobot(t, recoveryTestPrefix+"member_001", "team_r")
|
|
|
|
m := manager.New()
|
|
err := m.Start()
|
|
require.NoError(t, err)
|
|
defer m.Stop()
|
|
|
|
rec := getExecRecord(t, recoveryTestPrefix+"run_001")
|
|
require.NotNil(t, rec)
|
|
assert.Equal(t, "failed", rec["status"])
|
|
errMsg, _ := rec["error"].(string)
|
|
assert.Contains(t, errMsg, "server restart")
|
|
})
|
|
|
|
t.Run("keeps_waiting_on_restart", func(t *testing.T) {
|
|
cleanupRecoveryData(t)
|
|
defer cleanupRecoveryData(t)
|
|
|
|
insertRecoveryExec(t, recoveryTestPrefix+"wait_001", recoveryTestPrefix+"member_002", "team_r", "waiting")
|
|
insertRecoveryRobot(t, recoveryTestPrefix+"member_002", "team_r")
|
|
|
|
m := manager.New()
|
|
err := m.Start()
|
|
require.NoError(t, err)
|
|
defer m.Stop()
|
|
|
|
rec := getExecRecord(t, recoveryTestPrefix+"wait_001")
|
|
require.NotNil(t, rec)
|
|
assert.Equal(t, "waiting", rec["status"])
|
|
})
|
|
|
|
t.Run("keeps_confirming_on_restart", func(t *testing.T) {
|
|
cleanupRecoveryData(t)
|
|
defer cleanupRecoveryData(t)
|
|
|
|
insertRecoveryExec(t, recoveryTestPrefix+"conf_001", recoveryTestPrefix+"member_003", "team_r", "confirming")
|
|
insertRecoveryRobot(t, recoveryTestPrefix+"member_003", "team_r")
|
|
|
|
m := manager.New()
|
|
err := m.Start()
|
|
require.NoError(t, err)
|
|
defer m.Stop()
|
|
|
|
rec := getExecRecord(t, recoveryTestPrefix+"conf_001")
|
|
require.NotNil(t, rec)
|
|
assert.Equal(t, "confirming", rec["status"])
|
|
})
|
|
|
|
t.Run("marks_paused_as_failed_on_restart", func(t *testing.T) {
|
|
cleanupRecoveryData(t)
|
|
defer cleanupRecoveryData(t)
|
|
|
|
insertRecoveryExec(t, recoveryTestPrefix+"pause_001", recoveryTestPrefix+"member_004", "team_r", "paused")
|
|
insertRecoveryRobot(t, recoveryTestPrefix+"member_004", "team_r")
|
|
|
|
m := manager.New()
|
|
err := m.Start()
|
|
require.NoError(t, err)
|
|
defer m.Stop()
|
|
|
|
rec := getExecRecord(t, recoveryTestPrefix+"pause_001")
|
|
require.NotNil(t, rec)
|
|
assert.Equal(t, "failed", rec["status"])
|
|
})
|
|
|
|
t.Run("no_active_executions_starts_normally", func(t *testing.T) {
|
|
cleanupRecoveryData(t)
|
|
defer cleanupRecoveryData(t)
|
|
|
|
m := manager.New()
|
|
err := m.Start()
|
|
require.NoError(t, err)
|
|
defer m.Stop()
|
|
|
|
assert.True(t, m.IsStarted())
|
|
})
|
|
|
|
t.Run("updates_robot_status_to_idle_after_fail", func(t *testing.T) {
|
|
cleanupRecoveryData(t)
|
|
defer cleanupRecoveryData(t)
|
|
|
|
insertRecoveryExec(t, recoveryTestPrefix+"run_002", recoveryTestPrefix+"member_005", "team_r", "running")
|
|
insertRecoveryRobot(t, recoveryTestPrefix+"member_005", "team_r")
|
|
setRobotStatus(t, recoveryTestPrefix+"member_005", "working")
|
|
|
|
m := manager.New()
|
|
err := m.Start()
|
|
require.NoError(t, err)
|
|
defer m.Stop()
|
|
|
|
rec := getExecRecord(t, recoveryTestPrefix+"run_002")
|
|
require.NotNil(t, rec)
|
|
assert.Equal(t, "failed", rec["status"])
|
|
|
|
robot := getRobotRecord(t, recoveryTestPrefix+"member_005")
|
|
require.NotNil(t, robot)
|
|
assert.Equal(t, "idle", robot["robot_status"])
|
|
})
|
|
|
|
t.Run("keeps_robot_status_if_other_waiting", func(t *testing.T) {
|
|
cleanupRecoveryData(t)
|
|
defer cleanupRecoveryData(t)
|
|
|
|
insertRecoveryExec(t, recoveryTestPrefix+"run_003", recoveryTestPrefix+"member_006", "team_r", "running")
|
|
insertRecoveryExec(t, recoveryTestPrefix+"wait_003", recoveryTestPrefix+"member_006", "team_r", "waiting")
|
|
insertRecoveryRobot(t, recoveryTestPrefix+"member_006", "team_r")
|
|
setRobotStatus(t, recoveryTestPrefix+"member_006", "working")
|
|
|
|
m := manager.New()
|
|
err := m.Start()
|
|
require.NoError(t, err)
|
|
defer m.Stop()
|
|
|
|
// Running should be failed
|
|
rec := getExecRecord(t, recoveryTestPrefix+"run_003")
|
|
require.NotNil(t, rec)
|
|
assert.Equal(t, "failed", rec["status"])
|
|
|
|
// Waiting should remain
|
|
rec2 := getExecRecord(t, recoveryTestPrefix+"wait_003")
|
|
require.NotNil(t, rec2)
|
|
assert.Equal(t, "waiting", rec2["status"])
|
|
|
|
// Robot should NOT be set to idle because waiting exec still exists
|
|
robot := getRobotRecord(t, recoveryTestPrefix+"member_006")
|
|
require.NotNil(t, robot)
|
|
assert.NotEqual(t, "idle", robot["robot_status"],
|
|
"robot should not be idle when waiting execution exists")
|
|
})
|
|
|
|
t.Run("idempotent_on_double_restart", func(t *testing.T) {
|
|
cleanupRecoveryData(t)
|
|
defer cleanupRecoveryData(t)
|
|
|
|
insertRecoveryExec(t, recoveryTestPrefix+"run_004", recoveryTestPrefix+"member_007", "team_r", "running")
|
|
insertRecoveryRobot(t, recoveryTestPrefix+"member_007", "team_r")
|
|
|
|
// First start
|
|
m1 := manager.New()
|
|
err := m1.Start()
|
|
require.NoError(t, err)
|
|
m1.Stop()
|
|
|
|
rec := getExecRecord(t, recoveryTestPrefix+"run_004")
|
|
require.NotNil(t, rec)
|
|
assert.Equal(t, "failed", rec["status"])
|
|
|
|
// Second start — should not panic or error
|
|
m2 := manager.New()
|
|
err = m2.Start()
|
|
require.NoError(t, err)
|
|
defer m2.Stop()
|
|
|
|
rec2 := getExecRecord(t, recoveryTestPrefix+"run_004")
|
|
require.NotNil(t, rec2)
|
|
assert.Equal(t, "failed", rec2["status"])
|
|
})
|
|
}
|
|
|
|
// ==================== Helpers ====================
|
|
|
|
func insertRecoveryExec(t *testing.T, execID, memberID, teamID, status string) {
|
|
t.Helper()
|
|
mod := model.Select("__yao.agent.execution")
|
|
tableName := mod.MetaData.Table.Name
|
|
qb := capsule.Query()
|
|
|
|
now := time.Now()
|
|
err := qb.Table(tableName).Insert([]map[string]interface{}{
|
|
{
|
|
"execution_id": execID,
|
|
"member_id": memberID,
|
|
"team_id": teamID,
|
|
"trigger_type": "clock",
|
|
"status": status,
|
|
"phase": "run",
|
|
"start_time": now.Add(-1 * time.Hour),
|
|
},
|
|
})
|
|
require.NoError(t, err, "insert execution %s", execID)
|
|
}
|
|
|
|
func insertRecoveryRobot(t *testing.T, memberID, teamID string) {
|
|
t.Helper()
|
|
mod := model.Select("__yao.member")
|
|
tableName := mod.MetaData.Table.Name
|
|
qb := capsule.Query()
|
|
|
|
robotConfig := map[string]interface{}{
|
|
"identity": map[string]interface{}{"role": "Recovery Test Robot"},
|
|
"triggers": map[string]interface{}{
|
|
"clock": map[string]interface{}{"enabled": false},
|
|
},
|
|
}
|
|
configJSON, _ := json.Marshal(robotConfig)
|
|
|
|
err := qb.Table(tableName).Insert([]map[string]interface{}{
|
|
{
|
|
"member_id": memberID,
|
|
"team_id": teamID,
|
|
"member_type": "robot",
|
|
"display_name": "Recovery Test " + memberID,
|
|
"status": "active",
|
|
"role_id": "member",
|
|
"autonomous_mode": true,
|
|
"robot_status": "idle",
|
|
"robot_config": string(configJSON),
|
|
},
|
|
})
|
|
require.NoError(t, err, "insert robot %s", memberID)
|
|
}
|
|
|
|
func setRobotStatus(t *testing.T, memberID, status string) {
|
|
t.Helper()
|
|
mod := model.Select("__yao.member")
|
|
tableName := mod.MetaData.Table.Name
|
|
qb := capsule.Query()
|
|
_, err := qb.Table(tableName).Where("member_id", memberID).Update(map[string]interface{}{
|
|
"robot_status": status,
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func getExecRecord(t *testing.T, execID string) map[string]interface{} {
|
|
t.Helper()
|
|
mod := model.Select("__yao.agent.execution")
|
|
tableName := mod.MetaData.Table.Name
|
|
qb := capsule.Query()
|
|
|
|
rows, err := qb.Table(tableName).Where("execution_id", execID).Limit(1).Get()
|
|
require.NoError(t, err)
|
|
if len(rows) == 0 {
|
|
return nil
|
|
}
|
|
return map[string]interface{}(rows[0])
|
|
}
|
|
|
|
func getRobotRecord(t *testing.T, memberID string) map[string]interface{} {
|
|
t.Helper()
|
|
mod := model.Select("__yao.member")
|
|
tableName := mod.MetaData.Table.Name
|
|
qb := capsule.Query()
|
|
|
|
rows, err := qb.Table(tableName).Where("member_id", memberID).Limit(1).Get()
|
|
require.NoError(t, err)
|
|
if len(rows) == 0 {
|
|
return nil
|
|
}
|
|
return map[string]interface{}(rows[0])
|
|
}
|
|
|
|
func cleanupRecoveryData(t *testing.T) {
|
|
t.Helper()
|
|
|
|
// Clean executions
|
|
execMod := model.Select("__yao.agent.execution")
|
|
execTable := execMod.MetaData.Table.Name
|
|
qb := capsule.Query()
|
|
qb.Table(execTable).Where("execution_id", "like", recoveryTestPrefix+"%").Delete()
|
|
|
|
// Clean robots
|
|
memberMod := model.Select("__yao.member")
|
|
memberTable := memberMod.MetaData.Table.Name
|
|
qb.Table(memberTable).Where("member_id", "like", recoveryTestPrefix+"%").Delete()
|
|
// Also clean via model (soft delete)
|
|
memberMod.DeleteWhere(model.QueryParam{
|
|
Wheres: []model.QueryWhere{
|
|
{Column: "member_id", OP: "like", Value: recoveryTestPrefix + "%"},
|
|
},
|
|
})
|
|
}
|