Enhance Agent Documentation and Add Extract Command Functionality
- Updated README files to include new commands for running tests and extracting results for review. - Introduced the `yao agent extract` command to facilitate extraction of test results from JSONL files into Markdown or JSON formats. - Enhanced the `FormatAvailableResources` function to support localization and detailed information for agents and MCP tools. - Improved output formatting for better readability and usability in test result documentation.
This commit is contained in:
Max
parent
00b6b5ee0e
commit
6e68b42128
8 changed files with 847 additions and 12 deletions
|
|
@ -69,7 +69,14 @@ function Next(ctx: agent.Context, payload: agent.Payload): agent.Next {
|
||||||
### 3. Test (Optional)
|
### 3. Test (Optional)
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
|
# Run tests
|
||||||
yao agent test -i "Hello, how are you?"
|
yao agent test -i "Hello, how are you?"
|
||||||
|
|
||||||
|
# Run tests from JSONL file
|
||||||
|
yao agent test -i tests/inputs.jsonl -v
|
||||||
|
|
||||||
|
# Extract results for review
|
||||||
|
yao agent extract output-*.jsonl
|
||||||
```
|
```
|
||||||
|
|
||||||
### 4. Run
|
### 4. Run
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,15 @@
|
||||||
package standard
|
package standard
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/yaoapp/gou/mcp"
|
||||||
|
"github.com/yaoapp/yao/agent/assistant"
|
||||||
agentcontext "github.com/yaoapp/yao/agent/context"
|
agentcontext "github.com/yaoapp/yao/agent/context"
|
||||||
|
"github.com/yaoapp/yao/agent/i18n"
|
||||||
robottypes "github.com/yaoapp/yao/agent/robot/types"
|
robottypes "github.com/yaoapp/yao/agent/robot/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -115,6 +119,15 @@ func (f *InputFormatter) FormatRobotIdentity(robot *robottypes.Robot) string {
|
||||||
// This is critical for generating achievable goals - without knowing available tools,
|
// This is critical for generating achievable goals - without knowing available tools,
|
||||||
// the agent might generate goals that cannot be accomplished
|
// the agent might generate goals that cannot be accomplished
|
||||||
func (f *InputFormatter) FormatAvailableResources(robot *robottypes.Robot) string {
|
func (f *InputFormatter) FormatAvailableResources(robot *robottypes.Robot) string {
|
||||||
|
locale := "en" // default locale
|
||||||
|
if robot != nil && robot.Config != nil {
|
||||||
|
locale = robot.Config.GetDefaultLocale()
|
||||||
|
}
|
||||||
|
return f.FormatAvailableResourcesWithLocale(robot, locale)
|
||||||
|
}
|
||||||
|
|
||||||
|
// FormatAvailableResourcesWithLocale formats available resources with specific locale for i18n support
|
||||||
|
func (f *InputFormatter) FormatAvailableResourcesWithLocale(robot *robottypes.Robot, locale string) string {
|
||||||
if robot == nil || robot.Config == nil {
|
if robot == nil || robot.Config == nil {
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
@ -122,37 +135,153 @@ func (f *InputFormatter) FormatAvailableResources(robot *robottypes.Robot) strin
|
||||||
var sb strings.Builder
|
var sb strings.Builder
|
||||||
hasContent := false
|
hasContent := false
|
||||||
|
|
||||||
// Available Agents
|
// Available Agents - with detailed information (name, description)
|
||||||
if robot.Config.Resources != nil && len(robot.Config.Resources.Agents) > 0 {
|
if robot.Config.Resources != nil && len(robot.Config.Resources.Agents) > 0 {
|
||||||
if !hasContent {
|
if !hasContent {
|
||||||
sb.WriteString("## Available Resources\n\n")
|
sb.WriteString("## Available Resources\n\n")
|
||||||
hasContent = true
|
hasContent = true
|
||||||
}
|
}
|
||||||
sb.WriteString("### Agents\n")
|
sb.WriteString("### Agents\n")
|
||||||
sb.WriteString("These are the AI assistants you can delegate tasks to:\n")
|
sb.WriteString("These are the AI assistants you can delegate tasks to:\n\n")
|
||||||
for _, agent := range robot.Config.Resources.Agents {
|
for _, agentID := range robot.Config.Resources.Agents {
|
||||||
sb.WriteString(fmt.Sprintf("- **%s**\n", agent))
|
// Try to get agent details
|
||||||
|
ast, err := assistant.Get(agentID)
|
||||||
|
if err != nil {
|
||||||
|
// Fallback to just ID if agent not found
|
||||||
|
sb.WriteString(fmt.Sprintf("- **%s**\n", agentID))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get localized name and description
|
||||||
|
name := i18n.Translate(agentID, locale, ast.Name).(string)
|
||||||
|
description := ""
|
||||||
|
if ast.Description != "" {
|
||||||
|
description = i18n.Translate(agentID, locale, ast.Description).(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Format agent info
|
||||||
|
sb.WriteString(fmt.Sprintf("- **%s** (`%s`)\n", name, agentID))
|
||||||
|
if description != "" {
|
||||||
|
sb.WriteString(fmt.Sprintf(" - %s\n", description))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Available MCP Tools
|
// Available MCP Tools - with detailed tool information
|
||||||
if robot.Config.Resources != nil && len(robot.Config.Resources.MCP) > 0 {
|
if robot.Config.Resources != nil && len(robot.Config.Resources.MCP) > 0 {
|
||||||
if !hasContent {
|
if !hasContent {
|
||||||
sb.WriteString("## Available Resources\n\n")
|
sb.WriteString("## Available Resources\n\n")
|
||||||
hasContent = true
|
hasContent = true
|
||||||
}
|
}
|
||||||
sb.WriteString("### MCP Tools\n")
|
sb.WriteString("### MCP Tools\n")
|
||||||
sb.WriteString("These are the external tools and services you can use:\n")
|
sb.WriteString("These are the external tools and services you can use:\n\n")
|
||||||
for _, mcp := range robot.Config.Resources.MCP {
|
for _, mcpConfig := range robot.Config.Resources.MCP {
|
||||||
if len(mcp.Tools) > 0 {
|
// Try to get MCP client and list tools
|
||||||
sb.WriteString(fmt.Sprintf("- **%s**: %s\n", mcp.ID, strings.Join(mcp.Tools, ", ")))
|
client, err := mcp.Select(mcpConfig.ID)
|
||||||
|
if err != nil {
|
||||||
|
// Fallback to basic info if client not found
|
||||||
|
if len(mcpConfig.Tools) > 0 {
|
||||||
|
sb.WriteString(fmt.Sprintf("- **%s**: %s\n", mcpConfig.ID, strings.Join(mcpConfig.Tools, ", ")))
|
||||||
} else {
|
} else {
|
||||||
sb.WriteString(fmt.Sprintf("- **%s**: all tools available\n", mcp.ID))
|
sb.WriteString(fmt.Sprintf("- **%s**: all tools available\n", mcpConfig.ID))
|
||||||
}
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get client info for name and description
|
||||||
|
clientInfo := client.Info()
|
||||||
|
clientName := mcpConfig.ID
|
||||||
|
clientDesc := ""
|
||||||
|
if clientInfo != nil {
|
||||||
|
if clientInfo.Name != "" {
|
||||||
|
clientName = clientInfo.Name
|
||||||
|
}
|
||||||
|
if clientInfo.Description != "" {
|
||||||
|
clientDesc = clientInfo.Description
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write MCP header
|
||||||
|
if clientDesc != "" {
|
||||||
|
sb.WriteString(fmt.Sprintf("#### %s (`%s`)\n", clientName, mcpConfig.ID))
|
||||||
|
sb.WriteString(fmt.Sprintf("%s\n\n", clientDesc))
|
||||||
|
} else {
|
||||||
|
sb.WriteString(fmt.Sprintf("#### %s (`%s`)\n\n", clientName, mcpConfig.ID))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to list tools with context
|
||||||
|
ctx := context.Background()
|
||||||
|
toolsResp, err := client.ListTools(ctx, "")
|
||||||
|
if err != nil || toolsResp == nil {
|
||||||
|
// Fallback to configured tools
|
||||||
|
if len(mcpConfig.Tools) > 0 {
|
||||||
|
sb.WriteString("Available tools: ")
|
||||||
|
sb.WriteString(strings.Join(mcpConfig.Tools, ", "))
|
||||||
|
sb.WriteString("\n\n")
|
||||||
|
} else {
|
||||||
|
sb.WriteString("All tools available\n\n")
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Filter tools if specific tools are configured
|
||||||
|
toolsToShow := toolsResp.Tools
|
||||||
|
if len(mcpConfig.Tools) > 0 {
|
||||||
|
// Create a map for quick lookup
|
||||||
|
allowedTools := make(map[string]bool)
|
||||||
|
for _, t := range mcpConfig.Tools {
|
||||||
|
allowedTools[t] = true
|
||||||
|
}
|
||||||
|
// Filter tools
|
||||||
|
var filteredTools []struct {
|
||||||
|
Name string
|
||||||
|
Description string
|
||||||
|
}
|
||||||
|
for _, tool := range toolsResp.Tools {
|
||||||
|
if allowedTools[tool.Name] {
|
||||||
|
filteredTools = append(filteredTools, struct {
|
||||||
|
Name string
|
||||||
|
Description string
|
||||||
|
}{tool.Name, tool.Description})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Write filtered tools
|
||||||
|
if len(filteredTools) > 0 {
|
||||||
|
sb.WriteString("| Tool | Description |\n")
|
||||||
|
sb.WriteString("|------|-------------|\n")
|
||||||
|
for _, tool := range filteredTools {
|
||||||
|
desc := tool.Description
|
||||||
|
if len(desc) > 100 {
|
||||||
|
desc = desc[:97] + "..."
|
||||||
|
}
|
||||||
|
// Escape pipe characters in description
|
||||||
|
desc = strings.ReplaceAll(desc, "|", "\\|")
|
||||||
|
sb.WriteString(fmt.Sprintf("| `%s` | %s |\n", tool.Name, desc))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sb.WriteString("Configured tools: ")
|
||||||
|
sb.WriteString(strings.Join(mcpConfig.Tools, ", "))
|
||||||
|
}
|
||||||
|
} else if len(toolsToShow) > 0 {
|
||||||
|
// Show all available tools
|
||||||
|
sb.WriteString("| Tool | Description |\n")
|
||||||
|
sb.WriteString("|------|-------------|\n")
|
||||||
|
for _, tool := range toolsToShow {
|
||||||
|
desc := tool.Description
|
||||||
|
if len(desc) > 100 {
|
||||||
|
desc = desc[:97] + "..."
|
||||||
|
}
|
||||||
|
// Escape pipe characters in description
|
||||||
|
desc = strings.ReplaceAll(desc, "|", "\\|")
|
||||||
|
sb.WriteString(fmt.Sprintf("| `%s` | %s |\n", tool.Name, desc))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sb.WriteString("No tools available\n")
|
||||||
}
|
}
|
||||||
sb.WriteString("\n")
|
sb.WriteString("\n")
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Available Knowledge Base
|
// Available Knowledge Base
|
||||||
if robot.Config.KB != nil && len(robot.Config.KB.Collections) > 0 {
|
if robot.Config.KB != nil && len(robot.Config.KB.Collections) > 0 {
|
||||||
|
|
|
||||||
372
agent/robot/executor/standard/input_integration_test.go
Normal file
372
agent/robot/executor/standard/input_integration_test.go
Normal file
|
|
@ -0,0 +1,372 @@
|
||||||
|
package standard_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"github.com/yaoapp/yao/agent/robot/executor/standard"
|
||||||
|
"github.com/yaoapp/yao/agent/robot/types"
|
||||||
|
"github.com/yaoapp/yao/agent/testutils"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ============================================================================
|
||||||
|
// InputFormatter Integration Tests with Real Data
|
||||||
|
// These tests use the yao-dev-app environment with real assistants and MCPs
|
||||||
|
// ============================================================================
|
||||||
|
|
||||||
|
func TestFormatAvailableResourcesIntegration(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration test")
|
||||||
|
}
|
||||||
|
|
||||||
|
testutils.Prepare(t)
|
||||||
|
defer testutils.Clean(t)
|
||||||
|
|
||||||
|
formatter := standard.NewInputFormatter()
|
||||||
|
|
||||||
|
t.Run("formats_real_agents_with_details", func(t *testing.T) {
|
||||||
|
robot := &types.Robot{
|
||||||
|
MemberID: "test-robot-agents",
|
||||||
|
Config: &types.Config{
|
||||||
|
Identity: &types.Identity{
|
||||||
|
Role: "Test Robot",
|
||||||
|
Duties: []string{"Testing agent formatting"},
|
||||||
|
},
|
||||||
|
DefaultLocale: "en",
|
||||||
|
Resources: &types.Resources{
|
||||||
|
// Use real agents from yao-dev-app/assistants
|
||||||
|
Agents: []string{
|
||||||
|
"experts.data-analyst",
|
||||||
|
"experts.text-writer",
|
||||||
|
"experts.summarizer",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := formatter.FormatAvailableResources(robot)
|
||||||
|
|
||||||
|
// Verify structure
|
||||||
|
assert.Contains(t, result, "## Available Resources")
|
||||||
|
assert.Contains(t, result, "### Agents")
|
||||||
|
assert.Contains(t, result, "These are the AI assistants you can delegate tasks to:")
|
||||||
|
|
||||||
|
// Verify real agent details are included
|
||||||
|
// experts.data-analyst should show name and description
|
||||||
|
assert.Contains(t, result, "experts.data-analyst")
|
||||||
|
assert.Contains(t, result, "Data Analyst Expert") // Name from package.yao
|
||||||
|
|
||||||
|
// experts.text-writer
|
||||||
|
assert.Contains(t, result, "experts.text-writer")
|
||||||
|
|
||||||
|
// experts.summarizer
|
||||||
|
assert.Contains(t, result, "experts.summarizer")
|
||||||
|
|
||||||
|
// Verify important note is present
|
||||||
|
assert.Contains(t, result, "Only plan goals and tasks that can be accomplished")
|
||||||
|
|
||||||
|
t.Logf("Formatted agents result:\n%s", result)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("formats_real_mcp_with_tool_details", func(t *testing.T) {
|
||||||
|
robot := &types.Robot{
|
||||||
|
MemberID: "test-robot-mcp",
|
||||||
|
Config: &types.Config{
|
||||||
|
Identity: &types.Identity{
|
||||||
|
Role: "Test Robot",
|
||||||
|
Duties: []string{"Testing MCP formatting"},
|
||||||
|
},
|
||||||
|
DefaultLocale: "en",
|
||||||
|
Resources: &types.Resources{
|
||||||
|
// Use real MCPs from yao-dev-app/mcps
|
||||||
|
MCP: []types.MCPConfig{
|
||||||
|
{ID: "echo", Tools: []string{"ping", "status"}}, // Specific tools
|
||||||
|
{ID: "echo"}, // All tools
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := formatter.FormatAvailableResources(robot)
|
||||||
|
|
||||||
|
// Verify structure
|
||||||
|
assert.Contains(t, result, "## Available Resources")
|
||||||
|
assert.Contains(t, result, "### MCP Tools")
|
||||||
|
assert.Contains(t, result, "These are the external tools and services you can use:")
|
||||||
|
|
||||||
|
// Verify MCP details
|
||||||
|
assert.Contains(t, result, "echo")
|
||||||
|
|
||||||
|
// Verify important note is present
|
||||||
|
assert.Contains(t, result, "Only plan goals and tasks that can be accomplished")
|
||||||
|
|
||||||
|
t.Logf("Formatted MCP result:\n%s", result)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("formats_combined_resources", func(t *testing.T) {
|
||||||
|
robot := &types.Robot{
|
||||||
|
MemberID: "test-robot-combined",
|
||||||
|
Config: &types.Config{
|
||||||
|
Identity: &types.Identity{
|
||||||
|
Role: "Sales Analyst Robot",
|
||||||
|
Duties: []string{"Analyze sales data", "Generate reports"},
|
||||||
|
Rules: []string{"Be accurate", "Be concise"},
|
||||||
|
},
|
||||||
|
DefaultLocale: "en",
|
||||||
|
Resources: &types.Resources{
|
||||||
|
Agents: []string{
|
||||||
|
"experts.data-analyst",
|
||||||
|
"experts.summarizer",
|
||||||
|
},
|
||||||
|
MCP: []types.MCPConfig{
|
||||||
|
{ID: "echo", Tools: []string{"ping", "echo"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
KB: &types.KB{
|
||||||
|
Collections: []string{"sales-policies", "product-catalog"},
|
||||||
|
},
|
||||||
|
DB: &types.DB{
|
||||||
|
Models: []string{"sales", "customers", "orders"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := formatter.FormatAvailableResources(robot)
|
||||||
|
|
||||||
|
// Verify all sections are present
|
||||||
|
assert.Contains(t, result, "## Available Resources")
|
||||||
|
assert.Contains(t, result, "### Agents")
|
||||||
|
assert.Contains(t, result, "### MCP Tools")
|
||||||
|
assert.Contains(t, result, "### Knowledge Base")
|
||||||
|
assert.Contains(t, result, "### Database")
|
||||||
|
|
||||||
|
// Verify agents
|
||||||
|
assert.Contains(t, result, "experts.data-analyst")
|
||||||
|
assert.Contains(t, result, "experts.summarizer")
|
||||||
|
|
||||||
|
// Verify MCP
|
||||||
|
assert.Contains(t, result, "echo")
|
||||||
|
|
||||||
|
// Verify KB
|
||||||
|
assert.Contains(t, result, "sales-policies")
|
||||||
|
assert.Contains(t, result, "product-catalog")
|
||||||
|
|
||||||
|
// Verify DB
|
||||||
|
assert.Contains(t, result, "sales")
|
||||||
|
assert.Contains(t, result, "customers")
|
||||||
|
assert.Contains(t, result, "orders")
|
||||||
|
|
||||||
|
t.Logf("Formatted combined resources result:\n%s", result)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("handles_locale_zh", func(t *testing.T) {
|
||||||
|
robot := &types.Robot{
|
||||||
|
MemberID: "test-robot-zh",
|
||||||
|
Config: &types.Config{
|
||||||
|
Identity: &types.Identity{
|
||||||
|
Role: "测试机器人",
|
||||||
|
Duties: []string{"测试国际化"},
|
||||||
|
},
|
||||||
|
DefaultLocale: "zh-cn",
|
||||||
|
Resources: &types.Resources{
|
||||||
|
// Use agents that have zh-cn locales
|
||||||
|
Agents: []string{
|
||||||
|
"hello", // This agent has locales/zh-cn.yml
|
||||||
|
"mohe", // This agent also has locales/zh-cn.yml
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := formatter.FormatAvailableResourcesWithLocale(robot, "zh-cn")
|
||||||
|
|
||||||
|
// Verify structure
|
||||||
|
assert.Contains(t, result, "## Available Resources")
|
||||||
|
assert.Contains(t, result, "### Agents")
|
||||||
|
|
||||||
|
// Verify agents are listed
|
||||||
|
assert.Contains(t, result, "hello")
|
||||||
|
assert.Contains(t, result, "mohe")
|
||||||
|
|
||||||
|
t.Logf("Formatted zh-cn result:\n%s", result)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("gracefully_handles_missing_agents", func(t *testing.T) {
|
||||||
|
robot := &types.Robot{
|
||||||
|
MemberID: "test-robot-missing",
|
||||||
|
Config: &types.Config{
|
||||||
|
Identity: &types.Identity{
|
||||||
|
Role: "Test Robot",
|
||||||
|
},
|
||||||
|
Resources: &types.Resources{
|
||||||
|
Agents: []string{
|
||||||
|
"non-existent-agent",
|
||||||
|
"experts.data-analyst", // This one exists
|
||||||
|
"another-missing-agent",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := formatter.FormatAvailableResources(robot)
|
||||||
|
|
||||||
|
// Should not panic, should include fallback for missing agents
|
||||||
|
assert.Contains(t, result, "## Available Resources")
|
||||||
|
assert.Contains(t, result, "### Agents")
|
||||||
|
|
||||||
|
// Missing agents should still be listed with just ID
|
||||||
|
assert.Contains(t, result, "non-existent-agent")
|
||||||
|
assert.Contains(t, result, "another-missing-agent")
|
||||||
|
|
||||||
|
// Existing agent should have full details
|
||||||
|
assert.Contains(t, result, "experts.data-analyst")
|
||||||
|
assert.Contains(t, result, "Data Analyst Expert")
|
||||||
|
|
||||||
|
t.Logf("Formatted with missing agents:\n%s", result)
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("gracefully_handles_missing_mcp", func(t *testing.T) {
|
||||||
|
robot := &types.Robot{
|
||||||
|
MemberID: "test-robot-missing-mcp",
|
||||||
|
Config: &types.Config{
|
||||||
|
Identity: &types.Identity{
|
||||||
|
Role: "Test Robot",
|
||||||
|
},
|
||||||
|
Resources: &types.Resources{
|
||||||
|
MCP: []types.MCPConfig{
|
||||||
|
{ID: "non-existent-mcp", Tools: []string{"tool1", "tool2"}},
|
||||||
|
{ID: "echo"}, // This one exists
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := formatter.FormatAvailableResources(robot)
|
||||||
|
|
||||||
|
// Should not panic, should include fallback for missing MCP
|
||||||
|
assert.Contains(t, result, "## Available Resources")
|
||||||
|
assert.Contains(t, result, "### MCP Tools")
|
||||||
|
|
||||||
|
// Missing MCP should still be listed with fallback
|
||||||
|
assert.Contains(t, result, "non-existent-mcp")
|
||||||
|
assert.Contains(t, result, "tool1, tool2")
|
||||||
|
|
||||||
|
// Existing MCP should have details
|
||||||
|
assert.Contains(t, result, "echo")
|
||||||
|
|
||||||
|
t.Logf("Formatted with missing MCP:\n%s", result)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatAvailableResourcesTableFormat(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration test")
|
||||||
|
}
|
||||||
|
|
||||||
|
testutils.Prepare(t)
|
||||||
|
defer testutils.Clean(t)
|
||||||
|
|
||||||
|
formatter := standard.NewInputFormatter()
|
||||||
|
|
||||||
|
t.Run("mcp_tools_in_table_format", func(t *testing.T) {
|
||||||
|
robot := &types.Robot{
|
||||||
|
MemberID: "test-robot-table",
|
||||||
|
Config: &types.Config{
|
||||||
|
Identity: &types.Identity{
|
||||||
|
Role: "Test Robot",
|
||||||
|
},
|
||||||
|
Resources: &types.Resources{
|
||||||
|
MCP: []types.MCPConfig{
|
||||||
|
{ID: "echo"}, // All tools - should show table
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
result := formatter.FormatAvailableResources(robot)
|
||||||
|
|
||||||
|
// Check if table format is used when tools are available
|
||||||
|
// Table headers: | Tool | Description |
|
||||||
|
if strings.Contains(result, "| Tool | Description |") {
|
||||||
|
assert.Contains(t, result, "|------|-------------|")
|
||||||
|
t.Logf("MCP tools displayed in table format:\n%s", result)
|
||||||
|
} else {
|
||||||
|
// Fallback format
|
||||||
|
t.Logf("MCP tools displayed in fallback format:\n%s", result)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatClockContextWithRobotIntegration(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("Skipping integration test")
|
||||||
|
}
|
||||||
|
|
||||||
|
testutils.Prepare(t)
|
||||||
|
defer testutils.Clean(t)
|
||||||
|
|
||||||
|
formatter := standard.NewInputFormatter()
|
||||||
|
|
||||||
|
t.Run("full_context_for_inspiration", func(t *testing.T) {
|
||||||
|
// Create a realistic robot configuration
|
||||||
|
robot := &types.Robot{
|
||||||
|
MemberID: "sales-robot-001",
|
||||||
|
TeamID: "team-001",
|
||||||
|
DisplayName: "Sales Analyst Robot",
|
||||||
|
AutonomousMode: true,
|
||||||
|
Config: &types.Config{
|
||||||
|
Identity: &types.Identity{
|
||||||
|
Role: "Sales Analyst",
|
||||||
|
Duties: []string{"Monitor sales performance", "Generate daily reports", "Alert on anomalies"},
|
||||||
|
Rules: []string{"Only use approved data sources", "Maintain confidentiality"},
|
||||||
|
},
|
||||||
|
DefaultLocale: "en",
|
||||||
|
Resources: &types.Resources{
|
||||||
|
Agents: []string{
|
||||||
|
"experts.data-analyst",
|
||||||
|
"experts.summarizer",
|
||||||
|
},
|
||||||
|
MCP: []types.MCPConfig{
|
||||||
|
{ID: "echo", Tools: []string{"ping"}},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create clock context
|
||||||
|
clock := types.NewClockContext(time.Now(), "UTC")
|
||||||
|
|
||||||
|
// Format clock context (includes robot identity)
|
||||||
|
clockContent := formatter.FormatClockContext(clock, robot)
|
||||||
|
|
||||||
|
// Format available resources
|
||||||
|
resourcesContent := formatter.FormatAvailableResources(robot)
|
||||||
|
|
||||||
|
// Combine for full context (as done in inspiration.go)
|
||||||
|
fullContext := clockContent + "\n\n" + resourcesContent
|
||||||
|
|
||||||
|
// Verify full context contains all necessary information
|
||||||
|
require.NotEmpty(t, fullContext)
|
||||||
|
|
||||||
|
// Time context
|
||||||
|
assert.Contains(t, fullContext, "## Current Time Context")
|
||||||
|
assert.Contains(t, fullContext, "### Time Markers")
|
||||||
|
|
||||||
|
// Robot identity
|
||||||
|
assert.Contains(t, fullContext, "## Robot Identity")
|
||||||
|
assert.Contains(t, fullContext, "Sales Analyst")
|
||||||
|
assert.Contains(t, fullContext, "Monitor sales performance")
|
||||||
|
assert.Contains(t, fullContext, "Only use approved data sources")
|
||||||
|
|
||||||
|
// Available resources
|
||||||
|
assert.Contains(t, fullContext, "## Available Resources")
|
||||||
|
assert.Contains(t, fullContext, "### Agents")
|
||||||
|
assert.Contains(t, fullContext, "experts.data-analyst")
|
||||||
|
assert.Contains(t, fullContext, "### MCP Tools")
|
||||||
|
|
||||||
|
t.Logf("Full context for inspiration:\n%s", fullContext)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
@ -1428,6 +1428,51 @@ Simulates user behavior for dynamic mode testing.
|
||||||
yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent
|
yao agent test -i tests/dynamic.jsonl --simulator tests.simulator-agent
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Extract Command
|
||||||
|
|
||||||
|
Extract test results from output JSONL file to individual Markdown or JSON files for human review:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Extract to Markdown files (default)
|
||||||
|
yao agent extract output-20260127104118.jsonl
|
||||||
|
|
||||||
|
# Specify output directory
|
||||||
|
yao agent extract output.jsonl -o ./reports/
|
||||||
|
|
||||||
|
# Extract to JSON format
|
||||||
|
yao agent extract output.jsonl --format json
|
||||||
|
```
|
||||||
|
|
||||||
|
### Extract Command Options
|
||||||
|
|
||||||
|
| Flag | Description | Default |
|
||||||
|
| ---------- | ---------------------------------------- | ---------- |
|
||||||
|
| `-o` | Output directory | same as input |
|
||||||
|
| `--format` | Output format: `markdown`, `json` | `markdown` |
|
||||||
|
|
||||||
|
### Output Format (Markdown)
|
||||||
|
|
||||||
|
Each test result is extracted to a separate file:
|
||||||
|
|
||||||
|
```markdown
|
||||||
|
# T001-销售分析师-月末周五
|
||||||
|
|
||||||
|
**Status**: ✅ PASSED
|
||||||
|
|
||||||
|
**Duration**: 16743ms
|
||||||
|
|
||||||
|
## Input
|
||||||
|
(Full input content in markdown code block)
|
||||||
|
|
||||||
|
## Output
|
||||||
|
(Agent's response content)
|
||||||
|
```
|
||||||
|
|
||||||
|
This is useful for:
|
||||||
|
- Human review of agent outputs
|
||||||
|
- Comparing results across test runs
|
||||||
|
- Documentation and reporting
|
||||||
|
|
||||||
## Exit Codes
|
## Exit Codes
|
||||||
|
|
||||||
| Code | Description |
|
| Code | Description |
|
||||||
|
|
|
||||||
189
agent/test/extract.go
Normal file
189
agent/test/extract.go
Normal file
|
|
@ -0,0 +1,189 @@
|
||||||
|
package test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
jsoniter "github.com/json-iterator/go"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExtractOptions represents options for extracting test results
|
||||||
|
type ExtractOptions struct {
|
||||||
|
// InputFile is the path to the output JSONL file from test run
|
||||||
|
InputFile string
|
||||||
|
|
||||||
|
// OutputDir is the directory to write extracted files (default: same as input file)
|
||||||
|
OutputDir string
|
||||||
|
|
||||||
|
// Format is the output format: "markdown" (default), "json"
|
||||||
|
Format string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extractor extracts test results to individual files for review
|
||||||
|
type Extractor struct {
|
||||||
|
opts *ExtractOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewExtractor creates a new extractor
|
||||||
|
func NewExtractor(opts *ExtractOptions) *Extractor {
|
||||||
|
if opts.Format == "" {
|
||||||
|
opts.Format = "markdown"
|
||||||
|
}
|
||||||
|
if opts.OutputDir == "" {
|
||||||
|
opts.OutputDir = filepath.Dir(opts.InputFile)
|
||||||
|
}
|
||||||
|
return &Extractor{opts: opts}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract reads the test output file and extracts results to individual files
|
||||||
|
func (e *Extractor) Extract() ([]string, error) {
|
||||||
|
// Read the JSONL file
|
||||||
|
data, err := os.ReadFile(e.opts.InputFile)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to read input file: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the JSON (the output file is a single JSON object, not JSONL)
|
||||||
|
var report Report
|
||||||
|
if err := jsoniter.Unmarshal(data, &report); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to parse test report: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create output directory if it doesn't exist
|
||||||
|
if err := os.MkdirAll(e.opts.OutputDir, 0755); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create output directory: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var extractedFiles []string
|
||||||
|
|
||||||
|
// Extract each result
|
||||||
|
for _, result := range report.Results {
|
||||||
|
var filename string
|
||||||
|
var content string
|
||||||
|
|
||||||
|
switch e.opts.Format {
|
||||||
|
case "markdown":
|
||||||
|
filename = filepath.Join(e.opts.OutputDir, result.ID+".md")
|
||||||
|
content = e.formatMarkdown(result)
|
||||||
|
case "json":
|
||||||
|
filename = filepath.Join(e.opts.OutputDir, result.ID+".json")
|
||||||
|
jsonBytes, err := jsoniter.MarshalIndent(result, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return extractedFiles, fmt.Errorf("failed to marshal result %s: %w", result.ID, err)
|
||||||
|
}
|
||||||
|
content = string(jsonBytes)
|
||||||
|
default:
|
||||||
|
return nil, fmt.Errorf("unsupported format: %s", e.opts.Format)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.WriteFile(filename, []byte(content), 0644); err != nil {
|
||||||
|
return extractedFiles, fmt.Errorf("failed to write file %s: %w", filename, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extractedFiles = append(extractedFiles, filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
return extractedFiles, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatMarkdown formats a single test result as Markdown
|
||||||
|
func (e *Extractor) formatMarkdown(result *Result) string {
|
||||||
|
var sb strings.Builder
|
||||||
|
|
||||||
|
// Title
|
||||||
|
sb.WriteString(fmt.Sprintf("# %s\n\n", result.ID))
|
||||||
|
|
||||||
|
// Status badge
|
||||||
|
switch result.Status {
|
||||||
|
case StatusPassed:
|
||||||
|
sb.WriteString("**Status**: ✅ PASSED\n\n")
|
||||||
|
case StatusFailed:
|
||||||
|
sb.WriteString("**Status**: ❌ FAILED\n\n")
|
||||||
|
case StatusError:
|
||||||
|
sb.WriteString("**Status**: ⚠️ ERROR\n\n")
|
||||||
|
case StatusTimeout:
|
||||||
|
sb.WriteString("**Status**: ⏱️ TIMEOUT\n\n")
|
||||||
|
case StatusSkipped:
|
||||||
|
sb.WriteString("**Status**: ⏭️ SKIPPED\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Duration
|
||||||
|
sb.WriteString(fmt.Sprintf("**Duration**: %dms\n\n", result.DurationMs))
|
||||||
|
|
||||||
|
// Error (if any)
|
||||||
|
if result.Error != "" {
|
||||||
|
sb.WriteString("## Error\n\n")
|
||||||
|
sb.WriteString("```\n")
|
||||||
|
sb.WriteString(result.Error)
|
||||||
|
sb.WriteString("\n```\n\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Input
|
||||||
|
sb.WriteString("## Input\n\n")
|
||||||
|
sb.WriteString("```markdown\n")
|
||||||
|
sb.WriteString(formatInputAsString(result.Input))
|
||||||
|
sb.WriteString("\n```\n\n")
|
||||||
|
|
||||||
|
// Output
|
||||||
|
sb.WriteString("## Output\n\n")
|
||||||
|
output := formatOutputAsString(result.Output)
|
||||||
|
// Remove markdown code block wrapper if present
|
||||||
|
output = strings.TrimPrefix(output, "```markdown\n")
|
||||||
|
output = strings.TrimSuffix(output, "\n```")
|
||||||
|
output = strings.TrimSuffix(output, "```")
|
||||||
|
sb.WriteString(output)
|
||||||
|
sb.WriteString("\n")
|
||||||
|
|
||||||
|
return sb.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatInputAsString converts input to string format
|
||||||
|
func formatInputAsString(input interface{}) string {
|
||||||
|
switch v := input.(type) {
|
||||||
|
case string:
|
||||||
|
return v
|
||||||
|
case map[string]interface{}:
|
||||||
|
// Single message format
|
||||||
|
if content, ok := v["content"].(string); ok {
|
||||||
|
return content
|
||||||
|
}
|
||||||
|
// Fallback to JSON
|
||||||
|
jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ")
|
||||||
|
return string(jsonBytes)
|
||||||
|
case []interface{}:
|
||||||
|
// Array of messages - extract content from last user message
|
||||||
|
for i := len(v) - 1; i >= 0; i-- {
|
||||||
|
if msg, ok := v[i].(map[string]interface{}); ok {
|
||||||
|
if role, ok := msg["role"].(string); ok && role == "user" {
|
||||||
|
if content, ok := msg["content"].(string); ok {
|
||||||
|
return content
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Fallback to JSON
|
||||||
|
jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ")
|
||||||
|
return string(jsonBytes)
|
||||||
|
default:
|
||||||
|
jsonBytes, _ := jsoniter.MarshalIndent(input, "", " ")
|
||||||
|
return string(jsonBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatOutputAsString converts output to string format
|
||||||
|
func formatOutputAsString(output interface{}) string {
|
||||||
|
switch v := output.(type) {
|
||||||
|
case string:
|
||||||
|
return v
|
||||||
|
case map[string]interface{}, []interface{}:
|
||||||
|
jsonBytes, _ := jsoniter.MarshalIndent(v, "", " ")
|
||||||
|
return string(jsonBytes)
|
||||||
|
default:
|
||||||
|
if output == nil {
|
||||||
|
return "(no output)"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%v", output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -35,6 +35,11 @@ var langs = map[string]string{
|
||||||
"Error: agent (-n) is required when using direct message input and not in an agent directory": "错误: 使用直接消息输入且不在智能体目录时需要指定 -n 参数",
|
"Error: agent (-n) is required when using direct message input and not in an agent directory": "错误: 使用直接消息输入且不在智能体目录时需要指定 -n 参数",
|
||||||
"Hint: Make sure you're in a Yao application directory or specify --app flag": "提示: 确保在 Yao 应用目录中或使用 --app 参数指定",
|
"Hint: Make sure you're in a Yao application directory or specify --app flag": "提示: 确保在 Yao 应用目录中或使用 --app 参数指定",
|
||||||
"Error: invalid timeout format": "错误: 无效的超时格式",
|
"Error: invalid timeout format": "错误: 无效的超时格式",
|
||||||
|
// Extract command
|
||||||
|
"Extract test results to individual files for review": "提取测试结果到单独的文件供审查",
|
||||||
|
"Extract test results from output JSONL file to individual Markdown or JSON files": "从输出 JSONL 文件中提取测试结果到单独的 Markdown 或 JSON 文件",
|
||||||
|
"Output directory (default: same as input file)": "输出目录 (默认: 与输入文件相同)",
|
||||||
|
"Output format: markdown, json": "输出格式: markdown, json",
|
||||||
}
|
}
|
||||||
|
|
||||||
// L Language switch
|
// L Language switch
|
||||||
|
|
|
||||||
87
cmd/agent/extract.go
Normal file
87
cmd/agent/extract.go
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
package agent
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/fatih/color"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/yaoapp/yao/agent/test"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Extract command flags
|
||||||
|
var (
|
||||||
|
extractOutput string
|
||||||
|
extractFormat string
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExtractCmd is the agent extract command
|
||||||
|
var ExtractCmd = &cobra.Command{
|
||||||
|
Use: "extract <output-file.jsonl>",
|
||||||
|
Short: L("Extract test results to individual files for review"),
|
||||||
|
Long: L("Extract test results from output JSONL file to individual Markdown or JSON files"),
|
||||||
|
Args: cobra.ExactArgs(1),
|
||||||
|
Run: func(cmd *cobra.Command, args []string) {
|
||||||
|
inputFile := args[0]
|
||||||
|
|
||||||
|
// Resolve absolute path
|
||||||
|
absPath, err := filepath.Abs(inputFile)
|
||||||
|
if err != nil {
|
||||||
|
color.Red("Error: %s\n", err.Error())
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if file exists
|
||||||
|
if _, err := os.Stat(absPath); os.IsNotExist(err) {
|
||||||
|
color.Red("Error: file not found: %s\n", absPath)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build extract options
|
||||||
|
opts := &test.ExtractOptions{
|
||||||
|
InputFile: absPath,
|
||||||
|
OutputDir: extractOutput,
|
||||||
|
Format: extractFormat,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create extractor and run
|
||||||
|
extractor := test.NewExtractor(opts)
|
||||||
|
files, err := extractor.Extract()
|
||||||
|
if err != nil {
|
||||||
|
color.Red("Error: %s\n", err.Error())
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Print results
|
||||||
|
fmt.Println()
|
||||||
|
color.New(color.FgGreen, color.Bold).Println("═══════════════════════════════════════════════════════════════")
|
||||||
|
color.New(color.FgGreen, color.Bold).Println(" Extract Complete")
|
||||||
|
color.New(color.FgGreen, color.Bold).Println("═══════════════════════════════════════════════════════════════")
|
||||||
|
fmt.Println()
|
||||||
|
|
||||||
|
for _, file := range files {
|
||||||
|
color.New(color.FgGreen).Printf("✓ ")
|
||||||
|
fmt.Printf("Written: %s\n", filepath.Base(file))
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println()
|
||||||
|
color.New(color.FgWhite).Printf(" Total: ")
|
||||||
|
color.New(color.FgCyan).Printf("%d files\n", len(files))
|
||||||
|
|
||||||
|
if extractOutput != "" {
|
||||||
|
color.New(color.FgWhite).Printf(" Output: ")
|
||||||
|
color.New(color.FgCyan).Printf("%s\n", extractOutput)
|
||||||
|
} else {
|
||||||
|
color.New(color.FgWhite).Printf(" Output: ")
|
||||||
|
color.New(color.FgCyan).Printf("%s\n", filepath.Dir(absPath))
|
||||||
|
}
|
||||||
|
fmt.Println()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
// Extract command flags
|
||||||
|
ExtractCmd.Flags().StringVarP(&extractOutput, "output", "o", "", L("Output directory (default: same as input file)"))
|
||||||
|
ExtractCmd.Flags().StringVar(&extractFormat, "format", "markdown", L("Output format: markdown, json"))
|
||||||
|
}
|
||||||
|
|
@ -152,6 +152,7 @@ func init() {
|
||||||
|
|
||||||
// Agent
|
// Agent
|
||||||
agentCmd.AddCommand(agent.TestCmd)
|
agentCmd.AddCommand(agent.TestCmd)
|
||||||
|
agentCmd.AddCommand(agent.ExtractCmd)
|
||||||
|
|
||||||
rootCmd.AddCommand(
|
rootCmd.AddCommand(
|
||||||
versionCmd,
|
versionCmd,
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue