picoclaw/eval/cases/tool_calling.yaml

# Tool Calling Evaluation Cases
# Tests that the agent correctly selects and invokes the right tools.
# DragonScale uses progressive disclosure: the LLM may invoke tools directly
# (e.g. "read_file") or via the meta-tool "tool_call" with tool_name in args.

- description: "file read: agent reads a known fixture file and returns its content"
  vars:
    prompt: "Read the file eval_fixture.txt in the workspace and tell me the first line."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const hasRead = toolCalls.some(t => {
          if (t.tool === 'read_file') return true;
          if (t.tool === 'tool_call') {
            const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'read_file';
          }
          return false;
        });
        return { pass: hasRead, score: hasRead ? 1.0 : 0.0, reason: hasRead ? 'correctly used file read' : `did not read file (${toolCalls.length} tool calls: ${toolCalls.map(t=>t.tool).join(', ')})` };
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        const out = (trace.output || '').toLowerCase();
        const hasContent = out.includes('dragonscale eval fixture') || out.includes('hello from the eval harness');
        return { pass: hasContent, score: hasContent ? 1.0 : 0.3, reason: hasContent ? 'returned fixture content' : 'output did not contain expected fixture text' };

- description: "file write: agent creates a new file when asked"
  vars:
    prompt: "Create a file called eval_test_output.txt in the workspace with the text 'hello from eval'."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const hasWrite = toolCalls.some(t => {
          if (t.tool === 'write_file') return true;
          if (t.tool === 'tool_call') {
            const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'write_file';
          }
          return false;
        });
        return { pass: hasWrite, score: hasWrite ? 1.0 : 0.0, reason: hasWrite ? 'correctly used file write' : `did not write file (${toolCalls.length} tool calls: ${toolCalls.map(t=>t.tool).join(', ')})` };

- description: "shell exec: agent runs a shell command"
  vars:
    prompt: "Run the command 'echo dragonscale-eval-test' and tell me the output."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const hasExec = toolCalls.some(t => {
          if (t.tool === 'exec') return true;
          if (t.tool === 'tool_call') {
            const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'exec';
          }
          return false;
        });
        const outputContains = trace.output.includes('dragonscale-eval-test');
        return { pass: hasExec && outputContains, score: (hasExec ? 0.5 : 0) + (outputContains ? 0.5 : 0), reason: `exec=${hasExec}, output_correct=${outputContains} (${toolCalls.length} tool calls)` };

- description: "list directory: agent lists workspace contents"
  vars:
    prompt: "List the files and directories in my workspace root."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const hasListDir = toolCalls.some(t => {
          if (t.tool === 'list_dir') return true;
          if (t.tool === 'tool_call') {
            const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'list_dir';
          }
          return false;
        });
        return { pass: hasListDir, score: hasListDir ? 1.0 : 0.0, reason: hasListDir ? 'used list_dir' : `did not list directory (${toolCalls.length} tool calls: ${toolCalls.map(t=>t.tool).join(', ')})` };

- description: "edit file: agent edits an existing file"
  vars:
    prompt: "First write a file called edit_target.txt with 'hello world'. Then edit it to replace 'world' with 'dragonscale'. Read it back and confirm."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const getToolName = (t) => {
          if (t.tool !== 'tool_call') return t.tool;
          try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || t.tool; } catch(e) { return t.tool; }
        };
        const toolNames = toolCalls.map(getToolName);
        const hasEdit = toolNames.includes('edit_file');
        const hasWrite = toolNames.includes('write_file');
        const hasAnyModify = hasEdit || hasWrite;
        return { pass: hasAnyModify, score: hasEdit ? 1.0 : (hasWrite ? 0.7 : 0.0), reason: `edit=${hasEdit}, write=${hasWrite} (tools: ${toolNames.join(', ')})` };
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        const out = (trace.output || '').toLowerCase();
        const hasDragonScale = out.includes('dragonscale');
        return { pass: hasDragonScale, score: hasDragonScale ? 1.0 : 0.0, reason: hasDragonScale ? 'confirmed edit result' : 'did not confirm dragonscale in output' };

- description: "append file: agent appends to an existing file"
  vars:
    prompt: "Write 'line one' to append_test.txt. Then append 'line two' to the same file. Read it back and tell me both lines."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const getToolName = (t) => {
          if (t.tool !== 'tool_call') return t.tool;
          try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || t.tool; } catch(e) { return t.tool; }
        };
        const toolNames = toolCalls.map(getToolName);
        const hasAppend = toolNames.includes('append_file');
        const hasWrite = toolNames.filter(n => n === 'write_file').length >= 2;
        const modified = hasAppend || hasWrite;
        return { pass: modified, score: hasAppend ? 1.0 : (hasWrite ? 0.7 : 0.0), reason: `append=${hasAppend}, multi_write=${hasWrite} (tools: ${toolNames.join(', ')})` };

- description: "web search: agent searches the web"
  vars:
    prompt: "Search the web for 'dragonscale AI agent' and summarize what you find."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const getToolName = (t) => {
          if (t.tool !== 'tool_call') return t.tool;
          try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || t.tool; } catch(e) { return t.tool; }
        };
        const toolNames = toolCalls.map(getToolName);
        const hasWebSearch = toolNames.includes('web_search');
        return { pass: hasWebSearch, score: hasWebSearch ? 1.0 : 0.0, reason: hasWebSearch ? 'used web_search' : `no web search (tools: ${toolNames.join(', ')})` };

- description: "web fetch: agent fetches a URL"
  vars:
    prompt: "Fetch the contents of https://example.com and tell me the title of the page."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const getToolName = (t) => {
          if (t.tool !== 'tool_call') return t.tool;
          try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || t.tool; } catch(e) { return t.tool; }
        };
        const toolNames = toolCalls.map(getToolName);
        const hasFetch = toolNames.includes('web_fetch');
        return { pass: hasFetch, score: hasFetch ? 1.0 : 0.0, reason: hasFetch ? 'used web_fetch' : `no web_fetch (tools: ${toolNames.join(', ')})` };
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        const out = (trace.output || '').toLowerCase();
        const hasTitle = out.includes('example domain') || out.includes('example');
        return { pass: hasTitle, score: hasTitle ? 1.0 : 0.0, reason: hasTitle ? 'returned page content' : 'did not return page title' };