db4d98a1d4
Update Go module path and all import references across the codebase. Rename cmd/picoclaw/ → cmd/dragonscale/, pkg/pcerrors/ → pkg/dserrors/. Update all environment variable prefixes PICOCLAW_* → DRAGONSCALE_*. Remove stale translated READMEs (ja, pt-br, zh) and legacy community roadmap doc. Update CI workflows, Dockerfile, goreleaser, and Makefile to reference the new binary name and module path. BREAKING CHANGE: module path changed; all consumers must update imports
90 lines
4.9 KiB
YAML
90 lines
4.9 KiB
YAML
# Meta Tool Evaluation Cases
|
|
# tool_search and tool_call are always-on gateway tools.
|
|
|
|
- description: "meta tools: tool_search discovers file tools"
|
|
vars:
|
|
prompt: "Search for a tool that can read files."
|
|
assert:
|
|
- type: javascript
|
|
value: |
|
|
const trace = JSON.parse(output);
|
|
if (trace.error) return { pass: false, score: 0, reason: trace.error };
|
|
const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
|
|
const hasToolSearch = toolCalls.some(t => t.tool === 'tool_search');
|
|
return { pass: hasToolSearch, score: hasToolSearch ? 1.0 : 0.0, reason: hasToolSearch ? 'used tool_search to discover tools' : `did not use tool_search (tools: ${toolCalls.map(t=>t.tool).join(', ')})` };
|
|
- type: javascript
|
|
value: |
|
|
const trace = JSON.parse(output);
|
|
const out = (trace.output || '').toLowerCase();
|
|
const mentionsRead = out.includes('read_file') || out.includes('read') || out.includes('file');
|
|
return { pass: mentionsRead, score: mentionsRead ? 1.0 : 0.0, reason: mentionsRead ? 'mentioned file reading capability' : 'did not mention file tools' };
|
|
|
|
- description: "meta tools: tool_call dispatches read_file correctly"
|
|
vars:
|
|
prompt: "Read the file sample_data.txt in the workspace and tell me line 5."
|
|
assert:
|
|
- type: javascript
|
|
value: |
|
|
const trace = JSON.parse(output);
|
|
if (trace.error) return { pass: false, score: 0, reason: trace.error };
|
|
const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
|
|
const hasToolCall = toolCalls.some(t => {
|
|
if (t.tool === 'tool_call') {
|
|
const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'read_file';
|
|
}
|
|
return false;
|
|
});
|
|
const direct = toolCalls.some(t => t.tool === 'read_file');
|
|
const pass = hasToolCall || direct;
|
|
return { pass, score: hasToolCall ? 1.0 : (direct ? 0.8 : 0.0), reason: hasToolCall ? 'used tool_call to dispatch read_file' : (direct ? 'used direct read_file gateway' : `no read dispatch (tools: ${toolCalls.map(t=>t.tool).join(', ')})`) };
|
|
- type: javascript
|
|
value: |
|
|
const trace = JSON.parse(output);
|
|
const out = (trace.output || '');
|
|
const hasMarker = out.includes('dragonscale-fixture-marker-abc123') || out.includes('fixture');
|
|
return { pass: hasMarker, score: hasMarker ? 1.0 : 0.0, reason: hasMarker ? 'returned fixture content' : 'did not return expected file content' };
|
|
|
|
- description: "meta tools: tool_call dispatches exec correctly"
|
|
vars:
|
|
prompt: "Run the command 'echo progressive-test-marker' and tell me the output."
|
|
assert:
|
|
- type: javascript
|
|
value: |
|
|
const trace = JSON.parse(output);
|
|
if (trace.error) return { pass: false, score: 0, reason: trace.error };
|
|
const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
|
|
const hasToolCall = toolCalls.some(t => {
|
|
if (t.tool === 'tool_call') {
|
|
const _ta = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return _ta && _ta.tool_name === 'exec';
|
|
}
|
|
return false;
|
|
});
|
|
const direct = toolCalls.some(t => t.tool === 'exec');
|
|
const pass = hasToolCall || direct;
|
|
return { pass, score: hasToolCall ? 1.0 : (direct ? 0.8 : 0.0), reason: hasToolCall ? 'used tool_call for exec' : (direct ? 'used direct exec gateway' : `no exec dispatch (tools: ${toolCalls.map(t=>t.tool).join(', ')})`) };
|
|
- type: javascript
|
|
value: |
|
|
const trace = JSON.parse(output);
|
|
const out = (trace.output || '');
|
|
const hasMarker = out.includes('progressive-test-marker');
|
|
return { pass: hasMarker, score: hasMarker ? 1.0 : 0.0, reason: hasMarker ? 'output contains marker' : 'marker not in output' };
|
|
|
|
- description: "meta tools: multi-step via tool_call indirection"
|
|
vars:
|
|
prompt: "Create a file called progressive_test.txt with 'hello progressive', then read it back to confirm."
|
|
assert:
|
|
- type: javascript
|
|
value: |
|
|
const trace = JSON.parse(output);
|
|
if (trace.error) return { pass: false, score: 0, reason: trace.error };
|
|
const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
|
|
const getDispatchedName = (t) => {
|
|
if (t.tool === 'tool_call') {
|
|
try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || 'unknown'; } catch(e) { return 'unknown'; }
|
|
}
|
|
return t.tool;
|
|
};
|
|
const dispatched = toolCalls.map(getDispatchedName);
|
|
const hasWrite = dispatched.includes('write_file');
|
|
const hasRead = dispatched.includes('read_file');
|
|
return { pass: hasWrite && hasRead, score: (hasWrite ? 0.5 : 0) + (hasRead ? 0.5 : 0), reason: `write=${hasWrite}, read=${hasRead} (dispatched: ${dispatched.join(', ')})` };
|