picoclaw/eval/cases/edge_cases.yaml

# Edge Case Evaluation
# Tests error handling, ambiguous inputs, and boundary conditions.

- description: "missing file: graceful handling of non-existent file inside workspace"
  vars:
    prompt: "Read the file nonexistent_eval_file_xyz987.txt in the workspace and tell me what it says."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error && !trace.output) return { pass: false, score: 0, reason: 'crashed: ' + trace.error };
        const out = (trace.output || '').toLowerCase();
        const isDefault = out.includes('completed processing but have no response');
        if (isDefault) return { pass: false, score: 0.2, reason: 'fell back to default empty response instead of reporting error' };
        const mentionsError = out.includes('not found') || out.includes('error') ||
                              out.includes('does not exist') || out.includes("doesn't exist") ||
                              out.includes('no such file') || out.includes('cannot') || out.includes('unable');
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const getToolName = (t) => {
          if (t.tool !== 'tool_call') return t.tool;
          try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || t.tool; } catch(e) { return t.tool; }
        };
        const attemptedRead = toolCalls.map(getToolName).includes('read_file');
        const hasResponse = (trace.output || '').length > 0;
        const partialPass = attemptedRead && hasResponse;
        const pass = mentionsError || partialPass;
        return { pass, score: mentionsError ? 1.0 : (partialPass ? 0.5 : 0), reason: mentionsError ? 'gracefully reported error' : (partialPass ? 'read attempted, minimal response' : 'did not report file missing') };

- description: "empty prompt resilience"
  vars:
    prompt: "   "
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        const hasOutput = (trace.output || '').length > 0;
        const noError = !trace.error;
        const graceful = hasOutput && noError;
        return { pass: graceful, score: graceful ? 1.0 : 0.0, reason: graceful ? 'handled gracefully with informative output' : `error=${trace.error || 'none'}, output_len=${(trace.output||'').length}` };

- description: "ambiguous request: agent asks for clarification or makes reasonable assumption"
  vars:
    prompt: "Do the thing with the file."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error && !trace.output) return { pass: false, score: 0, reason: 'crashed' };
        const traceOutput = trace.output || '';
        const isDefaultFallback = traceOutput.includes('completed processing but have no response');
        if (isDefaultFallback) return { pass: false, score: 0.2, reason: 'fell back to default empty response' };
        const responded = traceOutput.length > 10;
        return { pass: responded, score: responded ? 1.0 : 0.0, reason: responded ? 'provided a response' : 'no meaningful response' };

- description: "unicode content: handles non-ASCII text"
  vars:
    prompt: "Write the text '你好世界 🌍 dragonscale' to a file called unicode_test.txt, then read it back."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const getToolName = (t) => {
          if (t.tool !== 'tool_call') return t.tool;
          try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || t.tool; } catch(e) { return t.tool; }
        };
        const toolNames = toolCalls.map(getToolName);
        const hasWrite = toolNames.includes('write_file');
        const hasRead = toolNames.includes('read_file');
        return { pass: hasWrite && hasRead, score: (hasWrite ? 0.5 : 0) + (hasRead ? 0.5 : 0), reason: `write=${hasWrite}, read=${hasRead}` };

- description: "large output handling: agent summarizes verbose results"
  vars:
    prompt: "Run 'ls -la /usr/bin | head -50' and give me a brief summary of what you see."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error) return { pass: false, score: 0, reason: trace.error };
        const out = (trace.output || '');
        const isDefault = out.includes('completed processing but have no response');
        if (isDefault) {
          const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
          const getToolName = (t) => {
            if (t.tool !== 'tool_call') return t.tool;
            try { const a = typeof t.args === 'string' ? JSON.parse(t.args) : t.args; return (a && a.tool_name) || t.tool; } catch(e) { return t.tool; }
          };
          const usedExec = toolCalls.map(getToolName).includes('exec');
          return { pass: usedExec, score: usedExec ? 0.5 : 0, reason: usedExec ? 'exec used but no summary' : 'fell back to default empty response' };
        }
        const hasSummary = out.length > 20;
        return { pass: hasSummary, score: hasSummary ? 1.0 : 0.0, reason: `output length: ${out.length}` };

- description: "special characters in filename: handles spaces and symbols"
  vars:
    prompt: "Create a file called 'test file (1).txt' with the content 'special chars test' and confirm success."
  assert:
    - type: javascript
      value: |
        const trace = JSON.parse(output);
        if (trace.error && !trace.output) return { pass: false, score: 0, reason: 'crashed: ' + trace.error };
        const toolCalls = trace.steps.filter(s => s.type === 'tool_call');
        const hasToolCall = toolCalls.length > 0;
        const out = (trace.output || '');
        const isDefault = out.includes('completed processing but have no response');
        if (isDefault) return { pass: false, score: 0.2, reason: 'fell back to default empty response' };
        return { pass: hasToolCall, score: hasToolCall ? 1.0 : 0.0, reason: hasToolCall ? 'attempted file operation' : 'no tool calls made' };