refactor(eval): streamline eval-runner and harness config

- eval/cmd/eval-runner/main.go: use pkg/eval/instrumentation, slim runner
- eval/cases/tool_calling.yaml: assertion updates
- eval/promptfooconfig.yaml: config tweak
- eval/README.md: doc update
- eval/scripts/compare.sh: comparison script updates

eval/README.md

@@ -130,4 +130,6 @@ python eval/scripts/generate_long_context_cases.py --count 12 --seed 20260221
 ## Environment Variables
 
 - `DRAGONSCALE_EVAL_CONFIG` - Optional overlay config path applied on top of user base config.
-- Base config discovery uses XDG first (`~/.config/dragonscale/config.json`), then legacy (`~/.dragonscale/config.json`), then XDG fallback if neither exists.
+- `DRAGONSCALE_EVAL_BASE_CONFIG` - Optional explicit base config path for eval runs.
+- `DRAGONSCALE_EVAL_HOST_HOME` - Path to a host-style home used as a fallback when container home paths are empty (used for host-mounted config discovery, typically `/host_home` in devcontainer).
+- Base config discovery uses XDG first (`~/.config/dragonscale/config.json`) with `DRAGONSCALE_EVAL_HOST_HOME`/`/host_home/.config/dragonscale/config.json` as a host-mount fallback.

eval/cases/tool_calling.yaml

@@ -104,8 +104,8 @@
       value: |
         const trace = JSON.parse(output);
         const out = (trace.output || '').toLowerCase();
-        const hasPicoclaw = out.includes('dragonscale');
+        const hasDragonScale = out.includes('dragonscale');
-        return { pass: hasPicoclaw, score: hasPicoclaw ? 1.0 : 0.0, reason: hasPicoclaw ? 'confirmed edit result' : 'did not confirm dragonscale in output' };
+        return { pass: hasDragonScale, score: hasDragonScale ? 1.0 : 0.0, reason: hasDragonScale ? 'confirmed edit result' : 'did not confirm dragonscale in output' };
 
 - description: "append file: agent appends to an existing file"
   vars:

eval/cmd/eval-runner/main.go

@@ -12,40 +12,11 @@ import (
 
 	fantasy "charm.land/fantasy"
 	"github.com/ZanzyTHEbar/dragonscale/pkg/config"
+	"github.com/ZanzyTHEbar/dragonscale/pkg/eval/instrumentation"
 	"github.com/ZanzyTHEbar/dragonscale/pkg/logger"
-	picoruntime "github.com/ZanzyTHEbar/dragonscale/pkg/runtime"
+	dragonruntime "github.com/ZanzyTHEbar/dragonscale/pkg/runtime"
 )
 
-// Trace is the structured output emitted by the eval runner.
-// Promptfoo parses this JSON to evaluate assertions.
-type Trace struct {
-	Output     string      `json:"output"`
-	Steps      []TraceStep `json:"steps"`
-	Metrics    Metrics     `json:"metrics"`
-	Error      string      `json:"error,omitempty"`
-	SessionKey string      `json:"session_key"`
-}
-
-type TraceStep struct {
-	Index    int             `json:"index"`
-	Type     string          `json:"type"`
-	Tool     string          `json:"tool,omitempty"`
-	Args     json.RawMessage `json:"args,omitempty"`
-	Result   string          `json:"result,omitempty"`
-	Duration int64           `json:"duration_ms,omitempty"`
-}
-
-type Metrics struct {
-	TotalDurationMs int64 `json:"total_duration_ms"`
-	StepCount       int   `json:"step_count"`
-	ToolCallCount   int   `json:"tool_call_count"`
-	InputTokens     int64 `json:"input_tokens"`
-	OutputTokens    int64 `json:"output_tokens"`
-	TotalTokens     int64 `json:"total_tokens"`
-	ReasoningTokens int64 `json:"reasoning_tokens"`
-	CacheReadTokens int64 `json:"cache_read_tokens"`
-}
-
 func main() {
 	logger.SetLevel(logger.ERROR)
 
@@ -74,20 +45,20 @@ func resolvePrompt() (string, error) {
 	return readPrompt()
 }
 
-func emptyPromptTrace(prompt string) *Trace {
+func emptyPromptTrace(prompt string) *instrumentation.Trace {
 	if strings.TrimSpace(prompt) != "" {
 		return nil
 	}
-	return &Trace{
+	return &instrumentation.Trace{
 		Output: "No prompt provided. Please provide a message.",
-		Metrics: Metrics{
+		Metrics: instrumentation.Metrics{
 			TotalDurationMs: 0,
 		},
 	}
 }
 
 func resolveEvalConfig() (*config.Config, error) {
-	return picoruntime.LoadEvalConfig(evalRunnerTimeout())
+	return dragonruntime.LoadEvalConfig(evalRunnerTimeout())
 }
 
 func readPrompt() (string, error) {
@@ -130,63 +101,55 @@ func parsePromptPayload(raw []byte) (string, bool) {
 	return payload.Prompt, true
 }
 
-func runEval(cfg *config.Config, prompt string) Trace {
+func runEval(cfg *config.Config, prompt string) instrumentation.Trace {
 	start := time.Now()
+	timeout := evalRunnerTimeout()
 
-	sessionKey := picoruntime.NewSessionKey("eval", start)
+	sessionKey := dragonruntime.NewSessionKey("eval", start)
-	runtime, initErr := newEvalRuntime(cfg)
+	runtime, initErr := newEvalRuntime(cfg, timeout)
 	if initErr != nil {
-		return Trace{Error: initErr.Error()}
+		return instrumentation.Trace{Error: initErr.Error()}
 	}
 	defer runtime.close()
 
-	result := picoruntime.RunPrompt(runtime.handle.Context(), runtime.handle, prompt, sessionKey)
+	result, duration := runEvalPrompt(runtime.handle, sessionKey, prompt, start)
+	return buildTrace(runtime.instrumentedModel, sessionKey, result, duration)
+}
 
+func runEvalPrompt(handle *dragonruntime.RuntimeHandle, sessionKey, prompt string, start time.Time) (dragonruntime.RunResult, time.Duration) {
+	result := dragonruntime.RunPrompt(handle.Context(), handle, prompt, sessionKey)
 	duration := result.Duration
 	if duration <= 0 {
 		duration = time.Since(start)
 	}
-
+	return result, duration
-	trace := Trace{
-		Output:     result.Output,
-		SessionKey: sessionKey,
-		Steps:      buildSteps(runtime.instrumentedModel),
-		Metrics: Metrics{
-			TotalDurationMs: duration.Milliseconds(),
-			StepCount:       len(runtime.instrumentedModel.calls),
-			InputTokens:     runtime.instrumentedModel.totalUsage.InputTokens,
-			OutputTokens:    runtime.instrumentedModel.totalUsage.OutputTokens,
-			TotalTokens:     runtime.instrumentedModel.totalUsage.TotalTokens,
-			ReasoningTokens: runtime.instrumentedModel.totalUsage.ReasoningTokens,
-			CacheReadTokens: runtime.instrumentedModel.totalUsage.CacheReadTokens,
-		},
 }
 
+func buildTrace(model *instrumentation.InstrumentedLanguageModel, sessionKey string, result dragonruntime.RunResult, duration time.Duration) instrumentation.Trace {
+	trace := instrumentation.Trace{
+		Output:     result.Output,
+		SessionKey: sessionKey,
+		Steps:      instrumentation.BuildSteps(model),
+		Metrics:    instrumentation.BuildMetrics(model, duration),
+	}
 	if result.Error != "" {
 		trace.Error = result.Error
 	}
-
-	for _, call := range runtime.instrumentedModel.calls {
-		for range call.toolCalls {
-			trace.Metrics.ToolCallCount++
-		}
-	}
-
 	return trace
 }
 
 type evalRuntime struct {
-	handle            *picoruntime.RuntimeHandle
+	handle            *dragonruntime.RuntimeHandle
-	instrumentedModel *instrumentedLanguageModel
+	instrumentedModel *instrumentation.InstrumentedLanguageModel
 }
 
-func newEvalRuntime(cfg *config.Config) (*evalRuntime, error) {
+func newEvalRuntime(cfg *config.Config, timeout time.Duration) (*evalRuntime, error) {
-	var instrumentedModel *instrumentedLanguageModel
+	var instrumentedModel *instrumentation.InstrumentedLanguageModel
-	handle, err := picoruntime.Bootstrap(context.Background(), cfg, picoruntime.BootstrapOptions{
+	handle, err := dragonruntime.Bootstrap(context.Background(), cfg, dragonruntime.BootstrapOptions{
-		Timeout:      evalRunnerTimeout(),
+		Timeout:      timeout,
-		OutboundMode: picoruntime.OutboundModeDrop,
+		OutboundMode: dragonruntime.OutboundModeDrop,
 		WrapModel: func(inner fantasy.LanguageModel) fantasy.LanguageModel {
-			instrumentedModel = &instrumentedLanguageModel{inner: inner}
+			instrumentedModel = instrumentation.Wrap(inner)
 			return instrumentedModel
 		},
 	})
@@ -210,41 +173,6 @@ func (r *evalRuntime) close() {
 	}
 }
 
-func buildSteps(model *instrumentedLanguageModel) []TraceStep {
-	var steps []TraceStep
-	idx := 0
-
-	for _, call := range model.calls {
-		steps = append(steps, TraceStep{
-			Index:    idx,
-			Type:     "llm_call",
-			Duration: call.duration.Milliseconds(),
-		})
-		idx++
-
-		for _, tc := range call.toolCalls {
-			argsRaw, _ := json.Marshal(tc.args)
-			steps = append(steps, TraceStep{
-				Index:  idx,
-				Type:   "tool_call",
-				Tool:   tc.name,
-				Args:   argsRaw,
-				Result: truncate(tc.result, 500),
-			})
-			idx++
-		}
-	}
-
-	return steps
-}
-
-func truncate(s string, maxLen int) string {
-	if len(s) <= maxLen {
-		return s
-	}
-	return s[:maxLen] + "...[truncated]"
-}
-
 func evalRunnerTimeout() time.Duration {
 	const defaultTimeout = 180 * time.Second
 	raw := strings.TrimSpace(os.Getenv("DRAGONSCALE_EVAL_TIMEOUT_MS"))
@@ -259,107 +187,10 @@ func evalRunnerTimeout() time.Duration {
 }
 
 func emitError(msg string) {
-	emitTrace(Trace{Error: msg})
+	emitTrace(instrumentation.Trace{Error: msg})
 }
 
-func emitTrace(trace Trace) {
+func emitTrace(trace instrumentation.Trace) {
 	out, _ := json.Marshal(trace)
 	fmt.Println(string(out))
 }
-
-type instrumentedCall struct {
-	duration  time.Duration
-	toolCalls []instrumentedToolCall
-	usage     fantasy.Usage
-}
-
-type instrumentedToolCall struct {
-	name   string
-	args   map[string]interface{}
-	result string
-}
-
-type instrumentedLanguageModel struct {
-	inner      fantasy.LanguageModel
-	calls      []instrumentedCall
-	totalUsage fantasy.Usage
-}
-
-var _ fantasy.LanguageModel = (*instrumentedLanguageModel)(nil)
-
-func (m *instrumentedLanguageModel) Provider() string { return m.inner.Provider() }
-func (m *instrumentedLanguageModel) Model() string    { return m.inner.Model() }
-
-func (m *instrumentedLanguageModel) GenerateObject(ctx context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
-	return m.inner.GenerateObject(ctx, call)
-}
-
-func (m *instrumentedLanguageModel) StreamObject(ctx context.Context, call fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) {
-	return m.inner.StreamObject(ctx, call)
-}
-
-func (m *instrumentedLanguageModel) Generate(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) {
-	start := time.Now()
-	resp, err := m.inner.Generate(ctx, call)
-	dur := time.Since(start)
-
-	ic := instrumentedCall{duration: dur}
-
-	if err == nil && resp != nil {
-		ic.usage = resp.Usage
-		m.totalUsage.InputTokens += resp.Usage.InputTokens
-		m.totalUsage.OutputTokens += resp.Usage.OutputTokens
-		m.totalUsage.TotalTokens += resp.Usage.TotalTokens
-		m.totalUsage.ReasoningTokens += resp.Usage.ReasoningTokens
-		m.totalUsage.CacheReadTokens += resp.Usage.CacheReadTokens
-
-		for _, tc := range resp.Content.ToolCalls() {
-			var args map[string]interface{}
-			_ = json.Unmarshal([]byte(tc.Input), &args)
-			ic.toolCalls = append(ic.toolCalls, instrumentedToolCall{
-				name: tc.ToolName,
-				args: args,
-			})
-		}
-	}
-
-	m.calls = append(m.calls, ic)
-	return resp, err
-}
-
-func (m *instrumentedLanguageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) {
-	start := time.Now()
-	stream, err := m.inner.Stream(ctx, call)
-	if err != nil {
-		m.calls = append(m.calls, instrumentedCall{duration: time.Since(start)})
-		return nil, err
-	}
-
-	ic := instrumentedCall{}
-
-	wrappedStream := func(yield func(fantasy.StreamPart) bool) {
-		stream(func(part fantasy.StreamPart) bool {
-			switch part.Type {
-			case fantasy.StreamPartTypeFinish:
-				ic.usage = part.Usage
-				m.totalUsage.InputTokens += part.Usage.InputTokens
-				m.totalUsage.OutputTokens += part.Usage.OutputTokens
-				m.totalUsage.TotalTokens += part.Usage.TotalTokens
-				m.totalUsage.ReasoningTokens += part.Usage.ReasoningTokens
-				m.totalUsage.CacheReadTokens += part.Usage.CacheReadTokens
-			case fantasy.StreamPartTypeToolCall:
-				var args map[string]interface{}
-				_ = json.Unmarshal([]byte(part.ToolCallInput), &args)
-				ic.toolCalls = append(ic.toolCalls, instrumentedToolCall{
-					name: part.ToolCallName,
-					args: args,
-				})
-			}
-			return yield(part)
-		})
-		ic.duration = time.Since(start)
-		m.calls = append(m.calls, ic)
-	}
-
-	return wrappedStream, nil
-}

eval/promptfooconfig.yaml

@@ -13,6 +13,7 @@ providers:
       timeout: 180000
     env:
       DRAGONSCALE_EVAL_CONFIG: "./configs/default.json"
+      DRAGONSCALE_EVAL_HOST_HOME: "/host_home"
 
 # Default assertions applied to every test case
 defaultTest:

eval/scripts/compare.sh

@@ -9,6 +9,14 @@ EVAL_DIR="$(dirname "$SCRIPT_DIR")"
 PROJECT_ROOT="$(dirname "$EVAL_DIR")"
 REPEAT=${1:-3}
 NPM_CMD="${EVAL_NPM_CMD:-npx}"
+read -r -a NPM_CMD_ARR <<< "${NPM_CMD}"
+export DEVCONTAINER_EXEC=""
+TEMP_CONFIG="$(mktemp "${SCRIPT_DIR}/promptfoo-compare-XXXXXX.yaml")"
+
+cleanup_compare_config() {
+  rm -f "$TEMP_CONFIG"
+}
+trap cleanup_compare_config EXIT INT TERM
 
 if [[ "${1:-}" == "--repeat" ]]; then
   REPEAT="${2:-3}"
@@ -22,7 +30,7 @@ cd "$PROJECT_ROOT"
 
 # 1. Build current branch eval-runner (instrumented wrapper)
 echo "[1/4] Building eval-runner from current branch..."
-make eval-build 2>&1 | tail -1
+make DEVCONTAINER_EXEC= eval-build 2>&1 | tail -1
 cp "$EVAL_DIR/bin/eval-runner" "$EVAL_DIR/bin/eval-runner-branch"
 
 # 2. Build main branch eval-runner
@@ -31,7 +39,7 @@ STASH_RESULT=$(git stash 2>&1)
 
 echo "[2/4] Building eval-runner from main branch..."
 git checkout main 2>/dev/null
-make eval-build 2>&1 | tail -1
+make DEVCONTAINER_EXEC= eval-build 2>&1 | tail -1
 cp "$EVAL_DIR/bin/eval-runner" "$EVAL_DIR/bin/eval-runner-main"
 
 # Restore working branch
@@ -46,8 +54,19 @@ cp "$EVAL_DIR/bin/eval-runner-branch" "$EVAL_DIR/bin/eval-runner"
 # 3. Update promptfoo config to enable comparison
 cd "$EVAL_DIR"
 
+# Resolve base config for promptfoo providers from explicit env if provided.
+EVAL_BASE_CONFIG="${DRAGONSCALE_EVAL_BASE_CONFIG:-}"
+EVAL_CONFIG="${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}"
+if [ -z "$EVAL_BASE_CONFIG" ] && [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then
+  echo "warning: DRAGONSCALE_EVAL_BASE_CONFIG not set; default config will be resolved by runtime" >&2
+fi
+
+if [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then
+  echo "DRAGONSCALE_EVAL_BASE_CONFIG=${EVAL_BASE_CONFIG}"
+fi
+
 # Create a temp config with both providers
-cat > promptfooconfig-compare.yaml <<'YAML'
+cat > "$TEMP_CONFIG" <<YAML
 description: "DragonScale A/B comparison (branch vs main)"
 
 providers:
@@ -56,13 +75,17 @@ providers:
     config:
       timeout: 120000
     env:
-      DRAGONSCALE_EVAL_CONFIG: "./configs/default.json"
+      DRAGONSCALE_EVAL_CONFIG: "${EVAL_CONFIG}"
+      DRAGONSCALE_EVAL_HOST_HOME: "/host_home"
+      DRAGONSCALE_EVAL_BASE_CONFIG: "${EVAL_BASE_CONFIG}"
   - id: "exec:./bin/eval-runner-main"
     label: "main"
     config:
       timeout: 120000
     env:
-      DRAGONSCALE_EVAL_CONFIG: "./configs/default.json"
+      DRAGONSCALE_EVAL_CONFIG: "${EVAL_CONFIG}"
+      DRAGONSCALE_EVAL_HOST_HOME: "/host_home"
+      DRAGONSCALE_EVAL_BASE_CONFIG: "${EVAL_BASE_CONFIG}"
 
 defaultTest:
   assert:
@@ -89,7 +112,7 @@ YAML
 
 # 4. Run comparison
 echo "[3/4] Running eval comparison (${REPEAT}x)..."
-"${NPM_CMD}" promptfoo eval -c promptfooconfig-compare.yaml --repeat "$REPEAT" --no-progress-bar
+"${NPM_CMD_ARR[@]}" promptfoo eval -c "$TEMP_CONFIG" --repeat "$REPEAT" --no-progress-bar
 
 echo ""
 echo "[4/4] Results saved to eval/results/comparison.json"