refactor(eval): use pkg/runtime in main and eval-runner; fix eval configs

cmd/picoclaw/main.go
- bootstrapAgentRuntime() wraps picoruntime.Bootstrap with OutboundModeNone
- agentCmd and gatewayCmd use bootstrapAgentRuntime instead of inline
  provider/model/bus/agentLoop construction
- loadConfig() delegates to picoruntime.LoadResolvedConfig
- Remove unused picofantasy import

eval/cmd/eval-runner/main.go
- newEvalRuntime() uses picoruntime.Bootstrap with OutboundModeDrop and
  a WrapModel hook to inject the instrumented language model
- resolveEvalConfig() delegates to picoruntime.LoadEvalConfig
- resolvePrompt/emptyPromptTrace/parsePromptPayload/emitTrace extracted
  as named helpers for clarity
- evalRunnerTimeout() reads PICOCLAW_EVAL_TIMEOUT_MS env var with a
  180 s default
- runEval() now calls picoruntime.RunPrompt and reads duration from
  RunResult.Duration

eval/configs/default.json
- Remove tools.progressive_disclosure (always-on, field no longer exists)
- Remove memory.enabled (always-on)
- Add tools.web.duckduckgo config
- Add agents.defaults.temperature: 0.1

eval/promptfooconfig.yaml
- Pass PICOCLAW_EVAL_CONFIG=./configs/default.json via provider env block
  so the overlay is applied regardless of how the runner is invoked

eval/cases/meta_tools.yaml
- read_file and exec dispatch assertions: accept direct gateway call (0.8)
  as well as tool_call dispatch (1.0); direct call is valid since those
  tools are now in the gateway set

eval/README.md
- Update PICOCLAW_EVAL_CONFIG description to reflect overlay semantics

Makefile
- Pass PICOCLAW_EVAL_CONFIG inline in the eval target for make eval

Makefile

@@ -225,7 +225,7 @@ eval-build: generate
 ## eval: Run the eval suite against the current build
 eval: eval-build eval-fixtures
 	@echo "Running eval suite..."
-	@cd eval && npx promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar
+	@cd eval && PICOCLAW_EVAL_CONFIG="./configs/default.json" npx promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar
 	@echo "Results: eval/results/latest.json"
 	@echo "View: cd eval && npx promptfoo view"
 

cmd/picoclaw/main.go

@@ -29,7 +29,6 @@ import (
 	"github.com/sipeed/picoclaw/pkg/config"
 	"github.com/sipeed/picoclaw/pkg/cron"
 	"github.com/sipeed/picoclaw/pkg/devices"
-	picofantasy "github.com/sipeed/picoclaw/pkg/fantasy"
 	"github.com/sipeed/picoclaw/pkg/health"
 	"github.com/sipeed/picoclaw/pkg/heartbeat"
 	"github.com/sipeed/picoclaw/pkg/itr"
@@ -37,6 +36,7 @@ import (
 	picomemory "github.com/sipeed/picoclaw/pkg/memory"
 	"github.com/sipeed/picoclaw/pkg/memory/delegate"
 	"github.com/sipeed/picoclaw/pkg/migrate"
+	picoruntime "github.com/sipeed/picoclaw/pkg/runtime"
 	"github.com/sipeed/picoclaw/pkg/security"
 	"github.com/sipeed/picoclaw/pkg/security/securebus"
 	"github.com/sipeed/picoclaw/pkg/skills"
@@ -439,27 +439,16 @@ func agentCmd() {
 		os.Exit(1)
 	}
 
-	fantasyProvider, err := picofantasy.CreateProvider(cfg)
-	if err != nil {
-		fmt.Printf("Error creating provider: %v\n", err)
-		os.Exit(1)
-	}
-
 	appCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
 	defer stop()
 
-	languageModel, err := fantasyProvider.LanguageModel(appCtx, picofantasy.ModelID(cfg))
+	rt, err := bootstrapAgentRuntime(appCtx, cfg)
-	if err != nil {
-		fmt.Printf("Error creating language model: %v\n", err)
-		os.Exit(1)
-	}
-
-	msgBus := bus.NewMessageBus()
-	agentLoop, err := agent.NewAgentLoop(appCtx, cfg, msgBus, languageModel)
 	if err != nil {
 		fmt.Printf("Error initializing agent: %v\n", err)
 		os.Exit(1)
 	}
+	defer rt.Close()
+	agentLoop := rt.AgentLoop()
 
 	closeBus := setupSecureBus(agentLoop)
 	defer closeBus()
@@ -586,27 +575,17 @@ func gatewayCmd() {
 		os.Exit(1)
 	}
 
-	fantasyProvider, err := picofantasy.CreateProvider(cfg)
-	if err != nil {
-		fmt.Printf("Error creating provider: %v\n", err)
-		os.Exit(1)
-	}
-
 	appCtx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	languageModel, err := fantasyProvider.LanguageModel(appCtx, picofantasy.ModelID(cfg))
+	rt, err := bootstrapAgentRuntime(appCtx, cfg)
-	if err != nil {
-		fmt.Printf("Error creating language model: %v\n", err)
-		os.Exit(1)
-	}
-
-	msgBus := bus.NewMessageBus()
-	agentLoop, err := agent.NewAgentLoop(appCtx, cfg, msgBus, languageModel)
 	if err != nil {
 		fmt.Printf("Error initializing agent: %v\n", err)
 		os.Exit(1)
 	}
+	defer rt.Close()
+	agentLoop := rt.AgentLoop()
+	msgBus := rt.MessageBus()
 
 	closeBus := setupSecureBus(agentLoop)
 	defer closeBus()
@@ -1555,7 +1534,15 @@ func setupCronTool(appCtx context.Context, agentLoop *agent.AgentLoop, msgBus *b
 }
 
 func loadConfig() (*config.Config, error) {
-	return config.LoadConfig(getConfigPath())
+	return picoruntime.LoadResolvedConfig(picoruntime.LoadConfigOptions{
+		BaseConfigPath: getConfigPath(),
+	})
+}
+
+func bootstrapAgentRuntime(appCtx context.Context, cfg *config.Config) (*picoruntime.RuntimeHandle, error) {
+	return picoruntime.Bootstrap(appCtx, cfg, picoruntime.BootstrapOptions{
+		OutboundMode: picoruntime.OutboundModeNone,
+	})
 }
 
 // setupSecureBus wires the Isolated Tool Runtime into an AgentLoop.

eval/README.md

@@ -114,4 +114,5 @@ Create a new YAML file in `eval/cases/` following this pattern:
 
 ## Environment Variables
 
-- `PICOCLAW_EVAL_CONFIG` - Path to picoclaw config.json for eval (defaults to `~/.picoclaw/config.json`)
+- `PICOCLAW_EVAL_CONFIG` - Optional overlay config path applied on top of user base config.
+- Base config discovery uses XDG first (`~/.config/picoclaw/config.json`), then legacy (`~/.picoclaw/config.json`), then XDG fallback if neither exists.

eval/cases/meta_tools.yaml

@@ -34,7 +34,9 @@
           }
           return false;
         });
-        return { pass: hasToolCall, score: hasToolCall ? 1.0 : 0.0, reason: hasToolCall ? 'used tool_call to dispatch read_file' : `no tool_call dispatch (tools: ${toolCalls.map(t=>t.tool).join(', ')})` };
+        const direct = toolCalls.some(t => t.tool === 'read_file');
+        const pass = hasToolCall || direct;
+        return { pass, score: hasToolCall ? 1.0 : (direct ? 0.8 : 0.0), reason: hasToolCall ? 'used tool_call to dispatch read_file' : (direct ? 'used direct read_file gateway' : `no read dispatch (tools: ${toolCalls.map(t=>t.tool).join(', ')})`) };
     - type: javascript
       value: |
         const trace = JSON.parse(output);
@@ -57,7 +59,9 @@
           }
           return false;
         });
-        return { pass: hasToolCall, score: hasToolCall ? 1.0 : 0.0, reason: hasToolCall ? 'used tool_call for exec' : `no tool_call dispatch to exec (tools: ${toolCalls.map(t=>t.tool).join(', ')})` };
+        const direct = toolCalls.some(t => t.tool === 'exec');
+        const pass = hasToolCall || direct;
+        return { pass, score: hasToolCall ? 1.0 : (direct ? 0.8 : 0.0), reason: hasToolCall ? 'used tool_call for exec' : (direct ? 'used direct exec gateway' : `no exec dispatch (tools: ${toolCalls.map(t=>t.tool).join(', ')})`) };
     - type: javascript
       value: |
         const trace = JSON.parse(output);

eval/cmd/eval-runner/main.go

@@ -6,15 +6,14 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
 	fantasy "charm.land/fantasy"
-	"github.com/sipeed/picoclaw/pkg/agent"
-	"github.com/sipeed/picoclaw/pkg/bus"
 	"github.com/sipeed/picoclaw/pkg/config"
-	picofantasy "github.com/sipeed/picoclaw/pkg/fantasy"
 	"github.com/sipeed/picoclaw/pkg/logger"
+	picoruntime "github.com/sipeed/picoclaw/pkg/runtime"
 )
 
 // Trace is the structured output emitted by the eval runner.
@@ -50,33 +49,45 @@ type Metrics struct {
 func main() {
 	logger.SetLevel(logger.ERROR)
 
-	prompt, err := readPrompt()
+	prompt, err := resolvePrompt()
 	if err != nil {
 		emitError(fmt.Sprintf("failed to read prompt: %v", err))
 		return
 	}
 
-	if strings.TrimSpace(prompt) == "" {
+	if empty := emptyPromptTrace(prompt); empty != nil {
-		trace := Trace{
+		emitTrace(*empty)
-			Output: "No prompt provided. Please provide a message.",
-			Metrics: Metrics{
-				TotalDurationMs: 0,
-			},
-		}
-		out, _ := json.Marshal(trace)
-		fmt.Println(string(out))
 		return
 	}
 
-	cfg, err := loadEvalConfig()
+	cfg, err := resolveEvalConfig()
 	if err != nil {
 		emitError(fmt.Sprintf("config error: %v", err))
 		return
 	}
 
 	trace := runEval(cfg, prompt)
-	out, _ := json.Marshal(trace)
+	emitTrace(trace)
-	fmt.Println(string(out))
+}
+
+func resolvePrompt() (string, error) {
+	return readPrompt()
+}
+
+func emptyPromptTrace(prompt string) *Trace {
+	if strings.TrimSpace(prompt) != "" {
+		return nil
+	}
+	return &Trace{
+		Output: "No prompt provided. Please provide a message.",
+		Metrics: Metrics{
+			TotalDurationMs: 0,
+		},
+	}
+}
+
+func resolveEvalConfig() (*config.Config, error) {
+	return picoruntime.LoadEvalConfig(evalRunnerTimeout())
 }
 
 func readPrompt() (string, error) {
@@ -87,11 +98,8 @@ func readPrompt() (string, error) {
 	// promptfoo exec: provider passes the prompt as the first positional argument
 	if len(os.Args) > 1 && !strings.HasPrefix(os.Args[1], "-") {
 		raw := os.Args[1]
-		var promptData struct {
+		if prompt, ok := parsePromptPayload([]byte(raw)); ok {
-			Prompt string `json:"prompt"`
+			return prompt, nil
-		}
-		if json.Unmarshal([]byte(raw), &promptData) == nil && promptData.Prompt != "" {
-			return promptData.Prompt, nil
 		}
 		return strings.TrimSpace(raw), nil
 	}
@@ -103,11 +111,8 @@ func readPrompt() (string, error) {
 		if err != nil {
 			return "", err
 		}
-		var promptData struct {
+		if prompt, ok := parsePromptPayload(data); ok {
-			Prompt string `json:"prompt"`
+			return prompt, nil
-		}
-		if json.Unmarshal(data, &promptData) == nil && promptData.Prompt != "" {
-			return promptData.Prompt, nil
 		}
 		return strings.TrimSpace(string(data)), nil
 	}
@@ -115,121 +120,53 @@ func readPrompt() (string, error) {
 	return "", fmt.Errorf("no prompt provided (use positional arg, --prompt, or pipe to stdin)")
 }
 
-// resolveBaseConfigPath returns the path to the user's config file, checking
+func parsePromptPayload(raw []byte) (string, bool) {
-// candidate paths in order and returning the first one that exists. Falls back
+	var payload struct {
-// to the XDG-standard path even if the file is absent, matching the main
+		Prompt string `json:"prompt"`
-// picoclaw binary's behaviour.
-func resolveBaseConfigPath() string {
-	// Prefer XDG standard path (~/.config/picoclaw/config.json).
-	if xdgPath, err := config.DefaultConfigPath(); err == nil {
-		if _, statErr := os.Stat(xdgPath); statErr == nil {
-			return xdgPath
 	}
+	if err := json.Unmarshal(raw, &payload); err != nil || payload.Prompt == "" {
+		return "", false
 	}
-	// Legacy path used before XDG migration.
+	return payload.Prompt, true
-	home, _ := os.UserHomeDir()
-	legacy := home + "/.picoclaw/config.json"
-	if _, err := os.Stat(legacy); err == nil {
-		return legacy
-	}
-	// Neither exists; return the XDG path so LoadConfig returns defaults.
-	if xdgPath, err := config.DefaultConfigPath(); err == nil {
-		return xdgPath
-	}
-	return home + "/.picoclaw/config.json"
-}
-
-func loadEvalConfig() (*config.Config, error) {
-	// Always load the user's base config first so provider API keys,
-	// model selection, and other credentials are preserved.
-	cfg, err := config.LoadConfig(resolveBaseConfigPath())
-	if err != nil {
-		return nil, fmt.Errorf("load base config: %w", err)
-	}
-
-	// If an eval-specific override file is set, merge it on top of the base
-	// config. Only fields present in the overlay are changed; API keys etc.
-	// from the base config are preserved.
-	if overlayPath := os.Getenv("PICOCLAW_EVAL_CONFIG"); overlayPath != "" {
-		if err := config.OverlayConfigFile(cfg, overlayPath); err != nil {
-			return nil, fmt.Errorf("load eval overlay: %w", err)
-		}
-	}
-
-	return cfg, nil
 }
 
 func runEval(cfg *config.Config, prompt string) Trace {
 	start := time.Now()
 
-	sessionKey := fmt.Sprintf("eval:%d", start.UnixNano())
+	sessionKey := picoruntime.NewSessionKey("eval", start)
-
+	runtime, initErr := newEvalRuntime(cfg)
-	fantasyProvider, err := picofantasy.CreateProvider(cfg)
+	if initErr != nil {
-	if err != nil {
+		return Trace{Error: initErr.Error()}
-		return Trace{Error: fmt.Sprintf("provider error: %v", err)}
 	}
+	defer runtime.close()
 
-	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	result := picoruntime.RunPrompt(runtime.handle.Context(), runtime.handle, prompt, sessionKey)
-	defer cancel()
 
-	languageModel, err := fantasyProvider.LanguageModel(ctx, picofantasy.ModelID(cfg))
+	duration := result.Duration
-	if err != nil {
+	if duration <= 0 {
-		return Trace{Error: fmt.Sprintf("model error: %v", err)}
+		duration = time.Since(start)
 	}
 
-	instrumentedModel := &instrumentedLanguageModel{
-		inner: languageModel,
-	}
-
-	msgBus := bus.NewMessageBus()
-
-	// Drain outbound messages to prevent blocking
-	outDone := make(chan struct{})
-	go func() {
-		defer close(outDone)
-		for {
-			_, ok := msgBus.SubscribeOutbound(ctx)
-			if !ok {
-				return
-			}
-		}
-	}()
-
-	agentLoop, err := agent.NewAgentLoop(ctx, cfg, msgBus, instrumentedModel)
-	if err != nil {
-		cancel()
-		<-outDone
-		return Trace{Error: fmt.Sprintf("agent loop init error: %v", err)}
-	}
-	defer agentLoop.Stop()
-
-	response, err := agentLoop.ProcessDirect(ctx, prompt, sessionKey)
-
-	cancel() // Signals context done, which stops outbound drain
-	<-outDone
-
-	duration := time.Since(start)
-
 	trace := Trace{
-		Output:     response,
+		Output:     result.Output,
 		SessionKey: sessionKey,
-		Steps:      buildSteps(instrumentedModel),
+		Steps:      buildSteps(runtime.instrumentedModel),
 		Metrics: Metrics{
 			TotalDurationMs: duration.Milliseconds(),
-			StepCount:       len(instrumentedModel.calls),
+			StepCount:       len(runtime.instrumentedModel.calls),
-			InputTokens:     instrumentedModel.totalUsage.InputTokens,
+			InputTokens:     runtime.instrumentedModel.totalUsage.InputTokens,
-			OutputTokens:    instrumentedModel.totalUsage.OutputTokens,
+			OutputTokens:    runtime.instrumentedModel.totalUsage.OutputTokens,
-			TotalTokens:     instrumentedModel.totalUsage.TotalTokens,
+			TotalTokens:     runtime.instrumentedModel.totalUsage.TotalTokens,
-			ReasoningTokens: instrumentedModel.totalUsage.ReasoningTokens,
+			ReasoningTokens: runtime.instrumentedModel.totalUsage.ReasoningTokens,
-			CacheReadTokens: instrumentedModel.totalUsage.CacheReadTokens,
+			CacheReadTokens: runtime.instrumentedModel.totalUsage.CacheReadTokens,
 		},
 	}
 
-	if err != nil {
+	if result.Error != "" {
-		trace.Error = err.Error()
+		trace.Error = result.Error
 	}
 
-	for _, call := range instrumentedModel.calls {
+	for _, call := range runtime.instrumentedModel.calls {
 		for range call.toolCalls {
 			trace.Metrics.ToolCallCount++
 		}
@@ -238,6 +175,41 @@ func runEval(cfg *config.Config, prompt string) Trace {
 	return trace
 }
 
+type evalRuntime struct {
+	handle            *picoruntime.RuntimeHandle
+	instrumentedModel *instrumentedLanguageModel
+}
+
+func newEvalRuntime(cfg *config.Config) (*evalRuntime, error) {
+	var instrumentedModel *instrumentedLanguageModel
+	handle, err := picoruntime.Bootstrap(context.Background(), cfg, picoruntime.BootstrapOptions{
+		Timeout:      evalRunnerTimeout(),
+		OutboundMode: picoruntime.OutboundModeDrop,
+		WrapModel: func(inner fantasy.LanguageModel) fantasy.LanguageModel {
+			instrumentedModel = &instrumentedLanguageModel{inner: inner}
+			return instrumentedModel
+		},
+	})
+	if err != nil {
+		return nil, err
+	}
+	if instrumentedModel == nil {
+		handle.Close()
+		return nil, fmt.Errorf("instrumented model wrapper not initialized")
+	}
+
+	return &evalRuntime{
+		handle:            handle,
+		instrumentedModel: instrumentedModel,
+	}, nil
+}
+
+func (r *evalRuntime) close() {
+	if r.handle != nil {
+		r.handle.Close()
+	}
+}
+
 func buildSteps(model *instrumentedLanguageModel) []TraceStep {
 	var steps []TraceStep
 	idx := 0
@@ -273,8 +245,24 @@ func truncate(s string, maxLen int) string {
 	return s[:maxLen] + "...[truncated]"
 }
 
+func evalRunnerTimeout() time.Duration {
+	const defaultTimeout = 180 * time.Second
+	raw := strings.TrimSpace(os.Getenv("PICOCLAW_EVAL_TIMEOUT_MS"))
+	if raw == "" {
+		return defaultTimeout
+	}
+	ms, err := strconv.Atoi(raw)
+	if err != nil || ms <= 0 {
+		return defaultTimeout
+	}
+	return time.Duration(ms) * time.Millisecond
+}
+
 func emitError(msg string) {
-	trace := Trace{Error: msg}
+	emitTrace(Trace{Error: msg})
+}
+
+func emitTrace(trace Trace) {
 	out, _ := json.Marshal(trace)
 	fmt.Println(string(out))
 }

eval/configs/default.json

@@ -1,15 +1,20 @@
 {
   "tools": {
-    "progressive_disclosure": false
+    "web": {
+      "duckduckgo": {
+        "enabled": true,
+        "max_results": 5
+      }
+    }
   },
   "memory": {
-    "enabled": true,
     "db_path": ":memory:"
   },
   "agents": {
     "defaults": {
       "restrict_to_sandbox": true,
-      "max_tool_iterations": 20
+      "max_tool_iterations": 20,
+      "temperature": 0.1
     }
   },
   "heartbeat": {

eval/promptfooconfig.yaml

@@ -11,6 +11,8 @@ providers:
     label: "picoclaw"
     config:
       timeout: 180000
+    env:
+      PICOCLAW_EVAL_CONFIG: "./configs/default.json"
 
 # Default assertions applied to every test case
 defaultTest: