diff --git a/eval/README.md b/eval/README.md index 89b5f49dd..0bb3a813b 100644 --- a/eval/README.md +++ b/eval/README.md @@ -130,4 +130,6 @@ python eval/scripts/generate_long_context_cases.py --count 12 --seed 20260221 ## Environment Variables - `DRAGONSCALE_EVAL_CONFIG` - Optional overlay config path applied on top of user base config. -- Base config discovery uses XDG first (`~/.config/dragonscale/config.json`), then legacy (`~/.dragonscale/config.json`), then XDG fallback if neither exists. +- `DRAGONSCALE_EVAL_BASE_CONFIG` - Optional explicit base config path for eval runs. +- `DRAGONSCALE_EVAL_HOST_HOME` - Path to a host-style home used as a fallback when container home paths are empty (used for host-mounted config discovery, typically `/host_home` in devcontainer). +- Base config discovery uses XDG first (`~/.config/dragonscale/config.json`) with `DRAGONSCALE_EVAL_HOST_HOME`/`/host_home/.config/dragonscale/config.json` as a host-mount fallback. diff --git a/eval/cases/tool_calling.yaml b/eval/cases/tool_calling.yaml index c6c44dde9..423e4bccf 100644 --- a/eval/cases/tool_calling.yaml +++ b/eval/cases/tool_calling.yaml @@ -104,8 +104,8 @@ value: | const trace = JSON.parse(output); const out = (trace.output || '').toLowerCase(); - const hasPicoclaw = out.includes('dragonscale'); - return { pass: hasPicoclaw, score: hasPicoclaw ? 1.0 : 0.0, reason: hasPicoclaw ? 'confirmed edit result' : 'did not confirm dragonscale in output' }; + const hasDragonScale = out.includes('dragonscale'); + return { pass: hasDragonScale, score: hasDragonScale ? 1.0 : 0.0, reason: hasDragonScale ? 'confirmed edit result' : 'did not confirm dragonscale in output' }; - description: "append file: agent appends to an existing file" vars: diff --git a/eval/cmd/eval-runner/main.go b/eval/cmd/eval-runner/main.go index 9ab9338ca..cd4dbf849 100644 --- a/eval/cmd/eval-runner/main.go +++ b/eval/cmd/eval-runner/main.go @@ -12,40 +12,11 @@ import ( fantasy "charm.land/fantasy" "github.com/ZanzyTHEbar/dragonscale/pkg/config" + "github.com/ZanzyTHEbar/dragonscale/pkg/eval/instrumentation" "github.com/ZanzyTHEbar/dragonscale/pkg/logger" - picoruntime "github.com/ZanzyTHEbar/dragonscale/pkg/runtime" + dragonruntime "github.com/ZanzyTHEbar/dragonscale/pkg/runtime" ) -// Trace is the structured output emitted by the eval runner. -// Promptfoo parses this JSON to evaluate assertions. -type Trace struct { - Output string `json:"output"` - Steps []TraceStep `json:"steps"` - Metrics Metrics `json:"metrics"` - Error string `json:"error,omitempty"` - SessionKey string `json:"session_key"` -} - -type TraceStep struct { - Index int `json:"index"` - Type string `json:"type"` - Tool string `json:"tool,omitempty"` - Args json.RawMessage `json:"args,omitempty"` - Result string `json:"result,omitempty"` - Duration int64 `json:"duration_ms,omitempty"` -} - -type Metrics struct { - TotalDurationMs int64 `json:"total_duration_ms"` - StepCount int `json:"step_count"` - ToolCallCount int `json:"tool_call_count"` - InputTokens int64 `json:"input_tokens"` - OutputTokens int64 `json:"output_tokens"` - TotalTokens int64 `json:"total_tokens"` - ReasoningTokens int64 `json:"reasoning_tokens"` - CacheReadTokens int64 `json:"cache_read_tokens"` -} - func main() { logger.SetLevel(logger.ERROR) @@ -74,20 +45,20 @@ func resolvePrompt() (string, error) { return readPrompt() } -func emptyPromptTrace(prompt string) *Trace { +func emptyPromptTrace(prompt string) *instrumentation.Trace { if strings.TrimSpace(prompt) != "" { return nil } - return &Trace{ + return &instrumentation.Trace{ Output: "No prompt provided. Please provide a message.", - Metrics: Metrics{ + Metrics: instrumentation.Metrics{ TotalDurationMs: 0, }, } } func resolveEvalConfig() (*config.Config, error) { - return picoruntime.LoadEvalConfig(evalRunnerTimeout()) + return dragonruntime.LoadEvalConfig(evalRunnerTimeout()) } func readPrompt() (string, error) { @@ -130,63 +101,55 @@ func parsePromptPayload(raw []byte) (string, bool) { return payload.Prompt, true } -func runEval(cfg *config.Config, prompt string) Trace { +func runEval(cfg *config.Config, prompt string) instrumentation.Trace { start := time.Now() + timeout := evalRunnerTimeout() - sessionKey := picoruntime.NewSessionKey("eval", start) - runtime, initErr := newEvalRuntime(cfg) + sessionKey := dragonruntime.NewSessionKey("eval", start) + runtime, initErr := newEvalRuntime(cfg, timeout) if initErr != nil { - return Trace{Error: initErr.Error()} + return instrumentation.Trace{Error: initErr.Error()} } defer runtime.close() - result := picoruntime.RunPrompt(runtime.handle.Context(), runtime.handle, prompt, sessionKey) + result, duration := runEvalPrompt(runtime.handle, sessionKey, prompt, start) + return buildTrace(runtime.instrumentedModel, sessionKey, result, duration) +} +func runEvalPrompt(handle *dragonruntime.RuntimeHandle, sessionKey, prompt string, start time.Time) (dragonruntime.RunResult, time.Duration) { + result := dragonruntime.RunPrompt(handle.Context(), handle, prompt, sessionKey) duration := result.Duration if duration <= 0 { duration = time.Since(start) } + return result, duration +} - trace := Trace{ +func buildTrace(model *instrumentation.InstrumentedLanguageModel, sessionKey string, result dragonruntime.RunResult, duration time.Duration) instrumentation.Trace { + trace := instrumentation.Trace{ Output: result.Output, SessionKey: sessionKey, - Steps: buildSteps(runtime.instrumentedModel), - Metrics: Metrics{ - TotalDurationMs: duration.Milliseconds(), - StepCount: len(runtime.instrumentedModel.calls), - InputTokens: runtime.instrumentedModel.totalUsage.InputTokens, - OutputTokens: runtime.instrumentedModel.totalUsage.OutputTokens, - TotalTokens: runtime.instrumentedModel.totalUsage.TotalTokens, - ReasoningTokens: runtime.instrumentedModel.totalUsage.ReasoningTokens, - CacheReadTokens: runtime.instrumentedModel.totalUsage.CacheReadTokens, - }, + Steps: instrumentation.BuildSteps(model), + Metrics: instrumentation.BuildMetrics(model, duration), } - if result.Error != "" { trace.Error = result.Error } - - for _, call := range runtime.instrumentedModel.calls { - for range call.toolCalls { - trace.Metrics.ToolCallCount++ - } - } - return trace } type evalRuntime struct { - handle *picoruntime.RuntimeHandle - instrumentedModel *instrumentedLanguageModel + handle *dragonruntime.RuntimeHandle + instrumentedModel *instrumentation.InstrumentedLanguageModel } -func newEvalRuntime(cfg *config.Config) (*evalRuntime, error) { - var instrumentedModel *instrumentedLanguageModel - handle, err := picoruntime.Bootstrap(context.Background(), cfg, picoruntime.BootstrapOptions{ - Timeout: evalRunnerTimeout(), - OutboundMode: picoruntime.OutboundModeDrop, +func newEvalRuntime(cfg *config.Config, timeout time.Duration) (*evalRuntime, error) { + var instrumentedModel *instrumentation.InstrumentedLanguageModel + handle, err := dragonruntime.Bootstrap(context.Background(), cfg, dragonruntime.BootstrapOptions{ + Timeout: timeout, + OutboundMode: dragonruntime.OutboundModeDrop, WrapModel: func(inner fantasy.LanguageModel) fantasy.LanguageModel { - instrumentedModel = &instrumentedLanguageModel{inner: inner} + instrumentedModel = instrumentation.Wrap(inner) return instrumentedModel }, }) @@ -210,41 +173,6 @@ func (r *evalRuntime) close() { } } -func buildSteps(model *instrumentedLanguageModel) []TraceStep { - var steps []TraceStep - idx := 0 - - for _, call := range model.calls { - steps = append(steps, TraceStep{ - Index: idx, - Type: "llm_call", - Duration: call.duration.Milliseconds(), - }) - idx++ - - for _, tc := range call.toolCalls { - argsRaw, _ := json.Marshal(tc.args) - steps = append(steps, TraceStep{ - Index: idx, - Type: "tool_call", - Tool: tc.name, - Args: argsRaw, - Result: truncate(tc.result, 500), - }) - idx++ - } - } - - return steps -} - -func truncate(s string, maxLen int) string { - if len(s) <= maxLen { - return s - } - return s[:maxLen] + "...[truncated]" -} - func evalRunnerTimeout() time.Duration { const defaultTimeout = 180 * time.Second raw := strings.TrimSpace(os.Getenv("DRAGONSCALE_EVAL_TIMEOUT_MS")) @@ -259,107 +187,10 @@ func evalRunnerTimeout() time.Duration { } func emitError(msg string) { - emitTrace(Trace{Error: msg}) + emitTrace(instrumentation.Trace{Error: msg}) } -func emitTrace(trace Trace) { +func emitTrace(trace instrumentation.Trace) { out, _ := json.Marshal(trace) fmt.Println(string(out)) } - -type instrumentedCall struct { - duration time.Duration - toolCalls []instrumentedToolCall - usage fantasy.Usage -} - -type instrumentedToolCall struct { - name string - args map[string]interface{} - result string -} - -type instrumentedLanguageModel struct { - inner fantasy.LanguageModel - calls []instrumentedCall - totalUsage fantasy.Usage -} - -var _ fantasy.LanguageModel = (*instrumentedLanguageModel)(nil) - -func (m *instrumentedLanguageModel) Provider() string { return m.inner.Provider() } -func (m *instrumentedLanguageModel) Model() string { return m.inner.Model() } - -func (m *instrumentedLanguageModel) GenerateObject(ctx context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { - return m.inner.GenerateObject(ctx, call) -} - -func (m *instrumentedLanguageModel) StreamObject(ctx context.Context, call fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) { - return m.inner.StreamObject(ctx, call) -} - -func (m *instrumentedLanguageModel) Generate(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) { - start := time.Now() - resp, err := m.inner.Generate(ctx, call) - dur := time.Since(start) - - ic := instrumentedCall{duration: dur} - - if err == nil && resp != nil { - ic.usage = resp.Usage - m.totalUsage.InputTokens += resp.Usage.InputTokens - m.totalUsage.OutputTokens += resp.Usage.OutputTokens - m.totalUsage.TotalTokens += resp.Usage.TotalTokens - m.totalUsage.ReasoningTokens += resp.Usage.ReasoningTokens - m.totalUsage.CacheReadTokens += resp.Usage.CacheReadTokens - - for _, tc := range resp.Content.ToolCalls() { - var args map[string]interface{} - _ = json.Unmarshal([]byte(tc.Input), &args) - ic.toolCalls = append(ic.toolCalls, instrumentedToolCall{ - name: tc.ToolName, - args: args, - }) - } - } - - m.calls = append(m.calls, ic) - return resp, err -} - -func (m *instrumentedLanguageModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { - start := time.Now() - stream, err := m.inner.Stream(ctx, call) - if err != nil { - m.calls = append(m.calls, instrumentedCall{duration: time.Since(start)}) - return nil, err - } - - ic := instrumentedCall{} - - wrappedStream := func(yield func(fantasy.StreamPart) bool) { - stream(func(part fantasy.StreamPart) bool { - switch part.Type { - case fantasy.StreamPartTypeFinish: - ic.usage = part.Usage - m.totalUsage.InputTokens += part.Usage.InputTokens - m.totalUsage.OutputTokens += part.Usage.OutputTokens - m.totalUsage.TotalTokens += part.Usage.TotalTokens - m.totalUsage.ReasoningTokens += part.Usage.ReasoningTokens - m.totalUsage.CacheReadTokens += part.Usage.CacheReadTokens - case fantasy.StreamPartTypeToolCall: - var args map[string]interface{} - _ = json.Unmarshal([]byte(part.ToolCallInput), &args) - ic.toolCalls = append(ic.toolCalls, instrumentedToolCall{ - name: part.ToolCallName, - args: args, - }) - } - return yield(part) - }) - ic.duration = time.Since(start) - m.calls = append(m.calls, ic) - } - - return wrappedStream, nil -} diff --git a/eval/promptfooconfig.yaml b/eval/promptfooconfig.yaml index d84db9c85..72875f990 100644 --- a/eval/promptfooconfig.yaml +++ b/eval/promptfooconfig.yaml @@ -13,6 +13,7 @@ providers: timeout: 180000 env: DRAGONSCALE_EVAL_CONFIG: "./configs/default.json" + DRAGONSCALE_EVAL_HOST_HOME: "/host_home" # Default assertions applied to every test case defaultTest: diff --git a/eval/scripts/compare.sh b/eval/scripts/compare.sh index c933355e4..e1de258ea 100755 --- a/eval/scripts/compare.sh +++ b/eval/scripts/compare.sh @@ -9,6 +9,14 @@ EVAL_DIR="$(dirname "$SCRIPT_DIR")" PROJECT_ROOT="$(dirname "$EVAL_DIR")" REPEAT=${1:-3} NPM_CMD="${EVAL_NPM_CMD:-npx}" +read -r -a NPM_CMD_ARR <<< "${NPM_CMD}" +export DEVCONTAINER_EXEC="" +TEMP_CONFIG="$(mktemp "${SCRIPT_DIR}/promptfoo-compare-XXXXXX.yaml")" + +cleanup_compare_config() { + rm -f "$TEMP_CONFIG" +} +trap cleanup_compare_config EXIT INT TERM if [[ "${1:-}" == "--repeat" ]]; then REPEAT="${2:-3}" @@ -22,7 +30,7 @@ cd "$PROJECT_ROOT" # 1. Build current branch eval-runner (instrumented wrapper) echo "[1/4] Building eval-runner from current branch..." -make eval-build 2>&1 | tail -1 +make DEVCONTAINER_EXEC= eval-build 2>&1 | tail -1 cp "$EVAL_DIR/bin/eval-runner" "$EVAL_DIR/bin/eval-runner-branch" # 2. Build main branch eval-runner @@ -31,7 +39,7 @@ STASH_RESULT=$(git stash 2>&1) echo "[2/4] Building eval-runner from main branch..." git checkout main 2>/dev/null -make eval-build 2>&1 | tail -1 +make DEVCONTAINER_EXEC= eval-build 2>&1 | tail -1 cp "$EVAL_DIR/bin/eval-runner" "$EVAL_DIR/bin/eval-runner-main" # Restore working branch @@ -46,8 +54,19 @@ cp "$EVAL_DIR/bin/eval-runner-branch" "$EVAL_DIR/bin/eval-runner" # 3. Update promptfoo config to enable comparison cd "$EVAL_DIR" +# Resolve base config for promptfoo providers from explicit env if provided. +EVAL_BASE_CONFIG="${DRAGONSCALE_EVAL_BASE_CONFIG:-}" +EVAL_CONFIG="${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}" +if [ -z "$EVAL_BASE_CONFIG" ] && [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then + echo "warning: DRAGONSCALE_EVAL_BASE_CONFIG not set; default config will be resolved by runtime" >&2 +fi + +if [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then + echo "DRAGONSCALE_EVAL_BASE_CONFIG=${EVAL_BASE_CONFIG}" +fi + # Create a temp config with both providers -cat > promptfooconfig-compare.yaml <<'YAML' +cat > "$TEMP_CONFIG" <