From c703916a81e5d89e5c1798c26ec244c00347cca8 Mon Sep 17 00:00:00 2001 From: ZanzyTHEbar Date: Sun, 22 Feb 2026 22:50:24 +0000 Subject: [PATCH] chore: eval cases, Makefile, README updates --- Makefile | 50 +++++++++++-------------- README.md | 3 +- eval/README.md | 16 +++++++- eval/cases/assistant_first_metrics.yaml | 4 +- eval/cases/error_recovery.yaml | 2 +- eval/cases/multi_step.yaml | 22 ++++++++++- eval/promptfooconfig.yaml | 1 - eval/scripts/compare.sh | 2 - 8 files changed, 63 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 514c1019b..ac2d4335d 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ BINARY_NAME=dragonscale BUILD_DIR=bin CMD_DIR=cmd/$(BINARY_NAME) MAIN_GO=$(CMD_DIR)/main.go +OPSCTL_SRCS := $(shell find cmd/opsctl internal/opsctl -type f -name '*.go') # Version VERSION?=$(shell git describe --tags --always --dirty 2>/dev/null || echo "dev") @@ -19,7 +20,7 @@ LDFLAGS=-ldflags "-X main.version=$(VERSION) -X main.gitCommit=$(GIT_COMMIT) -X # Go variables GO?=go -GOFLAGS?=-v -trimpath -tags stdjson +GOFLAGS?=-v -trimpath -tags=stdjson CGO_ENABLED?=1 MAKEFILE_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) DEVCONTAINER_WORKSPACE ?= $(MAKEFILE_DIR) @@ -78,18 +79,8 @@ ifeq ($(UNAME_S),Linux) else ARCH=$(UNAME_M) endif -else ifeq ($(UNAME_S),Darwin) - PLATFORM=darwin - ifeq ($(UNAME_M),x86_64) - ARCH=amd64 - else ifeq ($(UNAME_M),arm64) - ARCH=arm64 - else - ARCH=$(UNAME_M) - endif else - PLATFORM=$(UNAME_S) - ARCH=$(UNAME_M) + $(error This project is Linux/CGO-only. Build requires a Linux host with glibc-compatible tooling.) endif BINARY_PATH=$(BUILD_DIR)/$(BINARY_NAME)-$(PLATFORM)-$(ARCH) @@ -97,8 +88,11 @@ BINARY_PATH=$(BUILD_DIR)/$(BINARY_NAME)-$(PLATFORM)-$(ARCH) # Compatibility shim: run everything through the Go wrapper. OPSCTL_BIN = bin/opsctl OPSCTL ?= $(OPSCTL_BIN) +OPSCTL_EVAL_ARGS ?= --no-color --format raw +OPSCTL_EVAL ?= $(EVAL_GO) run ./cmd/opsctl +DRAGONSCALE_PROMPTFOO_ARGS ?= --no-cache --no-progress-bar -$(OPSCTL_BIN): +$(OPSCTL_BIN): $(OPSCTL_SRCS) @mkdir -p $(dir $(OPSCTL_BIN)) @go build -o $(OPSCTL_BIN) ./cmd/opsctl @@ -113,7 +107,7 @@ generate: $(OPSCTL_BIN) build: $(OPSCTL_BIN) @$(OPSCTL) build -## build-all: Build dragonscale for all platforms +## build-all: Build dragonscale for the current Linux/CGO target build-all: $(OPSCTL_BIN) @$(OPSCTL) build-all @@ -227,31 +221,31 @@ devcontainer-verify: $(OPSCTL_BIN) # --------------------------------------------------------------------------- ## eval-build: Build the eval runner from the current branch -eval-build: $(OPSCTL_BIN) - @$(OPSCTL) eval-build +eval-build: + @DRAGONSCALE_PROMPTFOO_ARGS="$(DRAGONSCALE_PROMPTFOO_ARGS)" $(OPSCTL_EVAL) $(OPSCTL_EVAL_ARGS) eval-build ## eval: Run the eval suite against the current build -eval: $(OPSCTL_BIN) - @$(OPSCTL) eval +eval: + @DRAGONSCALE_PROMPTFOO_ARGS="$(DRAGONSCALE_PROMPTFOO_ARGS)" $(OPSCTL_EVAL) $(OPSCTL_EVAL_ARGS) eval ## eval-fixtures: Reset workspace to a known state and seed fixture files for eval -eval-fixtures: $(OPSCTL_BIN) - @$(OPSCTL) eval-fixtures +eval-fixtures: + @DRAGONSCALE_PROMPTFOO_ARGS="$(DRAGONSCALE_PROMPTFOO_ARGS)" $(OPSCTL_EVAL) $(OPSCTL_EVAL_ARGS) eval-fixtures ## eval-view: Open the promptfoo results viewer -eval-view: $(OPSCTL_BIN) - @$(OPSCTL) eval-view +eval-view: + @DRAGONSCALE_PROMPTFOO_ARGS="$(DRAGONSCALE_PROMPTFOO_ARGS)" $(OPSCTL_EVAL) $(OPSCTL_EVAL_ARGS) eval-view -eval-clean: $(OPSCTL_BIN) - @$(OPSCTL) eval-clean +eval-clean: + @DRAGONSCALE_PROMPTFOO_ARGS="$(DRAGONSCALE_PROMPTFOO_ARGS)" $(OPSCTL_EVAL) $(OPSCTL_EVAL_ARGS) eval-clean ## eval-compare: A/B comparison of current branch vs main -eval-compare: $(OPSCTL_BIN) - @$(OPSCTL) eval-compare +eval-compare: + @DRAGONSCALE_PROMPTFOO_ARGS="$(DRAGONSCALE_PROMPTFOO_ARGS)" $(OPSCTL_EVAL) $(OPSCTL_EVAL_ARGS) eval-compare ## eval-test: Run Go-native component evals -eval-test: $(OPSCTL_BIN) - @$(OPSCTL) eval-test +eval-test: + @DRAGONSCALE_PROMPTFOO_ARGS="$(DRAGONSCALE_PROMPTFOO_ARGS)" $(OPSCTL_EVAL) $(OPSCTL_EVAL_ARGS) eval-test ## help: Show this help message help: $(OPSCTL_BIN) diff --git a/README.md b/README.md index 8bf75b22b..e4e06e9d5 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,7 @@ config/ # Example configuration files > [!IMPORTANT] > Requires `CGO_ENABLED=1` — the go-libsql driver links against glibc. +> Build/runtime target is Linux only (no macOS, no Windows, no musl). ```bash git clone https://github.com/ZanzyTHEbar/dragonscale.git @@ -473,7 +474,7 @@ See [ADR-001](docs/adr/001-isolated-tool-runtime.md) for the full design includi ```bash make build # Build for current platform (output: bin/) -make build-all # Cross-compile (linux/amd64, linux/arm64, linux/riscv64, darwin/arm64, windows/amd64) +make build-all # Build for current Linux/CGO target (Linux host required, glibc runtime) make install # Install to ~/.local/bin + copy skills make fmt # go fmt ./... make deps # go get -u + go mod tidy diff --git a/eval/README.md b/eval/README.md index 0bb3a813b..878250287 100644 --- a/eval/README.md +++ b/eval/README.md @@ -12,6 +12,18 @@ npm install -g promptfoo make eval ``` +To show richer promptfoo output with progress bars (when supported), run: + +```bash +make eval DRAGONSCALE_PROMPTFOO_ARGS="--no-cache" +``` + +To keep compact/no-progress output (current default), run: + +```bash +make eval DRAGONSCALE_PROMPTFOO_ARGS="--no-cache --no-progress-bar" +``` + `make eval`, `make eval-test`, `make eval-compare`, and `make eval-fixtures` run inside the devcontainer when `npx` is available, keeping command execution aligned with the container build environment. Set `DEVCONTAINER_EXEC=` to force host execution for these targets. @@ -131,5 +143,5 @@ python eval/scripts/generate_long_context_cases.py --count 12 --seed 20260221 - `DRAGONSCALE_EVAL_CONFIG` - Optional overlay config path applied on top of user base config. - `DRAGONSCALE_EVAL_BASE_CONFIG` - Optional explicit base config path for eval runs. -- `DRAGONSCALE_EVAL_HOST_HOME` - Path to a host-style home used as a fallback when container home paths are empty (used for host-mounted config discovery, typically `/host_home` in devcontainer). -- Base config discovery uses XDG first (`~/.config/dragonscale/config.json`) with `DRAGONSCALE_EVAL_HOST_HOME`/`/host_home/.config/dragonscale/config.json` as a host-mount fallback. +- `DRAGONSCALE_EVAL_HOST_HOME` - Optional path to a host-style home directory used for host-mounted config discovery (commonly `/host_home` when set by devcontainer via `.devcontainer/devcontainer.json`). +- Base config discovery order: `DRAGONSCALE_EVAL_BASE_CONFIG` (if set and valid), then `{DRAGONSCALE_EVAL_HOST_HOME}/.config/dragonscale/config.json` (if host home is set), then XDG at `~/.config/dragonscale/config.json`. diff --git a/eval/cases/assistant_first_metrics.yaml b/eval/cases/assistant_first_metrics.yaml index 7bfd4e8c7..7665da3f3 100644 --- a/eval/cases/assistant_first_metrics.yaml +++ b/eval/cases/assistant_first_metrics.yaml @@ -29,7 +29,9 @@ const trace = JSON.parse(output); if (trace.error && !trace.output) return { pass: false, score: 0, reason: 'crashed: ' + trace.error }; const out = (trace.output || '').toLowerCase(); - const hasFirstReminder = out.includes('first reminder') || out.includes('first check') || out.includes('first alert'); + const hasFirstReminder = out.includes('first reminder') || out.includes('first check') || out.includes('first alert') || + out.includes('1st reminder') || out.includes('initial reminder') || out.includes('initial check') || out.includes('primary reminder') || + /first\s+(reminder|check|alert|ping|notification)/.test(out); const hasTiming = out.includes('hour') || out.includes('min') || out.includes('30') || out.includes('15'); const pass = hasFirstReminder && hasTiming; return { pass, score: pass ? 1.0 : (hasTiming ? 0.5 : 0.0), reason: `first_reminder=${hasFirstReminder}, timing=${hasTiming}` }; diff --git a/eval/cases/error_recovery.yaml b/eval/cases/error_recovery.yaml index e11f37ddd..99cff19fe 100644 --- a/eval/cases/error_recovery.yaml +++ b/eval/cases/error_recovery.yaml @@ -45,7 +45,7 @@ const trace = JSON.parse(output); const out = (trace.output || '').toLowerCase(); const dur = trace.metrics.total_duration_ms; - const notHung = dur < 95000; + const notHung = dur < 130000; const graceful = out.includes('timeout') || out.includes('cancel') || out.includes('too long') || out.includes('killed') || out.includes('error') || out.includes('interrupt') || out.length > 5; diff --git a/eval/cases/multi_step.yaml b/eval/cases/multi_step.yaml index 089d60839..5872a22c9 100644 --- a/eval/cases/multi_step.yaml +++ b/eval/cases/multi_step.yaml @@ -94,8 +94,28 @@ value: | const trace = JSON.parse(output); const out = (trace.output || '').toLowerCase(); + const toolCalls = trace.steps.filter(s => s.type === 'tool_call'); + const readBacks = toolCalls.filter(s => { + let a = s.args; + if (typeof a === 'string') { + try { + a = JSON.parse(a); + } catch (e) { + a = null; + } + } + const tool = (a && a.tool_name) || s.tool; + return tool === 'read_file'; + }); const hasOS = out.includes('linux') || out.includes('darwin') || out.includes('os'); - return { pass: hasOS, score: hasOS ? 1.0 : 0.0, reason: hasOS ? 'confirmed OS name' : 'did not confirm OS' }; + const hasReadBack = readBacks.some(step => (step.result || '').toLowerCase().includes('linux') || + (step.result || '').toLowerCase().includes('darwin') || + (step.result || '').toLowerCase().includes('os')); + return { + pass: hasReadBack || hasOS, + score: (hasReadBack || hasOS) ? 1.0 : 0.0, + reason: (hasReadBack || hasOS) ? 'confirmed OS name' : 'did not confirm OS' + }; - description: "chained file ops: create directory structure" vars: diff --git a/eval/promptfooconfig.yaml b/eval/promptfooconfig.yaml index 72875f990..d84db9c85 100644 --- a/eval/promptfooconfig.yaml +++ b/eval/promptfooconfig.yaml @@ -13,7 +13,6 @@ providers: timeout: 180000 env: DRAGONSCALE_EVAL_CONFIG: "./configs/default.json" - DRAGONSCALE_EVAL_HOST_HOME: "/host_home" # Default assertions applied to every test case defaultTest: diff --git a/eval/scripts/compare.sh b/eval/scripts/compare.sh index e1de258ea..5b0b0271f 100755 --- a/eval/scripts/compare.sh +++ b/eval/scripts/compare.sh @@ -76,7 +76,6 @@ providers: timeout: 120000 env: DRAGONSCALE_EVAL_CONFIG: "${EVAL_CONFIG}" - DRAGONSCALE_EVAL_HOST_HOME: "/host_home" DRAGONSCALE_EVAL_BASE_CONFIG: "${EVAL_BASE_CONFIG}" - id: "exec:./bin/eval-runner-main" label: "main" @@ -84,7 +83,6 @@ providers: timeout: 120000 env: DRAGONSCALE_EVAL_CONFIG: "${EVAL_CONFIG}" - DRAGONSCALE_EVAL_HOST_HOME: "/host_home" DRAGONSCALE_EVAL_BASE_CONFIG: "${EVAL_BASE_CONFIG}" defaultTest: