refactor(opsctl): format, tasks, main enhancements and tests
This commit is contained in:
ZanzyTHEbar
parent
c2719ff30f
commit
e5e187bad1
6 changed files with 981 additions and 168 deletions
|
|
@ -5,6 +5,7 @@ import (
|
|||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
|
@ -70,10 +71,43 @@ func parseOutputMode(raw, json bool, mode string) (format.OutputMode, error) {
|
|||
}
|
||||
}
|
||||
|
||||
func resolveRootPath(root string) (string, error) {
|
||||
resolved := app.EnsureRoot(root)
|
||||
if strings.TrimSpace(resolved) == "" {
|
||||
return "", fmt.Errorf("invalid --root: resolved empty repository path")
|
||||
}
|
||||
|
||||
info, err := os.Stat(resolved)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid --root %q: %w", resolved, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("invalid --root %q: not a directory", resolved)
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func resolveWorkingDirectory(root, cwd string) (string, error) {
|
||||
clean := strings.TrimSpace(cwd)
|
||||
if clean == "" {
|
||||
return "", nil
|
||||
}
|
||||
if !filepath.IsAbs(clean) {
|
||||
clean = filepath.Join(root, clean)
|
||||
}
|
||||
info, err := os.Stat(clean)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("invalid --cwd %q: %w", clean, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return "", fmt.Errorf("invalid --cwd %q: not a directory", clean)
|
||||
}
|
||||
return clean, nil
|
||||
}
|
||||
|
||||
func newOpsApp(root string) *app.App {
|
||||
rootPath := app.EnsureRoot(root)
|
||||
ops := app.New(runner.OSRunner{}, rootPath)
|
||||
for _, t := range tasks.NewRegistry(rootPath) {
|
||||
ops := app.New(runner.OSRunner{}, root)
|
||||
for _, t := range tasks.NewRegistry(root) {
|
||||
ops.Register(t)
|
||||
}
|
||||
ops.Register(tasks.NewHelpTask(ops))
|
||||
|
|
@ -126,7 +160,10 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
|
|||
return &exitCodeError{code: 2, err: err}
|
||||
}
|
||||
|
||||
resolvedRoot := app.EnsureRoot(rootFlag)
|
||||
resolvedRoot, err := resolveRootPath(rootFlag)
|
||||
if err != nil {
|
||||
return &exitCodeError{code: 2, err: err}
|
||||
}
|
||||
ops := newOpsApp(resolvedRoot)
|
||||
|
||||
if len(cmdArgs) == 0 || cmdArgs[0] == "help" || cmdArgs[0] == "--help" || cmdArgs[0] == "-h" {
|
||||
|
|
@ -145,9 +182,14 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
|
|||
return &exitCodeError{code: 2, err: fmt.Errorf("unknown command: %s", taskName)}
|
||||
}
|
||||
|
||||
resolvedCwd, err := resolveWorkingDirectory(resolvedRoot, cwd)
|
||||
if err != nil {
|
||||
return &exitCodeError{code: 2, err: err}
|
||||
}
|
||||
|
||||
ctx := &app.Context{
|
||||
Root: resolvedRoot,
|
||||
Cwd: cwd,
|
||||
Cwd: resolvedCwd,
|
||||
Format: mode,
|
||||
Quiet: quiet,
|
||||
NoColor: noColor,
|
||||
|
|
@ -177,7 +219,7 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
|
|||
root.PersistentFlags().StringVar(&formatValue, "format", string(format.OutputText), "output mode: text, json, raw")
|
||||
root.PersistentFlags().BoolVar(&jsonMode, "json", false, "shortcut for --format json")
|
||||
root.PersistentFlags().BoolVar(&rawMode, "raw", false, "shortcut for --format raw")
|
||||
root.PersistentFlags().BoolVar(&noColor, "no-color", false, "disable color output (reserved)")
|
||||
root.PersistentFlags().BoolVar(&noColor, "no-color", false, "disable color output")
|
||||
root.PersistentFlags().BoolVar(&quiet, "quiet", false, "suppress non-error output")
|
||||
root.PersistentFlags().StringVar(&rootFlag, "root", "", "repository root")
|
||||
root.PersistentFlags().StringVar(&cwd, "cwd", "", "working directory for command execution")
|
||||
|
|
@ -188,7 +230,11 @@ func buildOpsctlCommand(stdout, stderr io.Writer, args []string) *cobra.Command
|
|||
_, _ = fmt.Fprintln(cobraCmd.ErrOrStderr(), err)
|
||||
return
|
||||
}
|
||||
rootPath := app.EnsureRoot(rootFlag)
|
||||
rootPath, err := resolveRootPath(rootFlag)
|
||||
if err != nil {
|
||||
_, _ = fmt.Fprintln(cobraCmd.ErrOrStderr(), err)
|
||||
return
|
||||
}
|
||||
printUsage(cobraCmd.OutOrStdout(), newOpsApp(rootPath), cobraCmd.Root().Name())
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -3,10 +3,12 @@ package main
|
|||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ZanzyTHEbar/dragonscale/internal/opsctl/format"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
|
|
@ -19,6 +21,10 @@ func TestParseOutputMode(t *testing.T) {
|
|||
require.NoError(t, err)
|
||||
require.Equal(t, "raw", string(mode))
|
||||
|
||||
mode, err = parseOutputMode(true, true, "text")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "raw", string(mode))
|
||||
|
||||
mode, err = parseOutputMode(false, true, "text")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "json", string(mode))
|
||||
|
|
@ -27,6 +33,26 @@ func TestParseOutputMode(t *testing.T) {
|
|||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestParseOutputModePriority(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
mode, err := parseOutputMode(true, true, "json")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, format.OutputRaw, mode)
|
||||
|
||||
mode, err = parseOutputMode(false, true, "text")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, format.OutputJSON, mode)
|
||||
|
||||
mode, err = parseOutputMode(true, false, "json")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, format.OutputRaw, mode)
|
||||
|
||||
mode, err = parseOutputMode(false, false, "json")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, format.OutputJSON, mode)
|
||||
}
|
||||
|
||||
func TestRunPrintsUsageWithoutArgs(t *testing.T) {
|
||||
out := &bytes.Buffer{}
|
||||
errOut := &bytes.Buffer{}
|
||||
|
|
@ -54,6 +80,27 @@ func TestRunInvalidFormat(t *testing.T) {
|
|||
require.Contains(t, errOut.String(), "invalid --format value \"bad\"")
|
||||
}
|
||||
|
||||
func TestRunInvalidRoot(t *testing.T) {
|
||||
out := &bytes.Buffer{}
|
||||
errOut := &bytes.Buffer{}
|
||||
root := filepath.Join(t.TempDir(), "missing")
|
||||
code := run([]string{"--root", root, "help"}, out, errOut)
|
||||
require.Equal(t, 2, code)
|
||||
require.Contains(t, errOut.String(), "invalid --root")
|
||||
require.Equal(t, "", out.String())
|
||||
}
|
||||
|
||||
func TestRunInvalidCwd(t *testing.T) {
|
||||
out := &bytes.Buffer{}
|
||||
errOut := &bytes.Buffer{}
|
||||
root := t.TempDir()
|
||||
missing := filepath.Join(root, "does-not-exist")
|
||||
code := run([]string{"--root", root, "--cwd", missing, "build"}, out, errOut)
|
||||
require.Equal(t, 2, code)
|
||||
require.Contains(t, errOut.String(), "invalid --cwd")
|
||||
require.Equal(t, "", out.String())
|
||||
}
|
||||
|
||||
func TestRunHelpAlias(t *testing.T) {
|
||||
out := &bytes.Buffer{}
|
||||
errOut := &bytes.Buffer{}
|
||||
|
|
@ -73,6 +120,22 @@ func TestRunHelpIncludesAllAlias(t *testing.T) {
|
|||
require.Contains(t, out.String(), "build")
|
||||
}
|
||||
|
||||
func TestRunHelpIncludesGlobalOutputFlags(t *testing.T) {
|
||||
out := &bytes.Buffer{}
|
||||
errOut := &bytes.Buffer{}
|
||||
code := run([]string{"--help"}, out, errOut)
|
||||
require.Equal(t, 0, code)
|
||||
require.Equal(t, "", errOut.String())
|
||||
require.Contains(t, out.String(), "--format")
|
||||
require.Contains(t, out.String(), "--json")
|
||||
require.Contains(t, out.String(), "--raw")
|
||||
require.Contains(t, out.String(), "--no-color")
|
||||
require.Contains(t, out.String(), "--quiet")
|
||||
require.Contains(t, out.String(), "--root")
|
||||
require.Contains(t, out.String(), "--cwd")
|
||||
require.Contains(t, out.String(), "--timeout")
|
||||
}
|
||||
|
||||
func TestMakefilePhonyTargetsAreAvailableInOpsctl(t *testing.T) {
|
||||
makeTargets := parseMakefilePhonyTargets(t)
|
||||
out := &bytes.Buffer{}
|
||||
|
|
@ -87,6 +150,25 @@ func TestMakefilePhonyTargetsAreAvailableInOpsctl(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestMakefileAllTargetIsAliasToBuild(t *testing.T) {
|
||||
makefile, err := findProjectMakefile()
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, []string{"build"}, parseMakeTargetPrereqs(t, makefile, "all"))
|
||||
}
|
||||
|
||||
func TestMakeAllTargetRunsOpsctlBuild(t *testing.T) {
|
||||
output := runMakeTargetWithFakeOpsctl(t, "all", nil)
|
||||
require.Contains(t, output, "cmd:build")
|
||||
}
|
||||
|
||||
func TestMakeEvalBuildForwardsEnvToOpsctl(t *testing.T) {
|
||||
output := runMakeTargetWithFakeOpsctl(t, "eval-build", map[string]string{
|
||||
"DRAGONSCALE_EVAL_CONFIG": "/tmp/eval/config.json",
|
||||
})
|
||||
require.Contains(t, output, "cmd:--no-color --format raw eval-build")
|
||||
require.Contains(t, output, "eval-config:/tmp/eval/config.json")
|
||||
}
|
||||
|
||||
func parseMakefilePhonyTargets(t *testing.T) []string {
|
||||
t.Helper()
|
||||
|
||||
|
|
@ -141,6 +223,70 @@ func parseMakefilePhonyTargets(t *testing.T) []string {
|
|||
return targets
|
||||
}
|
||||
|
||||
func parseMakeTargetPrereqs(t *testing.T, makefile, target string) []string {
|
||||
t.Helper()
|
||||
|
||||
raw, err := os.ReadFile(makefile)
|
||||
require.NoError(t, err)
|
||||
lines := strings.Split(string(raw), "\n")
|
||||
prefix := target + ":"
|
||||
|
||||
for _, line := range lines {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if strings.HasPrefix(trimmed, "#") || trimmed == "" {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(trimmed, prefix) {
|
||||
after := strings.TrimSpace(strings.TrimPrefix(trimmed, prefix))
|
||||
if idx := strings.Index(after, "#"); idx != -1 {
|
||||
after = strings.TrimSpace(after[:idx])
|
||||
}
|
||||
if after == "" {
|
||||
return nil
|
||||
}
|
||||
return strings.Fields(after)
|
||||
}
|
||||
}
|
||||
t.Fatalf("make target %q not found in %s", target, makefile)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runMakeTargetWithFakeOpsctl(t *testing.T, target string, env map[string]string) string {
|
||||
t.Helper()
|
||||
|
||||
makefile, err := findProjectMakefile()
|
||||
require.NoError(t, err)
|
||||
makeRoot := filepath.Dir(makefile)
|
||||
|
||||
fakeOpsctl := filepath.Join(t.TempDir(), "opsctl.sh")
|
||||
fakeScript := []byte(`#!/usr/bin/env sh
|
||||
printf 'cmd:%s\n' "$*"
|
||||
printf 'eval-config:%s\n' "${DRAGONSCALE_EVAL_CONFIG-}"
|
||||
`)
|
||||
require.NoError(t, os.WriteFile(fakeOpsctl, fakeScript, 0o755))
|
||||
|
||||
dummyOpsctl := filepath.Join(t.TempDir(), "opsctl")
|
||||
require.NoError(t, os.WriteFile(dummyOpsctl, []byte("#!/usr/bin/env sh\nexit 0\n"), 0o755))
|
||||
|
||||
cmd := exec.Command("make", "-s", "-f", makefile, target)
|
||||
cmd.Dir = makeRoot
|
||||
runEnv := append(
|
||||
os.Environ(),
|
||||
"OPSCTL="+fakeOpsctl,
|
||||
"OPSCTL_BIN="+dummyOpsctl,
|
||||
"OPSCTL_EVAL="+fakeOpsctl,
|
||||
"OPSCTL_EVAL_ARGS=--no-color --format raw",
|
||||
)
|
||||
for key, value := range env {
|
||||
runEnv = append(runEnv, key+"="+value)
|
||||
}
|
||||
cmd.Env = runEnv
|
||||
|
||||
output, err := cmd.CombinedOutput()
|
||||
require.NoError(t, err, "make %s failed: %s", target, string(output))
|
||||
return string(output)
|
||||
}
|
||||
|
||||
func findProjectMakefile() (string, error) {
|
||||
dir, err := os.Getwd()
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package format
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
|
@ -55,41 +56,103 @@ func NewTaskResult(name string, cmd string, result runner.CommandResult, started
|
|||
|
||||
func Render(w io.Writer, mode OutputMode, res TaskResult, quiet bool) error {
|
||||
res.Mode = mode
|
||||
if quiet && res.Success {
|
||||
return nil
|
||||
}
|
||||
switch mode {
|
||||
case OutputJSON:
|
||||
if quiet {
|
||||
return nil
|
||||
}
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(res)
|
||||
case OutputRaw:
|
||||
if quiet {
|
||||
return nil
|
||||
}
|
||||
prefix := statusPrefix(!res.NoColor, res.Success)
|
||||
_, _ = fmt.Fprintf(w, "%s %s (%dms)\n", prefix, res.Task, res.DurationMS)
|
||||
return nil
|
||||
default:
|
||||
if quiet {
|
||||
return nil
|
||||
}
|
||||
status := "ok"
|
||||
if !res.Success {
|
||||
status = "failed"
|
||||
}
|
||||
status = formatStatus(!res.NoColor, res.Success)
|
||||
_, _ = fmt.Fprintf(w, "%s: %s (exit=%d, duration=%dms)\n", res.Task, status, res.ExitCode, res.DurationMS)
|
||||
if strings.TrimSpace(res.Command) != "" {
|
||||
_, _ = fmt.Fprintf(w, " command: %s\n", normalizeCommand(res.Command))
|
||||
}
|
||||
if res.Error != "" {
|
||||
_, _ = fmt.Fprintf(w, " error: %s\n", res.Error)
|
||||
}
|
||||
if strings.TrimSpace(res.Stdout) != "" {
|
||||
_, _ = fmt.Fprintf(w, " stdout: %s\n", res.Stdout)
|
||||
_, _ = fmt.Fprintf(w, " stdout: %s\n", normalizeStdout(res.Stdout))
|
||||
}
|
||||
if strings.TrimSpace(res.Stderr) != "" {
|
||||
_, _ = fmt.Fprintf(w, " stderr: %s\n", res.Stderr)
|
||||
_, _ = fmt.Fprintf(w, " stderr: %s\n", normalizeStdout(res.Stderr))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeCommand(raw string) string {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return raw
|
||||
}
|
||||
lines := strings.Split(raw, "\n")
|
||||
compact := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
compact = append(compact, line)
|
||||
}
|
||||
return strings.Join(compact, " ")
|
||||
}
|
||||
|
||||
func normalizeStdout(raw string) string {
|
||||
return normalizeOutput(raw, true)
|
||||
}
|
||||
|
||||
func normalizeOutput(raw string, includeStructured bool) string {
|
||||
raw = strings.TrimRight(raw, "\n")
|
||||
if raw == "" {
|
||||
return raw
|
||||
}
|
||||
lines := strings.Split(raw, "\n")
|
||||
normalized := make([]string, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
normalized = append(normalized, normalizeLine(line, includeStructured))
|
||||
}
|
||||
return strings.Join(normalized, "\n")
|
||||
}
|
||||
|
||||
func normalizeLine(line string, includeStructured bool) string {
|
||||
if !includeStructured {
|
||||
return line
|
||||
}
|
||||
var payload any
|
||||
if err := json.Unmarshal([]byte(line), &payload); err != nil {
|
||||
return line
|
||||
}
|
||||
return normalizeJSONLine(payload)
|
||||
}
|
||||
|
||||
func normalizeJSONLine(payload any) string {
|
||||
var buf bytes.Buffer
|
||||
enc := json.NewEncoder(&buf)
|
||||
enc.SetIndent("", " ")
|
||||
if err := enc.Encode(payload); err != nil {
|
||||
return fmt.Sprint(payload)
|
||||
}
|
||||
return strings.TrimRight(buf.String(), "\n")
|
||||
}
|
||||
|
||||
func formatStatus(colored bool, success bool) string {
|
||||
if !colored {
|
||||
if !success {
|
||||
|
|
|
|||
|
|
@ -48,6 +48,21 @@ func TestRenderTextIncludesDetails(t *testing.T) {
|
|||
require.NotContains(t, output, "\x1b[")
|
||||
}
|
||||
|
||||
func TestRenderTextRespectsNoColorFalse(t *testing.T) {
|
||||
buf := &bytes.Buffer{}
|
||||
result := TaskResult{
|
||||
Task: "test",
|
||||
ExitCode: 0,
|
||||
DurationMS: 10,
|
||||
Success: true,
|
||||
NoColor: false,
|
||||
}
|
||||
|
||||
require.NoError(t, Render(buf, OutputText, result, false))
|
||||
output := buf.String()
|
||||
require.Contains(t, output, "\x1b[32mok\x1b[0m")
|
||||
}
|
||||
|
||||
func TestRenderRawHonorQuiet(t *testing.T) {
|
||||
buf := &bytes.Buffer{}
|
||||
result := TaskResult{Task: "run", ExitCode: 0, DurationMS: 42, Success: true}
|
||||
|
|
@ -59,3 +74,39 @@ func TestRenderRawHonorQuiet(t *testing.T) {
|
|||
require.True(t, strings.Contains(buf.String(), "run"))
|
||||
require.True(t, strings.Contains(buf.String(), "(42ms)"))
|
||||
}
|
||||
|
||||
func TestRenderTextIncludesCommandSummaryAndErrors(t *testing.T) {
|
||||
buf := &bytes.Buffer{}
|
||||
result := TaskResult{
|
||||
Task: "eval",
|
||||
Command: "if [ -n \"$X\" ]; then\necho ok\nfi",
|
||||
ExitCode: 1,
|
||||
Success: false,
|
||||
DurationMS: 55,
|
||||
Error: "command failed",
|
||||
Stdout: "ok\n",
|
||||
Stderr: `{"outcome":"success"}`,
|
||||
}
|
||||
require.NoError(t, Render(buf, OutputText, result, false))
|
||||
output := buf.String()
|
||||
require.Contains(t, output, "eval")
|
||||
require.Contains(t, output, "command:")
|
||||
require.Contains(t, output, "if [ -n")
|
||||
require.Contains(t, output, "error: command failed")
|
||||
require.Contains(t, output, "stdout: ok")
|
||||
require.Contains(t, output, `"outcome"`)
|
||||
}
|
||||
|
||||
func TestRenderTextHonorsQuietOnlyForSuccess(t *testing.T) {
|
||||
buf := &bytes.Buffer{}
|
||||
successResult := TaskResult{Task: "run", ExitCode: 0, DurationMS: 1, Success: true}
|
||||
require.NoError(t, Render(buf, OutputText, successResult, true))
|
||||
require.Equal(t, "", buf.String())
|
||||
|
||||
buf.Reset()
|
||||
failureResult := TaskResult{Task: "run", ExitCode: 1, DurationMS: 1, Success: false, Error: "boom"}
|
||||
require.NoError(t, Render(buf, OutputText, failureResult, true))
|
||||
require.Contains(t, buf.String(), "run")
|
||||
require.Contains(t, buf.String(), "exit=1")
|
||||
require.Contains(t, buf.String(), "error: boom")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import (
|
|||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
|
|
@ -25,6 +26,7 @@ type shellTask struct {
|
|||
description string
|
||||
script func(*app.Context) string
|
||||
env func(*app.Context) []string
|
||||
prepare func(*app.Context) error
|
||||
}
|
||||
|
||||
func (t shellTask) Name() string {
|
||||
|
|
@ -36,6 +38,17 @@ func (t shellTask) Description() string {
|
|||
}
|
||||
|
||||
func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (app.Result, error) {
|
||||
if t.prepare != nil {
|
||||
if err := t.prepare(c); err != nil {
|
||||
return app.Result{
|
||||
Task: t.name,
|
||||
Command: t.script(c),
|
||||
ExitCode: 1,
|
||||
Err: err.Error(),
|
||||
}, err
|
||||
}
|
||||
}
|
||||
|
||||
script := t.script(c)
|
||||
fullScript := applyDevcontainerWrapper(script, c)
|
||||
workDir := c.Root
|
||||
|
|
@ -54,7 +67,7 @@ func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (
|
|||
result, err := rnr.Run(ctx, spec)
|
||||
res := app.Result{
|
||||
Task: t.name,
|
||||
Command: script,
|
||||
Command: fullScript,
|
||||
Stdout: result.Stdout,
|
||||
Stderr: result.Stderr,
|
||||
Err: func() string {
|
||||
|
|
@ -71,6 +84,117 @@ func (t shellTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (
|
|||
return res, err
|
||||
}
|
||||
|
||||
type commandTask struct {
|
||||
name string
|
||||
description string
|
||||
specs func(*app.Context) []runner.CommandSpec
|
||||
env func(*app.Context) []string
|
||||
prepare func(*app.Context) error
|
||||
}
|
||||
|
||||
func (t commandTask) Name() string {
|
||||
return t.name
|
||||
}
|
||||
|
||||
func (t commandTask) Description() string {
|
||||
return t.description
|
||||
}
|
||||
|
||||
func (t commandTask) Run(ctx context.Context, rnr runner.Runner, c *app.Context) (app.Result, error) {
|
||||
if t.prepare != nil {
|
||||
if err := t.prepare(c); err != nil {
|
||||
return app.Result{
|
||||
Task: t.name,
|
||||
ExitCode: 1,
|
||||
Err: err.Error(),
|
||||
}, err
|
||||
}
|
||||
}
|
||||
|
||||
specs := t.specs(c)
|
||||
workDir := c.Root
|
||||
if c.Cwd != "" {
|
||||
workDir = c.Cwd
|
||||
}
|
||||
baseEnv := mergeEnv(c, t.env)
|
||||
|
||||
result := app.Result{
|
||||
Task: t.name,
|
||||
}
|
||||
commandParts := make([]string, 0, len(specs))
|
||||
for i, spec := range specs {
|
||||
spec.InheritOutput = c.Format == format.OutputRaw
|
||||
if spec.Dir == "" {
|
||||
spec.Dir = workDir
|
||||
}
|
||||
spec.Env = mergeEnvPairs(baseEnv, spec.Env)
|
||||
|
||||
commandParts = append(commandParts, describeCommandSpec(spec))
|
||||
commandResult, err := rnr.Run(ctx, spec)
|
||||
result.ExitCode = commandResult.ExitCode
|
||||
result.Stdout += commandResult.Stdout
|
||||
result.Stderr += commandResult.Stderr
|
||||
if i == 0 {
|
||||
result.Command = commandParts[i]
|
||||
} else {
|
||||
result.Command += " && " + commandParts[i]
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
result.Err = err.Error()
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func describeCommandSpec(spec runner.CommandSpec) string {
|
||||
if len(spec.Args) == 0 {
|
||||
return spec.Name
|
||||
}
|
||||
quotedArgs := make([]string, 0, len(spec.Args))
|
||||
for _, arg := range spec.Args {
|
||||
quotedArgs = append(quotedArgs, strconv.Quote(arg))
|
||||
}
|
||||
return spec.Name + " " + strings.Join(quotedArgs, " ")
|
||||
}
|
||||
|
||||
func mergeEnvPairs(base []string, extra []string) []string {
|
||||
seen := make(map[string]int)
|
||||
values := make(map[string]string)
|
||||
keys := make([]string, 0, len(base)+len(extra))
|
||||
|
||||
merge := func(items []string) {
|
||||
for _, item := range items {
|
||||
eq := strings.Index(item, "=")
|
||||
if eq <= 0 {
|
||||
continue
|
||||
}
|
||||
key := item[:eq]
|
||||
value := item[eq+1:]
|
||||
if _, ok := seen[key]; !ok {
|
||||
keys = append(keys, key)
|
||||
seen[key] = len(keys) - 1
|
||||
}
|
||||
values[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
merge(base)
|
||||
merge(extra)
|
||||
|
||||
env := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
value := values[key]
|
||||
if strings.TrimSpace(value) == "" {
|
||||
continue
|
||||
}
|
||||
env = append(env, fmt.Sprintf("%s=%s", key, value))
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
type helpTask struct {
|
||||
app *app.App
|
||||
}
|
||||
|
|
@ -134,12 +258,20 @@ func NewShellTask(name, description string, script func(*app.Context) string, en
|
|||
return shellTask{name: name, description: description, script: script, env: env}
|
||||
}
|
||||
|
||||
func NewCommandTask(name, description string, specs func(*app.Context) []runner.CommandSpec, env func(*app.Context) []string, prepare func(*app.Context) error) commandTask {
|
||||
return commandTask{name: name, description: description, specs: specs, env: env, prepare: prepare}
|
||||
}
|
||||
|
||||
func NewShellTaskWithPrepare(name, description string, script func(*app.Context) string, env func(*app.Context) []string, prepare func(*app.Context) error) shellTask {
|
||||
return shellTask{name: name, description: description, script: script, env: env, prepare: prepare}
|
||||
}
|
||||
|
||||
func NewRegistry(_ string) []app.Task {
|
||||
tasks := []app.Task{
|
||||
NewShellTask("all", "Build the dragonscale binary for current platform", buildScript, nil),
|
||||
NewShellTask("generate", "Run generate", generateScript, nil),
|
||||
NewShellTask("build", "Build the dragonscale binary for current platform", buildScript, nil),
|
||||
NewShellTask("build-all", "Build dragonscale for all platforms", buildAllScript, nil),
|
||||
NewShellTaskWithPrepare("build", "Build the dragonscale binary for current platform", buildScript, nil, validateBuildTaskEnv),
|
||||
NewShellTaskWithPrepare("build-all", "Build dragonscale for linux target with CGO", buildAllScript, nil, validateBuildTaskEnv),
|
||||
NewShellTask("install", "Install dragonscale to system and copy builtin skills", installScript, nil),
|
||||
NewShellTask("uninstall", "Uninstall dragonscale from system", uninstallScript, nil),
|
||||
NewShellTask("uninstall-all", "Uninstall dragonscale and workspace data", uninstallAllScript, nil),
|
||||
|
|
@ -147,7 +279,7 @@ func NewRegistry(_ string) []app.Task {
|
|||
NewShellTask("vet", "Run go vet for static analysis", staticGoScript("vet ./..."), nil),
|
||||
NewShellTask("test", "Run tests", staticGoScript("test ./..."), nil),
|
||||
NewShellTask("fmt", "Format Go code", staticGoScript("fmt ./..."), nil),
|
||||
NewShellTask("lint", "Run all linting checks", lintScript, nil),
|
||||
NewCommandTask("lint", "Run all linting checks", lintSpecs, nil, nil),
|
||||
NewShellTask("hooks", "Install git hooks", hooksScript, nil),
|
||||
NewShellTask("deps", "Download dependencies", staticGoScript("mod download && mod verify"), nil),
|
||||
NewShellTask("update-deps", "Update dependencies", staticGoScript("get -u ./... && mod tidy"), nil),
|
||||
|
|
@ -159,16 +291,16 @@ func NewRegistry(_ string) []app.Task {
|
|||
NewShellTask("fantasy-sync", "Full sync of vendored Fantasy SDK", fantasySyncScript, nil),
|
||||
NewShellTask("fantasy-patch", "Save local modifications as a patch", fantasyPatchScript, nil),
|
||||
NewShellTask("test-integration", "Run integration tests", staticGoScript("-tags integration -count=1 -timeout 120s -v ./pkg/memory/..."), nil),
|
||||
NewShellTask("check", "Run deps, formatting, linting and tests", checkScript, nil),
|
||||
NewCommandTask("check", "Run deps, formatting, linting and tests", checkSpecs, nil, nil),
|
||||
NewShellTask("run", "Build and run dragonscale", runScript, runTaskEnv),
|
||||
NewShellTask("devcontainer-build", "Build the local devcontainer image via npx", devcontainerBuildScript, nil),
|
||||
NewShellTask("devcontainer-up", "Start/update the local devcontainer", devcontainerUpScript, nil),
|
||||
NewShellTask("devcontainer-generate", "Run generation in devcontainer", devcontainerGenerateScript, nil),
|
||||
NewShellTask("devcontainer-verify", "Verify generated code in devcontainer", devcontainerVerifyScript, nil),
|
||||
NewShellTask("eval-build", "Build the eval runner from current branch", evalBuildScript, nil),
|
||||
NewShellTask("eval", "Run the eval suite", evalRunScript, nil),
|
||||
NewShellTask("eval-fixtures", "Prepare eval fixture workspace", evalFixturesScript, nil),
|
||||
NewShellTask("eval-view", "Open the promptfoo results viewer", evalViewScript, nil),
|
||||
NewCommandTask("devcontainer-build", "Build the local devcontainer image via npx", devcontainerBuildSpecs, nil, nil),
|
||||
NewCommandTask("devcontainer-up", "Start/update the local devcontainer", devcontainerUpSpecs, nil, nil),
|
||||
NewCommandTask("devcontainer-generate", "Run generation in devcontainer", devcontainerGenerateSpecs, nil, nil),
|
||||
NewCommandTask("devcontainer-verify", "Verify generated code in devcontainer", devcontainerVerifySpecs, nil, nil),
|
||||
NewCommandTask("eval-build", "Build the eval runner from current branch", evalBuildSpecs, nil, nil),
|
||||
NewCommandTask("eval", "Run the eval suite", evalRunSpecs, nil, nil),
|
||||
NewCommandTask("eval-fixtures", "Prepare eval fixture workspace", evalFixturesSpecs, nil, nil),
|
||||
NewCommandTask("eval-view", "Open the promptfoo results viewer", evalViewSpecs, nil, nil),
|
||||
NewShellTask("eval-clean", "Cleanup eval artifacts", simpleScript("rm -rf eval/results eval/bin"), nil),
|
||||
NewShellTask("eval-compare", "Run A/B comparison of current branch vs main", simpleScript("cd eval && DEVCONTAINER_EXEC= EVAL_NPM_CMD=$(npx --yes) ./scripts/compare.sh --repeat 3"), nil),
|
||||
NewShellTask("eval-test", "Run Go-native component evals", staticGoScript("-v ./eval/go_evals/..."), nil),
|
||||
|
|
@ -177,64 +309,52 @@ func NewRegistry(_ string) []app.Task {
|
|||
}
|
||||
|
||||
func defaultEnv(c *app.Context) []string {
|
||||
env := []string{
|
||||
fmt.Sprintf("GO=%s", cEnv(c, "GO", "go")),
|
||||
fmt.Sprintf("GOFLAGS=%s", cEnv(c, "GOFLAGS", "-v -trimpath -tags stdjson")),
|
||||
fmt.Sprintf("CGO_ENABLED=%s", cEnv(c, "CGO_ENABLED", "1")),
|
||||
fmt.Sprintf("BINARY_NAME=%s", cEnv(c, "BINARY_NAME", defaultBinaryName)),
|
||||
fmt.Sprintf("BUILD_DIR=%s", cEnv(c, "BUILD_DIR", defaultBuildDir)),
|
||||
fmt.Sprintf("CMD_DIR=%s", cEnv(c, "CMD_DIR", defaultCmdDir)),
|
||||
fmt.Sprintf("WORKSPACE_DIR=%s", cEnv(c, "WORKSPACE_DIR", filepath.Join(homeDir(), ".dragonscale", "workspace"))),
|
||||
pairs := map[string]string{
|
||||
"GO": cEnv(c, "GO", "go"),
|
||||
"GOFLAGS": cEnv(c, "GOFLAGS", "-v -trimpath -tags=stdjson"),
|
||||
"CGO_ENABLED": cEnv(c, "CGO_ENABLED", "1"),
|
||||
"BINARY_NAME": cEnv(c, "BINARY_NAME", defaultBinaryName),
|
||||
"BUILD_DIR": cEnv(c, "BUILD_DIR", defaultBuildDir),
|
||||
"CMD_DIR": cEnv(c, "CMD_DIR", defaultCmdDir),
|
||||
"WORKSPACE_DIR": cEnv(c, "WORKSPACE_DIR", filepath.Join(homeDir(), ".dragonscale", "workspace")),
|
||||
"GOOS": cEnv(c, "GOOS", "linux"),
|
||||
"GOARCH": cEnv(c, "GOARCH", runtime.GOARCH),
|
||||
}
|
||||
if value := cEnv(c, "GOOS", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("GOOS=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "GOARCH", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("GOARCH=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "PLATFORM", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("PLATFORM=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "ARCH", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("ARCH=%s", value))
|
||||
}
|
||||
if v := cEnv(c, "DEVCONTAINER_EXEC", ""); v != "" {
|
||||
env = append(env, "DEVCONTAINER_EXEC="+v)
|
||||
}
|
||||
if value := cEnv(c, "DRAGONSCALE_EVAL_HOST_HOME", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_HOST_HOME=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_BASE_CONFIG=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "DRAGONSCALE_EVAL_CONFIG", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_CONFIG=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "DRAGONSCALE_EVAL_DEBUG", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("DRAGONSCALE_EVAL_DEBUG=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "VERSION", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("VERSION=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "FANTASY_VERSION", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("FANTASY_VERSION=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "NAME", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("NAME=%s", value))
|
||||
}
|
||||
if value := cEnv(c, "ARGS", ""); value != "" {
|
||||
env = append(env, fmt.Sprintf("ARGS=%s", value))
|
||||
appendIfSet := func(key, value string) {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
pairs[key] = value
|
||||
}
|
||||
}
|
||||
appendIfSet("PLATFORM", cEnv(c, "PLATFORM", ""))
|
||||
appendIfSet("ARCH", cEnv(c, "ARCH", ""))
|
||||
appendIfSet("DEVCONTAINER_EXEC", cEnv(c, "DEVCONTAINER_EXEC", ""))
|
||||
appendIfSet("DRAGONSCALE_EVAL_HOST_HOME", cEnv(c, "DRAGONSCALE_EVAL_HOST_HOME", ""))
|
||||
appendIfSet("DRAGONSCALE_EVAL_BASE_CONFIG", cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", ""))
|
||||
appendIfSet("DRAGONSCALE_EVAL_CONFIG", cEnv(c, "DRAGONSCALE_EVAL_CONFIG", ""))
|
||||
appendIfSet("DRAGONSCALE_EVAL_DEBUG", cEnv(c, "DRAGONSCALE_EVAL_DEBUG", ""))
|
||||
appendIfSet("VERSION", cEnv(c, "VERSION", ""))
|
||||
appendIfSet("FANTASY_VERSION", cEnv(c, "FANTASY_VERSION", ""))
|
||||
appendIfSet("NAME", cEnv(c, "NAME", ""))
|
||||
appendIfSet("ARGS", cEnv(c, "ARGS", ""))
|
||||
if c.Root != "" {
|
||||
env = append(env, fmt.Sprintf("DRAGONSCALE_HOME=%s", filepath.Join(homeDir(), ".dragonscale")))
|
||||
appendIfSet("DRAGONSCALE_HOME", filepath.Join(homeDir(), ".dragonscale"))
|
||||
}
|
||||
env := make([]string, 0, len(pairs))
|
||||
for key, value := range pairs {
|
||||
if key == "" || value == "" {
|
||||
continue
|
||||
}
|
||||
env = append(env, fmt.Sprintf("%s=%s", key, value))
|
||||
}
|
||||
return env
|
||||
}
|
||||
|
||||
func cEnv(c *app.Context, key string, fallback string) string {
|
||||
if v, ok := c.ExtraEnv[key]; ok {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
if c != nil && c.ExtraEnv != nil {
|
||||
if v, ok := c.ExtraEnv[key]; ok {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
|
|
@ -270,15 +390,11 @@ func applyDevcontainerWrapper(script string, c *app.Context) string {
|
|||
return script
|
||||
}
|
||||
|
||||
workDir := c.Root
|
||||
if c.Cwd != "" {
|
||||
workDir = c.Cwd
|
||||
}
|
||||
if strings.TrimSpace(workDir) != "" {
|
||||
script = fmt.Sprintf("cd %s\n%s", strconv.Quote(workDir), script)
|
||||
}
|
||||
return fmt.Sprintf("%s -- bash -lc %s", execCmd, shellSingleQuote(script))
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s -- bash -lc %s", execCmd, strconv.Quote(script))
|
||||
func shellSingleQuote(input string) string {
|
||||
return "'" + strings.ReplaceAll(input, "'", `'"'"'`) + "'"
|
||||
}
|
||||
|
||||
func detectDevcontainerExec(root string) string {
|
||||
|
|
@ -331,61 +447,116 @@ func staticGoScript(rest string) func(*app.Context) string {
|
|||
}
|
||||
}
|
||||
|
||||
func generateScript(*app.Context) string {
|
||||
func generateScript(c *app.Context) string {
|
||||
cmdDir := cEnv(c, "CMD_DIR", defaultCmdDir)
|
||||
return scriptHeader() +
|
||||
"rm -rf ./$(CMD_DIR)/workspace 2>/dev/null || true\n" +
|
||||
fmt.Sprintf("rm -rf ./%s/workspace 2>/dev/null || true\n", cmdDir) +
|
||||
"$GO generate ./...\n"
|
||||
}
|
||||
|
||||
func buildScript(*app.Context) string {
|
||||
func buildScript(c *app.Context) string {
|
||||
version := "$(git describe --tags --always --dirty 2>/dev/null || echo \"dev\")"
|
||||
gitCommit := "$(git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\")"
|
||||
buildTime := "$(date +%FT%T%z)"
|
||||
goVersion := "$($GO version | awk '{print \"$3\"}')"
|
||||
goVersion := "$($GO version | awk '{print $3}')"
|
||||
buildDir := cEnv(c, "BUILD_DIR", defaultBuildDir)
|
||||
binaryName := cEnv(c, "BINARY_NAME", defaultBinaryName)
|
||||
cmdDir := cEnv(c, "CMD_DIR", defaultCmdDir)
|
||||
targetGOOS := cEnv(c, "GOOS", "linux")
|
||||
targetGOARCH := cEnv(c, "GOARCH", runtime.GOARCH)
|
||||
ldFlags := fmt.Sprintf("-X main.version=%s -X main.gitCommit=%s -X main.buildTime=%s -X main.goVersion=%s -s -w", version, gitCommit, buildTime, goVersion)
|
||||
return scriptHeader() +
|
||||
"mkdir -p $(BUILD_DIR)\n" +
|
||||
"GOOS=$($GO env GOOS)\n" +
|
||||
"GOARCH=$($GO env GOARCH)\n" +
|
||||
fmt.Sprintf("$GO build $GOFLAGS -ldflags \"%s\" -o $(BUILD_DIR)/$(BINARY_NAME)-${GOOS}-${GOARCH} ./$(CMD_DIR)\n", ldFlags) +
|
||||
"ln -sf $(BINARY_NAME)-$(GOOS)-$(GOARCH) $(BUILD_DIR)/$(BINARY_NAME)\n"
|
||||
fmt.Sprintf("mkdir -p %s\n", buildDir) +
|
||||
fmt.Sprintf("echo \"build: target=%s/%s/%s\"\n", targetGOOS, targetGOARCH, binaryName) +
|
||||
fmt.Sprintf("GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED $GO build $GOFLAGS -ldflags \"%s\" -o %s/%s-${GOOS}-${GOARCH} ./%s\n", ldFlags, buildDir, binaryName, cmdDir) +
|
||||
fmt.Sprintf("ln -sf %s-${GOOS}-${GOARCH} %s/%s\n", binaryName, buildDir, binaryName)
|
||||
}
|
||||
|
||||
func buildAllScript(*app.Context) string {
|
||||
func buildAllScript(c *app.Context) string {
|
||||
ldFlags := "-ldflags '-s -w'"
|
||||
return scriptHeader() +
|
||||
"mkdir -p $(BUILD_DIR)\n" +
|
||||
fmt.Sprintf("GOOS=linux GOARCH=amd64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-amd64 ./$(CMD_DIR)\n", ldFlags) +
|
||||
fmt.Sprintf("GOOS=linux GOARCH=arm64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-arm64 ./$(CMD_DIR)\n", ldFlags) +
|
||||
fmt.Sprintf("GOOS=linux GOARCH=loong64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-loong64 ./$(CMD_DIR)\n", ldFlags) +
|
||||
fmt.Sprintf("GOOS=linux GOARCH=riscv64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-linux-riscv64 ./$(CMD_DIR)\n", ldFlags) +
|
||||
fmt.Sprintf("GOOS=darwin GOARCH=arm64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-darwin-arm64 ./$(CMD_DIR)\n", ldFlags) +
|
||||
fmt.Sprintf("GOOS=windows GOARCH=amd64 $GO build $GOFLAGS %s -o $(BUILD_DIR)/$(BINARY_NAME)-windows-amd64.exe ./$(CMD_DIR)\n", ldFlags)
|
||||
buildDir := cEnv(c, "BUILD_DIR", defaultBuildDir)
|
||||
binaryName := cEnv(c, "BINARY_NAME", defaultBinaryName)
|
||||
cmdDir := cEnv(c, "CMD_DIR", defaultCmdDir)
|
||||
targetGOOS := cEnv(c, "GOOS", "linux")
|
||||
targetGOARCH := cEnv(c, "GOARCH", runtime.GOARCH)
|
||||
var script strings.Builder
|
||||
script.WriteString(scriptHeader())
|
||||
fmt.Fprintf(&script, "mkdir -p %s\n", buildDir)
|
||||
fmt.Fprintf(&script, "echo \"build-all: target=%s/%s output=%s/%s-%s-%s\"\n", targetGOOS, targetGOARCH, buildDir, binaryName, targetGOOS, targetGOARCH)
|
||||
fmt.Fprintf(&script, "OUTPUT_NAME=%s/%s-%s-%s\n", buildDir, binaryName, targetGOOS, targetGOARCH)
|
||||
script.WriteString("export CGO_ENABLED=1\n")
|
||||
fmt.Fprintf(&script, "GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED $GO build $GOFLAGS %s -o ${OUTPUT_NAME} ./%s\n", ldFlags, cmdDir)
|
||||
fmt.Fprintf(&script, "ln -sf ./%s %s/%s\n", "${OUTPUT_NAME}", buildDir, binaryName)
|
||||
return script.String()
|
||||
}
|
||||
|
||||
func validateBuildTaskEnv(c *app.Context) error {
|
||||
targetGOOS := cEnv(c, "GOOS", "linux")
|
||||
if targetGOOS != "linux" {
|
||||
return fmt.Errorf("unsupported GOOS=%s: this project only supports linux/gnu builds", targetGOOS)
|
||||
}
|
||||
|
||||
if cgoEnabled := strings.TrimSpace(cEnv(c, "CGO_ENABLED", "1")); cgoEnabled != "" && cgoEnabled != "1" {
|
||||
return fmt.Errorf("unsupported CGO_ENABLED=%s: builds require CGO_ENABLED=1", cgoEnabled)
|
||||
}
|
||||
|
||||
if err := detectGlibc(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var detectGlibc = detectGlibcAvailable
|
||||
|
||||
func detectGlibcAvailable() error {
|
||||
output, err := exec.Command("ldd", "--version").Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to verify libc: %w", err)
|
||||
}
|
||||
header := strings.ToLower(strings.SplitN(string(output), "\n", 2)[0])
|
||||
if strings.Contains(header, "glibc") || strings.Contains(header, "gnu c library") {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("unsupported libc: this project requires glibc")
|
||||
}
|
||||
|
||||
func installScript(*app.Context) string {
|
||||
prefix := filepath.Join(homeDir(), ".local")
|
||||
buildDir := cEnv(nil, "BUILD_DIR", defaultBuildDir)
|
||||
binaryName := cEnv(nil, "BINARY_NAME", defaultBinaryName)
|
||||
return scriptHeader() +
|
||||
"mkdir -p ${INSTALL_PREFIX:-" + prefix + "}/bin\n" +
|
||||
"cp $(BUILD_DIR)/$(BINARY_NAME) ${INSTALL_PREFIX:-" + prefix + "}/bin/$(BINARY_NAME)\n" +
|
||||
"chmod +x ${INSTALL_PREFIX:-" + prefix + "}/bin/$(BINARY_NAME)\n"
|
||||
fmt.Sprintf("cp %s/%s ${INSTALL_PREFIX:-%s}/bin/%s\n", buildDir, binaryName, prefix, binaryName) +
|
||||
fmt.Sprintf("chmod +x ${INSTALL_PREFIX:-%s}/bin/%s\n", prefix, binaryName)
|
||||
}
|
||||
|
||||
func uninstallScript(*app.Context) string {
|
||||
return "rm -f ${INSTALL_PREFIX:-" + filepath.Join(homeDir(), ".local") + "}/bin/$(BINARY_NAME)\n"
|
||||
binaryName := cEnv(nil, "BINARY_NAME", defaultBinaryName)
|
||||
prefix := filepath.Join(homeDir(), ".local")
|
||||
return "rm -f ${INSTALL_PREFIX:-" + prefix + "}/bin/" + binaryName + "\n"
|
||||
}
|
||||
|
||||
func uninstallAllScript(*app.Context) string {
|
||||
return "rm -rf $(DRAGONSCALE_HOME)\n"
|
||||
func uninstallAllScript(c *app.Context) string {
|
||||
dHome := filepath.Join(homeDir(), ".dragonscale")
|
||||
if v := cEnv(c, "DRAGONSCALE_HOME", ""); v != "" {
|
||||
dHome = v
|
||||
}
|
||||
return "rm -rf " + dHome + "\n"
|
||||
}
|
||||
|
||||
func cleanScript(*app.Context) string {
|
||||
return "rm -rf $(BUILD_DIR)\n"
|
||||
buildDir := cEnv(nil, "BUILD_DIR", defaultBuildDir)
|
||||
return "rm -rf " + buildDir + "\n"
|
||||
}
|
||||
|
||||
func lintScript(*app.Context) string {
|
||||
return scriptHeader() +
|
||||
"$GO fmt ./...\n$GO vet ./...\n$GO build ./...\n"
|
||||
func lintSpecs(c *app.Context) []runner.CommandSpec {
|
||||
goBinary := cEnv(c, "GO", "go")
|
||||
return []runner.CommandSpec{
|
||||
{Name: goBinary, Args: []string{"fmt", "./..."}},
|
||||
{Name: goBinary, Args: []string{"vet", "./..."}},
|
||||
{Name: goBinary, Args: []string{"build", "./..."}},
|
||||
}
|
||||
}
|
||||
|
||||
func hooksScript(*app.Context) string {
|
||||
|
|
@ -431,11 +602,15 @@ func fantasyPatchScript(c *app.Context) string {
|
|||
return simpleScript("./scripts/sync-fantasy.sh --save-patch " + cEnv(c, "NAME", ""))(c)
|
||||
}
|
||||
|
||||
func checkScript(*app.Context) string {
|
||||
return staticGoScript("mod download && mod verify")(&app.Context{}) +
|
||||
staticGoScript("fmt ./...")(&app.Context{}) +
|
||||
staticGoScript("vet ./...")(&app.Context{}) +
|
||||
staticGoScript("test ./...")(&app.Context{})
|
||||
func checkSpecs(c *app.Context) []runner.CommandSpec {
|
||||
goBinary := cEnv(c, "GO", "go")
|
||||
return []runner.CommandSpec{
|
||||
{Name: goBinary, Args: []string{"mod", "download"}},
|
||||
{Name: goBinary, Args: []string{"mod", "verify"}},
|
||||
{Name: goBinary, Args: []string{"fmt", "./..."}},
|
||||
{Name: goBinary, Args: []string{"vet", "./..."}},
|
||||
{Name: goBinary, Args: []string{"test", "./..."}},
|
||||
}
|
||||
}
|
||||
|
||||
func runTaskEnv(c *app.Context) []string {
|
||||
|
|
@ -446,7 +621,9 @@ func runTaskEnv(c *app.Context) []string {
|
|||
}
|
||||
|
||||
func runScript(c *app.Context) string {
|
||||
return scriptHeader() + "$(BUILD_DIR)/$(BINARY_NAME) " + strings.Join(quoteArgs(c.Argv), " ") + "\n"
|
||||
buildDir := cEnv(c, "BUILD_DIR", defaultBuildDir)
|
||||
binaryName := cEnv(c, "BINARY_NAME", defaultBinaryName)
|
||||
return scriptHeader() + buildDir + "/" + binaryName + " " + strings.Join(quoteArgs(c.Argv), " ") + "\n"
|
||||
}
|
||||
|
||||
func quoteArgs(args []string) []string {
|
||||
|
|
@ -457,52 +634,173 @@ func quoteArgs(args []string) []string {
|
|||
return quoted
|
||||
}
|
||||
|
||||
func devcontainerBuildScript(*app.Context) string {
|
||||
return "npx --yes @devcontainers/cli build --workspace-folder \"$(pwd)\"\n"
|
||||
func devcontainerRoot(c *app.Context) string {
|
||||
if c != nil && c.Root != "" {
|
||||
return c.Root
|
||||
}
|
||||
workspace := cValue(c, "DEVCONTAINER_WORKSPACE", ".")
|
||||
if workspace == "" {
|
||||
return "."
|
||||
}
|
||||
return workspace
|
||||
}
|
||||
|
||||
func devcontainerUpScript(*app.Context) string {
|
||||
return "npx --yes @devcontainers/cli up --workspace-folder \"$(pwd)\"\n"
|
||||
func devcontainerBuildSpecs(c *app.Context) []runner.CommandSpec {
|
||||
return []runner.CommandSpec{
|
||||
{
|
||||
Name: "npx",
|
||||
Args: []string{"--yes", "@devcontainers/cli", "build", "--workspace-folder", devcontainerRoot(c)},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func devcontainerGenerateScript(*app.Context) string {
|
||||
return "npx --yes @devcontainers/cli exec --workspace-folder \"$(pwd)\" -- bash -lc \"go generate ./pkg/itr ./pkg/tools && sqlc generate -f pkg/memory/sqlc/sqlc.yaml\"\n"
|
||||
func devcontainerUpSpecs(c *app.Context) []runner.CommandSpec {
|
||||
return []runner.CommandSpec{
|
||||
{
|
||||
Name: "npx",
|
||||
Args: []string{"--yes", "@devcontainers/cli", "up", "--workspace-folder", devcontainerRoot(c)},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func devcontainerVerifyScript(*app.Context) string {
|
||||
return "npx --yes @devcontainers/cli exec --workspace-folder \"$(pwd)\" -- bash -lc \"make flatc-check sqlc-check\"\n"
|
||||
func devcontainerGenerateSpecs(c *app.Context) []runner.CommandSpec {
|
||||
root := devcontainerRoot(c)
|
||||
goBinary := cEnv(c, "GO", "go")
|
||||
return []runner.CommandSpec{
|
||||
{
|
||||
Name: "npx",
|
||||
Args: []string{
|
||||
"--yes", "@devcontainers/cli", "exec",
|
||||
"--workspace-folder", root, "--",
|
||||
"bash", "-lc",
|
||||
fmt.Sprintf("%s generate ./pkg/itr ./pkg/tools && sqlc generate -f pkg/memory/sqlc/sqlc.yaml", goBinary),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func evalBuildScript(*app.Context) string {
|
||||
ldFlags := "-X main.version=$(git describe --tags --always --dirty 2>/dev/null || echo \"dev\") -X main.gitCommit=$(git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\") -X main.buildTime=$(date +%FT%T%z) -X main.goVersion=\"$($GO version | awk '{print \"$3\"}')\" -s -w"
|
||||
return scriptHeader() +
|
||||
"$GO generate ./...\n" +
|
||||
"mkdir -p eval/bin\n" +
|
||||
fmt.Sprintf("$GO build $GOFLAGS -ldflags \"%s\" -o eval/bin/eval-runner ./eval/cmd/eval-runner\n", ldFlags)
|
||||
func devcontainerVerifySpecs(c *app.Context) []runner.CommandSpec {
|
||||
root := devcontainerRoot(c)
|
||||
return []runner.CommandSpec{
|
||||
{
|
||||
Name: "npx",
|
||||
Args: []string{
|
||||
"--yes", "@devcontainers/cli", "exec",
|
||||
"--workspace-folder", root, "--",
|
||||
"bash", "-lc", "make flatc-check sqlc-check",
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func evalRunScript(*app.Context) string {
|
||||
return `if [ -n "${DRAGONSCALE_EVAL_BASE_CONFIG:-}" ] && [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then
|
||||
echo "DRAGONSCALE_EVAL_BASE_CONFIG=${DRAGONSCALE_EVAL_BASE_CONFIG}"
|
||||
fi
|
||||
if [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then
|
||||
echo "DRAGONSCALE_EVAL_CONFIG=${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}"
|
||||
fi
|
||||
DRAGONSCALE_EVAL_HOST_HOME=/host_home DRAGONSCALE_EVAL_CONFIG="${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}" npx --yes promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar
|
||||
`
|
||||
func evalBuildSpecs(c *app.Context) []runner.CommandSpec {
|
||||
goBinary := cEnv(c, "GO", "go")
|
||||
goFlags := strings.Fields(cEnv(c, "GOFLAGS", "-v -trimpath -tags=stdjson"))
|
||||
goVersion := "unknown"
|
||||
goVersionParts := strings.Fields(outputOrDefault(goBinary + " version"))
|
||||
if len(goVersionParts) >= 3 {
|
||||
goVersion = goVersionParts[2]
|
||||
}
|
||||
version := outputOrDefault("git describe --tags --always --dirty 2>/dev/null || echo \"dev\"")
|
||||
commit := outputOrDefault("git rev-parse --short=8 HEAD 2>/dev/null || echo \"dev\"")
|
||||
buildTime := outputOrDefault("date +%FT%T%z")
|
||||
|
||||
ldFlags := fmt.Sprintf(
|
||||
"-X main.version=%s -X main.gitCommit=%s -X main.buildTime=%s -X main.goVersion=%s -s -w",
|
||||
version, commit, buildTime, goVersion,
|
||||
)
|
||||
return []runner.CommandSpec{
|
||||
{Name: goBinary, Args: []string{"generate", "./..."}},
|
||||
{Name: "mkdir", Args: []string{"-p", "eval/bin"}},
|
||||
{
|
||||
Name: goBinary,
|
||||
Args: append(
|
||||
append(append([]string{"build"}, goFlags...), "-ldflags", ldFlags),
|
||||
"-o", filepath.Join("eval", "bin", "eval-runner"), "./eval/cmd/eval-runner",
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func evalFixturesScript(*app.Context) string {
|
||||
return scriptHeader() +
|
||||
"mkdir -p \"$HOME/.local/share/dragonscale/sandbox\"\n" +
|
||||
"rm -f \"$HOME/.local/share/dragonscale/sandbox/eval_test_output.txt\" \"$HOME/.local/share/dragonscale/sandbox/test_steps.txt\" \"$HOME/.local/share/dragonscale/sandbox/eval_checkpoint.txt\" \"$HOME/.local/share/dragonscale/sandbox/chain_test.txt\" \"$HOME/.local/share/dragonscale/sandbox/current_year.txt\" \"$HOME/.local/share/dragonscale/sandbox/result.txt\" \"$HOME/.local/share/dragonscale/sandbox/progressive_test.txt\" \"$HOME/.local/share/dragonscale/sandbox/os_name.txt\"\n" +
|
||||
"rm -rf \"$HOME/.local/share/dragonscale/sandbox/project\"\n" +
|
||||
"printf 'dragonscale eval fixture — hello from the eval harness\\nThis is line two of the fixture file.\\n' > \"$HOME/.local/share/dragonscale/sandbox/eval_fixture.txt\"\n" +
|
||||
"cp -f eval/fixtures/sample_data.txt \"$HOME/.local/share/dragonscale/sandbox/sample_data.txt\"\n" +
|
||||
"mkdir -p \"$HOME/.local/share/dragonscale/skills\"\n" +
|
||||
"cp -rf eval/fixtures/skills/* \"$HOME/.local/share/dragonscale/skills/\" 2>/dev/null || true\n"
|
||||
func evalRunSpecs(c *app.Context) []runner.CommandSpec {
|
||||
cfgPath := cEnv(c, "DRAGONSCALE_EVAL_CONFIG", "./configs/default.json")
|
||||
baseCfg := cEnv(c, "DRAGONSCALE_EVAL_BASE_CONFIG", "")
|
||||
debug := cEnv(c, "DRAGONSCALE_EVAL_DEBUG", "") != ""
|
||||
promptfooArgs := strings.Fields(cEnv(c, "DRAGONSCALE_PROMPTFOO_ARGS", "--no-cache --no-progress-bar"))
|
||||
if len(promptfooArgs) == 0 {
|
||||
promptfooArgs = []string{"--no-cache", "--no-progress-bar"}
|
||||
}
|
||||
|
||||
var specs []runner.CommandSpec
|
||||
if debug && strings.TrimSpace(baseCfg) != "" {
|
||||
specs = append(specs, runner.CommandSpec{
|
||||
Name: "echo",
|
||||
Args: []string{fmt.Sprintf("DRAGONSCALE_EVAL_BASE_CONFIG=%s", baseCfg)},
|
||||
})
|
||||
}
|
||||
if debug {
|
||||
specs = append(specs, runner.CommandSpec{
|
||||
Name: "echo",
|
||||
Args: []string{fmt.Sprintf("DRAGONSCALE_EVAL_CONFIG=%s", cfgPath)},
|
||||
})
|
||||
}
|
||||
args := append([]string{"--yes", "promptfoo", "eval", "--config", "promptfooconfig.yaml"}, promptfooArgs...)
|
||||
specs = append(specs, runner.CommandSpec{
|
||||
Name: "npx",
|
||||
Args: args,
|
||||
Dir: filepath.Join(c.Root, "eval"),
|
||||
Env: []string{fmt.Sprintf("DRAGONSCALE_EVAL_CONFIG=%s", cfgPath)},
|
||||
})
|
||||
return specs
|
||||
}
|
||||
|
||||
func evalViewScript(*app.Context) string {
|
||||
return "cd eval && npx --yes promptfoo view\n"
|
||||
func evalFixturesSpecs(c *app.Context) []runner.CommandSpec {
|
||||
sandbox := filepath.Join(homeDir(), ".local", "share", "dragonscale", "sandbox")
|
||||
project := filepath.Join(sandbox, "project")
|
||||
skills := filepath.Join(homeDir(), ".local", "share", "dragonscale", "skills")
|
||||
shared := filepath.Join(sandbox, "sample_data.txt")
|
||||
fixture := filepath.Join(sandbox, "eval_fixture.txt")
|
||||
files := []string{
|
||||
filepath.Join(sandbox, "eval_test_output.txt"),
|
||||
filepath.Join(sandbox, "test_steps.txt"),
|
||||
filepath.Join(sandbox, "eval_checkpoint.txt"),
|
||||
filepath.Join(sandbox, "chain_test.txt"),
|
||||
filepath.Join(sandbox, "current_year.txt"),
|
||||
filepath.Join(sandbox, "result.txt"),
|
||||
filepath.Join(sandbox, "progressive_test.txt"),
|
||||
filepath.Join(sandbox, "os_name.txt"),
|
||||
}
|
||||
sourceFixture := filepath.Join("eval", "fixtures", "sample_data.txt")
|
||||
|
||||
specs := []runner.CommandSpec{
|
||||
{Name: "mkdir", Args: []string{"-p", sandbox}},
|
||||
{Name: "rm", Args: append([]string{"-f"}, files...)},
|
||||
{Name: "rm", Args: []string{"-rf", project}},
|
||||
{Name: "mkdir", Args: []string{"-p", skills}},
|
||||
{Name: "bash", Args: []string{"-lc", fmt.Sprintf("printf '%%s\\n%%s\\n' \"dragonscale eval fixture — hello from the eval harness\" \"This is line two of the fixture file.\" > %q", fixture)}},
|
||||
{Name: "cp", Args: []string{"-f", sourceFixture, shared}},
|
||||
}
|
||||
specs = append(specs, runner.CommandSpec{
|
||||
Name: "bash",
|
||||
Args: []string{"-lc", "if [ -d eval/fixtures/skills ]; then cp -rf eval/fixtures/skills/. " + strconv.Quote(skills) + "; fi"},
|
||||
})
|
||||
return specs
|
||||
}
|
||||
|
||||
func evalViewSpecs(c *app.Context) []runner.CommandSpec {
|
||||
return []runner.CommandSpec{
|
||||
{
|
||||
Name: "npx",
|
||||
Args: []string{"--yes", "promptfoo", "view"},
|
||||
Dir: filepath.Join(c.Root, "eval"),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func outputOrDefault(command string) string {
|
||||
output, err := exec.Command("bash", "-lc", command).Output()
|
||||
if err != nil {
|
||||
return "dev"
|
||||
}
|
||||
return strings.TrimSpace(string(output))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -81,12 +81,14 @@ func TestQuoteArgs(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestBuildAllTaskPreservesGoEnvironmentForwarding(t *testing.T) {
|
||||
t.Parallel()
|
||||
origCheck := detectGlibc
|
||||
detectGlibc = func() error { return nil }
|
||||
t.Cleanup(func() { detectGlibc = origCheck })
|
||||
|
||||
ctx := &app.Context{
|
||||
Root: t.TempDir(),
|
||||
ExtraEnv: map[string]string{
|
||||
"GOOS": "freebsd",
|
||||
"GOOS": "linux",
|
||||
"GOARCH": "sparc64",
|
||||
"GOFLAGS": "-mod=mod",
|
||||
"DEVCONTAINER_EXEC": "echo devcontainer exec",
|
||||
|
|
@ -110,31 +112,100 @@ func TestBuildAllTaskPreservesGoEnvironmentForwarding(t *testing.T) {
|
|||
require.Len(t, fake.Calls, 1)
|
||||
|
||||
script := strings.Join(fake.Calls[0].Args, " ")
|
||||
require.Contains(t, script, "GOOS=linux GOARCH=amd64 $GO build")
|
||||
require.Contains(t, script, "GOOS=darwin GOARCH=arm64 $GO build")
|
||||
require.Contains(t, script, "GOOS=windows GOARCH=amd64 $GO build")
|
||||
require.Contains(t, script, "GOOS=linux GOARCH=arm64 $GO build")
|
||||
require.Contains(t, script, "GOOS=$GOOS")
|
||||
require.NotContains(t, script, "if [ \"$GOOS\" != \"linux\" ]; then")
|
||||
require.Contains(t, script, "CGO_ENABLED=1")
|
||||
require.NotContains(t, script, "CGO_BUILD=1")
|
||||
|
||||
joinedEnv := strings.Join(fake.Calls[0].Env, " ")
|
||||
require.Contains(t, joinedEnv, "GOOS=freebsd")
|
||||
require.Contains(t, joinedEnv, "GOOS=linux")
|
||||
require.Contains(t, joinedEnv, "GOARCH=sparc64")
|
||||
require.Contains(t, joinedEnv, "GOFLAGS=-mod=mod")
|
||||
require.Contains(t, joinedEnv, "DEVCONTAINER_EXEC=echo devcontainer exec")
|
||||
}
|
||||
|
||||
func TestEvalRunScriptPreservesEvalConfig(t *testing.T) {
|
||||
func TestBuildAllTaskDefaultsToHostTarget(t *testing.T) {
|
||||
origCheck := detectGlibc
|
||||
detectGlibc = func() error { return nil }
|
||||
t.Cleanup(func() { detectGlibc = origCheck })
|
||||
|
||||
ctx := &app.Context{
|
||||
Root: t.TempDir(),
|
||||
ExtraEnv: map[string]string{
|
||||
"GOFLAGS": "-mod=mod",
|
||||
"DEVCONTAINER_EXEC": "echo devcontainer exec",
|
||||
},
|
||||
}
|
||||
fake := &runner.FakeRunner{
|
||||
Result: runner.CommandResult{ExitCode: 0},
|
||||
}
|
||||
|
||||
var buildAll app.Task
|
||||
for _, task := range NewRegistry(ctx.Root) {
|
||||
if task.Name() == "build-all" {
|
||||
buildAll = task
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, buildAll)
|
||||
|
||||
_, err := buildAll.Run(context.Background(), fake, ctx)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.Calls, 1)
|
||||
|
||||
script := strings.Join(fake.Calls[0].Args, " ")
|
||||
require.Contains(t, script, "build-all: target=linux/")
|
||||
require.NotContains(t, script, "GOOS=$($GO env GOOS)")
|
||||
require.Contains(t, script, "CGO_ENABLED=1")
|
||||
require.NotContains(t, script, "CGO_BUILD=1")
|
||||
}
|
||||
|
||||
func TestBuildAllTaskRejectsNonLinuxTarget(t *testing.T) {
|
||||
origCheck := detectGlibc
|
||||
detectGlibc = func() error { return nil }
|
||||
t.Cleanup(func() { detectGlibc = origCheck })
|
||||
|
||||
ctx := &app.Context{
|
||||
Root: t.TempDir(),
|
||||
ExtraEnv: map[string]string{
|
||||
"GOOS": "freebsd",
|
||||
},
|
||||
}
|
||||
fake := &runner.FakeRunner{
|
||||
Result: runner.CommandResult{ExitCode: 0},
|
||||
}
|
||||
|
||||
var buildAll app.Task
|
||||
for _, task := range NewRegistry(ctx.Root) {
|
||||
if task.Name() == "build-all" {
|
||||
buildAll = task
|
||||
break
|
||||
}
|
||||
}
|
||||
require.NotNil(t, buildAll)
|
||||
|
||||
_, err := buildAll.Run(context.Background(), fake, ctx)
|
||||
require.Error(t, err)
|
||||
require.Len(t, fake.Calls, 0)
|
||||
}
|
||||
|
||||
func TestEvalRunSpecsPreservesEvalConfig(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
script := evalRunScript(&app.Context{
|
||||
specs := evalRunSpecs(&app.Context{
|
||||
ExtraEnv: map[string]string{
|
||||
"DRAGONSCALE_EVAL_DEBUG": "1",
|
||||
"DRAGONSCALE_EVAL_CONFIG": "./configs/override.json",
|
||||
},
|
||||
})
|
||||
|
||||
require.Contains(t, script, "DRAGONSCALE_EVAL_CONFIG=\"${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}\"")
|
||||
require.Contains(t, script, "npx --yes promptfoo eval --config promptfooconfig.yaml")
|
||||
require.NotContains(t, script, "DRAGONSCALE_EVAL_CONFIG=\"./configs/default.json\"")
|
||||
require.Len(t, specs, 2)
|
||||
require.Equal(t, "echo", specs[0].Name)
|
||||
require.Equal(t, []string{"DRAGONSCALE_EVAL_CONFIG=./configs/override.json"}, specs[0].Args)
|
||||
require.Equal(t, "npx", specs[1].Name)
|
||||
require.Contains(t, strings.Join(specs[1].Env, " "), "DRAGONSCALE_EVAL_CONFIG=./configs/override.json")
|
||||
require.NotContains(t, strings.Join(specs[1].Env, " "), "DRAGONSCALE_EVAL_CONFIG=./configs/default.json")
|
||||
require.Contains(t, strings.Join(specs[1].Args, " "), "--yes promptfoo eval --config promptfooconfig.yaml --no-cache --no-progress-bar")
|
||||
|
||||
compareScript, err := os.ReadFile(filepath.Clean(filepath.Join("..", "..", "..", "eval", "scripts", "compare.sh")))
|
||||
require.NoError(t, err)
|
||||
|
|
@ -142,18 +213,27 @@ func TestEvalRunScriptPreservesEvalConfig(t *testing.T) {
|
|||
require.Contains(t, content, "EVAL_CONFIG=\"${DRAGONSCALE_EVAL_CONFIG:-./configs/default.json}\"")
|
||||
require.Contains(t, content, "trap cleanup_compare_config EXIT INT TERM")
|
||||
require.Contains(t, content, "DRAGONSCALE_EVAL_CONFIG: \"${EVAL_CONFIG}\"")
|
||||
require.NotContains(t, content, "DRAGONSCALE_EVAL_HOST_HOME: \"/host_home\"")
|
||||
require.Contains(t, content, "TEMP_CONFIG")
|
||||
}
|
||||
|
||||
func TestEvalRunScriptUsesBaseConfigWhenSet(t *testing.T) {
|
||||
func TestEvalRunSpecsUsesBaseConfigWhenSetAndDebugEnabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
script := evalRunScript(&app.Context{
|
||||
specs := evalRunSpecs(&app.Context{
|
||||
ExtraEnv: map[string]string{
|
||||
"DRAGONSCALE_EVAL_BASE_CONFIG": "/tmp/base.json",
|
||||
"DRAGONSCALE_EVAL_DEBUG": "1",
|
||||
},
|
||||
})
|
||||
require.Contains(t, script, `if [ -n "${DRAGONSCALE_EVAL_BASE_CONFIG:-}" ] && [ -n "${DRAGONSCALE_EVAL_DEBUG:-}" ]; then`)
|
||||
|
||||
require.Len(t, specs, 3)
|
||||
require.Equal(t, "echo", specs[0].Name)
|
||||
require.Equal(t, []string{"DRAGONSCALE_EVAL_BASE_CONFIG=/tmp/base.json"}, specs[0].Args)
|
||||
require.Equal(t, "echo", specs[1].Name)
|
||||
require.Equal(t, []string{"DRAGONSCALE_EVAL_CONFIG=./configs/default.json"}, specs[1].Args)
|
||||
require.Equal(t, "npx", specs[2].Name)
|
||||
require.Equal(t, []string{"--yes", "promptfoo", "eval", "--config", "promptfooconfig.yaml", "--no-cache", "--no-progress-bar"}, specs[2].Args)
|
||||
}
|
||||
|
||||
func TestEvalCompareTaskDisablesNestedDevcontainerExecution(t *testing.T) {
|
||||
|
|
@ -248,6 +328,135 @@ func TestEvalCompareTaskPassesBaseConfigEnvToRunner(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestLintTaskUsesCommandSpecs(t *testing.T) {
|
||||
ctx := &app.Context{
|
||||
Root: t.TempDir(),
|
||||
}
|
||||
fake := &runner.FakeRunner{
|
||||
Result: runner.CommandResult{ExitCode: 0},
|
||||
}
|
||||
|
||||
task := findTaskByName(t, NewRegistry(ctx.Root), "lint")
|
||||
_, err := task.Run(context.Background(), fake, ctx)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.Calls, 3)
|
||||
require.Equal(t, "go", fake.Calls[0].Name)
|
||||
require.Equal(t, []string{"fmt", "./..."}, fake.Calls[0].Args)
|
||||
require.Equal(t, "go", fake.Calls[1].Name)
|
||||
require.Equal(t, []string{"vet", "./..."}, fake.Calls[1].Args)
|
||||
require.Equal(t, "go", fake.Calls[2].Name)
|
||||
require.Equal(t, []string{"build", "./..."}, fake.Calls[2].Args)
|
||||
require.NotContains(t, strings.Join(fake.Calls[0].Env, " "), "set -euo pipefail")
|
||||
}
|
||||
|
||||
func TestCheckTaskUsesCommandSpecs(t *testing.T) {
|
||||
ctx := &app.Context{
|
||||
Root: t.TempDir(),
|
||||
ExtraEnv: map[string]string{
|
||||
"GO": "go",
|
||||
},
|
||||
}
|
||||
fake := &runner.FakeRunner{
|
||||
Result: runner.CommandResult{ExitCode: 0},
|
||||
}
|
||||
|
||||
task := findTaskByName(t, NewRegistry(ctx.Root), "check")
|
||||
_, err := task.Run(context.Background(), fake, ctx)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.Calls, 5)
|
||||
require.Equal(t, []string{"mod", "download"}, fake.Calls[0].Args)
|
||||
require.Equal(t, []string{"mod", "verify"}, fake.Calls[1].Args)
|
||||
require.Equal(t, []string{"fmt", "./..."}, fake.Calls[2].Args)
|
||||
require.Equal(t, []string{"vet", "./..."}, fake.Calls[3].Args)
|
||||
require.Equal(t, []string{"test", "./..."}, fake.Calls[4].Args)
|
||||
}
|
||||
|
||||
func TestDevcontainerGenerateTaskUsesNpxExecCommand(t *testing.T) {
|
||||
ctx := &app.Context{
|
||||
Root: t.TempDir(),
|
||||
}
|
||||
fake := &runner.FakeRunner{
|
||||
Result: runner.CommandResult{ExitCode: 0},
|
||||
}
|
||||
|
||||
task := findTaskByName(t, NewRegistry(ctx.Root), "devcontainer-generate")
|
||||
_, err := task.Run(context.Background(), fake, ctx)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.Calls, 1)
|
||||
require.Equal(t, "npx", fake.Calls[0].Name)
|
||||
require.Contains(t, fake.Calls[0].Args, "@devcontainers/cli")
|
||||
require.Contains(t, fake.Calls[0].Args, "exec")
|
||||
require.Contains(t, fake.Calls[0].Args, "bash")
|
||||
require.Contains(t, strings.Join(fake.Calls[0].Args, " "), "go generate ./pkg/itr ./pkg/tools")
|
||||
}
|
||||
|
||||
func TestEvalTaskBuildsPromptfooCommandWithDefaultConfig(t *testing.T) {
|
||||
ctx := &app.Context{
|
||||
Root: t.TempDir(),
|
||||
ExtraEnv: map[string]string{
|
||||
"DRAGONSCALE_EVAL_DEBUG": "1",
|
||||
},
|
||||
}
|
||||
fake := &runner.FakeRunner{
|
||||
Result: runner.CommandResult{ExitCode: 0},
|
||||
}
|
||||
|
||||
task := findTaskByName(t, NewRegistry(ctx.Root), "eval")
|
||||
_, err := task.Run(context.Background(), fake, ctx)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.Calls, 2)
|
||||
require.Equal(t, "echo", fake.Calls[0].Name)
|
||||
require.Equal(t, "npx", fake.Calls[1].Name)
|
||||
require.Equal(t, filepath.Join(ctx.Root, "eval"), fake.Calls[1].Dir)
|
||||
joinedEnv := strings.Join(fake.Calls[1].Env, " ")
|
||||
require.Contains(t, joinedEnv, "DRAGONSCALE_EVAL_CONFIG=./configs/default.json")
|
||||
require.Contains(t, strings.Join(fake.Calls[1].Args, " "), "--no-progress-bar")
|
||||
}
|
||||
|
||||
func TestEvalViewTaskRunsInEvalDirectory(t *testing.T) {
|
||||
ctx := &app.Context{
|
||||
Root: t.TempDir(),
|
||||
}
|
||||
fake := &runner.FakeRunner{
|
||||
Result: runner.CommandResult{ExitCode: 0},
|
||||
}
|
||||
task := findTaskByName(t, NewRegistry(ctx.Root), "eval-view")
|
||||
_, err := task.Run(context.Background(), fake, ctx)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, fake.Calls, 1)
|
||||
require.Equal(t, filepath.Join(ctx.Root, "eval"), fake.Calls[0].Dir)
|
||||
require.Equal(t, "npx", fake.Calls[0].Name)
|
||||
require.Equal(t, []string{"--yes", "promptfoo", "view"}, fake.Calls[0].Args)
|
||||
}
|
||||
|
||||
func TestEvalFixturesTaskCreatesFixtureCommands(t *testing.T) {
|
||||
ctx := &app.Context{
|
||||
Root: t.TempDir(),
|
||||
}
|
||||
fake := &runner.FakeRunner{
|
||||
Result: runner.CommandResult{ExitCode: 0},
|
||||
}
|
||||
task := findTaskByName(t, NewRegistry(ctx.Root), "eval-fixtures")
|
||||
_, err := task.Run(context.Background(), fake, ctx)
|
||||
require.NoError(t, err)
|
||||
require.GreaterOrEqual(t, len(fake.Calls), 5)
|
||||
require.Equal(t, "mkdir", fake.Calls[0].Name)
|
||||
require.Equal(t, "rm", fake.Calls[1].Name)
|
||||
require.Equal(t, "bash", fake.Calls[4].Name)
|
||||
require.Equal(t, "cp", fake.Calls[5].Name)
|
||||
}
|
||||
|
||||
func findTaskByName(t *testing.T, tasks []app.Task, name string) app.Task {
|
||||
t.Helper()
|
||||
for _, task := range tasks {
|
||||
if task.Name() == name {
|
||||
return task
|
||||
}
|
||||
}
|
||||
t.Fatalf("task %q missing from registry", name)
|
||||
return nil
|
||||
}
|
||||
|
||||
func ptrString(value string) *string {
|
||||
return &value
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue