From 83bcec411d5eb391d41a1d0061881f35a25c4157 Mon Sep 17 00:00:00 2001 From: Kevin Tang <73975146+vt128@users.noreply.github.com> Date: Mon, 22 Jun 2026 14:03:33 +0800 Subject: [PATCH] [test] end-to-end golden suite: build binary, run samples, assert stdout/exit (STAR-53) The v0.1.0 cost-price audit's flagship P0 gap: no test built the real binary and ran scripts through it, so the wired modules were only "load-verified", never proven to work. Add an e2e package whose TestMain builds the starcli binary and runs sample scripts via exec.Command, asserting stdout and the exit code. Cases (through the real CLI): - print / basic output - the newly-wired pure domain modules actually produce results: emoji (emojize -> a glyph), yaml (decode), qrcode (encode().size) - fail() aborts non-zero with the message on stderr - cmd is disabled by default (run() -> non-zero + "disabled"); --allow-cmd actually runs a command (run("go version").success -> True) - --caps safe withholds http (non-zero + "withheld") - --check validates without running (valid -> 0 and the script's print never fires; invalid syntax -> non-zero) Full -race green; Docker golang:1.25 floor builds + runs the binary in-container. --- e2e/e2e_test.go | 112 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 e2e/e2e_test.go diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go new file mode 100644 index 0000000..5d5340b --- /dev/null +++ b/e2e/e2e_test.go @@ -0,0 +1,112 @@ +// Package e2e holds end-to-end tests that build the real starcli binary and run +// sample scripts through it, asserting stdout and the process exit code. This is +// the "build a binary -> run a .star -> compare stdout/exit" coverage the v0.1.0 +// cost-price audit flagged as missing: it proves the wired modules actually +// *work* through the CLI, not merely that they load. +package e2e + +import ( + "bytes" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// binPath is the freshly built starcli binary, set up once in TestMain. +var binPath string + +func TestMain(m *testing.M) { + dir, err := os.MkdirTemp("", "starcli-e2e") + if err != nil { + panic(err) + } + binPath = filepath.Join(dir, "starcli") + if isWindows() { + binPath += ".exe" + } + // Build from the repo root (the parent of this e2e package directory). + build := exec.Command("go", "build", "-o", binPath, ".") + build.Dir = ".." + if out, err := build.CombinedOutput(); err != nil { + _, _ = os.Stderr.WriteString("e2e: build failed: " + err.Error() + "\n" + string(out) + "\n") + os.RemoveAll(dir) + os.Exit(1) + } + code := m.Run() + os.RemoveAll(dir) + os.Exit(code) +} + +func isWindows() bool { return os.PathSeparator == '\\' } + +// runCLI runs the built binary with args and optional stdin, returning stdout, +// stderr, and the exit code. +func runCLI(t *testing.T, stdin string, args ...string) (stdout, stderr string, exit int) { + t.Helper() + cmd := exec.Command(binPath, args...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + var so, se bytes.Buffer + cmd.Stdout, cmd.Stderr = &so, &se + err := cmd.Run() + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + return so.String(), se.String(), ee.ExitCode() + } + t.Fatalf("run %v: %v", args, err) + } + return so.String(), se.String(), 0 +} + +func TestGolden(t *testing.T) { + const anyNonZero = -1 + cases := []struct { + name string + args []string + wantOut string // exact stdout, checked when non-empty + notOut string // stdout must NOT contain this, checked when non-empty + wantExit int // exact exit code; anyNonZero (-1) means "any non-zero" + errSub string // stderr must contain this, checked when non-empty + }{ + {name: "hello prints", args: []string{"-c", `print("hi", 6*7)`}, wantOut: "hi 42\n", wantExit: 0}, + // newly-wired pure domain modules actually run through the CLI: + {name: "emoji module runs", args: []string{"-c", `load("emoji","emojize"); print(emojize("hi :wave:"))`}, wantOut: "hi \U0001F44B\n", wantExit: 0}, + {name: "yaml module runs", args: []string{"-c", `load("yaml","decode"); print(decode("n: 7")["n"])`}, wantOut: "7\n", wantExit: 0}, + {name: "qrcode module runs", args: []string{"-c", `load("qrcode","encode"); print(encode("x").size > 0)`}, wantOut: "True\n", wantExit: 0}, + // error handling + exit codes: + {name: "fail aborts non-zero", args: []string{"-c", `fail("boom")`}, wantExit: anyNonZero, errSub: "boom"}, + // cmd execution gating through the real binary: + {name: "cmd disabled by default", args: []string{"-c", `load("cmd","run"); run("go version")`}, wantExit: anyNonZero, errSub: "disabled"}, + {name: "allow-cmd runs a command", args: []string{"--allow-cmd", "-c", `load("cmd","run"); print(run("go version").success)`}, wantOut: "True\n", wantExit: 0}, + // capability gate: + {name: "caps safe withholds http", args: []string{"--caps", "safe", "-c", `load("http","get")`}, wantExit: anyNonZero, errSub: "withheld"}, + // --check validates without running: + {name: "check valid does not run", args: []string{"--check", "-c", `print("RAN")`}, notOut: "RAN", wantExit: 0}, + {name: "check invalid is non-zero", args: []string{"--check", "-c", `x =`}, wantExit: anyNonZero}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + so, se, exit := runCLI(t, "", c.args...) + if c.wantExit == anyNonZero { + if exit == 0 { + t.Errorf("exit=0, want non-zero (stdout=%q stderr=%q)", so, se) + } + } else if exit != c.wantExit { + t.Errorf("exit=%d, want %d (stdout=%q stderr=%q)", exit, c.wantExit, so, se) + } + if c.wantOut != "" && so != c.wantOut { + t.Errorf("stdout=%q, want %q (stderr=%q)", so, c.wantOut, se) + } + if c.notOut != "" && strings.Contains(so, c.notOut) { + t.Errorf("stdout=%q must not contain %q", so, c.notOut) + } + if c.errSub != "" && !strings.Contains(se, c.errSub) { + t.Errorf("stderr=%q, want substring %q", se, c.errSub) + } + }) + } +}