123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551 | package tools
import (
"context"
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"slices"
"strings"
"testing"
"time"
"blinker/internal/policy"
"blinker/internal/sandbox"
"blinker/internal/testenv"
)
// The test binary stands in for blinker inside the sandbox.
func TestMain(m *testing.M) {
if code, ok := sandbox.RunHidden(os.Args); ok {
os.Exit(code)
}
if err := testenv.LeaveHome(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
os.Exit(m.Run())
}
// testPolicy gives home, then hides ~/other, makes the project readable
// without asking and writable once approved, src writable without asking,
// and hides .env files.
var testPolicy = []testRule{
{"~", policy.Write, policy.Write},
{"~/other", policy.None, policy.None},
{"**", policy.Read, policy.Write},
{"src", policy.Write, policy.Write},
{"**/.env*", policy.None, policy.None},
}
// testRule gives a path an access in each mode.
type testRule struct {
Path string
Safe, Unsafe policy.Perm
}
// savePolicies writes a project's two policies, with only these rules:
// the file rules' access in each mode, and the tool rules, an Allow in
// both, an Ask as an allow in the unsafe policy only, and a Deny in
// neither.
func savePolicies(t *testing.T, configDir, project string, rules []testRule, tools []policy.ToolRule) {
t.Helper()
for _, m := range []policy.Mode{policy.SafeMode, policy.UnsafeMode} {
f := &policy.File{Directory: project}
for _, r := range rules {
access := r.Safe
if m == policy.UnsafeMode {
access = r.Unsafe
}
f.Files = append(f.Files, policy.FileRule{Path: r.Path, Access: access})
}
for _, r := range tools {
if r.Access == policy.Allow || r.Access == policy.Ask && m == policy.UnsafeMode {
f.Tools = append(f.Tools, policy.ToolRule{Tool: r.Tool, Access: policy.Allow})
}
}
if err := policy.Save(filepath.Join(policy.ProjectDir(configDir, project), m.String()+".policy"), f); err != nil {
t.Fatal(err)
}
}
}
type fixture struct {
tb *Toolbox
project string
home string
asked []Question
answer func(Question) Answer
}
// setup makes a home and project, writes the given tool files (name to
// contents), allows the tools the rules allow, and starts a sandbox.
func setup(t *testing.T, toolFiles map[string]string, toolRules ...policy.ToolRule) *fixture {
t.Helper()
if _, err := exec.LookPath("bwrap"); err != nil {
t.Skip("bwrap not installed")
}
if err := exec.Command("bwrap", "--ro-bind", "/", "/", "true").Run(); err != nil {
t.Skip("bwrap cannot run here: ", err)
}
base, err := os.MkdirTemp(".", ".sandbox-test-")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(base) })
base, err = filepath.Abs(base)
if err != nil {
t.Fatal(err)
}
if base, err = filepath.EvalSymlinks(base); err != nil {
t.Fatal(err)
}
f := &fixture{home: filepath.Join(base, "home")}
f.project = filepath.Join(f.home, "proj")
files := map[string]string{
"proj/main.go": "package main\n\nfunc main() {}\n",
"proj/src/a.go": "package src\n",
"proj/.env": "SECRET=1\n",
"other/secret.txt": "secret\n",
}
for name, content := range files {
p := filepath.Join(f.home, name)
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
t.Setenv("HOME", f.home)
t.Chdir(f.project)
configDir := filepath.Join(f.home, ".config", "blinker")
for name, text := range toolFiles {
p := filepath.Join(configDir, "tools", name+".sh")
if err := os.MkdirAll(filepath.Dir(p), 0o700); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(p, []byte(text), 0o600); err != nil {
t.Fatal(err)
}
}
savePolicies(t, configDir, f.project, testPolicy, toolRules)
sb, err := sandbox.New(sandbox.Options{ConfigDir: configDir, ProxyPorts: []int{443}})
if err != nil {
t.Fatal(err)
}
t.Cleanup(sb.Close)
f.answer = func(q Question) Answer { return Answer{Option: q.Opts[0]} } // allow
f.tb = New(sb, func(_ context.Context, q Question) Answer {
f.asked = append(f.asked, q)
return f.answer(q)
}, 30*time.Second)
return f
}
func (f *fixture) run(t *testing.T, name string, args map[string]any) Result {
t.Helper()
data, err := json.Marshal(args)
if err != nil {
t.Fatal(err)
}
return f.tb.Run(context.Background(), name, string(data))
}
func TestRead(t *testing.T) {
f := setup(t, nil)
r := f.run(t, "Read", map[string]any{"path": "main.go"})
if r.Failed || !strings.Contains(r.Output, " 3\tfunc main() {}") || len(f.asked) != 0 {
t.Errorf("Read = %+v, asked %d", r, len(f.asked))
}
r = f.run(t, "Read", map[string]any{"path": "main.go", "offset": 2, "limit": 1})
if strings.Contains(r.Output, "package") || !strings.Contains(r.Output, "read on with offset 3") {
t.Errorf("offset/limit: %q", r.Output)
}
for _, p := range []string{".env", "~/other/secret.txt", "/etc/../root/x"} {
r = f.run(t, "Read", map[string]any{"path": p})
if !r.Failed || strings.Contains(r.Output, "SECRET") || strings.Contains(r.Output, "secret\n") {
t.Errorf("Read %s = %+v", p, r)
}
}
if len(f.asked) != 0 {
t.Error("forbidden reads asked the user")
}
}
func TestReadThroughSymlink(t *testing.T) {
f := setup(t, nil)
if err := os.Symlink(filepath.Join(f.home, "other", "secret.txt"), filepath.Join(f.project, "link")); err != nil {
t.Fatal(err)
}
r := f.run(t, "Read", map[string]any{"path": "link"})
if !r.Failed || strings.Contains(r.Output, "secret\n") {
t.Errorf("read through a symlink: %+v", r)
}
}
func TestWriteAsksWhenPolicySaysConfirm(t *testing.T) {
f := setup(t, nil)
// Safe: no question, so the result shows the diff.
r := f.run(t, "Write", map[string]any{"path": "src/b.go", "content": "package src\n"})
if r.Failed || len(f.asked) != 0 || !r.Diff || !strings.Contains(r.Detail, "+package src") {
t.Fatalf("safe write: %+v, asked %d", r, len(f.asked))
}
// Confirm, denied with a note.
if r := f.run(t, "Read", map[string]any{"path": "main.go"}); r.Failed {
t.Fatalf("Read = %+v", r)
}
f.answer = func(q Question) Answer { return Answer{Option: q.Opts[4], Note: "not now"} }
r = f.run(t, "Write", map[string]any{"path": "main.go", "content": "x\n"})
if !r.Failed || !strings.Contains(r.Output, "not now") || len(f.asked) != 1 || !strings.Contains(f.asked[0].Diff, "+x") {
t.Fatalf("denied write: %+v, asked %+v", r, f.asked)
}
if got, _ := os.ReadFile(filepath.Join(f.project, "main.go")); string(got) != "package main\n\nfunc main() {}\n" {
t.Errorf("denied write changed the file: %q", got)
}
// Edited by the user: their version is written, and the model hears
// how it differs from its own.
f.answer = func(q Question) Answer {
if len(q.Edits) != 1 || q.Edits[0].New != "proposed\n" || q.Edits[0].Name != "main.go" {
t.Errorf("edits = %+v", q.Edits)
}
return Answer{Option: q.Opts[2], Edited: []string{"edited\n"}}
}
r = f.run(t, "Patch", map[string]any{"path": "main.go", "diff": "@@\n-package main\n-\n-func main() {}\n+proposed\n"})
if r.Failed || !strings.Contains(r.Output, "-proposed") || !strings.Contains(r.Output, "+edited") || !strings.Contains(r.Summary, "Patch main.go (+1 -3, edited") ||
!r.Diff || !strings.Contains(r.Detail, "+edited") || strings.Contains(r.Detail, "proposed") {
t.Errorf("edited write: %+v", r)
}
if got, _ := os.ReadFile(filepath.Join(f.project, "main.go")); string(got) != "edited\n" {
t.Errorf("file = %q", got)
}
// Allowed for the session: asked once, then not again. The question
// showed the first diff, so its result does not repeat it; the second
// was not asked, so its result shows it.
f.answer = func(q Question) Answer { return Answer{Option: q.Opts[1]} }
r = f.run(t, "Write", map[string]any{"path": "main.go", "content": "one\n"})
if r.Failed || r.Detail != "" {
t.Errorf("approved write repeats the diff: %+v", r)
}
r = f.run(t, "Write", map[string]any{"path": "main.go", "content": "two\n"})
if r.Failed || len(f.asked) != 3 || !r.Diff || !strings.Contains(r.Detail, "+two") {
t.Errorf("remembered approval: %+v, asked %d times", r, len(f.asked))
}
if got, _ := os.ReadFile(filepath.Join(f.project, "main.go")); string(got) != "two\n" {
t.Errorf("file = %q", got)
}
// Forbidden: refused without asking.
r = f.run(t, "Write", map[string]any{"path": ".env.local", "content": "x"})
if !r.Failed || len(f.asked) != 3 {
t.Errorf("forbidden write: %+v", r)
}
}
// Write replaces only a file the model has read as it is now; Patch's
// hunks are checked against the file, so it needs no Read.
func TestWriteNeedsRead(t *testing.T) {
f := setup(t, nil)
path := filepath.Join(f.project, "src", "a.go")
write := func(content string) Result {
return f.run(t, "Write", map[string]any{"path": "src/a.go", "content": content})
}
if r := write("x\n"); !r.Failed || !strings.Contains(r.Output, "not read it") {
t.Errorf("unread write: %+v", r)
}
f.run(t, "Read", map[string]any{"path": "src/a.go", "limit": 1})
if r := write("one\n"); r.Failed {
t.Errorf("write after Read: %+v", r)
}
// Its own writes count as seen.
if r := write("two\n"); r.Failed {
t.Errorf("second write: %+v", r)
}
if err := os.WriteFile(path, []byte("user's\n"), 0o644); err != nil {
t.Fatal(err)
}
if r := write("three\n"); !r.Failed || !strings.Contains(r.Output, "changed since") {
t.Errorf("write over a change: %+v", r)
}
if r := f.run(t, "Patch", map[string]any{"path": "src/a.go", "diff": "@@\n-user's\n+patched\n"}); r.Failed {
t.Errorf("Patch after a change: %+v", r)
}
if r := write("four\n"); r.Failed {
t.Errorf("write after Patch: %+v", r)
}
f.tb.Forget()
if r := write("five\n"); !r.Failed {
t.Errorf("write after Forget: %+v", r)
}
// Creating a file needs no Read.
if r := f.run(t, "Write", map[string]any{"path": "src/new.go", "content": "new\n"}); r.Failed {
t.Errorf("new file: %+v", r)
}
if got, _ := os.ReadFile(path); string(got) != "four\n" {
t.Errorf("file = %q", got)
}
}
// A file that changes while the user decides is not written over.
func TestWriteRechecksAfterQuestion(t *testing.T) {
f := setup(t, nil)
path := filepath.Join(f.project, "main.go")
f.run(t, "Read", map[string]any{"path": "main.go"})
f.answer = func(q Question) Answer {
if err := os.WriteFile(path, []byte("user's\n"), 0o644); err != nil {
t.Fatal(err)
}
return Answer{Option: q.Opts[0]}
}
r := f.run(t, "Write", map[string]any{"path": "main.go", "content": "model's\n"})
if !r.Failed || !strings.Contains(r.Output, "while the user decided") {
t.Errorf("write over a change made during the question: %+v", r)
}
f.answer = func(q Question) Answer {
if err := os.WriteFile(filepath.Join(f.project, "late.go"), []byte("user's\n"), 0o644); err != nil {
t.Fatal(err)
}
return Answer{Option: q.Opts[0]}
}
r = f.run(t, "Write", map[string]any{"path": "late.go", "content": "model's\n"})
if !r.Failed || !strings.Contains(r.Output, "created while") {
t.Errorf("write over a file created during the question: %+v", r)
}
for _, name := range []string{"main.go", "late.go"} {
if got, _ := os.ReadFile(filepath.Join(f.project, name)); string(got) != "user's\n" {
t.Errorf("%s = %q", name, got)
}
}
}
func TestPatch(t *testing.T) {
f := setup(t, nil)
r := f.run(t, "Patch", map[string]any{"path": "src/a.go", "diff": "@@\n-package src\n+package src // patched\n"})
if r.Failed || !strings.HasPrefix(r.Summary, "Patch") {
t.Fatalf("Patch = %+v", r)
}
if got, _ := os.ReadFile(filepath.Join(f.project, "src", "a.go")); string(got) != "package src // patched\n" {
t.Errorf("file = %q", got)
}
r = f.run(t, "Patch", map[string]any{"path": "src/a.go", "diff": "@@\n-nothing like this\n+x\n"})
if !r.Failed || !strings.Contains(r.Output, "not found") {
t.Errorf("bad patch: %+v", r)
}
r = f.run(t, "Patch", map[string]any{"path": "src/missing.go", "diff": "@@\n-a\n+b\n"})
if !r.Failed || !strings.Contains(r.Output, "Write") {
t.Errorf("patching a missing file: %+v", r)
}
}
func TestBash(t *testing.T) {
f := setup(t, nil)
r := f.run(t, "Safe_Bash", map[string]any{"script": "echo hi > main.go"})
if !r.Failed {
t.Errorf("safe write denial: %q", r.Output)
}
// What a command prints is not taken as the sandbox's word.
r = f.run(t, "Safe_Bash", map[string]any{"script": "echo \"$HOME/.ssh/id_ed25519: Permission denied\"; exit 1"})
if !r.Failed || strings.Contains(r.Output, "[sandbox]") {
t.Errorf("printed denial blamed on the sandbox: %q", r.Output)
}
f.answer = denying
r = f.run(t, "Unsafe_Bash", map[string]any{"script": "echo hi > main.go"})
if !r.Failed || !strings.Contains(r.Output, "denied") || f.asked[0].Body != "echo hi > main.go" {
t.Errorf("denied unsafe: %+v", r)
}
f.answer = func(q Question) Answer { return Answer{Option: q.Opts[0]} }
r = f.run(t, "Unsafe_Bash", map[string]any{"script": "echo hi > main.go && cat main.go"})
if r.Failed || r.Output != "hi" {
t.Errorf("approved unsafe: %+v", r)
}
r = f.run(t, "Safe_Bash", map[string]any{"script": "sleep 5", "timeout_seconds": 0.2})
if !r.Failed || !strings.Contains(r.Output, "timed out") {
t.Errorf("timeout: %q", r.Output)
}
}
func TestBadArguments(t *testing.T) {
f := setup(t, nil)
for name, args := range map[string]string{"Read": `{"path": 3}`, "Write": `not json`, "Nope": `{}`} {
if r := f.tb.Run(context.Background(), name, args); !r.Failed {
t.Errorf("%s(%s) = %+v", name, args, r)
}
}
}
func TestClip(t *testing.T) {
if got := clip("short", 0); got != "short" {
t.Errorf("short output changed: %q", got)
}
long := strings.Repeat("a", maxOutputBytes) + strings.Repeat("b", 100)
if got := clip(long, 0); !strings.Contains(got, "[... 100 bytes omitted ...]") {
t.Errorf("long output: %q", got[maxOutputBytes/2-10:maxOutputBytes/2+40])
}
if got := clip("headtail", 1000); !strings.Contains(got, "[... 1000 bytes omitted ...]") || !strings.HasPrefix(got, "head") || !strings.HasSuffix(got, "tail") {
t.Errorf("output the capture dropped from: %q", got)
}
}
// A file the model may write once approved, but not read by itself, is
// not read for the write until the user allows that: before the write's
// own question, "No change" or a hunk that fails to match would tell the
// model what the file holds.
func TestWriteAsksBeforeReadingHiddenFile(t *testing.T) {
f := setup(t, nil)
if err := os.WriteFile(filepath.Join(f.project, "vault.txt"), []byte("guess\n"), 0o644); err != nil {
t.Fatal(err)
}
rules := append(append([]testRule{}, testPolicy...), testRule{"vault.txt", policy.None, policy.Write})
savePolicies(t, filepath.Join(f.home, ".config", "blinker"), f.project, rules, nil)
if err := f.tb.SB.Reload(); err != nil {
t.Fatal(err)
}
f.answer = func(q Question) Answer { return Answer{Option: Option{Label: "Deny"}} }
for _, call := range []struct {
tool string
args map[string]any
}{
{"Write", map[string]any{"path": "vault.txt", "content": "guess\n"}},
{"Patch", map[string]any{"path": "vault.txt", "diff": "@@ -1 +1 @@\n guess\n@@ -9 +9 @@\n-nothing\n+x\n"}},
{"Patch", map[string]any{"path": "missing.txt", "diff": "@@ -1 +1 @@\n-a\n+b\n"}},
} {
f.asked = nil
r := f.run(t, call.tool, call.args)
if !r.Failed || strings.Contains(r.Output, "No change") || strings.Contains(r.Output, "hunk") {
t.Errorf("%s %v = %+v", call.tool, call.args, r)
}
if call.args["path"] == "vault.txt" && (len(f.asked) != 1 || !strings.Contains(f.asked[0].Title, "read")) {
t.Errorf("%s %v asked %+v", call.tool, call.args, f.asked)
}
}
}
// Characters that would change how a path reads in an approval prompt
// are refused.
func TestPathCharacters(t *testing.T) {
f := setup(t, nil)
for _, p := range []string{"notes.txt\n@@ -1 +1 @@\n-old\n+harmless", "a\rb", "x\u202egnp.exe", "zero\u200bwidth", "c1\u009bx"} {
r := f.run(t, "Write", map[string]any{"path": p, "content": "evil\n"})
if !r.Failed || len(f.asked) != 0 {
t.Errorf("Write %q = %+v, asked %d", p, r, len(f.asked))
}
}
entries, err := os.ReadDir(f.project)
if err != nil {
t.Fatal(err)
}
if len(entries) != 3 { // main.go, src, .env
t.Errorf("the project has %d entries", len(entries))
}
}
// Huge offsets and limits are clamped, not overflowed.
func TestReadHugeOffsetLimit(t *testing.T) {
f := setup(t, nil)
const huge = 1<<63 - 1
r := f.run(t, "Read", map[string]any{"path": "main.go", "offset": 2, "limit": huge})
if r.Failed || !strings.Contains(r.Output, " 3\tfunc main() {}") || strings.Contains(r.Output, "-") || r.Summary != "Read main.go (lines 2-3 of 3)" {
t.Errorf("limit: %+v", r)
}
r = f.run(t, "Read", map[string]any{"path": "main.go", "offset": huge, "limit": 10})
if r.Failed || !strings.Contains(r.Output, "only 3 lines") {
t.Errorf("offset: %+v", r)
}
}
// A path in the project shows relative to it, even one whose name starts
// with "..", and one outside it shows in full.
func TestResolveShown(t *testing.T) {
f := setup(t, nil)
for path, want := range map[string]string{
"..notes": "..notes",
"src/../a.go": "a.go",
"../outside.go": filepath.Join(filepath.Dir(f.project), "outside.go"),
} {
if shown, _, err := f.tb.resolve(path); err != nil || shown != want {
t.Errorf("resolve(%q) shows %q, %v; want %q", path, shown, err, want)
}
}
}
// denying picks a question's plain denial.
func denying(q Question) Answer {
return Answer{Option: q.Opts[slices.IndexFunc(q.Opts, func(o Option) bool { return !o.Allow && !o.Explain })]}
}
// Previewing an Unsafe_Bash script shows what it would change, with its
// output, and makes the changes only if the user applies them; the model
// sees the output only then.
func TestPreviewScript(t *testing.T) {
f := setup(t, nil)
if err := f.tb.SB.CanPreview(); err != nil {
t.Skip(err)
}
main := filepath.Join(f.project, "main.go")
script := "sed -i s/main/MAIN/ main.go && echo done"
second := Answer{}
f.answer = func(q Question) Answer {
if i := slices.IndexFunc(q.Opts, func(o Option) bool { return o.Preview }); i >= 0 {
return Answer{Option: q.Opts[i]}
}
return second
}
second = Answer{Option: previewOpts[2], Note: "not that"}
r := f.run(t, "Unsafe_Bash", map[string]any{"script": script})
q := f.asked[len(f.asked)-1]
if !strings.Contains(q.Diff, "-package main") || !strings.Contains(q.Diff, "+package MAIN") || !strings.Contains(q.Body, "done") {
t.Errorf("the preview's question: %+v", q)
}
if !r.Failed || strings.Contains(r.Output, "done") || !strings.Contains(r.Output, "They said: not that") {
t.Errorf("discarded: %+v", r)
}
if data, _ := os.ReadFile(main); !strings.Contains(string(data), "package main") {
t.Errorf("discarded, main.go is %q", data)
}
second = Answer{Option: previewOpts[0]}
r = f.run(t, "Unsafe_Bash", map[string]any{"script": script})
if r.Failed || !strings.Contains(r.Output, "done") || !strings.Contains(r.Output, "the user applied") {
t.Errorf("applied: %+v", r)
}
if data, _ := os.ReadFile(main); !strings.Contains(string(data), "package MAIN") {
t.Errorf("applied, main.go is %q", data)
}
}
// A preview's question shows every change apply would make, names that
// could pass for part of it quoted; and when a preview fails, the model
// learns nothing of what the script made.
func TestPreviewShowsAll(t *testing.T) {
f := setup(t, nil)
if err := f.tb.SB.CanPreview(); err != nil {
t.Skip(err)
}
if err := os.MkdirAll(filepath.Join(f.project, "dir", "sub"), 0o755); err != nil {
t.Fatal(err)
}
f.answer = func(q Question) Answer {
if i := slices.IndexFunc(q.Opts, func(o Option) bool { return o.Preview }); i >= 0 {
return Answer{Option: q.Opts[i]}
}
return Answer{Option: previewOpts[1]}
}
f.run(t, "Unsafe_Bash", map[string]any{"script": `touch empty; rm -rf dir && ln -s main.go dir; echo x > "$(printf 'a\n+++ fake')"`})
q := f.asked[len(f.asked)-1]
for _, want := range []string{
"empty: a new empty file",
"dir: a symlink to main.go, in place of the directory there, and everything in it",
`"a\n+++ fake"`,
} {
if !strings.Contains(q.About+q.Diff, want) {
t.Errorf("the question leaves out %q:\n%s\n%s", want, q.About, q.Diff)
}
}
if strings.Contains(q.Diff, "\n+++ fake") {
t.Errorf("a name passed for a diff header:\n%s", q.Diff)
}
r := f.run(t, "Unsafe_Bash", map[string]any{"script": "mkfifo secret-name"})
if !r.Failed || strings.Contains(r.Output, "secret-name") || !strings.Contains(r.Detail, "secret-name") {
t.Errorf("a failed preview: %+v", r)
}
}
|