From ae8afa8070c6828d41db8b68dc28b190f54a27f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Boh=C3=A1=C4=8Dek?= Date: Sat, 26 Sep 2026 21:52:14 +0200 Subject: [PATCH] Add behaviour checks with stub agents --- checks/default.nix | 306 +++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 2 + 2 files changed, 308 insertions(+) create mode 100644 checks/default.nix diff --git a/checks/default.nix b/checks/default.nix new file mode 100644 index 0000000000000000000000000000000000000000..767a31e3f43dedb7e46196e8601c7b7475359f24 --- /dev/null +++ b/checks/default.nix @@ -0,0 +1,306 @@ +{ pkgs, self }: +let + inherit (pkgs) lib; + inherit (self.lib) mkEnvironment mkSkill mkSkillModule; + + # Stub agents record what the launcher handed them, so the checks can assert on + # the contract between launcher and agent without needing a login or network. + stub = + name: + pkgs.writeShellScriptBin name '' + log="$STUB_LOG" + : >"$log.args" + if [ $# -gt 0 ]; then printf '%s\n' "$@" >"$log.args"; fi + { + echo "CLAUDE_CONFIG_DIR=''${CLAUDE_CONFIG_DIR:-}" + echo "CODEX_HOME=''${CODEX_HOME:-}" + if command -v hello >/dev/null; then echo "HELLO=yes"; else echo "HELLO=no"; fi + } >"$log.env" + ''; + stubs = { + claude.package = stub "claude"; + codex.package = stub "codex"; + }; + + skillSrc = + name: + pkgs.writeTextDir "SKILL.md" '' + --- + name: ${name} + description: test skill ${name} + --- + ''; + # `leaf` needs the `hello` tool; `top` depends on `leaf`. + leaf = mkSkill { + inherit pkgs; + name = "leaf"; + src = skillSrc "leaf"; + runtimeInputs = [ pkgs.hello ]; + }; + top = mkSkill { + inherit pkgs; + name = "top"; + src = skillSrc "top"; + skills = [ leaf ]; + }; + + full = mkEnvironment { + inherit pkgs; + modules = [ + stubs + (mkSkillModule top) + { + claude.enable = true; + codex.enable = true; + modules.skills.top.enable = true; + # Names sort opposite to their order, so the check proves `order` wins. + instructions.a-second = { + text = "SECOND"; + order = 20; + }; + instructions.z-first = { + text = "FIRST"; + order = 10; + }; + claude.settings.model = "opus"; + codex.settings.model = "gpt-5"; + launchers = { + claude-work.agent = "claude"; + claude-pers.agent = "claude"; + codex-work.agent = "codex"; + # Same state dir as claude-work, used to test switching managed content. + claude-work-bare = { + agent = "claude"; + id = "claude-work"; + }; + }; + } + ]; + }; + + # Same launcher id as full's claude-work, but no skills, instructions or settings. + bare = mkEnvironment { + inherit pkgs; + modules = [ + stubs + { + claude.enable = true; + codex.enable = true; + launchers.claude-work.agent = "claude"; + launchers.codex-bare.agent = "codex"; + } + ]; + }; + + fails = + modules: !(builtins.tryEval (mkEnvironment { inherit pkgs modules; }).package.drvPath).success; + + otherLeaf = mkSkill { + inherit pkgs; + name = "leaf"; + src = skillSrc "leaf-other"; + }; + + runTest = + name: script: + pkgs.runCommand "agent-environments-${name}" { } '' + set -euo pipefail + export HOME=$TMPDIR/home + mkdir -p $HOME + export STUB_LOG=$TMPDIR/stub + fail() { echo "FAIL: $*" >&2; exit 1; } + ${script} + touch $out + ''; +in +{ + launch-contract = runTest "launch-contract" '' + ${full.launchers.claude-work}/bin/claude-work --resume x + s=$HOME/.local/agent-environments/claude-work + grep -qxF "CLAUDE_CONFIG_DIR=$s" $STUB_LOG.env || fail "CLAUDE_CONFIG_DIR not set to state dir" + grep -qxF "CODEX_HOME=" $STUB_LOG.env || fail "claude launcher leaked CODEX_HOME" + + # Instructions: order field wins over names, fragments separated by a blank line. + [ "$(cat $s/CLAUDE.md)" = "$(printf 'FIRST\n\nSECOND')" ] || fail "CLAUDE.md content: $(cat $s/CLAUDE.md)" + + # Skills: dependency closure linked, and the dependency's tool is on PATH. + [ -f $s/skills/top/SKILL.md ] || fail "top skill missing" + [ -f $s/skills/leaf/SKILL.md ] || fail "dependency skill leaf missing" + grep -qxF "HELLO=yes" $STUB_LOG.env || fail "skill runtime input not on PATH" + + # Settings layered via --settings, user args passed through after it. + [ "$(sed -n 1p $STUB_LOG.args)" = "--settings" ] || fail "missing --settings" + grep -q '"model": *"opus"' "$(sed -n 2p $STUB_LOG.args)" || fail "settings file content" + [ "$(sed -n '3,$p' $STUB_LOG.args)" = "$(printf -- '--resume\nx')" ] || fail "user args not passed through" + [ ! -e $s/settings.json ] || fail "launcher must not own settings.json" + + ${full.launchers.codex-work}/bin/codex-work exec hi + c=$HOME/.local/agent-environments/codex-work + grep -qxF "CODEX_HOME=$c" $STUB_LOG.env || fail "CODEX_HOME not set" + [ "$(cat $c/AGENTS.md)" = "$(printf 'FIRST\n\nSECOND')" ] || fail "AGENTS.md content" + [ -f $c/skills/leaf/SKILL.md ] || fail "codex skills missing" + grep -q 'model = "gpt-5"' $c/agent-environments.config.toml || fail "codex profile content" + [ "$(cat $STUB_LOG.args)" = "$(printf -- '--profile\nagent-environments\nexec\nhi')" ] || fail "codex args: $(cat $STUB_LOG.args)" + [ ! -e $c/config.toml ] || fail "launcher must not own config.toml" + ''; + + # Separate launchers from one environment must not share a login. + separate-logins = runTest "separate-logins" '' + ${full.launchers.claude-work}/bin/claude-work + a=$(grep CLAUDE_CONFIG_DIR $STUB_LOG.env) + ${full.launchers.claude-pers}/bin/claude-pers + b=$(grep CLAUDE_CONFIG_DIR $STUB_LOG.env) + [ "$a" != "$b" ] || fail "claude-work and claude-pers share $a" + [ "$b" = "CLAUDE_CONFIG_DIR=$HOME/.local/agent-environments/claude-pers" ] || fail "$b" + ''; + + # Without settings, no flags are injected and no files are managed. + no-settings = runTest "no-settings" '' + ${bare.launchers.claude-work}/bin/claude-work + [ ! -s $STUB_LOG.args ] || fail "unexpected args: $(cat $STUB_LOG.args)" + ${bare.launchers.codex-bare}/bin/codex-bare + [ ! -s $STUB_LOG.args ] || fail "unexpected codex args: $(cat $STUB_LOG.args)" + c=$HOME/.local/agent-environments/codex-bare + [ ! -e $c/AGENTS.md ] && [ ! -e $c/agent-environments.config.toml ] || fail "files managed without config" + ''; + + # Switching to an environment with fewer skills removes only what we linked. + cleanup-keeps-user-files = runTest "cleanup-keeps-user-files" '' + s=$HOME/.local/agent-environments/claude-work + mkdir -p $s/skills/.system $s/skills/mine + ln -s /nix/store $s/skills/user-link + ${full.launchers.claude-work}/bin/claude-work + [ -L $s/skills/top ] || fail "top not linked" + ${bare.launchers.claude-work}/bin/claude-work + [ ! -e $s/skills/top ] && [ ! -e $s/skills/leaf ] || fail "stale skills kept" + [ ! -e $s/CLAUDE.md ] || fail "stale CLAUDE.md kept" + [ -d $s/skills/.system ] && [ -d $s/skills/mine ] && [ -L $s/skills/user-link ] || fail "user files removed" + # And back again: the manifest must let us re-create the links. + ${full.launchers.claude-work-bare}/bin/claude-work-bare + [ -L $s/skills/top ] || fail "relink failed" + ''; + + # User-owned files at managed paths are never replaced. + refuses-unmanaged = runTest "refuses-unmanaged" '' + s=$HOME/.local/agent-environments/claude-work + mkdir -p $s/skills + echo mine >$s/CLAUDE.md + if ${full.launchers.claude-work}/bin/claude-work 2>err; then fail "launcher replaced user CLAUDE.md"; fi + grep -q "refusing to replace '$s/CLAUDE.md'" err || fail "unclear error: $(cat err)" + [ "$(cat $s/CLAUDE.md)" = mine ] || fail "user CLAUDE.md modified" + [ ! -e $STUB_LOG.args ] || fail "agent started despite conflict" + + rm $s/CLAUDE.md + ln -s /nix/store $s/skills/top + if ${full.launchers.claude-work}/bin/claude-work 2>err; then fail "launcher replaced user symlink"; fi + [ "$(readlink $s/skills/top)" = /nix/store ] || fail "user symlink modified" + + # A path we managed before, but which the user replaced with real data, is theirs now. + rm $s/skills/top + ${full.launchers.claude-work}/bin/claude-work + rm $s/CLAUDE.md && echo edited >$s/CLAUDE.md + if ${full.launchers.claude-work}/bin/claude-work 2>err; then fail "launcher replaced user file at managed path"; fi + [ "$(cat $s/CLAUDE.md)" = edited ] || fail "user file modified" + ''; + + # The launcher's own bookkeeping paths must not write through symlinks into + # files or directories outside the state dir. + no-writes-through-symlinks = runTest "no-writes-through-symlinks" '' + s=$HOME/.local/agent-environments/claude-work + mkdir -p $s/skills + echo precious >$TMPDIR/precious + ln -s $TMPDIR/precious $s/.agent-environments.lock + ${full.launchers.claude-work}/bin/claude-work + [ "$(cat $TMPDIR/precious)" = precious ] || fail "lock open truncated a symlinked file" + + p=$HOME/.local/agent-environments/claude-pers + mkdir -p $p $TMPDIR/shared-skills + ln -s $TMPDIR/shared-skills $p/skills + if ${full.launchers.claude-pers}/bin/claude-pers 2>err; then fail "managed links through a symlinked skills dir"; fi + [ -z "$(ls -A $TMPDIR/shared-skills)" ] || fail "wrote into the shared skills dir" + ''; + + # stateDir is data, not shell code: only a leading ~/ is expanded. + literal-state-dir = + let + env = mkEnvironment { + inherit pkgs; + modules = [ + stubs + { + claude.enable = true; + launchers.claude-odd = { + agent = "claude"; + stateDir = ''~/odd "dir"/$(touch injected)''; + }; + } + ]; + }; + in + runTest "literal-state-dir" '' + cd $TMPDIR + ${env.launchers.claude-odd}/bin/claude-odd + [ ! -e injected ] || fail "stateDir was executed as shell code" + grep -qxF "CLAUDE_CONFIG_DIR=$HOME/odd \"dir\"/\$(touch injected)" $STUB_LOG.env || fail "$(cat $STUB_LOG.env)" + ''; + + skill-without-skill-md = pkgs.testers.testBuildFailure (mkSkill { + inherit pkgs; + name = "broken"; + src = pkgs.writeTextDir "README.md" ""; + }); + + # Configuration mistakes are caught at evaluation time. + eval-errors = + assert fails [ + stubs + { + claude.enable = true; + launchers.x = { + agent = "claude"; + stateDir = "relative/dir"; + }; + } + ]; + assert fails [ + stubs + { launchers.x.agent = "claude"; } + ]; + assert fails [ + stubs + { + claude.enable = true; + launchers.x = { + agent = "claude"; + id = "../escape"; + }; + } + ]; + assert fails [ + stubs + { + claude.enable = true; + launchers."bad name".agent = "claude"; + } + ]; + assert fails [ + stubs + { + claude.enable = true; + skills.packages = [ + top + otherLeaf + ]; + launchers.x.agent = "claude"; + } + ]; + assert + !(builtins.tryEval + (mkSkill { + inherit pkgs; + name = "a/b"; + src = skillSrc "x"; + }).drvPath + ).success; + pkgs.runCommand "agent-environments-eval-errors" { } "touch $out"; +} diff --git a/flake.nix b/flake.nix index 67b00dbfe4eb9ca86b909a41d8895e6e844c9416..7d6ec46dc6888ac14153b3e20346e1b79766cec7 100644 --- a/flake.nix +++ b/flake.nix @@ -46,6 +46,8 @@ }).package; }); + checks = forAllSystems (pkgs: import ./checks { inherit pkgs self; }); + formatter = forAllSystems (pkgs: pkgs.nixfmt); }; }