~ruther/agent-environments

ae8afa8070c6828d41db8b68dc28b190f54a27f8 — František Boháček 2 months ago 14f4be6
Add behaviour checks with stub agents
2 files changed, 308 insertions(+), 0 deletions(-)

A checks/default.nix
M flake.nix
A checks/default.nix => checks/default.nix +306 -0
@@ 0,0 1,306 @@
{ pkgs, self }:
let
  inherit (pkgs) lib;
  inherit (self.lib) mkEnvironment mkSkill mkSkillModule;

  # Stub agents record what the launcher handed them, so the checks can assert on
  # the contract between launcher and agent without needing a login or network.
  stub =
    name:
    pkgs.writeShellScriptBin name ''
      log="$STUB_LOG"
      : >"$log.args"
      if [ $# -gt 0 ]; then printf '%s\n' "$@" >"$log.args"; fi
      {
        echo "CLAUDE_CONFIG_DIR=''${CLAUDE_CONFIG_DIR:-}"
        echo "CODEX_HOME=''${CODEX_HOME:-}"
        if command -v hello >/dev/null; then echo "HELLO=yes"; else echo "HELLO=no"; fi
      } >"$log.env"
    '';
  stubs = {
    claude.package = stub "claude";
    codex.package = stub "codex";
  };

  skillSrc =
    name:
    pkgs.writeTextDir "SKILL.md" ''
      ---
      name: ${name}
      description: test skill ${name}
      ---
    '';
  # `leaf` needs the `hello` tool; `top` depends on `leaf`.
  leaf = mkSkill {
    inherit pkgs;
    name = "leaf";
    src = skillSrc "leaf";
    runtimeInputs = [ pkgs.hello ];
  };
  top = mkSkill {
    inherit pkgs;
    name = "top";
    src = skillSrc "top";
    skills = [ leaf ];
  };

  full = mkEnvironment {
    inherit pkgs;
    modules = [
      stubs
      (mkSkillModule top)
      {
        claude.enable = true;
        codex.enable = true;
        modules.skills.top.enable = true;
        # Names sort opposite to their order, so the check proves `order` wins.
        instructions.a-second = {
          text = "SECOND";
          order = 20;
        };
        instructions.z-first = {
          text = "FIRST";
          order = 10;
        };
        claude.settings.model = "opus";
        codex.settings.model = "gpt-5";
        launchers = {
          claude-work.agent = "claude";
          claude-pers.agent = "claude";
          codex-work.agent = "codex";
          # Same state dir as claude-work, used to test switching managed content.
          claude-work-bare = {
            agent = "claude";
            id = "claude-work";
          };
        };
      }
    ];
  };

  # Same launcher id as full's claude-work, but no skills, instructions or settings.
  bare = mkEnvironment {
    inherit pkgs;
    modules = [
      stubs
      {
        claude.enable = true;
        codex.enable = true;
        launchers.claude-work.agent = "claude";
        launchers.codex-bare.agent = "codex";
      }
    ];
  };

  fails =
    modules: !(builtins.tryEval (mkEnvironment { inherit pkgs modules; }).package.drvPath).success;

  otherLeaf = mkSkill {
    inherit pkgs;
    name = "leaf";
    src = skillSrc "leaf-other";
  };

  runTest =
    name: script:
    pkgs.runCommand "agent-environments-${name}" { } ''
      set -euo pipefail
      export HOME=$TMPDIR/home
      mkdir -p $HOME
      export STUB_LOG=$TMPDIR/stub
      fail() { echo "FAIL: $*" >&2; exit 1; }
      ${script}
      touch $out
    '';
in
{
  launch-contract = runTest "launch-contract" ''
    ${full.launchers.claude-work}/bin/claude-work --resume x
    s=$HOME/.local/agent-environments/claude-work
    grep -qxF "CLAUDE_CONFIG_DIR=$s" $STUB_LOG.env || fail "CLAUDE_CONFIG_DIR not set to state dir"
    grep -qxF "CODEX_HOME=" $STUB_LOG.env || fail "claude launcher leaked CODEX_HOME"

    # Instructions: order field wins over names, fragments separated by a blank line.
    [ "$(cat $s/CLAUDE.md)" = "$(printf 'FIRST\n\nSECOND')" ] || fail "CLAUDE.md content: $(cat $s/CLAUDE.md)"

    # Skills: dependency closure linked, and the dependency's tool is on PATH.
    [ -f $s/skills/top/SKILL.md ] || fail "top skill missing"
    [ -f $s/skills/leaf/SKILL.md ] || fail "dependency skill leaf missing"
    grep -qxF "HELLO=yes" $STUB_LOG.env || fail "skill runtime input not on PATH"

    # Settings layered via --settings, user args passed through after it.
    [ "$(sed -n 1p $STUB_LOG.args)" = "--settings" ] || fail "missing --settings"
    grep -q '"model": *"opus"' "$(sed -n 2p $STUB_LOG.args)" || fail "settings file content"
    [ "$(sed -n '3,$p' $STUB_LOG.args)" = "$(printf -- '--resume\nx')" ] || fail "user args not passed through"
    [ ! -e $s/settings.json ] || fail "launcher must not own settings.json"

    ${full.launchers.codex-work}/bin/codex-work exec hi
    c=$HOME/.local/agent-environments/codex-work
    grep -qxF "CODEX_HOME=$c" $STUB_LOG.env || fail "CODEX_HOME not set"
    [ "$(cat $c/AGENTS.md)" = "$(printf 'FIRST\n\nSECOND')" ] || fail "AGENTS.md content"
    [ -f $c/skills/leaf/SKILL.md ] || fail "codex skills missing"
    grep -q 'model = "gpt-5"' $c/agent-environments.config.toml || fail "codex profile content"
    [ "$(cat $STUB_LOG.args)" = "$(printf -- '--profile\nagent-environments\nexec\nhi')" ] || fail "codex args: $(cat $STUB_LOG.args)"
    [ ! -e $c/config.toml ] || fail "launcher must not own config.toml"
  '';

  # Separate launchers from one environment must not share a login.
  separate-logins = runTest "separate-logins" ''
    ${full.launchers.claude-work}/bin/claude-work
    a=$(grep CLAUDE_CONFIG_DIR $STUB_LOG.env)
    ${full.launchers.claude-pers}/bin/claude-pers
    b=$(grep CLAUDE_CONFIG_DIR $STUB_LOG.env)
    [ "$a" != "$b" ] || fail "claude-work and claude-pers share $a"
    [ "$b" = "CLAUDE_CONFIG_DIR=$HOME/.local/agent-environments/claude-pers" ] || fail "$b"
  '';

  # Without settings, no flags are injected and no files are managed.
  no-settings = runTest "no-settings" ''
    ${bare.launchers.claude-work}/bin/claude-work
    [ ! -s $STUB_LOG.args ] || fail "unexpected args: $(cat $STUB_LOG.args)"
    ${bare.launchers.codex-bare}/bin/codex-bare
    [ ! -s $STUB_LOG.args ] || fail "unexpected codex args: $(cat $STUB_LOG.args)"
    c=$HOME/.local/agent-environments/codex-bare
    [ ! -e $c/AGENTS.md ] && [ ! -e $c/agent-environments.config.toml ] || fail "files managed without config"
  '';

  # Switching to an environment with fewer skills removes only what we linked.
  cleanup-keeps-user-files = runTest "cleanup-keeps-user-files" ''
    s=$HOME/.local/agent-environments/claude-work
    mkdir -p $s/skills/.system $s/skills/mine
    ln -s /nix/store $s/skills/user-link
    ${full.launchers.claude-work}/bin/claude-work
    [ -L $s/skills/top ] || fail "top not linked"
    ${bare.launchers.claude-work}/bin/claude-work
    [ ! -e $s/skills/top ] && [ ! -e $s/skills/leaf ] || fail "stale skills kept"
    [ ! -e $s/CLAUDE.md ] || fail "stale CLAUDE.md kept"
    [ -d $s/skills/.system ] && [ -d $s/skills/mine ] && [ -L $s/skills/user-link ] || fail "user files removed"
    # And back again: the manifest must let us re-create the links.
    ${full.launchers.claude-work-bare}/bin/claude-work-bare
    [ -L $s/skills/top ] || fail "relink failed"
  '';

  # User-owned files at managed paths are never replaced.
  refuses-unmanaged = runTest "refuses-unmanaged" ''
    s=$HOME/.local/agent-environments/claude-work
    mkdir -p $s/skills
    echo mine >$s/CLAUDE.md
    if ${full.launchers.claude-work}/bin/claude-work 2>err; then fail "launcher replaced user CLAUDE.md"; fi
    grep -q "refusing to replace '$s/CLAUDE.md'" err || fail "unclear error: $(cat err)"
    [ "$(cat $s/CLAUDE.md)" = mine ] || fail "user CLAUDE.md modified"
    [ ! -e $STUB_LOG.args ] || fail "agent started despite conflict"

    rm $s/CLAUDE.md
    ln -s /nix/store $s/skills/top
    if ${full.launchers.claude-work}/bin/claude-work 2>err; then fail "launcher replaced user symlink"; fi
    [ "$(readlink $s/skills/top)" = /nix/store ] || fail "user symlink modified"

    # A path we managed before, but which the user replaced with real data, is theirs now.
    rm $s/skills/top
    ${full.launchers.claude-work}/bin/claude-work
    rm $s/CLAUDE.md && echo edited >$s/CLAUDE.md
    if ${full.launchers.claude-work}/bin/claude-work 2>err; then fail "launcher replaced user file at managed path"; fi
    [ "$(cat $s/CLAUDE.md)" = edited ] || fail "user file modified"
  '';

  # The launcher's own bookkeeping paths must not write through symlinks into
  # files or directories outside the state dir.
  no-writes-through-symlinks = runTest "no-writes-through-symlinks" ''
    s=$HOME/.local/agent-environments/claude-work
    mkdir -p $s/skills
    echo precious >$TMPDIR/precious
    ln -s $TMPDIR/precious $s/.agent-environments.lock
    ${full.launchers.claude-work}/bin/claude-work
    [ "$(cat $TMPDIR/precious)" = precious ] || fail "lock open truncated a symlinked file"

    p=$HOME/.local/agent-environments/claude-pers
    mkdir -p $p $TMPDIR/shared-skills
    ln -s $TMPDIR/shared-skills $p/skills
    if ${full.launchers.claude-pers}/bin/claude-pers 2>err; then fail "managed links through a symlinked skills dir"; fi
    [ -z "$(ls -A $TMPDIR/shared-skills)" ] || fail "wrote into the shared skills dir"
  '';

  # stateDir is data, not shell code: only a leading ~/ is expanded.
  literal-state-dir =
    let
      env = mkEnvironment {
        inherit pkgs;
        modules = [
          stubs
          {
            claude.enable = true;
            launchers.claude-odd = {
              agent = "claude";
              stateDir = ''~/odd "dir"/$(touch injected)'';
            };
          }
        ];
      };
    in
    runTest "literal-state-dir" ''
      cd $TMPDIR
      ${env.launchers.claude-odd}/bin/claude-odd
      [ ! -e injected ] || fail "stateDir was executed as shell code"
      grep -qxF "CLAUDE_CONFIG_DIR=$HOME/odd \"dir\"/\$(touch injected)" $STUB_LOG.env || fail "$(cat $STUB_LOG.env)"
    '';

  skill-without-skill-md = pkgs.testers.testBuildFailure (mkSkill {
    inherit pkgs;
    name = "broken";
    src = pkgs.writeTextDir "README.md" "";
  });

  # Configuration mistakes are caught at evaluation time.
  eval-errors =
    assert fails [
      stubs
      {
        claude.enable = true;
        launchers.x = {
          agent = "claude";
          stateDir = "relative/dir";
        };
      }
    ];
    assert fails [
      stubs
      { launchers.x.agent = "claude"; }
    ];
    assert fails [
      stubs
      {
        claude.enable = true;
        launchers.x = {
          agent = "claude";
          id = "../escape";
        };
      }
    ];
    assert fails [
      stubs
      {
        claude.enable = true;
        launchers."bad name".agent = "claude";
      }
    ];
    assert fails [
      stubs
      {
        claude.enable = true;
        skills.packages = [
          top
          otherLeaf
        ];
        launchers.x.agent = "claude";
      }
    ];
    assert
      !(builtins.tryEval
        (mkSkill {
          inherit pkgs;
          name = "a/b";
          src = skillSrc "x";
        }).drvPath
      ).success;
    pkgs.runCommand "agent-environments-eval-errors" { } "touch $out";
}

M flake.nix => flake.nix +2 -0
@@ 46,6 46,8 @@
          }).package;
      });

      checks = forAllSystems (pkgs: import ./checks { inherit pkgs self; });

      formatter = forAllSystems (pkgs: pkgs.nixfmt);
    };
}