diff options
Diffstat (limited to 'plugins/security-guidance/tests')
56 files changed, 11471 insertions, 14 deletions
diff --git a/plugins/security-guidance/tests/conftest.py b/plugins/security-guidance/tests/conftest.py index 2cf4dd1c..b0df08de 100644 --- a/plugins/security-guidance/tests/conftest.py +++ b/plugins/security-guidance/tests/conftest.py @@ -1,10 +1,14 @@ import http.server +import io import json import os +import struct import subprocess import sys +import sysconfig import threading import time +import zipfile from pathlib import Path import pytest @@ -64,6 +68,15 @@ VULN_PY = ( ) +@pytest.fixture(autouse=True) +def _no_net_err_from_an_earlier_test(): + """A hook is one process per event; tests that call it in-process share + the module's record of the last failed call.""" + import _base + _base._USAGE["net_err"] = 0 + yield + + @pytest.fixture def workspace(tmp_path): ws = tmp_path / "ws" @@ -76,8 +89,10 @@ def workspace(tmp_path): class _Stub(http.server.BaseHTTPRequestHandler): def do_POST(self): n = int(self.headers.get("Content-Length") or 0) - if n: - self.rfile.read(n) + body = self.rfile.read(n).decode("utf-8", "replace") if n else "" + # Only the stub_api fixture keeps the request bodies. + if hasattr(self.server, "bodies"): + self.server.bodies.append(body) self.server.calls.append(self.path) if self.server.delay: time.sleep(self.server.delay) @@ -108,6 +123,7 @@ class _Stub(http.server.BaseHTTPRequestHandler): def stub_api(): srv = http.server.HTTPServer(("127.0.0.1", 0), _Stub) srv.calls = [] + srv.bodies = [] srv.status = 200 srv.delay = 0 srv.vulns = [] @@ -128,7 +144,8 @@ def hook_env(tmp_path, stub_api): "CLAUDE_PROJECT_DIR", "HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy", "ALL_PROXY", "all_proxy", "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX", - "CLAUDE_CODE_USE_FOUNDRY")} + "CLAUDE_CODE_USE_FOUNDRY", + "PLUGIN_ROOT", "SECURITY_GUIDANCE_PORTABLE_OUTPUT")} env.update(GIT_ENV) env.update({ "SECURITY_WARNINGS_STATE_DIR": str(state), @@ -136,6 +153,9 @@ def hook_env(tmp_path, stub_api): "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{stub_api.server_port}", "NO_PROXY": "*", "no_proxy": "*", "SG_AGENTIC_COMMIT_REVIEW": "0", + # Points the plugin at a claude that does not exist, so no hook test + # runs this machine's own claude. + "SG_AGENTIC_CLI_PATH": str(tmp_path / "no-claude"), "SECURITY_GUIDANCE_COMMIT_REVIEW": "on", "SG_PUSH_SWEEP": "on", "PYTHONDONTWRITEBYTECODE": "1", @@ -212,3 +232,114 @@ def ups_payload(cwd, session_id="s1"): "prompt": "hi", "cwd": str(cwd), } + + +# ── a local Anthropic-compatible endpoint the hook can be pointed at ────── + +class _Gateway(http.server.BaseHTTPRequestHandler): + """An Anthropic-compatible endpoint. With `required` set it answers 401 + unless that header arrives, like a LiteLLM proxy keyed on a custom header.""" + + def _handle(self): + n = int(self.headers.get("Content-Length") or 0) + if n: + self.rfile.read(n) + srv = self.server + srv.requests.append((self.command, self.path, {k.lower(): v for k, v in self.headers.items()})) + name, value = srv.required or (None, None) + if name and self.headers.get(name) != value: + return self._send(401, {"type": "error", "error": { + "type": "authentication_error", "message": "Invalid proxy server token"}}) + if srv.status != 200: + return self._send(srv.status, {"type": "error", "error": {"type": "api_error"}}) + if self.command == "GET": + return self._send(200, {"data": [{"id": "claude-opus-5-5"}], "has_more": False}) + text = (srv.reply if srv.reply is not None else json.dumps( + {"hasVulnerabilities": bool(srv.vulns), "vulnerabilities": srv.vulns})) + self._send(200, { + "id": "msg_stub", "type": "message", "role": "assistant", + "model": "stub", "stop_reason": "end_turn", + "content": [{"type": "text", "text": text}], + "usage": {"input_tokens": 1, "output_tokens": 1}, + }) + + do_GET = do_POST = do_HEAD = _handle + + def _send(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *a): + pass + + +@pytest.fixture +def gateway(): + srv = http.server.HTTPServer(("127.0.0.1", 0), _Gateway) + srv.requests, srv.required, srv.status, srv.reply, srv.vulns = [], None, 200, None, [] + threading.Thread(target=srv.serve_forever, daemon=True).start() + try: + yield srv + finally: + srv.shutdown() + + +@pytest.fixture +def env(hook_env, gateway): + e = dict(hook_env) + e["ANTHROPIC_BASE_URL"] = f"http://127.0.0.1:{gateway.server_port}" + e.pop("SECURITY_REVIEW_MODEL", None) + return e + + +def _windows_exe(script): + """`script` as the bytes of a Windows .exe: Windows starts neither a + `#!` file nor, without a shell, a .cmd with multi-line arguments. Built + the way pip builds a console script: its launcher stub, then the + interpreter to run, then a zip holding the script as __main__.py.""" + distlib = pytest.importorskip("pip._vendor.distlib") + stub = "t%s%s.exe" % ("64" if struct.calcsize("P") == 8 else "32", + "-arm" if sysconfig.get_platform() == "win-arm64" else "") + stub = os.path.join(os.path.dirname(distlib.__file__), stub) + if not os.path.isfile(stub): + pytest.skip(f"pip has no script launcher {os.path.basename(stub)}") + with open(stub, "rb") as f: + launcher = f.read() + python = f'"{sys.executable}"' if " " in sys.executable else sys.executable + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("__main__.py", script) + return launcher + f"#!{python}\n".encode("utf-8") + archive.getvalue() + + +def write_cli(tmp_path, name, script): + """A stand-in for `claude` in `tmp_path` that runs the Python `script`; + returns its path.""" + if os.name == "nt": + cli = tmp_path / f"{name}.exe" + cli.write_bytes(_windows_exe(script)) + else: + cli = tmp_path / name + cli.write_text(f"#!{sys.executable}\n{script}") + cli.chmod(0o755) + return cli + + +@pytest.fixture +def failing_inner_cli(tmp_path): + """An inner CLI that prints a warning on stderr and exits 1; the review + then falls back to a single-shot call.""" + return str(write_cli(tmp_path, "claude", ( + "import sys\n" + "sys.stderr.write('claude.ai connectors are disabled\\n')\n" + "sys.exit(1)\n"))) + + +@pytest.fixture +def agentic_env(env, failing_inner_cli): + pytest.importorskip("claude_agent_sdk") + return {**env, "SG_AGENTIC_COMMIT_REVIEW": "1", "SG_AGENTIC_CLI_PATH": failing_inner_cli} diff --git a/plugins/security-guidance/tests/test_add_failure_metric.py b/plugins/security-guidance/tests/test_add_failure_metric.py new file mode 100644 index 00000000..228b68b7 --- /dev/null +++ b/plugins/security-guidance/tests/test_add_failure_metric.py @@ -0,0 +1,473 @@ +"""`git add --intent-to-add` puts new files in the Stop review's diff. A user +who cannot write the object store gets a retry with a private object directory; +a failure that stays leaves the new files for a later review, unless the turn's +tracked files were reviewed, and tells Claude which files were left out. The +Stop row says what happened.""" +import json +import os +import shutil +import subprocess +import tempfile +from types import SimpleNamespace + +import pytest + +from conftest import ( + GIT_ENV, STUB_VULN, VULN_PY, edit_payload, git, make_repo, metrics_of, run_hook, + stop_payload, ups_payload, write_cli, +) + +import gitutil +import security_reminder_hook as hook + + +def _cannot_write_read_only_dirs(): + with tempfile.TemporaryDirectory() as d: + os.chmod(d, 0o555) + try: + return not os.access(d, os.W_OK) + finally: + os.chmod(d, 0o755) + + +@pytest.fixture +def read_only_objects(): + made = [] + + def lock(repo): + made.append(repo / ".git" / "objects") + subprocess.run(["chmod", "-R", "a-w", str(made[-1])], check=True) + + yield lock + for objects in made: + subprocess.run(["chmod", "-R", "u+w", str(objects)], check=True) + + +@pytest.mark.skipif(os.name == "nt" or not _cannot_write_read_only_dirs(), + reason="needs a user that file permissions apply to") +class TestObjectStoreNotWritable: + def test_turn_that_only_adds_a_file(self, tmp_path, hook_env, stub_api, read_only_objects): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + read_only_objects(repo) + before = sorted(p for p in (repo / ".git" / "objects").rglob("*")) + run_hook(ups_payload(repo), hook_env) + (repo / "handlers.py").write_text(VULN_PY) + m = metrics_of(run_hook(stop_payload(repo), hook_env)[1]) + assert m["files_reviewed"] == 1, m + assert m["add_fail"] == 400001 + assert stub_api.calls + assert sorted(p for p in (repo / ".git" / "objects").rglob("*")) == before + + def test_tracked_edit_and_new_files(self, tmp_path, hook_env, stub_api, read_only_objects): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + read_only_objects(repo) + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + (repo / "a.py").write_text("print('a')\n") + (repo / "b.py").write_text("print('b')\n") + m = metrics_of(run_hook(stop_payload(repo), hook_env)[1]) + assert not m.get("skipped"), m + assert m["files_reviewed"] == 3 + assert m["add_fail"] == 400002 + + def test_linked_worktree_and_packed_repo(self, tmp_path, hook_env, stub_api, read_only_objects): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + git(repo, "repack", "-adq") + wt = tmp_path / "wt" + git(repo, "worktree", "add", "-q", "-b", "side", str(wt)) + read_only_objects(repo) + run_hook(ups_payload(wt), hook_env) + (wt / "handlers.py").write_text(VULN_PY) + m = metrics_of(run_hook(stop_payload(wt), hook_env)[1]) + assert m["files_reviewed"] == 1, m + assert m["add_fail"] == 400001 + + def test_private_directory_is_gone_afterwards(self, tmp_path, monkeypatch, read_only_objects): + d = tmp_path / "tmproot" + d.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(d)) + for k, v in GIT_ENV.items(): + monkeypatch.setenv(k, v) + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + read_only_objects(repo) + (repo / "new.py").write_text("print('new')\n") + out = gitutil.get_git_diff(str(repo), "HEAD", paths=[str(repo / "new.py")], + untracked_paths=["new.py"]) + gitutil._add_failed["code"] = 0 + assert "new.py" in out + assert not [p for p in d.rglob("*") if gitutil._TEMP_INDEX_PREFIX in p.name] + + +class TestPathspecSettings: + @pytest.mark.parametrize("name", ["GIT_ICASE_PATHSPECS", "GIT_GLOB_PATHSPECS"]) + def test_new_file_is_reviewed(self, tmp_path, hook_env, stub_api, name): + # git refuses --literal-pathspecs together with these two. + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + env = {**hook_env, name: "1"} + run_hook(ups_payload(repo), env) + (repo / "handlers.py").write_text(VULN_PY) + m = metrics_of(run_hook(stop_payload(repo), env)[1]) + assert m["files_reviewed"] == 1, m + assert "add_fail" not in m + + +@pytest.mark.skipif(os.name == "nt", reason="the stand-in git is a shell script") +def _env_refusing_add(tmp_path, hook_env, unless_set=None): + """`hook_env` with a git that fails `add -N`, unless `unless_set` is in its + environment.""" + d = tmp_path / "shim" + d.mkdir() + write_cli(d, "git", ( + "import os, subprocess, sys\n" + f"if '--intent-to-add' in sys.argv and not os.environ.get({unless_set!r}):\n" + " sys.stderr.write('fatal: refused\\n')\n" + " sys.exit(128)\n" + f"sys.exit(subprocess.call([{shutil.which('git')!r}, *sys.argv[1:]]))\n")) + return {**hook_env, "PATH": f"{d}{os.pathsep}{hook_env['PATH']}"} + + +class TestAddThatKeepsFailing: + def test_new_file_is_reviewed_once_git_works(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + bad = _env_refusing_add(tmp_path, hook_env) + run_hook(ups_payload(repo), bad) + (repo / "handlers.py").write_text(VULN_PY) + run_hook(edit_payload(repo, str(repo / "handlers.py")), bad) + m = metrics_of(run_hook(stop_payload(repo), bad)[1]) + assert m["skip_reason"] == 6, m + assert m["add_fail"] == 200001 + assert not stub_api.calls + + run_hook(ups_payload(repo), hook_env) + m = metrics_of(run_hook(stop_payload(repo), hook_env)[1]) + assert m["files_reviewed"] == 1, m + assert stub_api.calls + + def test_file_no_edit_tool_wrote(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + bad = _env_refusing_add(tmp_path, hook_env) + run_hook(ups_payload(repo), bad) + (repo / "handlers.py").write_text(VULN_PY) + m = metrics_of(run_hook(stop_payload(repo), bad)[1]) + assert m["skip_reason"] == 6, m + + run_hook(ups_payload(repo), hook_env) + m = metrics_of(run_hook(stop_payload(repo), hook_env)[1]) + assert m["files_reviewed"] == 1, m + + def test_tracked_edit_is_still_reviewed(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + bad = _env_refusing_add(tmp_path, hook_env) + run_hook(ups_payload(repo), bad) + (repo / "app.py").write_text(VULN_PY) + (repo / "a.py").write_text("print('a')\n") + m = metrics_of(run_hook(stop_payload(repo), bad)[1]) + assert m["files_reviewed"] == 1, m + assert m["add_fail"] == 200001 + + +class TestNoticeForLeftOutFiles: + """A failure that stays is said once per set of files: Claude is woken with + their names, and the Stop that wake ends in stays quiet.""" + + @pytest.fixture + def bad(self, tmp_path, hook_env): + return _env_refusing_add(tmp_path, hook_env) + + @pytest.fixture + def repo(self, tmp_path): + return make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + + def test_turn_that_only_adds_a_file(self, repo, bad, stub_api): + run_hook(ups_payload(repo), bad) + (repo / "handlers.py").write_text(VULN_PY) + code, out, err = run_hook(stop_payload(repo), bad) + assert code == 2 + assert '"handlers.py"' in err + assert "Tell the user" in err + line = json.loads(next(x for x in out.splitlines() if x.startswith("{"))) + assert '"handlers.py"' in line["rewakeSummary"] + assert line["metrics"]["skip_reason"] == 6 + assert line["metrics"]["add_fail_told"] == 1 + assert line["decision"] == "block" + + # The Stop after the wake: same files, same failure, nothing more. + code, out, err = run_hook(stop_payload(repo), bad) + assert code == 0, err + assert err == "" + m = metrics_of(out) + assert m["skip_reason"] == 6 + assert "add_fail_told" not in m + + def test_new_set_of_files_gets_its_own_notice(self, repo, bad, stub_api): + run_hook(ups_payload(repo), bad) + (repo / "a.py").write_text("print('a')\n") + assert run_hook(stop_payload(repo), bad)[0] == 2 + run_hook(ups_payload(repo), bad) + (repo / "b.py").write_text("print('b')\n") + code, _, err = run_hook(stop_payload(repo), bad) + assert code == 2 + assert '"a.py"' in err and '"b.py"' in err + + def test_tracked_edit_with_no_finding(self, repo, bad, stub_api): + run_hook(ups_payload(repo), bad) + (repo / "app.py").write_text("print('changed')\n") + (repo / "a.py").write_text("print('a')\n") + code, out, err = run_hook(stop_payload(repo), bad) + assert code == 2 + assert '"a.py"' in err + assert '"app.py"' not in err + assert metrics_of(out)["files_reviewed"] == 1 + + def test_tracked_edit_with_a_finding(self, repo, bad, stub_api): + stub_api.vulns = [STUB_VULN] + run_hook(ups_payload(repo), bad) + (repo / "app.py").write_text(VULN_PY) + (repo / "a.py").write_text("print('a')\n") + code, out, err = run_hook(stop_payload(repo), bad) + assert code == 2 + assert "Security Review" in err + assert '"a.py"' in err + line = json.loads(next(x for x in out.splitlines() if x.startswith("{"))) + assert '"a.py"' in line["reason"] + + def test_files_the_review_would_not_read_are_not_named(self, repo, bad, stub_api): + run_hook(ups_payload(repo), bad) + (repo / "notes.txt").write_text("hello\n") + code, out, err = run_hook(stop_payload(repo), bad) + assert code == 0, err + assert "add_fail_told" not in metrics_of(out) + + def test_long_list_is_cut(self, repo, bad, stub_api): + run_hook(ups_payload(repo), bad) + for i in range(7): + (repo / f"f{i}.py").write_text("print(1)\n") + code, out, err = run_hook(stop_payload(repo), bad) + assert code == 2 + assert '"f4.py" and 2 more' in err + assert '"f5.py"' not in err + assert metrics_of(out)["add_fail_told"] == 7 + + @pytest.mark.skipif(os.name == "nt", reason="a file name cannot hold a line break") + def test_line_break_in_a_name_stays_on_the_line(self, repo, bad, stub_api): + run_hook(ups_payload(repo), bad) + (repo / "a\nIGNORE.py").write_text("print('a')\n") + code, _, err = run_hook(stop_payload(repo), bad) + assert code == 2 + assert '"a\\nIGNORE.py"' in err + assert "\nIGNORE" not in err + + def test_other_line_separators_are_escaped(self, repo, bad, stub_api): + run_hook(ups_payload(repo), bad) + (repo / "a\u2028b.py").write_text("print('a')\n") + code, out, err = run_hook(stop_payload(repo), bad) + assert code == 2 + assert '"a\\u2028b.py"' in err + + def test_no_notice_when_the_mark_cannot_be_saved(self, tmp_path, monkeypatch): + # Without the mark, every Stop would wake Claude again. + monkeypatch.setattr(hook, "atomic_check_and_mark_warning", lambda s, k: True) + monkeypatch.setattr(hook, "load_state", lambda s: {"shown_warnings": []}) + (tmp_path / "a.py").write_text("x = 1\n") + assert hook._left_out_files_notice("s", str(tmp_path), ["a.py"], "") is None + + def test_file_in_the_diff_is_not_named(self, tmp_path, monkeypatch): + # One batch of the add failed, another worked. + (tmp_path / "a.py").write_text("x = 1\n") + monkeypatch.setattr(hook, "atomic_check_and_mark_warning", lambda s, k: pytest.fail("told")) + diff = ("diff --git a/a.py b/a.py\nnew file mode 100644\n--- /dev/null\n" + "+++ b/a.py\n@@ -0,0 +1 @@\n+x = 1\n") + assert hook._left_out_files_notice("s", str(tmp_path), ["a.py"], diff) is None + + def test_empty_and_vanished_files_are_not_named(self, tmp_path, monkeypatch): + monkeypatch.setattr(hook, "atomic_check_and_mark_warning", lambda s, k: pytest.fail("told")) + (tmp_path / "__init__.py").write_text("") + assert hook._left_out_files_notice("s", str(tmp_path), ["__init__.py", "gone.py"], "") is None + + def test_tracked_edit_gets_reviewed_while_new_files_stay_out(self, repo, bad, stub_api): + # A failure on every turn must not stop later reviews of tracked files. + run_hook(ups_payload(repo), bad) + (repo / "a.py").write_text("print('a')\n") + assert run_hook(stop_payload(repo), bad)[0] == 2 + run_hook(ups_payload(repo), bad) + (repo / "app.py").write_text(VULN_PY) + code, out, _ = run_hook(stop_payload(repo), bad) + assert code == 0 + assert metrics_of(out)["files_reviewed"] == 1 + assert stub_api.calls + + def test_retry_that_works_says_nothing(self, repo, tmp_path, hook_env, stub_api): + env = _env_refusing_add(tmp_path, hook_env, unless_set="GIT_OBJECT_DIRECTORY") + run_hook(ups_payload(repo), env) + (repo / "handlers.py").write_text(VULN_PY) + code, out, err = run_hook(stop_payload(repo), env) + assert code == 0, err + m = metrics_of(out) + assert m["add_fail"] == 400001 + assert "add_fail_told" not in m + + +class TestStopRow: + def test_add_that_works_reports_nothing(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + run_hook(ups_payload(repo), hook_env) + (repo / "handlers.py").write_text(VULN_PY) + m = metrics_of(run_hook(stop_payload(repo), hook_env)[1]) + assert m["files_reviewed"] == 1 + assert "add_fail" not in m + + +class TestGetGitDiff: + @pytest.fixture(autouse=True) + def _env(self, tmp_path, monkeypatch): + d = tmp_path / "tmproot" + d.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(d)) + for k, v in GIT_ENV.items(): + monkeypatch.setenv(k, v) + yield + gitutil._add_failed["code"] = 0 + + @pytest.fixture + def repo(self, tmp_path): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + (repo / "new.py").write_text("print('new')\n") + return repo + + def _adds_fail_with(self, monkeypatch, outcome): + real = gitutil._run_git_on_review_paths + + def run(args, paths, timeout, **kwargs): + if args[0] != "add": + return real(args, paths, timeout, **kwargs) + if isinstance(outcome, Exception): + raise outcome + return outcome + + monkeypatch.setattr(gitutil, "_run_git_on_review_paths", run) + + def test_any_failed_batch_counts(self, repo, monkeypatch): + self._adds_fail_with(monkeypatch, [ + SimpleNamespace(returncode=0, stderr=b""), + SimpleNamespace(returncode=128, stderr=b"fatal: nope"), + ]) + gitutil.get_git_diff(str(repo), "HEAD", paths=[str(repo / "new.py")], + untracked_paths=["new.py"]) + assert gitutil._add_failed["code"] == 200001 + + def test_timeout_counts_and_keeps_the_tracked_diff(self, repo, monkeypatch): + self._adds_fail_with(monkeypatch, subprocess.TimeoutExpired("git", 10)) + (repo / "app.py").write_text("print('edited')\n") + out = gitutil.get_git_diff(str(repo), "HEAD", + paths=[str(repo / "new.py"), str(repo / "app.py")], + untracked_paths=["new.py"]) + assert "app.py" in out + assert "new.py" not in out + assert gitutil._add_failed == {"code": 300001, "left_out": True} + + def test_a_fallback_diff_that_works_clears_left_out(self, repo, monkeypatch): + self._adds_fail_with(monkeypatch, subprocess.TimeoutExpired("git", 10)) + gitutil.get_git_diff(str(repo), "HEAD", paths=[str(repo / "new.py")], + untracked_paths=["new.py"]) + monkeypatch.undo() + assert "new.py" in gitutil.get_git_diff( + str(repo), "HEAD", paths=[str(repo / "new.py")], untracked_paths=["new.py"]) + assert gitutil._add_failed == {"code": 300001, "left_out": False} + + def test_object_store_refusal_on_both_tries(self, repo, monkeypatch): + self._adds_fail_with(monkeypatch, [SimpleNamespace( + returncode=128, + stderr=b"fatal: cannot create an empty blob in the object database")]) + gitutil.get_git_diff(str(repo), "HEAD", paths=[str(repo / "new.py")], + untracked_paths=["new.py"]) + assert gitutil._add_failed["code"] == 100001 + + def test_retry_reads_the_repositorys_objects_and_writes_elsewhere(self, repo, monkeypatch): + seen = [] + real = gitutil._run_git_on_review_paths + + def run(args, paths, timeout, **kwargs): + if args[0] != "add": + return real(args, paths, timeout, **kwargs) + seen.append(kwargs["env"]) + return real(args, paths, timeout, **kwargs) if len(seen) > 1 else [ + SimpleNamespace(returncode=128, stderr=b"nope")] + + monkeypatch.setattr(gitutil, "_run_git_on_review_paths", run) + out = gitutil.get_git_diff(str(repo), "HEAD", paths=[str(repo / "new.py")], + untracked_paths=["new.py"]) + assert "new.py" in out + assert "GIT_OBJECT_DIRECTORY" not in seen[0] + assert seen[1]["GIT_OBJECT_DIRECTORY"].endswith(".objects") + assert os.path.samefile(seen[1]["GIT_ALTERNATE_OBJECT_DIRECTORIES"], + repo / ".git" / "objects") + assert gitutil._add_failed["code"] == 400001 + + def test_retry_timeout_keeps_the_tracked_diff(self, repo, monkeypatch): + (repo / "app.py").write_text("print('changed')\n") + real = gitutil._run_git_on_review_paths + adds = [] + + def run(args, paths, timeout, **kwargs): + if args[0] != "add": + return real(args, paths, timeout, **kwargs) + adds.append(args) + if len(adds) > 1: + raise subprocess.TimeoutExpired("git", 10) + return [SimpleNamespace(returncode=128, stderr=b"fatal: nope")] + + monkeypatch.setattr(gitutil, "_run_git_on_review_paths", run) + out = gitutil.get_git_diff(str(repo), "HEAD", untracked_paths=["new.py"]) + assert "app.py" in out + assert gitutil._add_failed["code"] == 200001 + + def test_a_later_add_that_works_keeps_the_failure(self, repo, monkeypatch): + gitutil._add_failed["code"] = 300001 + gitutil.get_git_diff(str(repo), "HEAD", paths=[str(repo / "new.py")], + untracked_paths=["new.py"]) + assert gitutil._add_failed["code"] == 300001 + + +class TestLeftovers: + def _private_dir(self, parent, name): + d = parent / name + (d / "e6").mkdir(parents=True) + blob = d / "e6" / "9de29" + blob.write_bytes(b"x") + blob.chmod(0o444) + return d + + def test_removed_with_its_copy(self, tmp_path): + copy = tmp_path / "security_hook_idx_pid1_abcdefgh" + copy.write_bytes(b"i") + self._private_dir(tmp_path, copy.name + ".objects") + gitutil._remove_copy(str(copy)) + assert not list(tmp_path.iterdir()) + + @pytest.mark.skipif(os.name == "nt", reason="no pid check on Windows") + def test_killed_hooks_directory_is_swept(self, tmp_path): + p = subprocess.Popen(["git", "--version"], stdout=subprocess.DEVNULL) + p.wait() + self._private_dir(tmp_path, f"security_hook_idx_pid{p.pid}_abcdefgh.objects") + gitutil._sweep_stale_indexes(str(tmp_path), None) + assert not list(tmp_path.iterdir()) + + def test_live_hooks_directory_stays(self, tmp_path): + self._private_dir(tmp_path, f"security_hook_idx_pid{os.getpid()}_abcdefgh.objects") + gitutil._sweep_stale_indexes(str(tmp_path), None) + assert list(tmp_path.iterdir()) + + def test_other_directories_stay(self, tmp_path): + (tmp_path / "security_hook_idx_pid1_abcdefgh.other").mkdir() + gitutil._sweep_stale_indexes(str(tmp_path), None) + assert list(tmp_path.iterdir()) + + @pytest.mark.skipif(os.name == "nt", reason="needs symlinks") + def test_link_at_the_private_directory_is_not_followed(self, tmp_path): + victim = tmp_path / "victim" + victim.mkdir() + (victim / "keep").write_text("x") + copy = tmp_path / "security_hook_idx_pid1_abcdefgh" + (tmp_path / (copy.name + ".objects")).symlink_to(victim) + gitutil._remove_copy(str(copy)) + assert (victim / "keep").exists() diff --git a/plugins/security-guidance/tests/test_agentic_auth_stop.py b/plugins/security-guidance/tests/test_agentic_auth_stop.py new file mode 100644 index 00000000..a2108cfc --- /dev/null +++ b/plugins/security-guidance/tests/test_agentic_auth_stop.py @@ -0,0 +1,148 @@ +"""The agentic reviewer stops as soon as its credentials are refused, instead +of waiting for the inner CLI to give up, and leaves every other run alone.""" +import os +import subprocess +import sys +import types + +import pytest + +import llm +from conftest import write_cli + + +class _Assistant: + def __init__(self, error=None): + self.content, self.error = [], error + + +class _System: + subtype = "api_retry" + + def __init__(self, status): + self.data = {"error_status": status} + + +@pytest.fixture +def sdk(monkeypatch, tmp_path): + """Replays `script.messages`, then a clean result, and counts what the + review read of them.""" + script = types.SimpleNamespace(messages=[], read=0) + + class ClaudeAgentOptions: + def __init__(self, **kwargs): + self.kwargs = kwargs + + class ResultMessage: + subtype, usage, total_cost_usd = "success", {}, 0.0 + structured_output = {"findings": []} + + async def query(prompt, options): + for m in script.messages: + script.read += 1 + yield m + yield ResultMessage() + + mod = types.ModuleType("claude_agent_sdk") + mod.ClaudeAgentOptions, mod.ResultMessage, mod.query = ClaudeAgentOptions, ResultMessage, query + mod.AssistantMessage = _Assistant + monkeypatch.setitem(sys.modules, "claude_agent_sdk", mod) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", "") + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(tmp_path / "state")) + monkeypatch.delenv("CLAUDE_CODE_REMOTE", raising=False) + return script + + +def _review(repo): + return llm.agentic_review(str(repo), [("a.py", "+x = 1\n")], ["a.py"])[2] + + +class TestStopsOnRefusedCredentials: + def test_first_auth_failure_ends_the_review(self, sdk, tmp_path): + sdk.messages = [_Assistant(error="authentication_failed") for _ in range(50)] + m = _review(tmp_path) + assert sdk.read == 1 + assert m["agentic_fallback"] == "investigate:_ReviewerAuthFailed" + + def test_repeated_401_retries_end_the_review(self, sdk, tmp_path): + sdk.messages = [_System(401) for _ in range(10)] + m = _review(tmp_path) + assert sdk.read == llm._AGENTIC_MAX_401_RETRIES + assert m["agentic_fallback"] == "investigate:_ReviewerAuthFailed" + + +class TestLeavesOtherRunsAlone: + def test_a_remote_session_rides_out_401_retries(self, sdk, tmp_path, monkeypatch): + monkeypatch.setenv("CLAUDE_CODE_REMOTE", "true") + sdk.messages = [_System(401) for _ in range(10)] + assert "agentic_fallback" not in _review(tmp_path) + + def test_a_login_renewed_after_one_401_goes_on(self, sdk, tmp_path): + sdk.messages = [_System(401), _Assistant()] + assert "agentic_fallback" not in _review(tmp_path) + + def test_401s_that_are_not_in_a_row_do_not_add_up(self, sdk, tmp_path): + sdk.messages = [_System(401), _System(401), _Assistant()] * 3 + assert "agentic_fallback" not in _review(tmp_path) + + def test_other_retries_do_not_count(self, sdk, tmp_path): + sdk.messages = [_System(s) for s in (529, 429, None, 500, 529, 529)] + assert "agentic_fallback" not in _review(tmp_path) + + def test_other_assistant_errors_do_not_stop_it(self, sdk, tmp_path): + sdk.messages = [_Assistant(error="rate_limit"), _Assistant(error="server_error")] + assert "agentic_fallback" not in _review(tmp_path) + + def test_a_message_without_the_field_is_read_as_before(self, sdk, tmp_path): + class Old: + content = [] + sys.modules["claude_agent_sdk"].AssistantMessage = Old + sdk.messages = [Old()] + assert "agentic_fallback" not in _review(tmp_path) + + +_REFUSED_CLI = """ +import json, os, sys, time +open(os.environ["SG_TEST_PID_FILE"], "w").write(str(os.getpid())) +def send(m): + sys.stdout.write(json.dumps(m) + chr(10)); sys.stdout.flush() +for line in sys.stdin: + m = json.loads(line) + if m.get("type") == "control_request": + send({"type": "control_response", "response": {"subtype": "success", + "request_id": m["request_id"], "response": {}}}) + elif m.get("type") == "user": + for _ in range(500): + send({"type": "assistant", "error": "authentication_failed", "message": { + "role": "assistant", "model": "m", "content": [{"type": "text", "text": "401"}]}}) + time.sleep(0.02) + time.sleep(60) +""" + +_DRIVER = """ +import json, llm, os, sys, time +start = time.time() +metrics = llm.agentic_review(sys.argv[1], [("a.py", "+x = 1")], ["a.py"])[2] +pid = int(open(os.environ["SG_TEST_PID_FILE"]).read()) +try: + os.kill(pid, 0); alive = True +except OSError: + alive = False +print(json.dumps({"fallback": metrics.get("agentic_fallback"), "alive": alive}), flush=True) +os._exit(0) +""" + + +def test_real_sdk_closes_the_cli_quietly(tmp_path): + """Stops the real SDK's child, and prints no traceback for the hook's + stderr, which reaches the session.""" + pytest.importorskip("claude_agent_sdk") + env = {**os.environ, "SG_TEST_PID_FILE": str(tmp_path / "pid"), + "SG_AGENTIC_CLI_PATH": str(write_cli(tmp_path, "claude", _REFUSED_CLI)), + "SECURITY_WARNINGS_STATE_DIR": str(tmp_path / "state"), + "PYTHONPATH": str(os.path.dirname(llm.__file__))} + r = subprocess.run([sys.executable, "-c", _DRIVER, str(tmp_path)], env=env, + capture_output=True, text=True, timeout=60) + assert "Traceback" not in r.stderr, r.stderr + assert '"fallback": "investigate:_ReviewerAuthFailed"' in r.stdout, (r.stdout, r.stderr) + assert '"alive": false' in r.stdout, r.stdout diff --git a/plugins/security-guidance/tests/test_base_url.py b/plugins/security-guidance/tests/test_base_url.py new file mode 100644 index 00000000..26317549 --- /dev/null +++ b/plugins/security-guidance/tests/test_base_url.py @@ -0,0 +1,87 @@ +"""A blank ANTHROPIC_BASE_URL means the default endpoint, as it does for the +Anthropic SDK and Claude Code, for every call the plugin makes.""" +import io +import json + +import pytest + +import llm +import security_reminder_hook as hook + +DEFAULT = "https://api.anthropic.com" +BLANK = ["", " ", "\t\n"] + + +@pytest.mark.parametrize("value", BLANK) +def test_blank_is_the_default_endpoint(monkeypatch, value): + monkeypatch.setenv("ANTHROPIC_BASE_URL", value) + assert llm._anthropic_base_url() == DEFAULT + + +@pytest.mark.parametrize("value, expected", [ + ("https://gw.example.com/", "https://gw.example.com"), + (" https://gw.example.com/base/ \n", "https://gw.example.com/base"), +]) +def test_a_real_value_is_kept(monkeypatch, value, expected): + monkeypatch.setenv("ANTHROPIC_BASE_URL", value) + assert llm._anthropic_base_url() == expected + + +def test_a_stray_slash_is_not_taken_for_blank(monkeypatch): + # A broken gateway setting must not quietly send code to the default host. + monkeypatch.setenv("ANTHROPIC_BASE_URL", "/") + assert llm._anthropic_base_url() == "" + + +def test_unset_is_the_default_endpoint(monkeypatch): + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + assert llm._anthropic_base_url() == DEFAULT + + +class _Reply(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +@pytest.mark.parametrize("value", BLANK) +def test_requests_go_to_the_default_endpoint(monkeypatch, value): + urls = [] + + def urlopen(request, timeout): + urls.append(request.full_url) + return _Reply(json.dumps({ + "data": [{"id": "claude-opus-4-8"}], + "content": [{"type": "text", "text": "{}"}], + "usage": {}, + }).encode()) + + for var in llm._PROVIDER_ENV_VARS: + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("ANTHROPIC_BASE_URL", value) + monkeypatch.setattr(llm, "ANTHROPIC_API_KEY", "key") + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", True) + monkeypatch.setattr(llm, "_urlopen", urlopen) + assert llm._probe_anthropic() + llm._fetch_newest_opus() + llm._call_claude("prompt", {"type": "object"}, thinking_budget=0, model="claude-x") + assert urls == [f"{DEFAULT}/", f"{DEFAULT}/v1/models?limit=100", f"{DEFAULT}/v1/messages"] + + +@pytest.mark.parametrize("value", BLANK) +def test_blank_is_not_reported_as_a_gateway(monkeypatch, value): + for var in llm._PROVIDER_ENV_VARS: + monkeypatch.delenv(var, raising=False) + monkeypatch.delenv("ANTHROPIC_CUSTOM_HEADERS", raising=False) + monkeypatch.setenv("ANTHROPIC_BASE_URL", value) + assert llm.gateway_cfg() == 0 + + +@pytest.mark.parametrize("value", BLANK) +def test_blank_gets_no_gateway_hint(monkeypatch, value): + for var in llm._PROVIDER_ENV_VARS: + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("ANTHROPIC_BASE_URL", value) + assert "ANTHROPIC_CUSTOM_HEADERS" not in hook._review_failure_reason(401)[2] diff --git a/plugins/security-guidance/tests/test_baseline_lookup.py b/plugins/security-guidance/tests/test_baseline_lookup.py new file mode 100644 index 00000000..d6d2e34d --- /dev/null +++ b/plugins/security-guidance/tests/test_baseline_lookup.py @@ -0,0 +1,145 @@ +import os + +import pytest + +from conftest import git, make_repo, run_hook, ups_payload + +import diffstate + +OLD = "import os\nos.system(cmd)\n" +FILE = "packages/app/main.py" + + +def write_payload(cwd, file_path, content, session_id="s1"): + return { + "session_id": session_id, + "hook_event_name": "PostToolUse", + "tool_name": "Write", + "tool_input": {"file_path": str(file_path), "content": content}, + "tool_response": {}, + "cwd": str(cwd), + } + + +@pytest.fixture +def repo(tmp_path): + repo = make_repo(tmp_path / "ws" / "mono") + # Bytes, so the committed text is OLD on every platform: write_text turns + # "\n" into "\r\n" on Windows. + for name in (FILE, "top.py"): + (repo / name).parent.mkdir(parents=True, exist_ok=True) + (repo / name).write_bytes(OLD.encode()) + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "add files") + return repo + + +def _symlink(link, target): + try: + link.symlink_to(target, target_is_directory=target.is_dir()) + except (OSError, NotImplementedError): + if os.name != "nt": + raise + pytest.skip("this Windows account cannot create symlinks") + + +@pytest.fixture +def linked(repo, tmp_path): + """The same repo, reached through a symlinked parent directory.""" + _symlink(tmp_path / "alias", tmp_path / "ws") + return tmp_path / "alias" / "mono" + + +@pytest.fixture +def baseline(repo, monkeypatch): + sha = git(repo, "rev-parse", "HEAD").strip() + monkeypatch.setattr(diffstate, "load_baseline_sha", lambda _sid: sha) + return sha + + +class TestBaselineFileContent: + """`git show <rev>:<path>` reads the path from the repo root, so the lookup + must not depend on where in the repo the session was started.""" + + @pytest.mark.parametrize("session_dir", ["", "packages", "packages/app"]) + def test_found_from_any_session_directory(self, repo, baseline, session_dir): + cwd = str(repo / session_dir) + assert diffstate.get_baseline_file_content("s1", str(repo / FILE), cwd) == OLD + assert diffstate.get_baseline_file_content("s1", str(repo / "top.py"), cwd) == OLD + + def test_native_separators_are_not_passed_to_git(self, repo, baseline, monkeypatch): + # What os.path.relpath returns on Windows. + monkeypatch.setattr(diffstate.os, "sep", "\\") + monkeypatch.setattr(diffstate.os.path, "relpath", + lambda _p, _start: "packages\\app\\main.py") + assert diffstate.get_baseline_file_content("s1", str(repo / FILE), str(repo)) == OLD + + def test_file_not_in_baseline(self, repo, baseline): + (repo / "packages" / "app" / "new.py").write_text(OLD) + for p in ("packages/app/new.py", "packages/gone/main.py"): + assert diffstate.get_baseline_file_content( + "s1", str(repo / p), str(repo / "packages")) is None + + def test_file_outside_the_session_repo(self, repo, baseline, tmp_path): + outside = tmp_path / "ws" / "main.py" + outside.write_text(OLD) + assert diffstate.get_baseline_file_content("s1", str(outside), str(repo)) is None + # A second checkout of the same repo shares its objects; its files are + # still not this session's baseline. + git(repo, "worktree", "add", "-q", str(tmp_path / "ws" / "other"), "-b", "other") + twin = tmp_path / "ws" / "other" / FILE + assert twin.read_bytes().replace(b"\r\n", b"\n") == OLD.encode() + for cwd in (repo, repo / "packages" / "app"): + assert diffstate.get_baseline_file_content("s1", str(twin), str(cwd)) is None + + @pytest.mark.parametrize("session_dir", ["", "packages/app"]) + def test_found_through_a_symlinked_directory(self, repo, baseline, linked, session_dir): + """macOS reaches /tmp and /var through symlinks, so the tool's path and + the session directory can spell the same repo differently.""" + real, alias = repo / session_dir, linked / session_dir + for cwd, path in ((real, linked / FILE), (alias, repo / FILE), (alias, linked / FILE)): + assert diffstate.get_baseline_file_content("s1", str(path), str(cwd)) == OLD + + def test_session_directory_is_a_symlink_into_the_repo(self, repo, baseline, tmp_path): + _symlink(tmp_path / "app", repo / "packages" / "app") + for path in (tmp_path / "app" / "main.py", repo / FILE): + assert diffstate.get_baseline_file_content( + "s1", str(path), str(tmp_path / "app")) == OLD + + def test_committed_content_not_the_working_copy(self, repo, baseline): + (repo / FILE).write_text("print('rewritten')\n") + assert diffstate.get_baseline_file_content( + "s1", str(repo / FILE), str(repo / "packages" / "app")) == OLD + + +@pytest.mark.parametrize("session_dir", ["", "packages/app"]) +def test_rewrite_through_a_symlinked_path_does_not_re_flag(repo, linked, hook_env, session_dir): + """The session runs in the real directory and the tool names the file + through the symlink, as under /tmp on macOS.""" + cwd = repo / session_dir + env = {**hook_env, "CLAUDE_PROJECT_DIR": str(cwd)} + run_hook(ups_payload(cwd), env) + + _rc, stdout, _err = run_hook(write_payload(cwd, linked / FILE, OLD + "print('x')\n"), env) + assert "additionalContext" not in stdout + + _rc, stdout, _err = run_hook( + write_payload(cwd, linked / FILE, OLD + "import pickle\npickle.loads(data)\n"), env) + assert "pickle" in stdout and "os.system" not in stdout + + +@pytest.mark.parametrize("session_dir", ["", "packages/app"]) +def test_rewrite_does_not_re_flag_code_that_was_already_there(repo, hook_env, session_dir): + """A full-file Write that keeps a pre-existing risky call is not a new + finding, whether the session runs at the repo root or below it.""" + cwd = repo / session_dir + env = {**hook_env, "CLAUDE_PROJECT_DIR": str(cwd)} + run_hook(ups_payload(cwd), env) + + _rc, stdout, _err = run_hook(write_payload(cwd, repo / FILE, OLD + "print('x')\n"), env) + assert "additionalContext" not in stdout + + _rc, stdout, _err = run_hook( + write_payload(cwd, repo / FILE, OLD + "import pickle\npickle.loads(data)\n"), env) + assert "additionalContext" in stdout + assert "pickle" in stdout and "os.system" not in stdout diff --git a/plugins/security-guidance/tests/test_baseline_snapshot.py b/plugins/security-guidance/tests/test_baseline_snapshot.py new file mode 100644 index 00000000..eaa0e5c2 --- /dev/null +++ b/plugins/security-guidance/tests/test_baseline_snapshot.py @@ -0,0 +1,257 @@ +"""The turn baseline is a commit of the tracked changes (#6854), where it used +to be `git stash create`. These pin the states where the two could differ.""" +import os +import subprocess + +import pytest + +import diffstate +import gitutil +from conftest import GIT_ENV, git, make_repo + + +@pytest.fixture(autouse=True) +def safe_env(monkeypatch): + for k, v in gitutil.git_config_env(gitutil.SAFE_GIT_CONFIG).items(): + monkeypatch.setenv(k, v) + + +def _tree(repo, sha): + return git(repo, "rev-parse", f"{sha}^{{tree}}").strip() + + +def _plain_stash_tree(repo): + """What `git stash create` would have produced, on the real index.""" + out = subprocess.run(["git", "stash", "create"], cwd=repo, capture_output=True, + text=True, env={**os.environ, **GIT_ENV}) + sha = out.stdout.strip() + return _tree(repo, sha) if sha else _tree(repo, "HEAD") + + +def _capture(repo): + sha = diffstate.capture_git_baseline(str(repo)) + return sha, diffstate.last_baseline_fail + + +def _repo(tmp_path, name="r"): + return make_repo(tmp_path / name, {"a.txt": "a\n", "b.txt": "b\n", "d/c.txt": "c\n"}) + + +def test_unstaged_and_staged_changes_are_in_the_snapshot(tmp_path): + repo = _repo(tmp_path) + (repo / "a.txt").write_text("staged\n") + git(repo, "add", "a.txt") + (repo / "a.txt").write_text("staged\nunstaged\n") + (repo / "b.txt").write_text("also dirty\n") + sha, fail = _capture(repo) + assert fail == 0 + assert git(repo, "show", f"{sha}:a.txt") == "staged\nunstaged\n" + assert git(repo, "show", f"{sha}:b.txt") == "also dirty\n" + assert git(repo, "rev-parse", f"{sha}^").strip() == git(repo, "rev-parse", "HEAD").strip() + + +@pytest.mark.parametrize("state", [ + "unstaged", "staged", "staged-plus-unstaged", "worktree-delete", "staged-delete", + "staged-rename", "staged-new", "mode-change", "symlink", "untracked-only", "stale-stat", +]) +def test_same_tree_as_stash_create(tmp_path, state): + repo = _repo(tmp_path, state) + if state == "unstaged": + (repo / "a.txt").write_text("x\n") + elif state == "staged": + (repo / "a.txt").write_text("x\n"); git(repo, "add", "a.txt") + elif state == "staged-plus-unstaged": + (repo / "a.txt").write_text("x\n"); git(repo, "add", "a.txt") + (repo / "a.txt").write_text("y\n") + elif state == "worktree-delete": + (repo / "a.txt").unlink() + elif state == "staged-delete": + git(repo, "rm", "-q", "a.txt") + elif state == "staged-rename": + git(repo, "mv", "a.txt", "z.txt") + elif state == "staged-new": + (repo / "n.txt").write_text("n\n"); git(repo, "add", "n.txt") + elif state == "mode-change": + os.chmod(repo / "a.txt", 0o755) + elif state == "symlink": + (repo / "a.txt").unlink(); os.symlink("b.txt", repo / "a.txt") + elif state == "untracked-only": + (repo / "u.txt").write_text("u\n") + elif state == "stale-stat": + t = os.stat(repo / "a.txt").st_mtime + 2 + os.utime(repo / "a.txt", (t, t)) + # Capture FIRST: `git stash create` refreshes the real index, which would + # cancel the stale stat entry the last state is about. + sha, fail = _capture(repo) + assert fail == 0 + assert _tree(repo, sha) == _plain_stash_tree(repo) + + +def test_clean_tree_is_head(tmp_path): + repo = _repo(tmp_path) + assert _capture(repo) == (git(repo, "rev-parse", "HEAD").strip(), 0) + + +def test_conflicted_merge_snapshots_the_conflicted_files(tmp_path): + """`stash create` refused here and the whole conflicted file was then + reviewed as the turn's work. `add -u` resolves the conflict in the index + COPY, so the baseline holds what the work tree had, markers and all, and + the user's own index keeps its unmerged entries.""" + repo = _repo(tmp_path) + git(repo, "checkout", "-q", "-b", "other") + (repo / "a.txt").write_text("theirs\n"); git(repo, "commit", "-qam", "theirs") + git(repo, "checkout", "-q", "main") + (repo / "a.txt").write_text("ours\n"); git(repo, "commit", "-qam", "ours") + subprocess.run(["git", "merge", "other"], cwd=repo, capture_output=True, + env={**os.environ, **GIT_ENV}) + assert "UU" in git(repo, "status", "--porcelain") + conflicted = (repo / "a.txt").read_text() + sha, fail = _capture(repo) + assert fail == 0 + assert git(repo, "show", f"{sha}:a.txt") == conflicted + assert "UU" in git(repo, "status", "--porcelain") + assert git(repo, "ls-files", "-u") + + +def test_intent_to_add_file_is_snapshotted_with_its_content(tmp_path): + """`git add -N` marks a file to be committed without staging content. + `stash create` refused on such an index and the baseline fell back to + HEAD, so the file's pre-existing content was reviewed as the turn's work. + The snapshot holds that content, so only what the turn changes is.""" + repo = _repo(tmp_path) + (repo / "n.txt").write_text("secret = 1\n") + git(repo, "add", "-N", "n.txt") + sha, fail = _capture(repo) + assert fail == 0 + assert sha != git(repo, "rev-parse", "HEAD").strip() + assert git(repo, "show", f"{sha}:n.txt") == "secret = 1\n" + # The path is still named in the review set, as it was before, but its + # content is in the baseline now, so the reviewer is sent nothing for it + # where `stash create` failing sent the whole file. + + +def test_subdirectory_cwd_snapshots_the_whole_tree(tmp_path): + repo = _repo(tmp_path) + (repo / "a.txt").write_text("top changed\n") + (repo / "d" / "c.txt").write_text("below changed\n") + sha, fail = _capture(repo / "d") + assert fail == 0 + assert git(repo, "show", f"{sha}:a.txt") == "top changed\n" + assert git(repo, "show", f"{sha}:d/c.txt") == "below changed\n" + + +def test_nested_repository_stays_out_of_the_snapshot(tmp_path): + """A directory with its own .git is another repository: #6837 keeps its + gitlink out of the index copy, so it must stay out of the tree too.""" + repo = _repo(tmp_path) + nested = repo / "vendor" + make_repo(nested, {"x.txt": "x\n"}) + git(repo, "add", "vendor") + git(repo, "commit", "-qm", "add nested") + (nested / "x.txt").write_text("changed inside\n") + (repo / "a.txt").write_text("changed outside\n") + sha, fail = _capture(repo) + assert fail == 0 + assert git(repo, "show", f"{sha}:a.txt") == "changed outside\n" + assert "vendor" not in git(repo, "ls-tree", "--name-only", sha) + + +def test_clean_tree_with_a_nested_repository_is_head(tmp_path): + """The index copy has no gitlink, so its tree can never equal HEAD's. + Without a second look that would mint a commit object on every prompt in + a repo with submodules, and the baseline would be a prunable dangling + commit instead of HEAD.""" + repo = _repo(tmp_path) + make_repo(repo / "vendor", {"x.txt": "x\n"}) + git(repo, "add", "vendor") + git(repo, "commit", "-qm", "add nested") + head = git(repo, "rev-parse", "HEAD").strip() + assert _capture(repo) == (head, 0) + + # A change inside the nested repository is still not this repo's work. + (repo / "vendor" / "x.txt").write_text("changed inside\n") + assert _capture(repo) == (head, 0) + + # A real change outside it still gets a snapshot. + (repo / "a.txt").write_text("changed outside\n") + sha, fail = _capture(repo) + assert fail == 0 and sha != head + assert git(repo, "show", f"{sha}:a.txt") == "changed outside\n" + + +def test_snapshot_leaves_the_real_index_and_worktree_alone(tmp_path): + repo = _repo(tmp_path) + (repo / "a.txt").write_text("dirty\n") + before_index = (repo / ".git" / "index").read_bytes() + before_status = git(repo, "status", "--porcelain") + before_stash = git(repo, "stash", "list") + _capture(repo) + assert (repo / ".git" / "index").read_bytes() == before_index + assert git(repo, "status", "--porcelain") == before_status + assert git(repo, "stash", "list") == before_stash + assert [p for p in os.listdir(repo / ".git") if p.startswith("index.")] == [] + + +def test_snapshot_commit_is_unreferenced(tmp_path): + repo = _repo(tmp_path) + (repo / "a.txt").write_text("dirty\n") + sha, _ = _capture(repo) + assert sha not in git(repo, "log", "--all", "--format=%H") + assert git(repo, "rev-parse", "HEAD").strip() != sha + + +def test_partial_clone_snapshots_without_fetching(tmp_path): + """A blobless clone has no blob for an unchanged file. git counts a + promisor object as present, so write-tree neither fetches nor fails; + this pins that, since the snapshot must never make a network call.""" + origin = _repo(tmp_path, "origin") + git(origin, "config", "uploadpack.allowFilter", "true") + for i in range(3): + (origin / f"big{i}.txt").write_text(f"content {i}\n") + git(origin, "add", "-A") + git(origin, "commit", "-qm", "more") + clone = tmp_path / "clone" + subprocess.run(["git", "clone", "-q", "--filter=blob:none", "--no-local", + str(origin), str(clone)], check=True, capture_output=True, + env={**os.environ, **GIT_ENV}) + # Nothing can be fetched from here on: a snapshot that tried would fail. + git(clone, "config", "remote.origin.url", "/nonexistent-origin") + (clone / "a.txt").write_text("dirty\n") + sha, fail = _capture(clone) + assert fail == 0 and sha + assert git(clone, "show", f"{sha}:a.txt") == "dirty\n" + + +def test_literal_pathspecs_in_the_environment(tmp_path, monkeypatch): + """A user who exports GIT_LITERAL_PATHSPECS gets it in the hook's + environment, and a magic pathspec would then fail outright and drop the + baseline to HEAD on every prompt of the session.""" + repo = _repo(tmp_path) + monkeypatch.setenv("GIT_LITERAL_PATHSPECS", "1") + (repo / "a.txt").write_text("dirty\n") + sha, fail = _capture(repo) + assert fail == 0 + assert git(repo, "show", f"{sha}:a.txt") == "dirty\n" + + +def test_no_identity_configured_still_snapshots(tmp_path, monkeypatch): + repo = _repo(tmp_path) + (repo / "a.txt").write_text("dirty\n") + for var in ("GIT_AUTHOR_NAME", "GIT_AUTHOR_EMAIL", + "GIT_COMMITTER_NAME", "GIT_COMMITTER_EMAIL", "EMAIL"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("GIT_CONFIG_GLOBAL", os.devnull) + monkeypatch.setenv("GIT_CONFIG_SYSTEM", os.devnull) + sha, fail = _capture(repo) + assert fail == 0 and sha + assert "security-guidance" in git(repo, "show", "-s", "--format=%an", sha) + + +def test_signing_configured_does_not_sign_the_snapshot(tmp_path): + repo = _repo(tmp_path) + git(repo, "config", "commit.gpgsign", "true") + git(repo, "config", "gpg.program", "/nonexistent-gpg") + (repo / "a.txt").write_text("dirty\n") + sha, fail = _capture(repo) + assert fail == 0 and sha + assert git(repo, "show", "-s", "--format=%G?", sha).strip() in ("N", "") diff --git a/plugins/security-guidance/tests/test_bootstrap_spawn_cwd.py b/plugins/security-guidance/tests/test_bootstrap_spawn_cwd.py new file mode 100644 index 00000000..2325cd9b --- /dev/null +++ b/plugins/security-guidance/tests/test_bootstrap_spawn_cwd.py @@ -0,0 +1,75 @@ +"""The SDK bootstrap's Python subprocesses must not import from the repository. + +`python -c` and `python -m` put the working directory first on the import +path, and hooks run in the user's repository, so every interpreter the +bootstrap starts runs from a directory the plugin owns. +""" +import ast +import os +import sys +from pathlib import Path + +import pytest + +import ensure_agent_sdk + +from conftest import HOOKS_DIR, make_repo + + +def _spawns(tree): + for node in ast.walk(tree): + if (isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and node.func.attr in ("run", "Popen", "call", "check_call", "check_output") + and isinstance(node.func.value, ast.Name) and node.func.value.id == "subprocess"): + yield node + + +def test_every_spawn_sets_a_plugin_owned_cwd(): + tree = ast.parse((HOOKS_DIR / "ensure_agent_sdk.py").read_text()) + calls = list(_spawns(tree)) + assert len(calls) >= 6 + for call in calls: + cwd = [k for k in call.keywords if k.arg == "cwd"] + assert cwd, f"subprocess call on line {call.lineno} inherits the hook's working directory" + assert ast.unparse(cwd[0].value) == "_spawn_cwd()", call.lineno + + +def test_spawn_cwd_is_the_state_dir(tmp_path, monkeypatch): + state = tmp_path / "state" + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(state)) + assert ensure_agent_sdk._spawn_cwd() == str(state) + assert state.is_dir() + + +def test_relative_state_dir_is_made_absolute(tmp_path, monkeypatch): + # Children run in the state dir, so a relative one would be resolved twice. + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", "rel") + assert ensure_agent_sdk._spawn_cwd() == str(tmp_path / "rel") + + +@pytest.mark.skipif(sys.platform == "win32", reason="symlinked interpreter") +@pytest.mark.skipif(sys.version_info < (3, 10), reason="bootstrap stops before the probe") +def test_health_check_ignores_module_in_repo(tmp_path, monkeypatch): + repo = tmp_path / "repo" + make_repo(repo) + marker = tmp_path / "imported-from-repo" + (repo / "claude_agent_sdk.py").write_text( + f"open({str(marker)!r}, 'w').write('x')\n") + + state = tmp_path / "state" + venv_bin = state / "agent-sdk-venv" / ("Scripts" if sys.platform == "win32" else "bin") + venv_bin.mkdir(parents=True) + venv_py = venv_bin / ("python.exe" if sys.platform == "win32" else "python") + os.symlink(sys.executable, venv_py) + + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(state)) + monkeypatch.chdir(repo) + monkeypatch.setattr(ensure_agent_sdk, "_sdk_on_syspath", lambda: False) + # No rebuild if the probe fails: this test only covers the probe. + ensure_agent_sdk._write_cooldown(state) + + outcome, _, _ = ensure_agent_sdk.main() + + assert outcome in (ensure_agent_sdk.NOOP_VENV, ensure_agent_sdk.SKIP_COOLDOWN) + assert not marker.exists() diff --git a/plugins/security-guidance/tests/test_custom_headers.py b/plugins/security-guidance/tests/test_custom_headers.py new file mode 100644 index 00000000..fe019d52 --- /dev/null +++ b/plugins/security-guidance/tests/test_custom_headers.py @@ -0,0 +1,95 @@ +"""ANTHROPIC_CUSTOM_HEADERS reach every call the plugin makes.""" +from conftest import ( + STUB_VULN, VULN_PY, edit_payload, metrics_of, run_hook, stop_payload, ups_payload, +) + +import llm + +GATEWAY_KEY = ("x-litellm-api-key", "Bearer sk-gateway") + + +class TestCustomHeaders: + def test_parsed_like_claude_code(self, monkeypatch): + monkeypatch.setenv("ANTHROPIC_CUSTOM_HEADERS", + "x-litellm-api-key: Bearer sk-1\r\n" + "X-Tenant: acme:eu \n" + "no colon here\n" + ": no name\n" + "\n" + "X-Empty:") + assert llm._custom_headers() == [ + ("x-litellm-api-key", "Bearer sk-1"), ("X-Tenant", "acme:eu"), ("X-Empty", ""), + ] + + def test_unsendable_lines_are_skipped(self, monkeypatch): + monkeypatch.setenv("ANTHROPIC_CUSTOM_HEADERS", + "bad name: x\nX-Ok: 1\nX-Uni: caf\u00e9\u2603\nX-Cr: a\rb") + assert llm._custom_headers() == [("X-Ok", "1")] + + def test_sent_with_the_plugin_credentials(self, monkeypatch): + monkeypatch.setattr(llm, "ANTHROPIC_AUTH_TOKEN", "oauth") + monkeypatch.setenv("ANTHROPIC_CUSTOM_HEADERS", "x-litellm-api-key: k") + h = llm._build_auth_headers(use_token=True) + assert h["x-litellm-api-key"] == "k" + assert h["Authorization"] == "Bearer oauth" + + def test_custom_line_wins_over_the_same_header(self, monkeypatch): + monkeypatch.setattr(llm, "ANTHROPIC_AUTH_TOKEN", "oauth") + monkeypatch.setenv("ANTHROPIC_CUSTOM_HEADERS", "authorization: Bearer gw") + h = llm._build_auth_headers(use_token=True) + assert [v for k, v in h.items() if k.lower() == "authorization"] == ["Bearer gw"] + + def test_custom_beta_lines_are_not_sent(self, monkeypatch): + monkeypatch.setenv("ANTHROPIC_CUSTOM_HEADERS", "anthropic-beta: extra-1\nX-Ok: 1") + h = llm._build_auth_headers(use_token=False) + assert h["anthropic-beta"] == "structured-outputs-2025-11-13" + assert h["X-Ok"] == "1" + + def test_without_custom_headers_only_the_plugin_headers_are_sent(self, monkeypatch): + monkeypatch.delenv("ANTHROPIC_CUSTOM_HEADERS", raising=False) + assert set(llm._build_auth_headers(use_token=False)) == { + "Content-Type", "anthropic-version", "x-api-key", "anthropic-beta"} + + def test_stop_review_reaches_a_header_keyed_gateway(self, env, gateway, workspace): + """Every plugin call (model lookup and review) carries the gateway + key, and the review finds the vuln.""" + _, repo = workspace + gateway.required = GATEWAY_KEY + gateway.vulns = [STUB_VULN] + e = {**env, "ANTHROPIC_CUSTOM_HEADERS": f"{GATEWAY_KEY[0]}: {GATEWAY_KEY[1]}"} + run_hook(ups_payload(repo), e) + (repo / "app.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py", VULN_PY), e) + rc, so, se = run_hook(stop_payload(repo), e) + assert rc == 2, (so, se) + assert "command_injection" in se + assert metrics_of(so)["gateway_cfg"] == 3 # custom base URL and custom headers + calls = [(m, p.split("?")[0]) for m, p, _ in gateway.requests] + assert ("GET", "/v1/models") in calls and ("POST", "/v1/messages") in calls + for method, path, headers in gateway.requests: + if path.startswith("/v1/"): + assert headers.get(GATEWAY_KEY[0]) == GATEWAY_KEY[1], (method, path) + + def test_gateway_cfg_is_a_number_with_no_names_or_values(self, monkeypatch): + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("ANTHROPIC_CUSTOM_HEADERS", raising=False) + assert llm.gateway_cfg() == 0 + for same_host in ("https://api.anthropic.com/", "https://API.anthropic.com:443"): + monkeypatch.setenv("ANTHROPIC_BASE_URL", same_host) + assert llm.gateway_cfg() == 0 + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://gw.example.com") + assert llm.gateway_cfg() == 1 + monkeypatch.setenv("ANTHROPIC_CUSTOM_HEADERS", "anthropic-beta: extra-1") + assert llm.gateway_cfg() == 1 # beta lines are not sent + monkeypatch.setenv("ANTHROPIC_CUSTOM_HEADERS", "X-Ok: 1") + assert llm.gateway_cfg() == 3 + monkeypatch.delenv("ANTHROPIC_BASE_URL") + assert llm.gateway_cfg() == 2 + + def test_gateway_cfg_is_left_out_when_no_api_call_was_made(self, env, gateway, workspace): + """Only rows that made API calls carry gateway_cfg.""" + _, repo = workspace + rc, so, se = run_hook(stop_payload(repo), env) + assert rc == 0, (so, se) + row = metrics_of(so) + assert row["skipped"] is True and "gateway_cfg" not in row diff --git a/plugins/security-guidance/tests/test_diff_parsing.py b/plugins/security-guidance/tests/test_diff_parsing.py new file mode 100644 index 00000000..7665a064 --- /dev/null +++ b/plugins/security-guidance/tests/test_diff_parsing.py @@ -0,0 +1,99 @@ +import os +import subprocess + +from conftest import GIT_ENV, git, make_repo + +import gitutil + +VULN = "import os\nos.system(input())\n" + + +def staged_diff(repo): + r = subprocess.run( + [*gitutil.GIT_CMD, "diff", "--cached", "--no-color"], cwd=repo, + capture_output=True, env={**os.environ, **GIT_ENV}, + ) + assert r.returncode == 0, r.stderr + return r.stdout.decode("utf-8", errors="replace") + + +def stage(tmp_path, files): + repo = make_repo(tmp_path / "repo") + for name, content in files.items(): + (repo / name).write_text(content) + git(repo, "add", "-A") + return staged_diff(repo) + + +class TestEmbeddedHeaderText: + SOURCE = ( + 'HEADER = """\n' + "diff --git a/other.py b/other.py\n" + '"""\n' + "import os\n" + "os.system(input()) # AFTER_EMBEDDED_HEADER\n" + ) + + def test_content_after_embedded_header_text_is_kept(self, tmp_path): + files = dict(gitutil.parse_diff_into_files(stage(tmp_path, {"embed.py": self.SOURCE}))) + assert list(files) == ["embed.py"] + assert "AFTER_EMBEDDED_HEADER" in files["embed.py"] + + def test_embedded_header_text_adds_no_path(self, tmp_path): + diff = stage(tmp_path, {"embed.py": self.SOURCE}) + assert gitutil.extract_file_paths_from_diff(diff) == ["embed.py"] + + def test_following_file_is_still_parsed(self, tmp_path): + diff = stage(tmp_path, {"embed.py": self.SOURCE, "z_next.py": VULN}) + assert [p for p, _ in gitutil.parse_diff_into_files(diff)] == ["embed.py", "z_next.py"] + + +class TestQuotedPaths: + def test_names_git_quotes_are_reviewed(self, tmp_path): + names = ['we"ird.py', "tab\there.py", "back\\slash.py", "café.py", "plain.py"] + diff = stage(tmp_path, {n: VULN for n in names}) + assert '"a/we\\"ird.py"' in diff # git did quote it + parsed = gitutil.parse_diff_into_files(diff) + assert sorted(p for p, _ in parsed) == sorted(names) + assert all("os.system" in c for _, c in parsed) + assert sorted(gitutil.extract_file_paths_from_diff(diff)) == sorted(names) + + def test_rename_to_a_quoted_name(self, tmp_path): + repo = make_repo(tmp_path / "repo", {"plain.py": VULN * 20}) + git(repo, "mv", "plain.py", 'qu"ote.py') + (repo / 'qu"ote.py').write_text(VULN * 20 + "eval(input())\n") + git(repo, "add", "-A") + diff = staged_diff(repo) + assert 'a/plain.py "b/qu\\"ote.py"' in diff + assert [p for p, _ in gitutil.parse_diff_into_files(diff)] == ['qu"ote.py'] + + def test_octal_escapes_decode_as_utf8(self): + header = '"a/caf\\303\\251.py" "b/caf\\303\\251.py"' + assert gitutil._parse_diff_header(header) == ("café.py", "café.py") + + def test_unterminated_quote_is_skipped(self): + assert gitutil._parse_diff_header('"a/x.py b/x.py') is None + + +class TestCommitOutput: + def test_git_show_preamble_is_ignored(self, tmp_path): + repo = make_repo(tmp_path / "repo") + (repo / "app.py").write_text(VULN) + git(repo, "add", "-A") + git(repo, "commit", "-q", "-m", "diff --git a/fake.py b/fake.py") + r = subprocess.run( + [*gitutil.GIT_CMD, "show", "-p", "--no-color", "HEAD", "--"], cwd=repo, + capture_output=True, env={**os.environ, **GIT_ENV}, + ) + parsed = gitutil.parse_diff_into_files(r.stdout.decode()) + assert [p for p, _ in parsed] == ["app.py"] + + +def test_unparsed_headers_are_counted_for_telemetry(): + import gitutil + gitutil._unparsed_diff_headers.clear() + diff = 'diff --git "a/x.py b/x.py\n@@ -0,0 +1 @@\n+x = 1\n' + assert gitutil.parse_diff_into_files(diff) == [] + assert gitutil.extract_file_paths_from_diff(diff) == [] + assert len(gitutil._unparsed_diff_headers) == 1 + gitutil._unparsed_diff_headers.clear() diff --git a/plugins/security-guidance/tests/test_diff_prefix_config.py b/plugins/security-guidance/tests/test_diff_prefix_config.py new file mode 100644 index 00000000..ce2a8017 --- /dev/null +++ b/plugins/security-guidance/tests/test_diff_prefix_config.py @@ -0,0 +1,70 @@ +"""A user's diff prefix settings must not hide files from the reviews.""" +import os +import subprocess + +import pytest + +from conftest import ( + GIT_ENV, VULN_PY, commit_file, bash_payload, git, make_repo, metrics_of, + run_hook, stop_payload, ups_payload, +) + +import gitutil + +PREFIX_CONFIGS = [ + ("diff.mnemonicPrefix", "true"), + ("diff.noprefix", "true"), + ("diff.srcPrefix", "x/"), + ("diff.dstPrefix", "y/"), +] + + +@pytest.fixture +def git_env(monkeypatch): + for k, v in GIT_ENV.items(): + monkeypatch.setenv(k, v) + + +@pytest.mark.parametrize("key,value", PREFIX_CONFIGS) +class TestParsers: + def test_work_tree_diff(self, tmp_path, git_env, key, value): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + git(repo, "config", key, value) + (repo / "app.py").write_text("x = 1\ny = 2\n") + diff = gitutil.get_git_diff(str(repo), "HEAD") + assert [f for f, _ in gitutil.parse_diff_into_files(diff)] == ["app.py"] + assert gitutil.extract_file_paths_from_diff(diff) == ["app.py"] + + def test_commit_diff(self, tmp_path, git_env, key, value): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + git(repo, "config", key, value) + commit_file(repo, "app.py", "x = 1\ny = 2\n") + show = subprocess.run( + [*gitutil.GIT_CMD, "show", "-p", "--no-color", "HEAD", "--"], + cwd=repo, capture_output=True, + ).stdout.decode("utf-8", errors="replace") + assert [f for f, _ in gitutil.parse_diff_into_files(show)] == ["app.py"] + diff = gitutil._git_diff_range(str(repo), "HEAD~1") + assert [f for f, _ in gitutil.parse_diff_into_files(diff)] == ["app.py"] + + +@pytest.mark.parametrize("key,value", PREFIX_CONFIGS[:2]) +class TestWholeHook: + def test_stop_review_runs(self, tmp_path, hook_env, stub_api, key, value): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + git(repo, "config", key, value) + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + rc, so, se = run_hook(stop_payload(repo), hook_env) + m = metrics_of(so) + assert m.get("skip_reason") is None, m + assert stub_api.calls + + +class TestWholeHookCommit: + def test_commit_review_runs_with_noprefix(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + git(repo, "config", "diff.noprefix", "true") + sha, out = commit_file(repo, "app.py", VULN_PY) + rc, so, se = run_hook(bash_payload(repo, "git commit -m change", stdout=out), hook_env) + assert metrics_of(so).get("skip_reason") != 30 diff --git a/plugins/security-guidance/tests/test_findings_delivery.py b/plugins/security-guidance/tests/test_findings_delivery.py new file mode 100644 index 00000000..0989ed4b --- /dev/null +++ b/plugins/security-guidance/tests/test_findings_delivery.py @@ -0,0 +1,88 @@ +"""Findings reach Claude on the channel Claude Code reads for an asyncRewake +hook (`stderr || stdout`, never additionalContext).""" +import json +import sys +import types + +from conftest import ( + STUB_VULN, VULN_PY, bash_payload, commit_file, metrics_of, run_hook, +) + +import llm +import security_reminder_hook as hook + + +def _rewake_body(stdout, stderr): + """What Claude Code shows Claude for an exit-2 asyncRewake hook: stderr, + else stdout.""" + return stderr or stdout + + +def _commit(repo, env, content=VULN_PY, session_id="s1"): + sha, out = commit_file(repo, "app.py", content) + rc, so, se = run_hook(bash_payload(repo, "git commit -m change", out, + session_id=session_id), env) + return sha, rc, so, se + + +class TestAgenticReviewerOn: + def test_findings_not_replaced_by_the_inner_cli_warning(self, agentic_env, gateway, workspace): + _, repo = workspace + gateway.vulns = [STUB_VULN] + sha, rc, so, se = _commit(repo, agentic_env) + assert rc == 2, (so, se) + assert metrics_of(so)["agentic_fallback"] == 2 + assert metrics_of(so)["child_stderr"] >= 1 # the warning was counted, not shown + body = _rewake_body(so, se) + assert "command_injection" in body + assert "connectors are disabled" not in body + assert "Fatal error in message reader" not in body # the SDK's own log line + + +class TestFindingsReachClaude: + def test_commit_findings_are_the_rewake_body(self, env, gateway, workspace): + _, repo = workspace + gateway.vulns = [STUB_VULN] + sha, rc, so, se = _commit(repo, env) + assert rc == 2 + body = _rewake_body(so, se) + assert "Security Review: Potential vulnerabilities detected" in body + assert "command_injection" in body + out = json.loads(so.strip().splitlines()[-1]) + assert out["metrics"]["vulns_found"] == 1 + assert out["rewakeSummary"].startswith("Commit security review found") + assert "hookSpecificOutput" not in out # not repeated where the sync path would show it twice + + def test_stop_findings_also_go_in_decision_and_reason(self, capsys): + hook.emit_metrics({"vulns_found": 1}, additional_context="FINDINGS", hook_event_name="Stop") + cap = capsys.readouterr() + assert cap.err == "FINDINGS" + out = json.loads(cap.out) + assert out["decision"] == "block" and out["reason"] == "FINDINGS" + + def test_agentic_review_captures_the_inner_cli_stderr(self, monkeypatch, tmp_path, capfd): + spawned = [] + + class ClaudeAgentOptions: + def __init__(self, **kwargs): + self.kwargs = kwargs + spawned.append(kwargs) + + class ResultMessage: + subtype, usage, total_cost_usd = "success", {}, 0.0 + structured_output = {"findings": []} + + async def query(prompt, options): + # What the inner CLI prints on stderr; with a callback set the SDK + # pipes it to the callback instead of the hook's own stderr. + options.kwargs["stderr"]("claude.ai connectors are disabled") + yield ResultMessage() + + sdk = types.ModuleType("claude_agent_sdk") + sdk.ClaudeAgentOptions, sdk.ResultMessage, sdk.query = ClaudeAgentOptions, ResultMessage, query + sdk.AssistantMessage = type("AssistantMessage", (), {}) + monkeypatch.setitem(sys.modules, "claude_agent_sdk", sdk) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", "") + llm.agentic_review(str(tmp_path), [("a.py", "+x = 1\n")], ["a.py"]) + assert callable(spawned[0].get("stderr")) + assert "connectors" not in capfd.readouterr().err diff --git a/plugins/security-guidance/tests/test_gateway_sign_in.py b/plugins/security-guidance/tests/test_gateway_sign_in.py new file mode 100644 index 00000000..7b6e9fc4 --- /dev/null +++ b/plugins/security-guidance/tests/test_gateway_sign_in.py @@ -0,0 +1,396 @@ +"""Reviews in a session signed in to a Claude apps gateway with /login, where +Claude Code gives hooks no credential.""" +import asyncio +import json +import os +import sys +import types + +import pytest + +from conftest import ( + VULN_PY, bash_payload, commit_file, edit_payload, git, metrics_of, run_hook, + stop_payload, ups_payload, write_cli, +) + +import _base +import llm + +SIGNED_IN = json.dumps({"loggedIn": True, "authMethod": "third_party", "apiProvider": "gateway"}) +NOT_SIGNED_IN = json.dumps({"loggedIn": False, "authMethod": "none", "apiProvider": "gateway"}) + + +def make_cli(tmp_path, auth_status, *, name="claude", hang=False, linger=False): + """A fake `claude` that logs each call to <name>.calls as its arguments, the + SECURITY_GUIDANCE_DISABLE it got and its working directory, prints + `auth_status` for `auth status --json`, and fails anything else. `hang` + waits 5 s before answering; `linger` leaves a process running that holds + stdout open after the CLI exits.""" + calls = tmp_path / f"{name}.calls" + script = f"""import os, subprocess, sys, time +with open({str(calls)!r}, "a", encoding="utf-8") as f: + f.write(" ".join(sys.argv[1:]) + " | SGD=" + os.environ.get("SECURITY_GUIDANCE_DISABLE", "") + + " | CWD=" + os.getcwd() + "\\n") +if sys.argv[1:3] != ["auth", "status"]: + sys.stderr.write("review refused\\n") + sys.exit(1) +if {hang!r}: + time.sleep(5) +sys.stdout.write({auth_status!r}) +sys.stdout.flush() +if {linger!r}: + subprocess.Popen([sys.executable, "-c", "import time; time.sleep(5)"]) +""" + return write_cli(tmp_path, name, script), calls + + +def calls_of(calls): + return calls.read_text(encoding="utf-8").splitlines() if calls.exists() else [] + + +def review_calls(calls): + return [c for c in calls_of(calls) if "--input-format stream-json" in c] + + +def model_of(call): + args = call.split(" | ")[0].split() + return args[args.index("--model") + 1] + + +@pytest.fixture +def no_credential(monkeypatch, tmp_path): + # CLAUDE_CODE_REMOTE_SESSION_ID makes every session share one state file. + for var in ("SECURITY_REVIEW_MODEL", "CLAUDE_CODE_EXECPATH", "CLAUDE_CODE_REMOTE_SESSION_ID", + *llm._PROVIDER_ENV_VARS): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(tmp_path / "no-claude")) + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(tmp_path / "state")) + monkeypatch.setattr(_base, "DEBUG_LOG_FILE", str(tmp_path / "log.txt")) + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", False) + new_hook_process(monkeypatch) + + +def new_hook_process(monkeypatch): + monkeypatch.setattr(llm, "_gateway_cli", None) + monkeypatch.setattr(llm, "_gateway_check_unanswered", False) + monkeypatch.setattr(llm, "_gateway_review_timed_out", False) + + +class TestGatewaySignInCheck: + @pytest.mark.parametrize("auth_status, signed_in", [ + pytest.param(SIGNED_IN, True, id="signed-in"), + pytest.param(NOT_SIGNED_IN, False, id="gateway-named-but-not-signed-in"), + pytest.param(json.dumps({"loggedIn": True, "apiProvider": "firstParty"}), False, + id="first-party"), + pytest.param(json.dumps({"loggedIn": "true", "apiProvider": "gateway"}), False, + id="logged-in-not-a-boolean"), + pytest.param("Logged in via gateway", False, id="not-json"), + pytest.param("[]", False, id="not-an-object"), + pytest.param("[" * 60000, False, id="too-deeply-nested"), + ]) + def test_reviews_only_for_a_signed_in_gateway_session( + self, no_credential, monkeypatch, tmp_path, auth_status, signed_in): + cli, calls = make_cli(tmp_path, auth_status) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + assert llm.can_review("s1") is signed_in + assert [c.split(" | ")[0] for c in calls_of(calls)] == ["auth status --json"] + + def test_cli_that_cannot_start_means_no_review(self, no_credential): + assert llm.can_review("s1") is False + + def test_no_cli_found_means_no_review(self, no_credential, monkeypatch): + monkeypatch.setattr(llm, "_user_cli_path", lambda: None) + assert llm.can_review("s1") is False + + def test_relative_cli_path_is_not_run(self, no_credential, monkeypatch, tmp_path): + cli, calls = make_cli(tmp_path, SIGNED_IN) + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", f"./{cli.name}") + assert llm.can_review("s1") is False + assert calls_of(calls) == [] + + def test_asked_from_the_home_directory(self, no_credential, monkeypatch, tmp_path): + cli, calls = make_cli(tmp_path, SIGNED_IN) + home, repo = tmp_path / "home", tmp_path / "repo" + home.mkdir() + repo.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) # "~" on Windows + monkeypatch.chdir(repo) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + assert llm.can_review("s1") + assert calls_of(calls)[0].endswith(f"CWD={home}") + + def test_hung_cli_means_no_review(self, no_credential, monkeypatch, tmp_path): + cli, _ = make_cli(tmp_path, SIGNED_IN, hang=True) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + monkeypatch.setattr(llm, "_GATEWAY_CHECK_TIMEOUT_S", 1) + assert llm.can_review("s1") is False + + def test_answer_is_read_while_a_process_it_started_holds_stdout( + self, no_credential, monkeypatch, tmp_path): + cli, _ = make_cli(tmp_path, SIGNED_IN, linger=True) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + monkeypatch.setattr(llm, "_GATEWAY_CHECK_TIMEOUT_S", 3) + assert llm.can_review("s1") is True + + def test_asked_once_per_session(self, no_credential, monkeypatch, tmp_path): + cli, calls = make_cli(tmp_path, SIGNED_IN) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + assert llm.can_review("s1") + new_hook_process(monkeypatch) + assert llm.can_review("s1") + assert len(calls_of(calls)) == 1 + new_hook_process(monkeypatch) + assert llm.can_review("s2") + assert len(calls_of(calls)) == 2 + + def test_no_answer_is_asked_again(self, no_credential, monkeypatch, tmp_path): + cli, calls = make_cli(tmp_path, "") + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + assert llm.can_review("s1") is False + make_cli(tmp_path, SIGNED_IN) + new_hook_process(monkeypatch) + assert llm.can_review("s1") is True + assert len(calls_of(calls)) == 2 + + def test_a_different_cli_is_asked_again(self, no_credential, monkeypatch, tmp_path): + first, _ = make_cli(tmp_path, SIGNED_IN, name="first") + second, second_calls = make_cli(tmp_path, NOT_SIGNED_IN, name="second") + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(first)) + assert llm.can_review("s1") + new_hook_process(monkeypatch) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(second)) + assert llm.can_review("s1") is False + assert len(calls_of(second_calls)) == 1 + + def test_credential_in_env_never_asks_the_cli(self, no_credential, monkeypatch, tmp_path): + cli, calls = make_cli(tmp_path, SIGNED_IN) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", True) + assert llm.can_review("s1") + assert calls_of(calls) == [] + + def test_telemetry_names_the_route(self, no_credential, monkeypatch, tmp_path): + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("ANTHROPIC_CUSTOM_HEADERS", raising=False) + cli, _ = make_cli(tmp_path, SIGNED_IN) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + assert llm.gateway_cfg() == 0 + assert llm.can_review("s1") + assert llm.gateway_cfg() == 4 + + def test_essential_traffic_only_still_reviews(self, no_credential, monkeypatch, tmp_path): + cli, _ = make_cli(tmp_path, SIGNED_IN) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + monkeypatch.setenv("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "1") + assert llm.can_review("s1") + + +class FakeChild: + """Stands in for the child CLI's review; `reply` is what each call returns.""" + + def __init__(self): + self.prompts = [] + self.reply = {"hasVulnerabilities": False, "vulnerabilities": []} + + def __call__(self, prompt, schema, *, max_tokens=16000, model=None, cli_path=None): + self.prompts.append(prompt) + if self.reply is None: + llm._last_call_claude_http_error = -1 + return self.reply + + +class TestGatewayReviewOutcome: + @pytest.fixture + def child(self, no_credential, monkeypatch, tmp_path): + cli, _ = make_cli(tmp_path, SIGNED_IN) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + fake = FakeChild() + monkeypatch.setattr(llm, "_call_claude_via_sdk", fake) + return fake + + def test_review_is_sent_to_the_child(self, child): + assert llm.can_review("s1") + assert llm.analyze_code_security([("a.py", "+x = 1\n")], is_diff=True) == (None, []) + assert not llm.last_review_failed() + assert "+x = 1" in child.prompts[0] + assert child.prompts[0].startswith(llm._CODE_REVIEW_RUBRIC) + + @pytest.mark.parametrize("reply", [ + pytest.param(None, id="child-failed"), + pytest.param({"findings": []}, id="no-verdict"), + ]) + def test_child_without_a_verdict_is_a_failed_review(self, child, reply): + child.reply = reply + assert llm.can_review("s1") + assert llm.analyze_code_security([("a.py", "+x = 1\n")], is_diff=True) == (None, []) + assert llm.last_review_failed() + + def test_failed_review_is_tried_on_sonnet_unless_it_timed_out(self, child, monkeypatch): + child.reply = None + assert llm.can_review("s1") + llm.analyze_code_security([("a.py", "+x = 1\n")], is_diff=True) + assert len(child.prompts) == 2 + monkeypatch.setattr(llm, "_gateway_review_timed_out", True) + llm.analyze_code_security([("a.py", "+x = 1\n")], is_diff=True) + assert len(child.prompts) == 3 and llm.last_review_failed() + + def test_not_signed_in_sends_nothing(self, child, monkeypatch, tmp_path): + cli, _ = make_cli(tmp_path, NOT_SIGNED_IN, name="signed-out") + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", str(cli)) + assert not llm.can_review("s1") + assert llm.analyze_code_security([("a.py", "+x = 1\n")], is_diff=True) == (None, []) + assert not llm.last_review_failed() + assert child.prompts == [] + + +def test_timed_out_review_logs_what_the_cli_retried(no_credential, monkeypatch, tmp_path): + """The CLI retries a refused sign-in by itself until the plugin's time + limit; the retries are in its output, not on stderr.""" + class Retry: + subtype, data = "api_retry", {"error_status": 401} + + async def query(prompt, options): + yield Retry() + raise asyncio.TimeoutError() + + mod = types.ModuleType("claude_agent_sdk") + mod.query = query + mod.ClaudeAgentOptions = lambda **kwargs: kwargs + mod.AssistantMessage = type("AssistantMessage", (), {}) + mod.ResultMessage = type("ResultMessage", (), {}) + monkeypatch.setitem(sys.modules, "claude_agent_sdk", mod) + cli, _ = make_cli(tmp_path, SIGNED_IN) + schema = {"type": "object", "properties": {}} + assert llm._call_claude_via_sdk("review this", schema, cli_path=str(cli)) is None + assert llm._gateway_review_timed_out is True + assert ("gateway sdk-single-turn: timeout after 120s; the CLI's API requests were " + "retried after status 401") in (tmp_path / "log.txt").read_text() + # Not for the cloud-provider route, where the model can be the reason. + monkeypatch.setattr(llm, "_gateway_review_timed_out", False) + assert llm._call_claude_via_sdk("review this", schema) is None + assert llm._gateway_review_timed_out is False + + +def without_credential(hook_env): + return {k: v for k, v in hook_env.items() + if k not in ("ANTHROPIC_API_KEY", "SECURITY_REVIEW_MODEL", + "CLAUDE_CODE_REMOTE_SESSION_ID", *llm._PROVIDER_ENV_VARS)} + + +class TestHooksInASignedInSession: + """The hooks with the real Agent SDK and a fake CLI that signs in but + refuses every review.""" + + @pytest.fixture + def session(self, hook_env, tmp_path): + pytest.importorskip("claude_agent_sdk") + env = without_credential(hook_env) + cli, calls = make_cli(tmp_path, SIGNED_IN) + env["SG_AGENTIC_CLI_PATH"] = str(cli) + return env, calls + + def test_commit_review_goes_through_the_cli_and_fails_loudly(self, session, workspace): + env, calls = session + _, repo = workspace + sha, out = commit_file(repo, "app.py", VULN_PY) + rc, so, se = run_hook(bash_payload(repo, "git commit -m change", out), env) + assert rc == 2 + assert metrics_of(so)["review_failed"] is True + assert ("did not complete: the review through Claude Code on the Claude apps " + "gateway returned no result.") in se + reviewed = repo / ".git" / "sg-reviewed-shas" + assert not reviewed.exists() or sha not in reviewed.read_text() + made = calls_of(calls) + assert made[0].startswith("auth status --json") + reviews = review_calls(calls) + assert [model_of(r) for r in reviews] == ["opus", "sonnet"] + # Every claude process the plugin starts (the sign-in check and the + # reviews) has this plugin switched off. The Agent SDK's own + # `claude -v` version check starts no session, so it is not checked. + assert " | SGD=1 | " in made[0] + assert all(" | SGD=1 | " in r for r in reviews) + # The single-shot review gets no tools, as on Bedrock and Vertex. + assert all(" --tools --strict-mcp-config" in r for r in reviews) + assert metrics_of(so)["gateway_cfg"] & 4 + + def test_pinned_model_is_sent_as_given(self, session, workspace): + env, calls = session + env["SECURITY_REVIEW_MODEL"] = "claude-sonnet-5-5" + _, repo = workspace + _, out = commit_file(repo, "app.py", VULN_PY) + rc, so, se = run_hook(bash_payload(repo, "git commit -m change", out), env) + assert rc == 2 + assert [model_of(r) for r in review_calls(calls)] == ["claude-sonnet-5-5"] + assert "SECURITY_REVIEW_MODEL (claude-sonnet-5-5) may also name a model" in se + + def test_agentic_commit_review_runs(self, session, workspace): + env, calls = session + env["SG_AGENTIC_COMMIT_REVIEW"] = "1" + env["SG_AGENTIC_MAX_TURNS"] = "7" + _, repo = workspace + _, out = commit_file(repo, "app.py", VULN_PY) + rc, so, se = run_hook(bash_payload(repo, "git commit -m change", out), env) + assert rc == 2 and metrics_of(so)["review_failed"] is True + reviews = review_calls(calls) + assert any("--max-turns 7" in r for r in reviews) + # The agentic reviewer gets the file-reading tools and nothing else. + assert any(" --tools Read,Grep,Glob --strict-mcp-config" in r for r in reviews) + + def test_stop_review_goes_through_the_cli(self, session, workspace): + env, calls = session + ws, repo = workspace + assert run_hook(ups_payload(repo), env)[0] == 0 + (repo / "app.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py", VULN_PY), env) + rc, so, se = run_hook(stop_payload(repo), env) + assert metrics_of(so)["review_failed"] is True + assert review_calls(calls) + + def test_push_review_goes_through_the_cli(self, session, workspace, tmp_path): + env, calls = session + ws, repo = workspace + remote = tmp_path / "remote.git" + git(ws, "init", "-q", "--bare", str(remote)) + git(repo, "remote", "add", "origin", str(remote)) + git(repo, "push", "-q", "-u", "origin", "main") + base = git(repo, "rev-parse", "HEAD").strip() + sha, _ = commit_file(repo, "app.py", VULN_PY) + git(repo, "push", "-q", "origin", "main") + push_stdout = f"To {remote}\n {base[:7]}..{sha[:7]} main -> main\n" + rc, so, se = run_hook(bash_payload(repo, "git push origin main", push_stdout), env) + m = metrics_of(so) + assert rc == 2 and m["push_sweep"] is True and m["review_failed"] is True, m + assert review_calls(calls) + + def test_not_signed_in_skips_and_asks_once(self, hook_env, tmp_path, workspace): + env = without_credential(hook_env) + cli, calls = make_cli(tmp_path, NOT_SIGNED_IN) + env["SG_AGENTIC_CLI_PATH"] = str(cli) + _, repo = workspace + for n in range(2): + _, out = commit_file(repo, "app.py", VULN_PY + f"# {n}\n") + rc, so, se = run_hook(bash_payload(repo, "git commit -m change", out), env) + assert rc == 0 and metrics_of(so)["skip_reason"] == 22 + assert len(calls_of(calls)) == 1 + + def test_unanswered_sign_in_check_keeps_the_turn_for_the_next_stop( + self, hook_env, tmp_path, workspace): + pytest.importorskip("claude_agent_sdk") + env = without_credential(hook_env) + cli, calls = make_cli(tmp_path, "") + env["SG_AGENTIC_CLI_PATH"] = str(cli) + _, repo = workspace + assert run_hook(ups_payload(repo), env)[0] == 0 + (repo / "app.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py", VULN_PY), env) + rc, so, _ = run_hook(stop_payload(repo), env) + assert metrics_of(so)["skip_reason"] == 3 and not review_calls(calls) + # The CLI answers in the next turn: the first turn's edit is reviewed. + make_cli(tmp_path, SIGNED_IN) + assert run_hook(ups_payload(repo), env)[0] == 0 + rc, so, _ = run_hook(stop_payload(repo), env) + assert metrics_of(so)["review_failed"] is True + assert review_calls(calls) diff --git a/plugins/security-guidance/tests/test_git_index_locks.py b/plugins/security-guidance/tests/test_git_index_locks.py new file mode 100644 index 00000000..6faaddd3 --- /dev/null +++ b/plugins/security-guidance/tests/test_git_index_locks.py @@ -0,0 +1,317 @@ +import os +import shutil +import subprocess +import tempfile +import time + +import pytest + +from conftest import ( + GIT_ENV, VULN_PY, commit_file, bash_payload, edit_payload, git, make_repo, + metrics_of, run_hook, stop_payload, ups_payload, +) + +import diffstate +import gitutil + + +@pytest.fixture +def tmproot(tmp_path, monkeypatch): + """A temp dir of our own, and the environment restored afterwards + (apply_safe_git_env writes to os.environ).""" + d = tmp_path / "tmproot" + d.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(d)) + saved = dict(os.environ) + os.environ.update(GIT_ENV) + yield d + os.environ.clear() + os.environ.update(saved) + + +def _leftovers(root): + """Everything left under `root` but the slot files, which stay.""" + return sorted( + os.path.join(dirpath, f) + for dirpath, _, files in os.walk(root) for f in files + if not (f.startswith("slot") and f.endswith(".lock")) + ) + + +def _index_copies_left(root): + """Index copies and the files git writes next to them. The whole hook may + also start the detached SDK bootstrap, which uses the temp dir too.""" + return [p for p in _leftovers(root) + if os.path.basename(p).startswith(gitutil._TEMP_INDEX_PREFIX)] + + +def _leftovers_of_copies(root): + return _index_copies_left(root) + + +def _index_litter(git_dir): + return sorted(n for n in os.listdir(git_dir) if n.startswith("index") and n != "index") + + +def _index_sig(git_dir): + st = os.stat(os.path.join(git_dir, "index")) + return (st.st_ino, st.st_mtime_ns, st.st_size) + + +def _make_stat_stale(path): + """Move a tracked file's mtime, content unchanged: a stat mismatch that + git would normally write back to the index it read.""" + t = os.stat(path).st_mtime + 2 + os.utime(path, (t, t)) + + +def _is_snapshot_add(args): + """The `git add -u` that stages the turn baseline's tracked changes.""" + return list(args[-2:]) == ["add", "-u"] + + +class TestBaselineCapture: + def test_snapshot_gets_a_throwaway_index(self, tmp_path, tmproot, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("x = 2\n") + before = _index_sig(repo / ".git") + seen = [] + real_run = subprocess.run + + def spy(args, **kw): + if _is_snapshot_add(args): + seen.append(kw["env"]["GIT_INDEX_FILE"]) + return real_run(args, **kw) + + monkeypatch.setattr(subprocess, "run", spy) + assert diffstate.capture_git_baseline(str(repo)) + assert len(seen) == 1 + assert os.path.realpath(seen[0]).startswith(os.path.realpath(tmproot) + os.sep) + assert _index_sig(repo / ".git") == before + assert _index_litter(repo / ".git") == [] + assert _leftovers(tmproot) == [] + + def test_killed_snapshot_leaves_nothing(self, tmp_path, tmproot, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("x = 2\n") + real_run = subprocess.run + + def killed(args, **kw): + if _is_snapshot_add(args): + idx = kw["env"]["GIT_INDEX_FILE"] + for suffix in (".lock", ".stash.4242", ".stash.4242.lock"): + with open(idx + suffix, "wb") as f: + f.write(b"partial") + raise subprocess.TimeoutExpired(args, kw["timeout"]) + return real_run(args, **kw) + + monkeypatch.setattr(subprocess, "run", killed) + assert diffstate.capture_git_baseline(str(repo)) is None + assert _index_litter(repo / ".git") == [] + assert _leftovers(tmproot) == [] + + def test_succeeds_while_index_lock_exists(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("x = 2\n") + head = git(repo, "rev-parse", "HEAD").strip() + (repo / ".git" / "index.lock").touch() + sha = diffstate.capture_git_baseline(str(repo)) + assert sha and sha != head + assert git(repo, "show", f"{sha}:app.py") == "x = 2\n" + assert _index_litter(repo / ".git") == ["index.lock"] + + def test_dirty_tree_matches_a_plain_stash_create(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("staged\n") + git(repo, "add", "app.py") + (repo / "app.py").write_text("staged\nunstaged\n") + gitutil.apply_safe_git_env() + ours = diffstate.capture_git_baseline(str(repo)) + plain = git(repo, "stash", "create").strip() + assert git(repo, "show", f"{ours}:app.py") == "staged\nunstaged\n" + assert (git(repo, "rev-parse", f"{ours}^{{tree}}") + == git(repo, "rev-parse", f"{plain}^{{tree}}")) + + def test_clean_tree_returns_head(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo") + head = git(repo, "rev-parse", "HEAD").strip() + assert diffstate.capture_git_baseline(str(repo)) == head + + def test_missing_index_returns_head(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo") + head = git(repo, "rev-parse", "HEAD").strip() + (repo / ".git" / "index").unlink() + assert diffstate.capture_git_baseline(str(repo)) == head + assert _index_litter(repo / ".git") == [] + + def test_linked_worktree_uses_its_own_index(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + wt = tmp_path / "wt" + git(repo, "worktree", "add", "-q", "-b", "side", str(wt)) + assert (wt / ".git").is_file() + wt_git_dir = repo / ".git" / "worktrees" / "wt" + (wt / "app.py").write_text("x = 3\n") + (wt_git_dir / "index.lock").touch() + sha = diffstate.capture_git_baseline(str(wt)) + assert git(wt, "show", f"{sha}:app.py") == "x = 3\n" + assert _index_litter(wt_git_dir) == ["index.lock"] + assert _index_litter(repo / ".git") == [] + assert _leftovers(tmproot) == [] + + +class TestRacilyCleanEdit: + def test_same_size_edit_in_the_second_the_index_was_written(self, tmp_path, tmproot): + """Every stat field git caches still matches, so git only notices by + comparing the file's mtime with the index file's own mtime.""" + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + git(repo, "config", "core.trustctime", "false") + then = int(time.time()) - 100 + os.utime(repo / "app.py", (then, then)) + git(repo, "update-index", "--refresh") + (repo / "app.py").write_text("x = 2\n") + os.utime(repo / "app.py", (then, then)) + os.utime(repo / ".git" / "index", (then, then)) + gitutil.apply_safe_git_env() + sha = diffstate.capture_git_baseline(str(repo)) + assert git(repo, "show", f"{sha}:app.py") == "x = 2\n" + assert gitutil._git_name_only(str(repo), "HEAD") == {"app.py"} + assert "+x = 2" in gitutil.get_git_diff(str(repo), "HEAD", untracked_paths=[]) + + +class TestReadOnlyGitLeavesTheRealIndexAlone: + def test_env_carries_optional_locks_off(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo") + os.environ.pop("GIT_OPTIONAL_LOCKS", None) + gitutil.apply_safe_git_env() + assert os.environ["GIT_OPTIONAL_LOCKS"] == "0" + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + assert env["GIT_OPTIONAL_LOCKS"] == "0" + + def test_status(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "new.py").write_text("n = 1\n") + _make_stat_stale(repo / "app.py") + before = _index_sig(repo / ".git") + gitutil.apply_safe_git_env() + assert gitutil._git_status_porcelain(str(repo)) == (set(), {"new.py"}) + assert _index_sig(repo / ".git") == before + + def test_work_tree_diff(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n", "b.py": "b = 1\n"}) + (repo / "b.py").write_text("b = 2\n") + _make_stat_stale(repo / "app.py") + before = _index_sig(repo / ".git") + head = git(repo, "rev-parse", "HEAD").strip() + gitutil.apply_safe_git_env() + # app.py: stat changed, content did not. The name list may hold it, + # the review set must not. + assert "b.py" in gitutil._git_name_only(str(repo), "HEAD") + paths, _, _, _, _ = diffstate.compute_v2_review_set(str(repo), head, head) + assert paths == [os.path.join(os.path.realpath(repo), "b.py")] + assert _index_sig(repo / ".git") == before + assert _leftovers(tmproot) == [] + + def test_diffs_need_no_index_copy(self, tmp_path, tmproot, monkeypatch): + """Every copy is index-sized, so only `add -N` and the snapshot get one.""" + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n", "b.py": "b = 1\n"}) + first = git(repo, "rev-parse", "HEAD").strip() + commit_file(repo, "app.py", "x = 2\n") + (repo / "b.py").write_text("b = 2\n") + _make_stat_stale(repo / "app.py") + before = _index_sig(repo / ".git") + + def no_copy(*a, **kw): + raise AssertionError("copied the index") + + monkeypatch.setattr(gitutil, "_temp_index", no_copy) + assert gitutil._git_name_only(str(repo), f"{first}..HEAD") == {"app.py"} + assert "b.py" in gitutil._git_name_only(str(repo), "HEAD") + diff = gitutil.get_git_diff(str(repo), "HEAD", untracked_paths=[]) + assert "+b = 2" in diff and "app.py" not in diff + assert _index_sig(repo / ".git") == before + + def test_untracked_files_still_reach_the_diff(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "new.py").write_text("n = 1\n") + before = _index_sig(repo / ".git") + diff = gitutil.get_git_diff(str(repo), "HEAD", untracked_paths=["new.py"]) + assert "+n = 1" in diff + assert gitutil._git_name_only(str(repo), "HEAD", include_untracked=True) == {"new.py"} + assert _index_sig(repo / ".git") == before + assert _leftovers_of_copies(tmproot) == [] + + +@pytest.mark.skipif(os.name == "nt", reason="uses a /bin/sh git shim") +class TestWholeHook: + def _shim(self, tmp_path, hook_env): + """Put a logging `git` first on PATH; returns (env, read_log).""" + real_git = shutil.which("git") + bindir = tmp_path / "bin" + bindir.mkdir() + log = tmp_path / "git.log" + shim = bindir / "git" + shim.write_text( + "#!/bin/sh\n" + f'printf "%s\\t%s\\t%s\\n" "$GIT_OPTIONAL_LOCKS" "$GIT_INDEX_FILE" "$*" >> "{log}"\n' + f'exec "{real_git}" "$@"\n' + ) + shim.chmod(0o755) + tmproot = tmp_path / "tmproot" + tmproot.mkdir() + env = dict(hook_env) + env.pop("GIT_OPTIONAL_LOCKS", None) + env["PATH"] = f"{bindir}{os.pathsep}{env['PATH']}" + env["TMPDIR"] = str(tmproot) + + prefix = " ".join(gitutil.GIT_CMD[1:]) + " " + + def read_log(): + """[(optional_locks, index_file, [subcommand, *args])] per call.""" + calls = [] + for ln in log.read_text().splitlines(): + locks, index_file, argv = ln.split("\t", 2) + if argv.startswith(prefix): + argv = argv[len(prefix):] + calls.append((locks, index_file, argv.split())) + return calls + + return env, tmproot, read_log + + def test_every_git_call_of_a_turn(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n", "wip.py": "w = 1\n"}) + env, tmproot, read_log = self._shim(tmp_path, hook_env) + (repo / "wip.py").write_text("w = 2\n") + _make_stat_stale(repo / "app.py") + before = _index_sig(repo / ".git") + + run_hook(ups_payload(repo), env) + (repo / "app.py").write_text(VULN_PY) + (repo / "new.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py", VULN_PY), env) + run_hook(edit_payload(repo, repo / "new.py", VULN_PY), env) + rc, so, se = run_hook(stop_payload(repo), env) + m = metrics_of(so) + assert m.get("skip_reason") is None and m["files_reviewed"] == 2, m + + calls = read_log() + assert {"write-tree", "status", "diff", "add"} <= {sub[0] for _, _, sub in calls} + assert [c for c in calls if c[0] != "0"] == [] + for _, index_file, sub in calls: + if sub[0] in ("write-tree", "commit-tree", "add"): + assert index_file.startswith(str(tmproot)), sub + assert _index_sig(repo / ".git") == before + assert _index_litter(repo / ".git") == [] + assert _index_copies_left(tmproot) == [] + + def test_every_git_call_of_a_commit_review(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + env, tmproot, read_log = self._shim(tmp_path, hook_env) + sha, out = commit_file(repo, "app.py", VULN_PY) + before = _index_sig(repo / ".git") + rc, so, se = run_hook(bash_payload(repo, "git commit -m change", stdout=out), env) + assert metrics_of(so).get("skip_reason") is None + calls = read_log() + assert calls and [c for c in calls if c[0] != "0"] == [] + assert _index_sig(repo / ".git") == before + assert _index_litter(repo / ".git") == [] + assert _index_copies_left(tmproot) == [] diff --git a/plugins/security-guidance/tests/test_git_program.py b/plugins/security-guidance/tests/test_git_program.py new file mode 100644 index 00000000..9249c6e4 --- /dev/null +++ b/plugins/security-guidance/tests/test_git_program.py @@ -0,0 +1,115 @@ +"""On Windows the hooks name git by its full path, found on PATH, so a git.exe +in the project is not the one that starts.""" +import json +import os +import subprocess +import sys + +import pytest + +from conftest import HOOK_SCRIPT, make_repo, ups_payload, write_cli + +import gitutil + +windows_only = pytest.mark.skipif(os.name != "nt", reason="Windows program lookup") + + +def _git_in(d): + d.mkdir(parents=True, exist_ok=True) + (d / "git.exe").write_bytes(b"") + return str(d / "git.exe") + + +def test_bare_name_outside_windows(): + assert gitutil._git_program(windows=False) == "git" + + +def test_first_git_on_path(tmp_path, monkeypatch): + first, second = tmp_path / "a", tmp_path / "b" + first.mkdir() + expected = _git_in(second) + _git_in(tmp_path / "c") + monkeypatch.setenv("PATH", os.pathsep.join([str(first), str(second), str(tmp_path / "c")])) + assert gitutil._git_program(windows=True) == expected + + +def test_working_directory_and_relative_entries_are_left_out(tmp_path, monkeypatch): + project = tmp_path / "project" + _git_in(project) + _git_in(project / "bin") + expected = _git_in(tmp_path / "installed") + monkeypatch.chdir(project) + monkeypatch.delenv("CLAUDE_PROJECT_DIR", raising=False) + monkeypatch.setenv("PATH", os.pathsep.join( + ["", ".", "bin", str(project), str(project) + os.sep, str(tmp_path / "installed")])) + assert gitutil._git_program(windows=True) == expected + + +def test_folder_inside_the_project_is_left_out(tmp_path, monkeypatch): + project = tmp_path / "project" + _git_in(project) + _git_in(project / ".venv" / "Scripts") + (project / "pkg").mkdir() + expected = _git_in(tmp_path / "installed") + monkeypatch.chdir(project / "pkg") + monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(project)) + monkeypatch.setenv("PATH", os.pathsep.join( + [str(project), str(project / ".venv" / "Scripts"), str(tmp_path / "installed")])) + assert gitutil._git_program(windows=True) == expected + + +def test_project_above_the_home_folder_keeps_the_users_tools(tmp_path, monkeypatch): + home = tmp_path / "me" + expected = _git_in(home / "scoop" / "shims") + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("CLAUDE_PROJECT_DIR", raising=False) + monkeypatch.setenv("PATH", str(home / "scoop" / "shims")) + assert gitutil._git_program(windows=True) == expected + + +def test_similar_folder_name_stays(tmp_path, monkeypatch): + project = tmp_path / "project" + project.mkdir() + expected = _git_in(tmp_path / "project-tools") + monkeypatch.chdir(project) + monkeypatch.delenv("CLAUDE_PROJECT_DIR", raising=False) + monkeypatch.setenv("PATH", str(tmp_path / "project-tools")) + assert gitutil._git_program(windows=True) == expected + + +def test_quoted_entry(tmp_path, monkeypatch): + expected = _git_in(tmp_path / "installed") + monkeypatch.setenv("PATH", f'"{tmp_path / "installed"}"') + assert gitutil._git_program(windows=True) == expected + + +def test_no_git_on_path_names_nothing_that_exists(tmp_path, monkeypatch): + project = tmp_path / "project" + _git_in(project) + monkeypatch.chdir(project) + monkeypatch.setenv("PATH", str(project)) + program = gitutil._git_program(windows=True) + assert os.path.isabs(program) and not os.path.exists(program) + + +def test_working_directory_that_is_gone(tmp_path, monkeypatch): + expected = _git_in(tmp_path / "installed") + monkeypatch.setattr(gitutil.os, "getcwd", lambda: (_ for _ in ()).throw(OSError())) + monkeypatch.setenv("PATH", str(tmp_path / "installed")) + assert gitutil._git_program(windows=True) == expected + + +@windows_only +def test_git_exe_in_the_project_does_not_start(tmp_path, env): + marker = tmp_path / "started" + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + write_cli(repo, "git", f"open({str(marker)!r}, 'a').write('x')\n") + + r = subprocess.run( + [sys.executable, str(HOOK_SCRIPT)], input=json.dumps(ups_payload(repo)), + capture_output=True, text=True, env=env, cwd=repo, timeout=120) + + assert r.returncode == 0, r.stderr + assert not marker.exists() diff --git a/plugins/security-guidance/tests/test_inflight_reviews.py b/plugins/security-guidance/tests/test_inflight_reviews.py new file mode 100644 index 00000000..fa998501 --- /dev/null +++ b/plugins/security-guidance/tests/test_inflight_reviews.py @@ -0,0 +1,494 @@ +"""A commit still under review is not sent through a second review.""" +import contextlib +import json +import os +import subprocess +import sys +import threading +import time + +import pytest + +from conftest import ( + HOOK_SCRIPT, bash_payload, commit_file, git, make_repo, metrics_of, run_hook, +) + +import diffstate + +_HOOKS_DIR = os.path.dirname(diffstate.__file__) + +# Locks the way a review does on this platform: flock, or LockFileEx on Windows. +_HOLD = ( + "import os, sys, time\n" + "sys.path.insert(0, sys.argv[3])\n" + "import diffstate\n" + "fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT, 0o600)\n" + "assert diffstate._flock_nb(fd, exclusive=True)\n" + "print('held', flush=True)\n" + "time.sleep(float(sys.argv[2]))\n" +) + + +@contextlib.contextmanager +def _held_by_another_review(repo, sha, seconds=60): + """A marker on `sha` that a live process holds, as a running review does.""" + marker = repo / ".git" / f"sg-inflight-{sha}" + p = subprocess.Popen( + [sys.executable, "-c", _HOLD, str(marker), str(seconds), _HOOKS_DIR], + stdout=subprocess.PIPE, text=True) + try: + assert p.stdout.readline().strip() == "held" + yield marker + finally: + p.kill() + p.wait() + p.stdout.close() + + +@contextlib.contextmanager +def _hook_in_background(payload, env): + p = subprocess.Popen( + [sys.executable, str(HOOK_SCRIPT)], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, env=env, + ) + try: + p.stdin.write(json.dumps(payload)) + p.stdin.close() + yield p + finally: + if p.poll() is None: + p.kill() + p.wait() + p.stdout.close() + + +def _repo_with_remote(tmp_path): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + remote = tmp_path / "remote.git" + git(tmp_path, "init", "-q", "--bare", str(remote)) + git(repo, "remote", "add", "origin", str(remote)) + git(repo, "push", "-q", "-u", "origin", "main") + return repo, remote + + +def _markers(repo): + return [n for n in os.listdir(repo / ".git") if n.startswith("sg-inflight-")] + + +def _wait_until(cond, what, timeout=20): + end = time.time() + timeout + while time.time() < end: + if cond(): + return + time.sleep(0.05) + raise AssertionError(f"timed out waiting for {what}") + + +def _reviews(stub_api): + return [c for c in stub_api.calls if c.endswith("/v1/messages")] + + +def _commit_and_push(repo, remote): + base = git(repo, "rev-parse", "HEAD").strip() + sha, commit_out = commit_file(repo, "handlers.py", "def h(x):\n return x\n") + git(repo, "push", "-q", "origin", "main") + push_out = f"To {remote}\n {base[:7]}..{sha[:7]} main -> main\n" + return base, sha, commit_out, push_out + + +def test_push_right_after_commit_runs_one_review(tmp_path, hook_env, stub_api): + repo, remote = _repo_with_remote(tmp_path) + base, sha, commit_out, push_out = _commit_and_push(repo, remote) + stub_api.delay = 6 + with _hook_in_background(bash_payload(repo, "git commit -m change", commit_out), + hook_env) as commit: + _wait_until(lambda: _markers(repo), "the commit review's marker") + rc, so, _ = run_hook(bash_payload(repo, "git push origin main", push_out), hook_env) + commit.wait(timeout=60) + + m = metrics_of(so) + assert rc == 0 and m["push_sweep"] is True + assert m["pushed"] == 1 and m["unreviewed"] == 0 + assert m.get("inflight_wait_s", 0) >= 1 + assert len(_reviews(stub_api)) == 1 + assert sha in diffstate._load_reviewed_shas(str(repo)) + assert _markers(repo) == [] + + +def test_push_reviews_commit_whose_review_died(tmp_path, hook_env, stub_api): + repo, remote = _repo_with_remote(tmp_path) + base, sha, _, push_out = _commit_and_push(repo, remote) + with _held_by_another_review(repo, sha): + pass # killed: its marker file stays, its lock does not + assert _markers(repo) == [f"sg-inflight-{sha}"] + rc, so, _ = run_hook(bash_payload(repo, "git push origin main", push_out), hook_env) + + m = metrics_of(so) + assert m["unreviewed"] == 1 and "inflight_wait_s" not in m + assert len(_reviews(stub_api)) == 1 + assert _markers(repo) == [] + + +def test_push_reviews_commit_whose_review_ended_without_a_verdict(tmp_path, hook_env, stub_api): + repo, remote = _repo_with_remote(tmp_path) + base, sha, _, push_out = _commit_and_push(repo, remote) + with _held_by_another_review(repo, sha, seconds=3): + rc, so, _ = run_hook(bash_payload(repo, "git push origin main", push_out), hook_env) + + m = metrics_of(so) + assert m["unreviewed"] == 1 + assert m["files_reviewed"] == 1 and len(_reviews(stub_api)) == 1 + assert sha in diffstate._load_reviewed_shas(str(repo)) + + +def test_push_stops_waiting_after_the_limit(tmp_path, hook_env, stub_api): + repo, remote = _repo_with_remote(tmp_path) + base, sha, _, push_out = _commit_and_push(repo, remote) + env = {**hook_env, "SG_INFLIGHT_WAIT_S": "2"} + with _held_by_another_review(repo, sha): + rc, so, _ = run_hook(bash_payload(repo, "git push origin main", push_out), env) + + m = metrics_of(so) + assert m["unreviewed"] == 1 and m["inflight_wait_s"] >= 2 + assert len(_reviews(stub_api)) == 1 + + +def test_a_push_that_waited_ends_within_the_hook_limit(tmp_path, monkeypatch): + """The wait for a commit review and the race share one limit; a race that + restarts it runs the hook past Claude Code's timeout, with no notice.""" + import security_reminder_hook as hook + + repo, remote = _repo_with_remote(tmp_path) + _, sha, _, _ = _commit_and_push(repo, remote) + monkeypatch.setattr(hook, "_RACE_MAX_WAIT_S", 1.0) + monkeypatch.setattr(hook, "_RACE_END_S", 1.5) + monkeypatch.delenv("SG_AGENTIC_NO_RACE", raising=False) + monkeypatch.setenv("SG_AGENTIC_RACE_DELAY_S", "0") + release = threading.Event() + monkeypatch.setattr(hook, "agentic_review", lambda *a, **k: release.wait(20)) + monkeypatch.setattr(hook, "analyze_code_security", lambda *a, **k: release.wait(20)) + + started = time.monotonic() + try: + with _held_by_another_review(repo, sha): + _, waited = diffstate._reviewed_after_inflight(str(repo), [sha], 1.0, poll_s=0.05) + _, _, m = hook._agentic_review_with_race( + str(repo), [], [], [], spent_s=waited) + finally: + release.set() + assert 1.0 <= waited < 1.5 + assert 1.5 <= time.monotonic() - started < 2.4 + assert m["review_failed"] is True and m["api_error"] == hook._RACE_TIMED_OUT + + +def test_the_push_handler_counts_its_wait(tmp_path, agentic_env, stub_api): + repo, remote = _repo_with_remote(tmp_path) + _, sha, _, push_out = _commit_and_push(repo, remote) + log = tmp_path / "race.log" + env = {**agentic_env, "SG_INFLIGHT_WAIT_S": "2", "SG_AGENTIC_RACE_DELAY_S": "60", + "SECURITY_GUIDANCE_DEBUG_LOG": str(log)} + with _held_by_another_review(repo, sha): + _, so, _ = run_hook(bash_payload(repo, "git push origin main", push_out), env) + assert metrics_of(so)["inflight_wait_s"] >= 2 + assert "already spent, fallback starts after 5" in log.read_text() + + +def test_the_fallback_starts_sooner_after_a_wait(tmp_path, monkeypatch): + import security_reminder_hook as hook + + monkeypatch.delenv("SG_AGENTIC_NO_RACE", raising=False) + monkeypatch.setenv("SG_AGENTIC_RACE_DELAY_S", "180") + release = threading.Event() + monkeypatch.setattr(hook, "agentic_review", lambda *a, **k: release.wait(20)) + monkeypatch.setattr(hook, "analyze_code_security", lambda *a, **k: ("guidance", [])) + monkeypatch.setattr(hook.llm, "last_review_failed", lambda: False) + try: + g, _, m = hook._agentic_review_with_race(str(tmp_path), [], [], [], spent_s=240.0) + finally: + release.set() + assert g == "guidance" + assert (m["race_winner"], m["race_delay_s"], m["race_started"]) == (2, 0, 1) + + +def test_push_does_not_wait_when_an_earlier_commit_needs_review_anyway(tmp_path, hook_env, stub_api): + repo, remote = _repo_with_remote(tmp_path) + base = git(repo, "rev-parse", "HEAD").strip() + commit_file(repo, "outside.py", "def o(x):\n return x\n") + sha, _ = commit_file(repo, "handlers.py", "def h(x):\n return x\n") + git(repo, "push", "-q", "origin", "main") + push_out = f"To {remote}\n {base[:7]}..{sha[:7]} main -> main\n" + with _held_by_another_review(repo, sha): + rc, so, _ = run_hook(bash_payload(repo, "git push origin main", push_out), hook_env) + + m = metrics_of(so) + assert m["pushed"] == 2 and m["unreviewed"] == 2 and "inflight_wait_s" not in m + assert m["files_reviewed"] == 2 + + +def test_push_does_not_record_the_range_when_head_moves_during_the_wait(tmp_path, hook_env, stub_api): + repo, remote = _repo_with_remote(tmp_path) + base, sha, _, push_out = _commit_and_push(repo, remote) + env = {**hook_env, "SG_INFLIGHT_WAIT_S": "10"} + with _held_by_another_review(repo, sha): + with _hook_in_background(bash_payload(repo, "git push origin main", push_out), + env) as push: + time.sleep(5) + git(repo, "checkout", "-q", "-b", "elsewhere", base) + so = push.stdout.read() + push.wait(timeout=60) + + m = metrics_of(so) + assert m["skip_reason"] == 44 and m.get("inflight_wait_s", 0) >= 1 + assert _reviews(stub_api) == [] + assert sha not in diffstate._load_reviewed_shas(str(repo)) + + +def test_next_quiet_commit_leaves_a_running_review_its_commit(tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + commit_file(repo, "a.py", "def a(x):\n return x\n") + stub_api.delay = 6 + with _hook_in_background(bash_payload(repo, "git commit -q -m a"), hook_env) as first: + _wait_until(lambda: _markers(repo), "the first review's marker") + commit_file(repo, "b.py", "def b(x):\n return x\n") + rc, so, _ = run_hook(bash_payload(repo, "git commit -q -m b"), hook_env) + first.wait(timeout=60) + + m = metrics_of(so) + assert m["sha_via_reflog"] is True and m["reflog_shas_n"] == 1 + assert m["files_reviewed"] == 1 + assert _markers(repo) == [] + + +def test_marker_is_not_taken_from_a_live_review(tmp_path): + repo = make_repo(tmp_path / "repo") + sha = git(repo, "rev-parse", "HEAD").strip() + with _held_by_another_review(repo, sha) as marker: + diffstate._mark_inflight(str(repo), [sha]) + assert diffstate._inflight_held == {} + assert diffstate._load_inflight_shas(str(repo), [sha]) == {sha} + assert diffstate._load_inflight_shas(str(repo), [sha]) == set() + assert marker.exists() # fresh leftovers stay; old ones are removed + old = time.time() - diffstate._INFLIGHT_GC_AGE_S - 5 + os.utime(marker, (old, old)) + diffstate._mark_inflight(str(repo), []) + assert not marker.exists() + + +def test_no_marker_is_left_where_files_cannot_be_locked(tmp_path, monkeypatch): + repo = make_repo(tmp_path / "repo") + sha = git(repo, "rev-parse", "HEAD").strip() + + def no_locks(fd, exclusive): + raise OSError(37, "No locks available") + monkeypatch.setattr(diffstate, "_flock_nb", no_locks) + diffstate._mark_inflight(str(repo), [sha]) + assert _markers(repo) == [] and diffstate._inflight_held == {} + (repo / ".git" / f"sg-inflight-{sha}").write_text("") + assert diffstate._load_inflight_shas(str(repo), [sha]) == set() + + +def test_own_marker_is_held_until_released(tmp_path): + repo = make_repo(tmp_path / "repo") + sha = git(repo, "rev-parse", "HEAD").strip() + diffstate._mark_inflight(str(repo), [sha, "not-a-sha"]) + marker = repo / ".git" / f"sg-inflight-{sha}" + try: + assert _markers(repo) == [marker.name] + assert diffstate._load_inflight_shas(str(repo), [sha]) == set() + probe = ( + "import sys; sys.path.insert(0, sys.argv[1]); import diffstate; " + "print(sorted(diffstate._load_inflight_shas(sys.argv[2], [sys.argv[3]])))" + ) + seen = subprocess.run( + [sys.executable, "-c", probe, _HOOKS_DIR, str(repo), sha], + capture_output=True, text=True).stdout.strip() + assert seen == str([sha]) + reviewed, waited = diffstate._reviewed_after_inflight(str(repo), [sha], 5) + assert reviewed == set() and waited == 0.0 + finally: + diffstate._release_inflight() + assert not marker.exists() + + +def test_marker_is_closed_before_it_is_deleted_where_open_files_cannot_be(tmp_path, monkeypatch): + # The order used on Windows, where deleting an open file is refused. + repo = make_repo(tmp_path / "repo") + sha = git(repo, "rev-parse", "HEAD").strip() + marker = repo / ".git" / f"sg-inflight-{sha}" + monkeypatch.setattr(diffstate, "_UNLINK_BEFORE_CLOSE", False) + open_at_delete, refusals = [], [2] + + def delete_like_windows(p, fd): + try: + os.fstat(fd) + open_at_delete.append(True) + except OSError: + open_at_delete.append(False) + if refusals[0]: + # Another review asking about the marker has it open just now. + refusals[0] -= 1 + raise PermissionError(13, "The process cannot access the file") + real_unlink(p) + + real_unlink = os.unlink + try: + diffstate._mark_inflight(str(repo), [sha]) + (fd,) = diffstate._inflight_held.values() + monkeypatch.setattr(diffstate.os, "unlink", lambda p: delete_like_windows(p, fd)) + diffstate._release_inflight() + finally: + monkeypatch.setattr(diffstate.os, "unlink", real_unlink) + diffstate._release_inflight() + assert open_at_delete == [False, False, False] and not marker.exists() + + # An old leftover goes the same way. + marker.write_text("") + old = time.time() - diffstate._INFLIGHT_GC_AGE_S - 5 + os.utime(marker, (old, old)) + diffstate._mark_inflight(str(repo), []) + assert not marker.exists() + + +def test_no_markers_in_a_python_without_file_locks(tmp_path, monkeypatch): + if os.name == "nt": + pytest.skip("Windows locks with LockFileEx") + repo = make_repo(tmp_path / "repo") + sha = git(repo, "rev-parse", "HEAD").strip() + monkeypatch.setitem(sys.modules, "fcntl", None) + diffstate._mark_inflight(str(repo), [sha]) + assert _markers(repo) == [] and diffstate._inflight_held == {} + (repo / ".git" / f"sg-inflight-{sha}").write_text("") + assert diffstate._load_inflight_shas(str(repo), [sha]) == set() + + +def test_one_marker_that_cannot_be_claimed_does_not_stop_the_others(tmp_path, monkeypatch): + repo = make_repo(tmp_path / "repo") + first = git(repo, "rev-parse", "HEAD").strip() + second, _ = commit_file(repo, "b.py", "b = 1\n") + third, _ = commit_file(repo, "c.py", "c = 1\n") + # Cannot be opened at all. + (repo / ".git" / f"sg-inflight-{first}").mkdir() + # Its last holder deletes it between this review's open and its lock. + real_flock_nb = diffstate._flock_nb + + def holder_ends_first(fd, exclusive): + taken = real_flock_nb(fd, exclusive) + gone = repo / ".git" / f"sg-inflight-{second}" + if gone.exists() and os.name != "nt": + gone.unlink() + return taken + monkeypatch.setattr(diffstate, "_flock_nb", holder_ends_first) + try: + diffstate._mark_inflight(str(repo), [first, second, third]) + held = [os.path.basename(p) for p in diffstate._inflight_held] + assert f"sg-inflight-{third}" in held and f"sg-inflight-{first}" not in held + if os.name != "nt": + assert held == [f"sg-inflight-{third}"] + finally: + diffstate._release_inflight() + + +class _FakeKernel32: + """LockFileEx as kernel32 has it, for the code path only Windows runs.""" + + def __init__(self): + self.calls, self.result, self.error = [], 1, 0 + + def LockFileEx(handle, flags, reserved, low, high, overlapped): + ov = overlapped._obj + self.calls.append((handle, flags, low, high, ov.Offset, ov.OffsetHigh)) + return self.result + self.LockFileEx = LockFileEx + + +@pytest.fixture +def kernel32(monkeypatch): + import ctypes + import types + fake = _FakeKernel32() + monkeypatch.setattr(ctypes, "WinDLL", lambda name, use_last_error=False: fake, raising=False) + monkeypatch.setattr(ctypes, "get_last_error", lambda: fake.error, raising=False) + monkeypatch.setitem(sys.modules, "msvcrt", + types.SimpleNamespace(get_osfhandle=lambda fd: 1000 + fd)) + monkeypatch.setattr(diffstate, "_win_lock_file_ex", None) + return fake + + +def test_windows_lock_call(kernel32): + assert diffstate._win_lock_nb(7, exclusive=True) is True + assert diffstate._win_lock_nb(7, exclusive=False) is True + # Fail-immediately, exclusive or shared, one byte far past the end of the + # file so that reading the marker is never refused. + assert kernel32.calls == [(1007, 0x3, 1, 0, 0, 0x7FFFFFFF), (1007, 0x1, 1, 0, 0, 0x7FFFFFFF)] + kernel32.result, kernel32.error = 0, 33 + assert diffstate._win_lock_nb(7, exclusive=True) is False + for other_error in (5, 6, 997): + kernel32.error = other_error + with pytest.raises(OSError): + diffstate._win_lock_nb(7, exclusive=True) + + +def test_windows_lock_is_off_where_the_call_is_missing(monkeypatch): + import ctypes + monkeypatch.delattr(ctypes, "WinDLL", raising=False) + monkeypatch.setattr(diffstate, "_win_lock_file_ex", None) + with pytest.raises(OSError): + diffstate._win_lock_nb(7, exclusive=True) + + +_SHORT_LIMIT = ( + "import sys\n" + "sys.path.insert(0, sys.argv[1])\n" + "import security_reminder_hook as hook\n" + "hook._RACE_END_S = float(sys.argv[2])\n" + "hook.main()\n" +) + + +def _run_with_limit(payload, env, limit_s): + started = time.monotonic() + r = subprocess.run( + [sys.executable, "-c", _SHORT_LIMIT, _HOOKS_DIR, str(limit_s)], + input=json.dumps(payload), capture_output=True, text=True, env=env, timeout=30) + return time.monotonic() - started, r + + +def test_single_shot_after_a_fast_agentic_failure_ends_in_time( + tmp_path, hook_env, failing_inner_cli, stub_api): + """The agentic run fails before the fallback starts, and the single-shot + review that follows hangs: the session still hears before the hook limit.""" + pytest.importorskip("claude_agent_sdk") + repo, _ = _repo_with_remote(tmp_path) + _, commit_out = commit_file(repo, "handlers.py", "def h(x):\n return x\n") + stub_api.delay = 6 + env = {**hook_env, "SG_AGENTIC_COMMIT_REVIEW": "1", "SG_AGENTIC_CLI_PATH": failing_inner_cli, + "SG_AGENTIC_RACE_DELAY_S": "60"} + took, r = _run_with_limit(bash_payload(repo, "git commit -m change", commit_out), env, 4) + assert 4 <= took < 10, r.stderr + assert r.returncode == 2 and "did not finish in time" in r.stderr + assert metrics_of(r.stdout)["agentic_fallback"] == 2 + + +def test_single_shot_after_a_push_wait_ends_in_time( + tmp_path, hook_env, failing_inner_cli, stub_api): + pytest.importorskip("claude_agent_sdk") + repo, remote = _repo_with_remote(tmp_path) + _, sha, _, push_out = _commit_and_push(repo, remote) + stub_api.delay = 10 + env = {**hook_env, "SG_AGENTIC_COMMIT_REVIEW": "1", "SG_AGENTIC_CLI_PATH": failing_inner_cli, + "SG_INFLIGHT_WAIT_S": "4", "SG_AGENTIC_RACE_DELAY_S": "60"} + with _held_by_another_review(repo, sha): + took, r = _run_with_limit(bash_payload(repo, "git push origin main", push_out), env, 8) + assert 8 <= took < 11.5, r.stderr # 12 s if the wait did not count + assert r.returncode == 2 and "did not finish in time" in r.stderr + assert metrics_of(r.stdout)["inflight_wait_s"] >= 4 + + +def test_single_shot_with_the_agentic_review_off_ends_in_time(tmp_path, hook_env, stub_api): + repo, _ = _repo_with_remote(tmp_path) + _, commit_out = commit_file(repo, "handlers.py", "def h(x):\n return x\n") + stub_api.delay = 6 + env = {**hook_env, "SG_AGENTIC_COMMIT_REVIEW": "0"} + took, r = _run_with_limit(bash_payload(repo, "git commit -m change", commit_out), env, 4) + assert 4 <= took < 10, r.stderr + assert r.returncode == 2 and "did not finish in time" in r.stderr diff --git a/plugins/security-guidance/tests/test_launcher_alias_own_path.py b/plugins/security-guidance/tests/test_launcher_alias_own_path.py new file mode 100644 index 00000000..2dd133d0 --- /dev/null +++ b/plugins/security-guidance/tests/test_launcher_alias_own_path.py @@ -0,0 +1,372 @@ +"""sg-python.sh stops starting a packaged Windows alias on every hook. + +When the name that answers is an alias under ...\\Microsoft\\WindowsApps and +its interpreter is a regular file elsewhere, the hook runs with that file and +the path is remembered for a day, so later hooks are one process and none +through the alias. See anthropics/claude-code#98929. + +Aliases and interpreters are shell wrappers, so these tests need a POSIX +shell; the launcher's Windows branch is switched on with OSTYPE. +""" +import os +import shutil +import subprocess +import sys +import time + +import pytest + +from conftest import HOOKS_DIR, metrics_of, run_hook, stop_payload + +SHIM = HOOKS_DIR / "sg-python.sh" +BASH = shutil.which("bash") or "/bin/bash" +REAL = os.path.realpath(sys.executable) +VERSION = f"{sys.version_info[0]}.{sys.version_info[1]}" +try: + BASH_MAJOR = int(subprocess.run([BASH, "-c", "echo ${BASH_VERSINFO[0]}"], capture_output=True, + text=True, timeout=30).stdout.strip() or 0) +except (OSError, ValueError, subprocess.SubprocessError): + BASH_MAJOR = 0 + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX shell wrappers") + +HOOK = ( + "import os, sys\n" + "print('direct=' + os.environ.get('SG_LAUNCHER_DIRECT', ''))\n" + "print('exe=' + os.environ.get('SG_NAME', ''))\n" + "print('argv=' + ' '.join(sys.argv[1:]))\n" + "print('stdin=' + sys.stdin.read())\n" +) + +# A packaged alias: answers the probe like Windows Python does (version, then +# sys.executable, CRLF), and runs the hook itself when asked to. +ALIAS = """#!{bash} +[ "${{0##*/}}" = "py" ] && [ "${{1:-}}" = "-3" ] && shift +if [ "${{1:-}}" = "-c" ]; then + echo "${{0##*/}} probe" >> '{log}'; printf '{version}\\r\\n{own}\\r\\n'; exit 0 +fi +echo "${{0##*/}} run" >> '{log}' +SG_NAME=alias exec '{real}' "$@" +""" +# The interpreter behind it: the real one, logging its starts. +OWN = """#!{bash} +if [ "${{1:-}}" = "-c" ]; then + echo "own probe" >> '{log}' + [ {rc} -eq 0 ] || exit {rc} +else + echo "own run" >> '{log}' +fi +SG_NAME=own exec '{real}' "$@" +""" + + +class Box: + def __init__(self, tmp_path): + self.root = tmp_path + self.apps = tmp_path / "AppData" / "Local" / "Microsoft" / "WindowsApps" + self.own = tmp_path / "AppData" / "Local" / "Python" / "pythoncore-3-64" / "python.exe" + self.state = tmp_path / "state" + self.log = tmp_path / "log" + self.hooks = tmp_path / "plugin" / "hooks" + for d in (self.apps, self.own.parent, self.state, self.hooks): + d.mkdir(parents=True) + self.log.write_text("") + self.script = self.hooks / "hook.py" + self.script.write_text(HOOK) + self.memo = self.state / "launcher-python" + + def write(self, path, template, **kw): + kw = {"version": VERSION, "own": self.own, "rc": 0, **kw} + path.write_text(template.format(bash=BASH, log=self.log, real=REAL, **kw)) + path.chmod(0o755) + + def alias(self, name="py", **kw): + self.write(self.apps / name, ALIAS, **kw) + + def interpreter(self, **kw): + self.write(self.own, OWN, **kw) + + def launch(self, *args, script=None, ostype="msys", path=None, **extra_env): + env = {"PATH": path or str(self.apps), "HOME": str(self.root), "OSTYPE": ostype, + "SECURITY_WARNINGS_STATE_DIR": str(self.state)} + env.update(extra_env) + self.log.write_text("") + return subprocess.run([BASH, str(SHIM), str(script or self.script), *args], input="payload", + capture_output=True, text=True, timeout=60, env=env) + + def run(self, *args, **kw): + r = self.launch(*args, **kw) + assert r.returncode == 0 and r.stderr == "", r.stderr + out = dict(line.split("=", 1) for line in r.stdout.splitlines()) + return out, self.starts() + + def starts(self): + return self.log.read_text().splitlines() + + +@pytest.fixture +def box(tmp_path): + if BASH_MAJOR < 5: + pytest.skip("the remembered route needs bash 5") + b = Box(tmp_path) + b.alias() + b.interpreter() + return b + + +def test_first_hook_checks_the_interpreter_and_runs_it(box): + out, starts = box.run("extra") + assert starts == ["py probe", "own probe", "own run"] + assert (out["exe"], out["direct"], out["argv"], out["stdin"]) == ("own", "2", "extra", "payload") + assert box.memo.read_text().splitlines()[3] == str(box.own) + + +def test_later_hooks_are_one_process_and_never_the_alias(box): + box.run() + for _ in range(3): + out, starts = box.run("extra") + assert starts == ["own run"] + assert (out["exe"], out["direct"], out["argv"], out["stdin"]) == ("own", "1", "extra", "payload") + + +def test_state_folder_follows_the_config_dir(box): + (box.root / "cfg" / "security").mkdir(parents=True) + env = {"SECURITY_WARNINGS_STATE_DIR": "", "CLAUDE_CONFIG_DIR": str(box.root / "cfg")} + box.run(**env) + assert (box.root / "cfg" / "security" / "launcher-python").exists() + assert box.run(**env)[1] == ["own run"] + (box.root / ".claude" / "security").mkdir(parents=True) + env["CLAUDE_CONFIG_DIR"] = "" + box.run(**env) + assert (box.root / ".claude" / "security" / "launcher-python").exists() + assert not box.memo.exists() + + +def test_changed_path_is_searched_again(box): + box.run() + other = box.root / "other" + other.mkdir() + out, starts = box.run(path=f"{box.apps}{os.pathsep}{other}") + assert starts == ["py probe", "own probe", "own run"] and out["direct"] == "2" + + +def test_another_hook_folder_is_checked_again(box): + box.run() + moved = box.root / "plugin-next" / "hooks" + moved.mkdir(parents=True) + (moved / "hook.py").write_text(HOOK) + out, starts = box.run(script=moved / "hook.py") + assert starts == ["py probe", "own probe", "own run"] + # Same folder, another hook script: still the remembered route. + (moved / "other.py").write_text(HOOK) + assert box.run(script=moved / "other.py")[1] == ["own run"] + + +@pytest.mark.parametrize("shift", [-86400 - 60, 3600]) +def test_entry_is_trusted_for_a_day(box, shift): + box.run() + lines = box.memo.read_text().splitlines() + lines[1] = str(int(time.time()) + shift) + box.memo.write_text("\n".join(lines) + "\n") + out, starts = box.run() + assert starts == ["py probe", "own probe", "own run"] + assert abs(int(box.memo.read_text().splitlines()[1]) - time.time()) < 120 + + +@pytest.mark.parametrize("damage", [ + lambda lines: lines[:4], # cut short while being written + lambda lines: ["sg-launcher-python 0"] + lines[1:], # another format + lambda lines: lines[:1] + ["soon"] + lines[2:], + lambda lines: lines[:1] + ["9" * 30] + lines[2:], + lambda lines: lines[:3] + ["/bin/sh"] + lines[4:], # not an interpreter + lambda lines: lines[:3] + ["tools/python.exe"] + lines[4:], # not absolute + lambda lines: lines[:3] + [lines[4] + "/python.exe"] + lines[4:], # a folder (made below) + lambda lines: lines[:5] + ["more"], + lambda lines: [], +]) +def test_damaged_entry_is_ignored_and_replaced(box, damage): + box.run() + (box.hooks / "python.exe").mkdir() + good = box.memo.read_text() + box.memo.write_text("".join(line + "\n" for line in damage(good.splitlines()))) + out, starts = box.run() + assert starts == ["py probe", "own probe", "own run"] and out["stdin"] == "payload" + assert box.memo.read_text().splitlines()[2:] == good.splitlines()[2:] + + +def test_remembered_alias_path_is_never_started(box): + box.run() + lines = box.memo.read_text().splitlines() + inner = box.apps / "python.exe" + box.write(inner, OWN) + lines[3] = str(inner) + box.memo.write_text("\n".join(lines) + "\n") + assert box.run()[1] == ["py probe", "own probe", "own run"] + + +def test_removed_interpreter_falls_back_to_the_alias(box): + box.run() + box.own.unlink() + out, starts = box.run() + assert starts == ["py probe", "py run"] + assert (out["exe"], out["direct"], out["stdin"]) == ("alias", "", "payload") + + +def test_missing_hook_script_is_not_run_from_memory(box): + box.run() + box.script.unlink() + box.launch() + # The full search ran: the remembered interpreter was not started first. + assert box.starts()[0] == "py probe" + + +@pytest.mark.parametrize("code", [1, 2]) +def test_failed_hook_on_the_remembered_path_empties_the_entry(box, code): + box.run() + failing = box.hooks / "fails.py" + failing.write_text(f"import sys\nsys.stderr.write('findings')\nsys.exit({code})\n") + r = box.launch(script=failing) + # Its exit code and output pass through unchanged, once. + assert (r.returncode, r.stderr, box.starts()) == (code, "findings", ["own run"]) + assert box.memo.read_text() == "" + out, starts = box.run() + assert starts == ["py probe", "own probe", "own run"] and out["direct"] == "2" + assert box.run()[1] == ["own run"] + + +def test_virtual_environment_keeps_the_alias(box): + box.run() + out, starts = box.run(VIRTUAL_ENV=str(box.root / "venv")) + assert starts == ["py probe", "py run"] and (out["exe"], out["direct"]) == ("alias", "") + + +def test_tilde_and_userprofile_resolve_like_the_hooks_do(box): + profile = box.root / "profile" + (profile / "st").mkdir(parents=True) + env = {"SECURITY_WARNINGS_STATE_DIR": "~/st", "USERPROFILE": str(profile)} + box.run(**env) + assert (profile / "st" / "launcher-python").exists() + assert box.run(**env)[1] == ["own run"] + (profile / ".claude" / "security").mkdir(parents=True) + env["SECURITY_WARNINGS_STATE_DIR"] = "" + box.run(**env) + assert (profile / ".claude" / "security" / "launcher-python").exists() + + +def test_unwritable_entry_is_silent(box): + box.memo.mkdir() + for _ in range(2): + out, starts = box.run() # run() asserts an empty stderr + assert starts == ["py probe", "own probe", "own run"] + + +def test_windows_form_path_goes_through_cygpath(box): + # OSTYPE=cygwin has no /<drive> mounts: the answer is converted by cygpath. + box.write(box.apps / "cygpath", """#!{bash} +echo "cygpath $1" >> '{log}' +[ "$1" = "-u" ] && [ "$2" = 'C:\\Py\\python.exe' ] && {{ echo '{own}'; exit 0; }} +echo "$2" +""") + box.alias(own="C:\\\\Py\\\\python.exe") + out, starts = box.run(ostype="cygwin") + assert [s for s in starts if not s.startswith("cygpath -w")] == [ + "py probe", "cygpath -u", "own probe", "own run"] + assert (out["exe"], out["direct"]) == ("own", "2") + assert box.memo.read_text().splitlines()[3] == str(box.own) + assert [s for s in box.run(ostype="cygwin")[1] if not s.startswith("cygpath -w")] == ["own run"] + + +@pytest.mark.parametrize("name", ["pypy3.exe", "python"]) +def test_interpreter_by_another_name_is_neither_checked_nor_remembered(box, name): + other = box.own.parent / name + box.write(other, OWN) + box.alias(own=other) + out, starts = box.run() + assert starts == ["py probe", "py run"] and not box.memo.exists() + + +def test_interpreter_that_cannot_read_the_script_is_not_used(box): + box.interpreter(rc=3) + out, starts = box.run() + assert starts == ["py probe", "own probe", "py run"] + assert (out["exe"], out["direct"]) == ("alias", "") and not box.memo.exists() + + +def test_other_version_behind_the_alias_is_not_used(box): + box.alias(version="3.10" if VERSION != "3.10" else "3.11") + out, starts = box.run() + assert starts == ["py probe", "own probe", "py run"] + assert (out["exe"], out["direct"]) == ("alias", "") and not box.memo.exists() + + +def test_interpreter_inside_the_alias_folder_is_left_alone(box): + # Microsoft Store Python: sys.executable is itself under WindowsApps. + inner = box.apps / "PythonSoftwareFoundation.Python.3" / "python.exe" + inner.parent.mkdir() + box.write(inner, OWN) + box.alias(own=inner) + out, starts = box.run() + assert starts == ["py probe", "py run"] and out["direct"] == "" and not box.memo.exists() + + +def test_regular_install_is_left_alone(box): + regular = box.root / "Python312" + regular.mkdir() + box.write(regular / "python", ALIAS) + out, starts = box.run(path=str(regular)) + assert starts == ["python probe", "python run"] and out["direct"] == "" and not box.memo.exists() + + +@pytest.mark.parametrize("ostype", ["linux-gnu", "darwin23", ""]) +def test_other_systems_are_left_alone(box, ostype): + box.run() + out, starts = box.run(ostype=ostype) + assert starts == ["py probe", "py run"] and out["direct"] == "" + + +def test_missing_state_folder_is_not_created(box): + gone = box.root / "nowhere" + out, starts = box.run(SECURITY_WARNINGS_STATE_DIR=str(gone)) + assert starts == ["py probe", "own probe", "own run"] and out["exe"] == "own" + assert not gone.exists() + + +def test_versioned_alias_is_remembered_too(box): + (box.apps / "py").unlink() + box.alias("python3.12", version="3.12") + box.write(box.own, ALIAS.replace("SG_NAME=alias", "SG_NAME=own").replace("${{0##*/}}", "own"), + version="3.12") + out, starts = box.run() + assert starts == ["python3.12 probe", "own probe", "own run"] and out["direct"] == "2" + assert box.run()[1] == ["own run"] + + +def test_newer_python_answering_with_two_lines_is_still_chosen(box): + # Pass 2b reads the probe without `ask`. + (box.apps / "py").unlink() + box.alias("python3.14", version="3.14") + out, starts = box.run() + assert starts == ["python3.14 probe", "python3.14 run"] and out["exe"] == "alias" + + +@pytest.mark.parametrize("value,expected", [("1", 1), ("2", 2), ("", None), ("3", None), ("yes", None)]) +def test_stop_hook_reports_the_direct_route(hook_env, workspace, value, expected): + _, repo = workspace + rc, out, err = run_hook(stop_payload(repo), {**hook_env, "SG_LAUNCHER_DIRECT": value}) + m = metrics_of(out) + assert m is not None, err + assert m.get("launcher_direct") == expected + + +def test_direct_route_key_is_the_last_one(hook_env, workspace): + """Claude Code keeps the first 20 keys, so this one never displaces another.""" + _, repo = workspace + rc, out, err = run_hook(stop_payload(repo), {**hook_env, "SG_LAUNCHER_DIRECT": "1"}) + assert list(metrics_of(out))[-1] == "launcher_direct" + + +def test_value_is_read_once(monkeypatch): + import _base + monkeypatch.setenv("SG_LAUNCHER_DIRECT", "1") + assert _base.launcher_direct_metric() == {"launcher_direct": 1} + assert "SG_LAUNCHER_DIRECT" not in os.environ and _base.launcher_direct_metric() == {} diff --git a/plugins/security-guidance/tests/test_launcher_newer_python.py b/plugins/security-guidance/tests/test_launcher_newer_python.py new file mode 100644 index 00000000..fc152b73 --- /dev/null +++ b/plugins/security-guidance/tests/test_launcher_newer_python.py @@ -0,0 +1,55 @@ +"""sg-python.sh finds Python releases newer than its versioned list. + +The versioned names it prefers stop at python3.13. A machine whose `python3` +is below 3.10 (stock macOS: 3.9) and whose only newer interpreter is +python3.14 used to end up on 3.9, which cannot load the agent SDK, so the +commit reviewer was off without a word. See #4907, #5024. + +Interpreters are emulated with shell scripts, so these tests need a POSIX +shell. +""" +import sys + +import pytest + +from test_launcher_process_count import PYTHON, _add, _run, _starts, box # noqa: F401 + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX shell wrappers") + + +def test_newer_release_beats_a_python3_below_3_10(box): + _add(box, "python3", PYTHON, version="3.9") + _add(box, "python3.14", PYTHON, version="3.14") + r = _run(box, "hook.py") + assert "ran:python3.14" in r.stdout, r.stdout + r.stderr + assert _starts(box) == ["python3 probe", "python3.14 probe", "python3.14 hook.py"] + + +def test_newer_release_is_found_when_it_is_the_only_python(box): + _add(box, "python3.15", PYTHON, version="3.15") + r = _run(box, "hook.py") + assert "ran:python3.15" in r.stdout, r.stdout + r.stderr + assert _starts(box) == ["python3.15 probe", "python3.15 hook.py"] + + +@pytest.mark.parametrize("working", ["python3.12", "python3", "python"]) +def test_an_interpreter_that_already_works_is_kept(box, working): + _add(box, working, PYTHON, version="3.12") + _add(box, "python3.14", PYTHON, version="3.14") + r = _run(box, "hook.py") + assert f"ran:{working}" in r.stdout, r.stdout + r.stderr + assert "python3.14 probe" not in _starts(box) + + +def test_oldest_of_the_newer_releases_is_tried_first(box): + _add(box, "python3.16", PYTHON, version="3.16") + _add(box, "python3.14", PYTHON, version="3.14") + r = _run(box, "hook.py") + assert "ran:python3.14" in r.stdout, r.stdout + r.stderr + + +def test_no_python_at_all_still_starts_nothing(box): + r = _run(box, "hook.py") + assert r.returncode == 1 + assert "python3.14" in r.stderr + assert _starts(box) == [] diff --git a/plugins/security-guidance/tests/test_launcher_probe_timing.py b/plugins/security-guidance/tests/test_launcher_probe_timing.py new file mode 100644 index 00000000..69b867a6 --- /dev/null +++ b/plugins/security-guidance/tests/test_launcher_probe_timing.py @@ -0,0 +1,196 @@ +"""sg-python.sh times its interpreter probes and the hooks report slow ones. + +Measurement only: the launcher still waits for every probe and still picks +the interpreter it picked before. + +The slow interpreter is a shell wrapper around the real one, so these tests +need a POSIX shell. +""" +import json +import os +import shutil +import subprocess +import sys + +import pytest + +from conftest import HOOKS_DIR, edit_payload, metrics_of, run_hook, stop_payload + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX shell wrappers") + +SHIM = HOOKS_DIR / "sg-python.sh" +BASH = shutil.which("bash") or "/bin/bash" +REAL = os.path.realpath(sys.executable) + + +def _bash_major(): + try: + return int(subprocess.run([BASH, "-c", "echo ${BASH_VERSINFO[0]}"], + capture_output=True, text=True, timeout=30).stdout) + except (OSError, ValueError, subprocess.SubprocessError): + return 0 + + +BASH5 = sys.platform != "win32" and _bash_major() >= 5 + +# Prints what the launcher handed over, then its stdin. +HOOK = ( + "import os, sys\n" + "print('max=' + os.environ.get('SG_LAUNCHER_PROBE_MAX_MS', ''))\n" + "print('total=' + os.environ.get('SG_LAUNCHER_PROBE_MS', ''))\n" + "print('name=' + os.environ.get('SG_NAME', ''))\n" + "print('stdin=' + sys.stdin.read())\n" +) + +WRAPPER = """#!{bash} +# Answers the launcher's `-c` probe after DELAY seconds; RC != 0 fails it. +if [ "${{1:-}}" = "-c" ]; then + sleep {delay} + [ {rc} -eq 0 ] || exit {rc} +fi +SG_NAME={name} exec '{real}' "$@" +""" + + +def _python(bindir, name, delay=0, rc=0): + bindir.mkdir(parents=True, exist_ok=True) + p = bindir / name + p.write_text(WRAPPER.format(bash=BASH, delay=delay, rc=rc, name=name, real=REAL)) + p.chmod(0o755) + + +def _run(bindir, script, extra_env=None, shim=SHIM): + env = {"PATH": f"{bindir}{os.pathsep}/usr/bin{os.pathsep}/bin", "HOME": os.environ.get("HOME", "/")} + env.update(extra_env or {}) + r = subprocess.run([BASH, str(shim), str(script)], input="payload", capture_output=True, + text=True, timeout=60, env=env) + assert r.returncode == 0, r.stderr + return dict(line.split("=", 1) for line in r.stdout.splitlines()) + + +@pytest.fixture +def script(tmp_path): + p = tmp_path / "hook.py" + p.write_text(HOOK) + return p + + +@pytest.fixture +def bindir(tmp_path): + """A PATH entry with only what the tests put there (plus sleep).""" + return tmp_path / "bin" + + +def _only_these(bindir): + # Hide the system interpreters: PATH is bindir plus /usr/bin:/bin, so every + # name the launcher tries gets an entry here, failing unless a test set it. + for name in ("python3.13", "python3.12", "python3.11", "python3.10", "python3", "python", "py"): + if not (bindir / name).exists(): + _python(bindir, name, rc=1) + + +@pytest.mark.skipif(not BASH5, reason="millisecond clock needs bash 5") +def test_fast_probes_are_timed_in_milliseconds(bindir, script): + _python(bindir, "python3") + _only_these(bindir) + out = _run(bindir, script) + assert out["name"] == "python3" and out["stdin"] == "payload" + assert 0 < int(out["max"]) <= int(out["total"]) + + +def test_slow_probe_is_waited_for_and_measured(bindir, script): + _python(bindir, "python3", delay=2.2) + _python(bindir, "python") + _only_these(bindir) + out = _run(bindir, script) + # Still the first name that answers, however long it took. + assert out["name"] == "python3" and out["stdin"] == "payload" + assert 2000 <= int(out["max"]) < 10000 + assert int(out["total"]) >= int(out["max"]) + + +def test_slow_failing_probe_counts_towards_the_wait(bindir, script): + _python(bindir, "python3", delay=2.2, rc=49) + _python(bindir, "python") + _only_these(bindir) + out = _run(bindir, script) + assert out["name"] == "python" + assert int(out["max"]) >= 2000 + + +def test_inherited_values_are_not_reported_again(bindir, script): + _python(bindir, "python3") + _only_these(bindir) + out = _run(bindir, script, {"SG_LAUNCHER_PROBE_MAX_MS": "999999", "SG_LAUNCHER_PROBE_MS": "999999"}) + assert out["max"] != "999999" and out["total"] != "999999" + + +@pytest.fixture +def whole_second_shim(tmp_path): + """The launcher as bash 3.2 runs it: no EPOCHREALTIME, only SECONDS.""" + src = SHIM.read_text() + assert 'case "${EPOCHREALTIME:-}" in' in src + p = tmp_path / "sg-python-coarse.sh" + p.write_text(src.replace('case "${EPOCHREALTIME:-}" in', 'case "" in')) + return p + + +def test_whole_second_clock_ignores_fast_probes(bindir, script, whole_second_shim): + _python(bindir, "python3") + _only_these(bindir) + out = _run(bindir, script, shim=whole_second_shim) + assert out["name"] == "python3" + # A probe of a few ms can straddle a second; that is not reported as 1 s. + assert out["max"] == "" and out["total"] == "" + + +def test_whole_second_clock_measures_slow_probes(bindir, script, whole_second_shim): + _python(bindir, "python3", delay=2.2) + _only_these(bindir) + out = _run(bindir, script, shim=whole_second_shim) + assert out["name"] == "python3" + assert int(out["max"]) in (2000, 3000) and int(out["total"]) >= int(out["max"]) + + +@pytest.mark.parametrize("slowest,total,expected", [ + ("4200", "4350", {"launcher_probe_ms": 4350, "launcher_probe_max_ms": 4200}), + ("900", "1500", {"launcher_probe_ms": 1500, "launcher_probe_max_ms": 900}), + ("999", "999", {}), + ("86400000", "86400000", {"launcher_probe_ms": 3600000, "launcher_probe_max_ms": 3600000}), + ("5000", "5", {}), + ("", "", {}), + ("junk", "5", {}), +]) +def test_stop_hook_reports_only_slow_probes(hook_env, workspace, slowest, total, expected): + _, repo = workspace + env = {**hook_env, "SG_LAUNCHER_PROBE_MAX_MS": slowest, "SG_LAUNCHER_PROBE_MS": total} + rc, out, err = run_hook(stop_payload(repo), env) + m = metrics_of(out) + assert m is not None, err + assert {k: v for k, v in m.items() if k.startswith("launcher_probe")} == expected + + +def test_launcher_keys_come_last(hook_env, workspace): + """Claude Code keeps the first 20 keys, so these never displace another.""" + _, repo = workspace + env = {**hook_env, "SG_LAUNCHER_PROBE_MAX_MS": "4200", "SG_LAUNCHER_PROBE_MS": "4350"} + rc, out, err = run_hook(stop_payload(repo), env) + assert list(metrics_of(out))[-2:] == ["launcher_probe_ms", "launcher_probe_max_ms"] + + +def test_edit_hook_reports_slow_probes_on_pattern_rows(hook_env, workspace): + _, repo = workspace + env = {**hook_env, "SG_LAUNCHER_PROBE_MAX_MS": "4200", "SG_LAUNCHER_PROBE_MS": "4350"} + payload = edit_payload(repo, repo / "app.py", new_string="import os\nos.system('ls ' + x)\n") + rc, out, err = run_hook(payload, env) + m = metrics_of(out) + assert m is not None and "rule_id" in m, (out, err) + assert m["launcher_probe_ms"] == 4350 and m["launcher_probe_max_ms"] == 4200 + + +def test_values_are_read_once(monkeypatch): + import _base + monkeypatch.setenv("SG_LAUNCHER_PROBE_MAX_MS", "7000") + monkeypatch.setenv("SG_LAUNCHER_PROBE_MS", "7100") + assert _base.launcher_probe_metrics() == {"launcher_probe_ms": 7100, "launcher_probe_max_ms": 7000} + assert "SG_LAUNCHER_PROBE_MS" not in os.environ and _base.launcher_probe_metrics() == {} diff --git a/plugins/security-guidance/tests/test_launcher_process_count.py b/plugins/security-guidance/tests/test_launcher_process_count.py new file mode 100644 index 00000000..f22fd680 --- /dev/null +++ b/plugins/security-guidance/tests/test_launcher_process_count.py @@ -0,0 +1,158 @@ +"""sg-python.sh starts as few processes as it can before it runs the hook. + +Under Git Bash every command substitution is a new Windows process, and the +launcher runs on every hook, so probing interpreters that are not installed, +probing the same one twice, or calling cygpath for a plain drive path is paid +on each Edit, prompt and Stop. See #6470. + +Interpreters and cygpath are emulated with shell scripts, so these tests need +a POSIX shell. +""" +import shutil +import subprocess +import sys + +import pytest + +from conftest import HOOKS_DIR + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX shell wrappers") + +SHIM = HOOKS_DIR / "sg-python.sh" +BASH = shutil.which("bash") or "/bin/bash" + +# Logs each start; answers the probe with a fixed version, else prints its argv. +PYTHON = """#!{bash} +if [ "${{1:-}}" = "-c" ] || [ "${{1:-}} ${{2:-}}" = "-3 -c" ]; then + echo "${{0##*/}} probe" >> '{log}'; printf '{version}{eol}'; exit 0 +fi +echo "${{0##*/}} $*" >> '{log}' +printf 'ran:%s\\n' "${{0##*/}}"; printf 'arg:%s\\n' "$@" +""" +# ${x^^} in the launcher's Git Bash branch needs bash 4; stock macOS has 3.2. +BASH_MAJOR = int(subprocess.run([BASH, "-c", "echo ${BASH_VERSINFO[0]}"], capture_output=True, + text=True).stdout.strip() or 0) + +# The Microsoft Store placeholder: on PATH, starts, does nothing. +STUB = """#!{bash} +echo "${{0##*/}} probe" >> '{log}' +exit 49 +""" +CYGPATH = """#!{bash} +echo "cygpath $*" >> '{log}' +echo "CYGPATH($2)" +""" + + +@pytest.fixture +def box(tmp_path): + (tmp_path / "bin").mkdir() + (tmp_path / "log").write_text("") + return tmp_path + + +def _add(box, name, template, version="", crlf=False): + p = box / "bin" / name + p.write_text(template.format(bash=BASH, log=box / "log", version=version, + eol="\\r\\n" if crlf else "\\n")) + p.chmod(0o755) + + +def _run(box, *args, ostype="linux-gnu", trace=False): + cmd = [BASH] + (["-x"] if trace else []) + [str(SHIM), *args] + return subprocess.run(cmd, input="", capture_output=True, text=True, timeout=60, + env={"PATH": str(box / "bin"), "HOME": "/", "OSTYPE": ostype}) + + +def _starts(box): + return (box / "log").read_text().splitlines() + + +def test_only_installed_interpreters_are_probed(box): + _add(box, "python", PYTHON, "3.12") + r = _run(box, "hook.py", trace=True) + assert r.stdout.splitlines() == ["ran:python", "arg:hook.py"] + # xtrace marks what runs in a subshell with "++": only the one probe did. + probes = [line for line in r.stderr.splitlines() if line.startswith("++ probe ")] + assert probes == ["++ probe python"] + assert _starts(box) == ["python probe", "python hook.py"] + + +def test_store_placeholder_still_falls_through_to_python(box): + _add(box, "python3", STUB) + _add(box, "python", PYTHON, "3.13") + r = _run(box, "hook.py") + assert r.stdout.splitlines() == ["ran:python", "arg:hook.py"] + assert _starts(box) == ["python3 probe", "python probe", "python hook.py"] + + +def test_python_below_3_10_is_probed_once(box): + _add(box, "python3", PYTHON, "3.9") + r = _run(box, "hook.py") + assert r.stdout.splitlines() == ["ran:python3", "arg:hook.py"] + assert _starts(box) == ["python3 probe", "python3 hook.py"] + + +def test_first_older_python_wins_when_none_is_3_10(box): + _add(box, "python3", PYTHON, "3.8") + _add(box, "python", PYTHON, "3.9") + r = _run(box, "hook.py") + assert r.stdout.splitlines() == ["ran:python3", "arg:hook.py"] + + +def test_newer_python_under_a_later_name_beats_an_older_python3(box): + _add(box, "python3", PYTHON, "3.9") + _add(box, "python", PYTHON, "3.12") + r = _run(box, "hook.py") + assert r.stdout.splitlines() == ["ran:python", "arg:hook.py"] + + +def test_py_launcher_is_still_a_candidate(box): + _add(box, "py", PYTHON, "3.12") + r = _run(box, "hook.py") + assert r.stdout.splitlines() == ["ran:py", "arg:-3", "arg:hook.py"] + assert _starts(box) == ["py probe", "py -3 hook.py"] + + +def test_crlf_version_output_does_not_cost_further_probes(box): + # Native Windows Python may print CRLF; 3.12 must still read as 3.10+. + _add(box, "python", PYTHON, "3.12", crlf=True) + _add(box, "py", PYTHON, "3.12", crlf=True) + r = _run(box, "hook.py") + assert r.stdout.splitlines() == ["ran:python", "arg:hook.py"] + assert _starts(box) == ["python probe", "python hook.py"] + + +@pytest.mark.skipif(BASH_MAJOR < 4, reason="needs bash 4+, as Git Bash has") +@pytest.mark.parametrize("posix, native", [ + ("/c/Users/me/.claude/hooks/hook.py", "C:\\Users\\me\\.claude\\hooks\\hook.py"), + ("/d/a b/hook.py", "D:\\a b\\hook.py"), + ("/c/", "C:\\"), +]) +def test_drive_paths_are_converted_without_cygpath(box, posix, native): + _add(box, "cygpath", CYGPATH) + _add(box, "python", PYTHON, "3.12") + rest = ["--flag", "rel/path", "C:\\already\\native", "C:/d/mixed"] + r = _run(box, posix, *rest, ostype="msys") + assert r.stdout.splitlines() == ["ran:python", "arg:" + native] + ["arg:" + a for a in rest] + assert not [s for s in _starts(box) if s.startswith("cygpath")] + + +@pytest.mark.parametrize("ostype, posix", [ + ("msys", "/tmp/hook.py"), # a mount point, not a drive + ("msys", "/usr/bin"), + ("msys", "/c"), + ("msys", "//server/share/hook.py"), + ("cygwin", "/c/Users/me/hook.py"), # drives live under /cygdrive there +]) +def test_other_absolute_paths_still_go_through_cygpath(box, ostype, posix): + _add(box, "cygpath", CYGPATH) + _add(box, "python", PYTHON, "3.12") + r = _run(box, posix, ostype=ostype) + assert r.stdout.splitlines() == ["ran:python", "arg:CYGPATH(%s)" % posix] + + +def test_paths_are_left_alone_without_cygpath(box): + _add(box, "python3", PYTHON, "3.12") + r = _run(box, "/c/Users/me/hook.py", ostype="msys") + assert r.stdout.splitlines() == ["ran:python3", "arg:/c/Users/me/hook.py"] diff --git a/plugins/security-guidance/tests/test_launcher_unreadable_script.py b/plugins/security-guidance/tests/test_launcher_unreadable_script.py new file mode 100644 index 00000000..d963eaa0 --- /dev/null +++ b/plugins/security-guidance/tests/test_launcher_unreadable_script.py @@ -0,0 +1,192 @@ +"""sg-python.sh never hands the hook script to an interpreter that cannot see it. + +On Windows, python/python3/py can be packaged launcher aliases whose +interpreter answers `-c` but has no view of the plugin directory. Running the +hook with one fails with CPython's "can't open file" and exit code 2, which an +asyncRewake hook reports to the model as review findings, on every Stop. + +The aliases are emulated with shell wrappers around the real interpreter, so +these tests need a POSIX shell. +""" +import json +import os +import shutil +import subprocess +import sys + +import pytest + +from conftest import HOOKS_DIR, metrics_of, run_hook, stop_payload + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX shell wrappers") + +SHIM = HOOKS_DIR / "sg-python.sh" +BASH = shutil.which("bash") or "/bin/bash" +REAL = os.path.realpath(sys.executable) + +# Echoes its stdin and exits 2, like a hook that has findings to report. +HOOK = ( + "import os, sys\n" + "print('fallback=' + os.environ.get('SG_LAUNCHER_FALLBACK', ''))\n" + "sys.stderr.write('ran:' + sys.stdin.read())\nsys.exit(2)\n" +) + +ALIAS = """#!{bash} +# A packaged alias: `-c` works, files under HIDE are invisible. +HIDE='{hide}'; REAL='{real}' +if [ "${{1:-}}" = "-c" ]; then + code="$2"; shift 2 + out=$("$REAL" -c "import os as _o, sys as _s +_f = _o.path.isfile +_o.path.isfile = lambda p, _f=_f: False if str(p).startswith('$HIDE') else _f(p) +if '{sealed}': _s.executable = '$0' +$code" "$@"); rc=$? + [ -n "$out" ] && printf '%s{eol}' "$out" + exit $rc +fi +case "${{1:-}}" in + "$HIDE"/*) echo "python: can't open file '$1': [Errno 2] No such file or directory" >&2; exit 2 ;; +esac +exec "$REAL" "$@" +""" + + +def _alias(bindir, name, hide, sealed=False, crlf=False): + bindir.mkdir(parents=True, exist_ok=True) + p = bindir / name + p.write_text(ALIAS.format(bash=BASH, hide=hide, real=REAL, sealed="1" if sealed else "", + eol="\\r\\n" if crlf else "\\n")) + p.chmod(0o755) + + +def _run(path_dirs, script, payload='{"k": 1}'): + return subprocess.run( + [BASH, str(SHIM), str(script)], input=payload, capture_output=True, text=True, timeout=60, + env={"PATH": os.pathsep.join(str(d) for d in path_dirs), "HOME": os.environ.get("HOME", "/")}, + ) + + +@pytest.fixture +def plugin(tmp_path): + d = tmp_path / "plugin" / "hooks" + d.mkdir(parents=True) + (d / "hook.py").write_text(HOOK) + return d + + +def test_alias_is_replaced_by_the_interpreter_it_reports(plugin, tmp_path): + for name in ("python3", "python"): + _alias(tmp_path / "alias", name, plugin) + r = _run([tmp_path / "alias"], plugin / "hook.py") + assert "can't open file" not in r.stderr + # The hook ran, read its payload, and its own exit code came through. + assert r.stderr == 'ran:{"k": 1}' + assert r.returncode == 2 + assert r.stdout.strip() == "fallback=1" + + +def test_regular_interpreter_is_not_marked_as_fallback(plugin, tmp_path): + (tmp_path / "real").mkdir() + os.symlink(REAL, tmp_path / "real" / "python3") + r = subprocess.run( + [BASH, str(SHIM), str(plugin / "hook.py")], input="{}", capture_output=True, text=True, timeout=60, + env={"PATH": str(tmp_path / "real"), "HOME": "/", "SG_LAUNCHER_FALLBACK": "1"}, + ) + assert r.returncode == 2 + assert r.stdout.strip() == "fallback=" + + +def test_regular_install_later_on_path_is_used(plugin, tmp_path): + _alias(tmp_path / "alias", "python3", plugin, sealed=True) + (tmp_path / "real").mkdir() + os.symlink(REAL, tmp_path / "real" / "python3") + r = _run([tmp_path / "alias", tmp_path / "real"], plugin / "hook.py") + assert r.stderr == 'ran:{"k": 1}' + assert r.returncode == 2 + + +def test_regular_install_under_another_name_is_used(plugin, tmp_path): + _alias(tmp_path / "alias", "python3", plugin, sealed=True) + (tmp_path / "real").mkdir() + os.symlink(REAL, tmp_path / "real" / "python") + r = _run([tmp_path / "alias", tmp_path / "real"], plugin / "hook.py") + assert r.stderr == 'ran:{"k": 1}' + assert r.returncode == 2 + + +def test_no_interpreter_can_see_the_script_is_not_reported_as_findings(plugin, tmp_path): + for name in ("python3", "python"): + _alias(tmp_path / "alias", name, plugin, sealed=True) + r = _run([tmp_path / "alias"], plugin / "hook.py") + assert r.returncode == 1 + assert "can't open file" not in r.stderr + assert "cannot read the hook script" in r.stderr + assert str(plugin / "hook.py") in r.stderr + assert metrics_of(r.stdout) == {"launcher_unreadable": 1} + + +def test_missing_script_is_not_reported_as_findings(plugin, tmp_path): + (tmp_path / "real").mkdir() + os.symlink(REAL, tmp_path / "real" / "python3") + r = _run([tmp_path / "real"], plugin / "gone.py") + assert r.returncode == 1 + assert "can't open file" not in r.stderr + + +def test_no_python_keeps_its_own_message(plugin, tmp_path): + (tmp_path / "empty").mkdir() + r = _run([tmp_path / "empty"], plugin / "hook.py") + assert r.returncode == 1 + assert "no working Python 3 interpreter found" in r.stderr + + +def test_unusable_python_keeps_the_no_interpreter_message(plugin, tmp_path): + # Runs, but is not Python 3: nothing says the script is unreadable. + d = tmp_path / "old" + d.mkdir() + (d / "python").write_text("#!%s\nexit 1\n" % BASH) + (d / "python").chmod(0o755) + r = _run([d], plugin / "hook.py") + assert r.returncode == 1 + assert "no working Python 3 interpreter found" in r.stderr + + +def test_alias_with_crlf_output_is_still_resolved(plugin, tmp_path): + # Native Windows Python prints CRLF; the reported path must survive that. + _alias(tmp_path / "alias", "python3", plugin, crlf=True) + r = _run([tmp_path / "alias"], plugin / "hook.py") + assert r.stderr == 'ran:{"k": 1}' + assert r.returncode == 2 + + +@pytest.mark.skipif(shutil.which("git") is None, reason="needs git") +def test_real_hook_runs_through_an_alias_only_path(tmp_path): + for name in ("python3", "python"): + _alias(tmp_path / "alias", name, HOOKS_DIR) + payload = json.dumps({"hook_event_name": "UserPromptSubmit", "session_id": "launcher-test", + "cwd": str(tmp_path), "prompt": "hello"}) + # git only: its directory usually holds a regular python3 as well. + git_dir = tmp_path / "git" + git_dir.mkdir() + os.symlink(shutil.which("git"), git_dir / "git") + r = subprocess.run( + [BASH, str(SHIM), str(HOOKS_DIR / "security_reminder_hook.py")], + input=payload, capture_output=True, text=True, timeout=120, + env={"PATH": os.pathsep.join([str(tmp_path / "alias"), str(git_dir)]), "HOME": str(tmp_path), + "SECURITY_GUIDANCE_DEBUG_LOG": str(tmp_path / "debug.log")}, + ) + assert r.returncode == 0, r.stderr + assert "hook_event=UserPromptSubmit" in (tmp_path / "debug.log").read_text() + + +@pytest.mark.parametrize("flag", ["1", None]) +def test_hook_reports_launcher_fallback(workspace, hook_env, flag): + env = dict(hook_env) + env.pop("SG_LAUNCHER_FALLBACK", None) + if flag: + env["SG_LAUNCHER_FALLBACK"] = flag + rc, out, _ = run_hook(stop_payload(workspace), env) + assert rc == 0 + m = metrics_of(out) + assert m is not None + assert m.get("launcher_fallback") == (1 if flag else None) diff --git a/plugins/security-guidance/tests/test_nested_repos.py b/plugins/security-guidance/tests/test_nested_repos.py new file mode 100644 index 00000000..8357abf1 --- /dev/null +++ b/plugins/security-guidance/tests/test_nested_repos.py @@ -0,0 +1,473 @@ +import os +import subprocess +import tempfile + +import pytest + +from conftest import ( + GIT_ENV, STUB_VULN, VULN_PY, git, make_repo, metrics_of, run_hook, + stop_payload, ups_payload, write_cli, +) + +import diffstate +import gitutil + + +def _add_nested_repo(parent, rel, marker): + """A directory with its own `.git` whose config names a filter command, + the way an unpacked archive or a vendored checkout can. The command leaves + a line in `marker` each time git runs it.""" + nested = make_repo(parent / rel, { + "lib.py": "print('one')\n", + ".gitattributes": "* filter=probe\n", + }) + git(nested, "config", "filter.probe.clean", + f"sh -c 'echo ran >> \"{marker.as_posix()}\"; cat'") + # Same size, so git has to read the file to tell whether it changed. + (nested / "lib.py").write_text("print('two')\n") + old = os.stat(nested / ".git" / "index").st_mtime - 3600 + os.utime(nested / ".git" / "index", (old, old)) + return nested + + +@pytest.fixture +def outer(tmp_path): + return make_repo(tmp_path / "outer", {"app.py": "print('hi')\n"}) + + +@pytest.fixture +def marker(tmp_path): + return tmp_path / "filter-ran" + + +def _turn(repo, hook_env, stub_api, during): + stub_api.vulns = [STUB_VULN] + assert run_hook(ups_payload(repo), hook_env)[0] == 0 + during() + return run_hook(stop_payload(repo), hook_env) + + +def test_untracked_nested_repo_filter_not_run(outer, marker, hook_env, stub_api): + def during(): + _add_nested_repo(outer, "vendor/pkg", marker) + (outer / "app.py").write_text(VULN_PY) + + rc, out, err = _turn(outer, hook_env, stub_api, during) + + assert not marker.exists(), marker.read_text() + # The turn's own edit is still reviewed. + assert stub_api.calls, (rc, out, err) + assert "app.py" in out + err + + +def test_untracked_nested_repo_alone_is_not_reviewed(outer, marker, hook_env, stub_api): + rc, out, err = _turn( + outer, hook_env, stub_api, + lambda: _add_nested_repo(outer, "vendor/pkg", marker)) + + assert not marker.exists(), marker.read_text() + assert not stub_api.calls + assert (metrics_of(out) or {}).get("skipped") is True + + +@pytest.fixture +def git_env(tmp_path, monkeypatch): + """For calling gitutil in this process: a temp dir of our own, and the + environment restored afterwards.""" + d = tmp_path / "tmproot" + d.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(d)) + saved = dict(os.environ) + os.environ.update(GIT_ENV) + yield + os.environ.clear() + os.environ.update(saved) + + +def _record_nested_repo(outer, marker, gitmodules=None): + nested = _add_nested_repo(outer, "vendor/pkg", marker) + if gitmodules: + (outer / ".gitmodules").write_text(gitmodules) + git(outer, "add", "-A") + git(outer, "commit", "-q", "-m", "add vendor/pkg") + if marker.exists(): + marker.unlink() + return nested + + +def test_untracked_nested_repo_is_not_a_changed_file(outer, marker, git_env): + _add_nested_repo(outer, "vendor/pkg", marker) + (outer / "new.py").write_text("x = 1\n") + + tracked, untracked = gitutil._git_status_porcelain(str(outer)) + + assert (tracked, untracked) == (set(), {"new.py"}) + + +def test_intent_to_add_skips_nested_repos(outer, marker, git_env): + _add_nested_repo(outer, "vendor/pkg", marker) + (outer / "new.py").write_text("x = 1\n") + + with gitutil._temp_index(str(outer), ["new.py", "vendor/pkg/"]) as env: + assert env is not None + listed = subprocess.run( + ["git", "ls-files", "-s"], cwd=outer, capture_output=True, + text=True, env=env, + ).stdout + + assert "new.py" in listed + assert "160000" not in listed + assert not marker.exists() + + +# A repository can ask for one submodule to be checked whatever the default. +GITMODULES_IGNORE_NONE = ( + '[submodule "pkg"]\n\tpath = vendor/pkg\n\turl = ./vendor/pkg\n\tignore = none\n' +) + + +@pytest.mark.parametrize("gitmodules", [None, GITMODULES_IGNORE_NONE]) +def test_status_and_diff_leave_out_recorded_nested_repo(outer, marker, git_env, gitmodules): + nested = _record_nested_repo(outer, marker, gitmodules) + (nested / "lib.py").write_text("print('abc')\n") + (outer / "app.py").write_text(VULN_PY) + + tracked, untracked = gitutil._git_status_porcelain(str(outer)) + diff = gitutil.get_git_diff(str(outer), "HEAD", untracked_paths=[]) + names = gitutil._git_name_only(str(outer), "HEAD") + + assert tracked == {"app.py"} + assert names == {"app.py"} + assert "app.py" in diff and "vendor/pkg" not in diff + assert not marker.exists() + + +def test_commit_range_diff_does_not_enter_nested_repo(outer, marker, git_env, tmp_path): + nested = _record_nested_repo(outer, marker, GITMODULES_IGNORE_NONE) + ext_marker = tmp_path / "ext-diff-ran" + git(nested, "config", "diff.external", + f"sh -c 'echo ran >> \"{ext_marker.as_posix()}\"'") + base = git(outer, "rev-parse", "HEAD").strip() + git(nested, "commit", "-q", "-am", "two") + git(outer, "commit", "-q", "-am", "move vendor/pkg") + for m in (marker, ext_marker): + if m.exists(): + m.unlink() + # The user's own setting asks for a submodule's changes inline. + os.environ.update({"GIT_CONFIG_COUNT": "1", "GIT_CONFIG_KEY_0": "diff.submodule", + "GIT_CONFIG_VALUE_0": "diff"}) + + out = gitutil._git_diff_range(str(outer), base, "HEAD") + + assert not ext_marker.exists() and not marker.exists() + assert "lib.py" not in (out or "") + + +def _record_during_turn(outer, marker, commit): + _add_nested_repo(outer, "vendor/pkg", marker) + (outer / "app.py").write_text(VULN_PY) + git(outer, "add", "-A") + if commit: + git(outer, "commit", "-q", "-m", "wip") + marker.unlink(missing_ok=True) # only what the hooks run counts + + +@pytest.mark.parametrize("finding", [False, True], ids=["no-finding", "finding"]) +@pytest.mark.parametrize("commit", [False, True], ids=["staged", "committed"]) +def test_nested_repo_recorded_during_the_turn_filter_not_run( + outer, marker, hook_env, stub_api, commit, finding): + stub_api.vulns = [STUB_VULN] if finding else [] + ran_at = [] + + def hook(name, payload): + rc, out, err = run_hook(payload, hook_env) + # 2 is a Stop that hands a finding back. + assert rc == (2 if finding and name.startswith("stop") else 0), (name, out, err) + if marker.exists(): + ran_at.append(name) + marker.unlink() + + hook("prompt 1", ups_payload(outer)) + _record_during_turn(outer, marker, commit) + hook("stop 1", stop_payload(outer)) + hook("prompt 2", ups_payload(outer)) + (outer / "app.py").write_text(VULN_PY + "# one more edit\n") + hook("stop 2", stop_payload(outer)) + hook("prompt 3", ups_payload(outer)) + + assert ran_at == [] + assert stub_api.calls # the turn's own edit is still reviewed + + +@pytest.mark.skipif(os.name == "nt", reason="'*' is not allowed in Windows file names") +@pytest.mark.parametrize("commit", [False, True], ids=["staged", "committed"]) +def test_glob_shaped_file_name_beside_recorded_nested_repo( + outer, marker, hook_env, stub_api, commit): + assert run_hook(ups_payload(outer), hook_env)[0] == 0 + _record_during_turn(outer, marker, commit) + (outer / "vendor" / "p*").write_text("x = 1\n") + (outer / "new.py").write_text(VULN_PY) + + rc, out, err = run_hook(stop_payload(outer), hook_env) + + assert not marker.exists(), marker.read_text() + assert stub_api.calls, (rc, out, err) + + +def _index_copies(): + return [f for _, _, files in os.walk(tempfile.gettempdir()) for f in files + if f.startswith(gitutil._TEMP_INDEX_PREFIX)] + + +def _index_entries(repo, env=None): + return subprocess.run( + ["git", "ls-files", "-s"], cwd=repo, capture_output=True, text=True, + env=env, check=True, + ).stdout + + +@pytest.mark.parametrize("subdir", ["", "src"]) +def test_index_copy_has_no_gitlinks_and_real_index_is_unchanged( + outer, marker, git_env, subdir): + (outer / "src").mkdir() + (outer / "src" / "a.py").write_text("x = 1\n") + _add_nested_repo(outer, "src/third party/dep", marker) + _record_nested_repo(outer, marker) + real = outer / ".git" / "index" + before = (real.read_bytes(), real.stat().st_mtime_ns) + + with gitutil._temp_index(str(outer / subdir), untracked_paths=[]) as env: + assert env is not None + assert len(_index_copies()) == 1 + copy = _index_entries(outer, env) + copy_mtime = os.stat(env["GIT_INDEX_FILE"]).st_mtime_ns + + assert "160000" not in copy + assert "app.py" in copy and "src/a.py" in copy + assert copy_mtime == before[1] + assert (real.read_bytes(), real.stat().st_mtime_ns) == before + assert _index_entries(outer).count("160000") == 2 + assert not marker.exists() + + +def test_baseline_and_diff_with_recorded_nested_repo(outer, marker, git_env): + nested = _record_nested_repo(outer, marker) + (nested / "lib.py").write_text("print('abc')\n") + (outer / "app.py").write_text(VULN_PY) + (outer / "new.py").write_text("x = 1\n") + + baseline = diffstate.capture_git_baseline(str(outer)) + diff = gitutil.get_git_diff(str(outer), "HEAD", untracked_paths=["new.py"]) + since = gitutil.get_git_diff(str(outer), baseline, untracked_paths=["new.py"]) + # The snapshot has no entry for the nested repository; the real index does. + tracked_since = gitutil.get_git_diff(str(outer), baseline, untracked_paths=[]) + names_since = gitutil._git_name_only(str(outer), baseline) + + # The baseline is a snapshot with the edit in it, not HEAD. + assert baseline and baseline != git(outer, "rev-parse", "HEAD").strip() + assert "app.py" in diff and "new.py" in diff and "vendor/pkg" not in diff + assert "new.py" in since and "app.py" not in since and "vendor/pkg" not in since + assert tracked_since == "" and "vendor/pkg" not in names_since + assert not marker.exists() + + +def test_whole_tree_index_copy_with_recorded_nested_repo(outer, marker, git_env): + nested = _record_nested_repo(outer, marker) + (nested / "lib.py").write_text("print('abc')\n") + (outer / "new.py").write_text("x = 1\n") + + with gitutil._temp_index(str(outer)) as env: + assert env is not None + copy = _index_entries(outer, env) + + assert "new.py" in copy and "160000" not in copy + assert not marker.exists() + + +def test_no_index_copy_when_gitlinks_cannot_be_removed( + outer, marker, git_env, monkeypatch): + _record_nested_repo(outer, marker) + (outer / "app.py").write_text(VULN_PY) + real_run = subprocess.run + + def run(cmd, *a, **kw): + if "update-index" in cmd: + return subprocess.CompletedProcess(cmd, 1, b"", b"no") + return real_run(cmd, *a, **kw) + + monkeypatch.setattr(gitutil.subprocess, "run", run) + + with gitutil._temp_index(str(outer), untracked_paths=[]) as env: + assert env is None + # Without the copy the baseline is HEAD. + assert diffstate.capture_git_baseline(str(outer)) == git(outer, "rev-parse", "HEAD").strip() + assert diffstate.last_baseline_fail == 2 + assert not marker.exists() + assert _index_copies() == [] + + +@pytest.mark.skipif(os.name == "nt", reason="':' and '*' are not allowed in Windows file names") +def test_untracked_file_name_is_not_read_as_a_pattern(outer, git_env): + name = ":(glob)**" + (outer / name).write_text("x = 1\n") + (outer / "other.py").write_text("x = 2\n") + + with gitutil._temp_index(str(outer), [name]) as env: + copy = _index_entries(outer, env) + + assert name in copy and "other.py" not in copy + + +def test_gitlink_without_a_repository_stays_in_the_copy(outer, marker, git_env): + """A submodule that is not checked out: an empty directory.""" + nested = _record_nested_repo(outer, marker) + # Moved, not deleted: git's object files are read-only on Windows. + nested.rename(outer.parent / "moved-away") + nested.mkdir() + (outer / "app.py").write_text(VULN_PY) + + with gitutil._temp_index(str(outer), untracked_paths=[]) as env: + copy = _index_entries(outer, env) + baseline = diffstate.capture_git_baseline(str(outer)) + + assert "160000" in copy + assert baseline and baseline != git(outer, "rev-parse", "HEAD").strip() + assert diffstate.last_baseline_fail == 0 + + +def test_gitlinks_listed_by_a_git_without_sparse_listing(outer, marker, git_env, monkeypatch): + _record_nested_repo(outer, marker) + real_popen = subprocess.Popen + seen = [] + + def popen(cmd, *a, **kw): + if "ls-files" in cmd and "-z" in cmd: # the plugin's own listing + seen.append("--sparse" in cmd) + if "--sparse" in cmd: + cmd = [*gitutil.GIT_CMD, "ls-files", "--no-such-option"] + return real_popen(cmd, *a, **kw) + + monkeypatch.setattr(gitutil.subprocess, "Popen", popen) + monkeypatch.setattr(gitutil, "_ls_files_sparse", True) + + for _ in range(2): + with gitutil._temp_index(str(outer), untracked_paths=[]) as env: + assert "160000" not in _index_entries(outer, env) + + assert seen == [True, False, False] + + +def test_no_index_copy_when_the_listing_cannot_be_watched(outer, marker, git_env, monkeypatch): + _record_nested_repo(outer, marker) + + def no_thread(self): + raise RuntimeError("can't start new thread") + + monkeypatch.setattr(gitutil.threading.Timer, "start", no_thread) + + with gitutil._temp_index(str(outer), untracked_paths=[]) as env: + assert env is None + assert _index_copies() == [] + + +def _listings(monkeypatch): + """Counts the plugin's own `git ls-files` runs.""" + real_popen = subprocess.Popen + seen = [] + + def popen(cmd, *a, **kw): + if "ls-files" in cmd and "-z" in cmd: + seen.append(cmd) + return real_popen(cmd, *a, **kw) + + monkeypatch.setattr(gitutil.subprocess, "Popen", popen) + return seen + + +def test_index_without_gitlinks_is_not_listed(outer, git_env, monkeypatch): + for i in range(300): + (outer / f"f{i}.py").write_text(f"x = {i}\n") + git(outer, "add", "-A") + seen = _listings(monkeypatch) + + with gitutil._temp_index(str(outer), untracked_paths=[]) as env: + assert env is not None + + assert seen == [] + + +@pytest.mark.parametrize("setup", [ + [], + [("config", "index.version", "4")], + [("config", "core.splitIndex", "true")], + [("config", "feature.manyFiles", "true")], +], ids=["default", "index-v4", "split-index", "many-files"]) +def test_gitlink_found_in_every_index_format(outer, marker, git_env, monkeypatch, setup): + for args in setup: + git(outer, *args) + for i in range(50): + (outer / f"f{i}.py").write_text(f"x = {i}\n") + git(outer, "add", "-A") + git(outer, "commit", "-q", "-m", "files") + nested = _record_nested_repo(outer, marker) + # One more index write after the gitlink went in. + (outer / "later.py").write_text("x = 1\n") + git(outer, "add", "later.py") + marker.unlink(missing_ok=True) + (nested / "lib.py").write_text("print('abc')\n") + (outer / "app.py").write_text(VULN_PY) + seen = _listings(monkeypatch) + + with gitutil._temp_index(str(outer), untracked_paths=[]) as env: + copy = _index_entries(outer, env) + baseline = diffstate.capture_git_baseline(str(outer)) + + assert seen and "160000" not in copy and "later.py" in copy + assert baseline != git(outer, "rev-parse", "HEAD").strip() + assert not marker.exists() + + +def _signed_head(repo, tmp_path, marker): + """Make HEAD a commit that carries a signature, with log.showSignature on + and gpg.program naming a stand-in that leaves a line in `marker`.""" + program = write_cli(tmp_path, "signature-check", ( + "import sys\n" + f"open({str(marker)!r}, 'a').write('ran\\n')\n" + "sys.exit(1)\n")) + head = git(repo, "rev-parse", "HEAD").strip() + tree = git(repo, "rev-parse", "HEAD^{tree}").strip() + who = "t <t@example.com> 1800000000 +0000" + commit = ( + f"tree {tree}\nparent {head}\nauthor {who}\ncommitter {who}\n" + "gpgsig -----BEGIN PGP SIGNATURE-----\n \n xx\n" + " -----END PGP SIGNATURE-----\n\nsigned\n") + sha = subprocess.run( + ["git", "hash-object", "-t", "commit", "-w", "--stdin"], cwd=repo, + input=commit.encode(), capture_output=True, check=True, + ).stdout.decode().strip() + git(repo, "update-ref", "--create-reflog", "-m", "commit: signed", "HEAD", sha) + git(repo, "config", "log.showSignature", "true") + git(repo, "config", "gpg.program", str(program)) + return sha + + +def test_reading_the_reflog_does_not_check_signatures(outer, marker, tmp_path, git_env): + sha = _signed_head(outer, tmp_path, marker) + # The stand-in does run for a plain `git log` in this repository. + git(outer, "log", "-1") + assert marker.exists() + marker.unlink() + + fresh, _stale = gitutil._git_reflog_recent_commits(str(outer), max_age_s=10**10) + + assert sha in fresh + assert not marker.exists(), marker.read_text() + + +def test_reviewer_environment_turns_signature_display_off(outer, marker, tmp_path, git_env): + _signed_head(outer, tmp_path, marker) + + gitutil.apply_safe_git_env() + git(outer, "log", "-1") + + assert not marker.exists(), marker.read_text() diff --git a/plugins/security-guidance/tests/test_network_failure_class.py b/plugins/security-guidance/tests/test_network_failure_class.py new file mode 100644 index 00000000..896a251f --- /dev/null +++ b/plugins/security-guidance/tests/test_network_failure_class.py @@ -0,0 +1,356 @@ +"""A call that got no HTTP response reports why, as `net_err`.""" +import asyncio +import errno +import http.client +import socket +import ssl +import sys +import types +import urllib.error + +import pytest + +from conftest import VULN_PY, metrics_of, run_hook, stop_payload, ups_payload + +import _base +import llm +import security_reminder_hook as hook + + +def _cert_error(code): + e = ssl.SSLCertVerificationError(1, "certificate verify failed") + e.verify_code = code + return e + + +class TestClass: + @pytest.mark.parametrize("exc, expected", [ + (urllib.error.URLError(_cert_error(20)), (llm.NET_ERR_CERT_VERIFY, 20)), + (urllib.error.URLError(_cert_error(19)), (llm.NET_ERR_CERT_VERIFY, 19)), + (urllib.error.URLError(_cert_error(0)), (llm.NET_ERR_CERT_VERIFY, 0)), + (urllib.error.URLError(ssl.SSLError(1, "wrong version number")), (llm.NET_ERR_TLS, 0)), + # A default context that cannot be built is raised bare, before any connection. + (ssl.SSLError(1, "[ASN1] nested asn1 error"), (llm.NET_ERR_TLS, 0)), + (urllib.error.URLError(socket.gaierror(-2, "Name or service not known")), (llm.NET_ERR_DNS, 0)), + (urllib.error.URLError(ConnectionRefusedError(errno.ECONNREFUSED, "refused")), (llm.NET_ERR_CONNECT, 0)), + (urllib.error.URLError(OSError(errno.ENETUNREACH, "unreachable")), (llm.NET_ERR_CONNECT, 0)), + (http.client.RemoteDisconnected("closed without response"), (llm.NET_ERR_CONNECT, 0)), + (TimeoutError("The read operation timed out"), (llm.NET_ERR_TIMEOUT, 0)), + (urllib.error.URLError(socket.timeout("timed out")), (llm.NET_ERR_TIMEOUT, 0)), + (urllib.error.URLError(OSError("Tunnel connection failed: 403 Forbidden")), (llm.NET_ERR_PROXY, 0)), + (urllib.error.URLError(OSError("Tunnel connection failed: 504 Gateway Timed Out")), + (llm.NET_ERR_PROXY, 0)), + (urllib.error.URLError("unknown url type"), (llm.NET_ERR_OTHER, 0)), + (http.client.IncompleteRead(b"x"), (llm.NET_ERR_OTHER, 0)), + (http.client.BadStatusLine("x"), (llm.NET_ERR_OTHER, 0)), + (ValueError("step timed out"), (0, 0)), + (UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid start byte"), (0, 0)), + ]) + def test_class_and_verify_code(self, exc, expected): + assert llm._net_error_class(exc) == expected + + def test_code_packs_class_verify_code_and_flags(self, monkeypatch): + monkeypatch.setattr(llm, "_net_error_flags", lambda tls: 5 if tls else 2) + assert llm._net_error_code(urllib.error.URLError(_cert_error(20))) == 1 + 2000 + 50000 + assert llm._net_error_code(urllib.error.URLError(socket.gaierror(-2, "x"))) == 3 + 20000 + + def test_no_code_and_no_setup_checks_for_an_exception_that_is_not_a_network_failure( + self, monkeypatch): + monkeypatch.setattr(llm, "_net_error_flags", lambda tls: pytest.fail("checked")) + assert llm._net_error_code(ValueError("x")) == 0 + + def test_a_setup_check_that_raises_keeps_the_class(self, monkeypatch): + def boom(tls): + raise RuntimeError("x") + monkeypatch.setattr(llm, "_net_error_flags", boom) + assert llm._net_error_code(urllib.error.URLError(_cert_error(20))) == 2001 + + +class TestFlags: + @pytest.fixture(autouse=True) + def _clean(self, monkeypatch, tmp_path): + for name in ("NODE_EXTRA_CA_CERTS", "SSL_CERT_FILE", "SSL_CERT_DIR"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://api.example.com") + monkeypatch.setattr(llm.urllib.request, "getproxies", lambda: {}) + monkeypatch.setattr(llm.sys, "platform", "darwin") + self.cafile = tmp_path / "cert.pem" + self.cafile.write_text("-----BEGIN CERTIFICATE-----\n") + self.capath = tmp_path / "certs" + self.capath.mkdir() + + def _paths(self, monkeypatch, cafile, capath): + monkeypatch.setattr(ssl, "get_default_verify_paths", lambda: ssl.DefaultVerifyPaths( + cafile, capath, "SSL_CERT_FILE", str(self.cafile), "SSL_CERT_DIR", str(self.capath))) + + def test_no_ca_file_and_an_empty_directory(self, monkeypatch): + self._paths(monkeypatch, None, str(self.capath)) + assert llm._net_error_flags(True) == 1 + self._paths(monkeypatch, None, None) + assert llm._net_error_flags(True) == 1 + + def test_an_empty_ca_file_counts_as_none(self, monkeypatch): + self.cafile.write_text("") + self._paths(monkeypatch, str(self.cafile), None) + assert llm._net_error_flags(True) == 1 + + def test_a_ca_file_or_a_filled_directory_is_not_flagged(self, monkeypatch): + self._paths(monkeypatch, str(self.cafile), str(self.capath)) + assert llm._net_error_flags(True) == 0 + (self.capath / "0a1b2c3d.0").write_text("x") + self._paths(monkeypatch, None, str(self.capath)) + assert llm._net_error_flags(True) == 0 + + def test_windows_is_not_flagged_for_its_files(self, monkeypatch): + monkeypatch.setattr(llm.sys, "platform", "win32") + self._paths(monkeypatch, None, None) + assert llm._net_error_flags(True) == 0 + + def test_tls_variables(self, monkeypatch): + self._paths(monkeypatch, str(self.cafile), None) + monkeypatch.setenv("NODE_EXTRA_CA_CERTS", "/x/ca.pem") + assert llm._net_error_flags(True) == 4 + monkeypatch.setenv("SSL_CERT_DIR", "/x") + assert llm._net_error_flags(True) == 12 + # Not read for a failure that is not about TLS. + assert llm._net_error_flags(False) == 0 + + def test_proxy_for_the_url_scheme(self, monkeypatch): + monkeypatch.setattr(llm.urllib.request, "proxy_bypass", lambda host: False) + monkeypatch.setattr(llm.urllib.request, "getproxies", lambda: {"https": "http://proxy:8080"}) + assert llm._net_error_flags(False) == 2 + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://gateway.internal:4000") + assert llm._net_error_flags(False) == 0 + monkeypatch.setattr(llm.urllib.request, "getproxies", lambda: {"http": "http://proxy:8080"}) + assert llm._net_error_flags(False) == 2 + + def test_proxy_bypass_is_asked_with_the_port(self, monkeypatch): + asked = [] + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://gw.internal:8443/base") + monkeypatch.setattr(llm.urllib.request, "getproxies", lambda: {"https": "http://proxy:8080"}) + monkeypatch.setattr(llm.urllib.request, "proxy_bypass", lambda host: asked.append(host) or True) + assert llm._net_error_flags(False) == 0 + assert asked == ["gw.internal:8443"] + + +@pytest.fixture +def direct(monkeypatch, tmp_path): + """In-process calls with a credential, no real state dir, no model + lookup, no waiting between retries and a plain machine setup.""" + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(tmp_path / "state")) + for var in llm._PROVIDER_ENV_VARS: + monkeypatch.delenv(var, raising=False) + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", True) + monkeypatch.setattr(llm, "ANTHROPIC_API_KEY", "k") + monkeypatch.setattr(llm, "default_review_model", lambda: "claude-opus-5-5") + monkeypatch.setattr(llm, "debug_log", lambda *a, **k: None) + monkeypatch.setattr(llm, "_net_error_flags", lambda tls: 0) + monkeypatch.setattr("time.sleep", lambda s: None) + monkeypatch.setitem(_base._USAGE, "http_err_count", 0) + monkeypatch.setitem(_base._USAGE, "http_err_last", 0) + + +class _Reply: + def __init__(self, text): + self._body = ('{"content": [{"type": "text", "text": %s}], "usage": {}}' + % __import__("json").dumps(text)).encode() + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def read(self): + return self._body + + +class TestRecorded: + def _urlopen(self, monkeypatch, *outcomes): + """Each call takes the next outcome (the last one repeats): an + exception is raised, anything else is the reply text.""" + calls = [] + + def urlopen(*a, **k): + outcome = outcomes[min(len(calls), len(outcomes) - 1)] + calls.append(outcome) + if isinstance(outcome, BaseException): + raise outcome + return _Reply(outcome) + monkeypatch.setattr(llm.urllib.request, "urlopen", urlopen) + return calls + + def test_call_that_never_connects(self, direct, monkeypatch): + self._urlopen(monkeypatch, urllib.error.URLError(_cert_error(20))) + assert llm._call_claude("p", {"type": "object"}, model="m") is None + assert llm._last_call_claude_http_error == -1 + assert _base._net_error_metric() == 2001 + + def test_http_error_records_nothing(self, direct, monkeypatch): + self._urlopen(monkeypatch, urllib.error.HTTPError("u", 400, "bad", {}, None)) + assert llm._call_claude("p", {"type": "object"}, model="m") is None + assert llm._last_call_claude_http_error == 400 + assert _base._net_error_metric() == 0 + + def test_a_later_call_that_gets_a_response_clears_it(self, direct, monkeypatch): + refused = urllib.error.URLError(ConnectionRefusedError(errno.ECONNREFUSED, "refused")) + self._urlopen(monkeypatch, refused, refused, refused, '{"ok": true}') + assert llm._call_claude("p", {"type": "object"}, model="m") is None + assert _base._net_error_metric() == llm.NET_ERR_CONNECT + assert llm._call_claude("p", {"type": "object"}, model="fallback") == {"ok": True} + assert _base._net_error_metric() == 0 + + def test_a_later_call_refused_with_a_status_clears_it(self, direct, monkeypatch): + refused = urllib.error.URLError(ConnectionRefusedError(errno.ECONNREFUSED, "refused")) + self._urlopen(monkeypatch, refused, refused, refused, + urllib.error.HTTPError("u", 400, "bad", {}, None)) + llm._call_claude("p", {"type": "object"}, model="m") + llm._call_claude("p", {"type": "object"}, model="fallback") + assert llm._last_call_claude_http_error == 400 + assert _base._net_error_metric() == 0 + + def test_tls_context_that_cannot_be_built(self, direct, monkeypatch): + """Raised bare by the HTTP client, so the review sees an exception + and no api_error.""" + self._urlopen(monkeypatch, ssl.SSLError(1, "[ASN1] nested asn1 error")) + assert llm.analyze_code_security([("a.py", "+x = 1\n")]) == (None, []) + assert llm.last_review_failed() and llm.last_review_status() is None + assert _base._net_error_metric() == llm.NET_ERR_TLS + + def test_failure_on_the_windows_store_context_is_recorded(self, direct, monkeypatch): + """The context built for a Windows store with an unreadable entry is + the one the call uses, and a failure on it is still classified.""" + context = ssl.create_default_context() + monkeypatch.setattr(llm, "_store_tls_context", context) + used = [] + + def urlopen(*a, **k): + used.append(k.get("context")) + raise urllib.error.URLError(_cert_error(20)) + monkeypatch.setattr(llm.urllib.request, "urlopen", urlopen) + assert llm._call_claude("p", {"type": "object"}, model="m") is None + assert used and all(c is context for c in used) + assert _base._net_error_metric() == 2001 + + def test_store_check_that_fails_is_recorded_as_tls(self, direct, monkeypatch): + """The store check raises and Python's own default context cannot be + built either: the call raises past the retry loop.""" + def check(): + raise ssl.SSLError(1, "[ASN1] nested asn1 error") + monkeypatch.setattr(llm, "_store_tls_context", llm._STORE_TLS_UNSET) + monkeypatch.setattr(llm, "_context_without_unreadable_store_certs", check) + self._urlopen(monkeypatch, ssl.SSLError(1, "[ASN1] nested asn1 error")) + assert llm.analyze_code_security([("a.py", "+x = 1\n")]) == (None, []) + assert llm.last_review_failed() + assert _base._net_error_metric() == llm.NET_ERR_TLS + + def test_reply_that_is_not_json_records_nothing(self, direct, monkeypatch): + self._urlopen(monkeypatch, "<html>sign in</html>") + assert llm.analyze_code_security([("a.py", "+x = 1\n")]) == (None, []) + assert llm.last_review_failed() + assert _base._net_error_metric() == 0 + + def test_only_a_positive_code_is_kept(self): + _base._record_net_error(None) + _base._record_net_error(0) + assert _base._net_error_metric() == 0 + _base._record_net_error(4) + _base._record_net_error(20003) + assert _base._net_error_metric() == 20003 + + def test_probe_that_cannot_connect(self, direct, monkeypatch): + self._urlopen(monkeypatch, urllib.error.URLError(socket.gaierror(-2, "x"))) + assert llm._probe_anthropic() is False + assert _base._net_error_metric() == llm.NET_ERR_DNS + + +class TestInnerCli: + @pytest.fixture + def sdk(self, direct, monkeypatch): + mod = types.ModuleType("claude_agent_sdk") + for name in ("AssistantMessage", "ClaudeAgentOptions", "ResultMessage"): + setattr(mod, name, type(name, (), {"__init__": lambda self, **k: None})) + mod.query = None + monkeypatch.setitem(sys.modules, "claude_agent_sdk", mod) + monkeypatch.setattr(llm, "_send_sdk_logs_to_debug_log", lambda: None) + return mod + + def _run(self, monkeypatch, exc): + def run(start, stderr_lines): + raise exc + monkeypatch.setattr(llm, "_run_inner_cli", run) + assert llm._call_claude_via_sdk("p", {"type": "object"}) is None + assert llm._last_call_claude_http_error == -1 + return _base._net_error_metric() + + def test_cli_failed(self, sdk, monkeypatch): + assert self._run(monkeypatch, RuntimeError("exit code 1")) == llm.NET_ERR_INNER_CLI + + def test_cli_timed_out(self, sdk, monkeypatch): + assert self._run(monkeypatch, asyncio.TimeoutError()) == llm.NET_ERR_INNER_CLI_TIMEOUT + + def test_no_sdk(self, direct, monkeypatch): + monkeypatch.setitem(sys.modules, "claude_agent_sdk", None) + monkeypatch.setattr(llm, "_inject_agent_sdk_venv_into_syspath", lambda d: None) + assert llm._call_claude_via_sdk("p", {"type": "object"}) is None + assert _base._net_error_metric() == llm.NET_ERR_NO_SDK + + +class TestEmittedRow: + @pytest.fixture(autouse=True) + def _clean(self, monkeypatch): + monkeypatch.setattr(hook, "_TAIL_METRICS", {}) + monkeypatch.setattr(hook, "_log_usage", lambda usage: None) + monkeypatch.setattr(hook.llm, "gateway_cfg", lambda: 0) + + def _row(self, capsys): + return __import__("json").loads(capsys.readouterr().out)["metrics"] + + def test_ahead_of_the_rows_own_keys(self, capsys, monkeypatch): + monkeypatch.setitem(_base._USAGE, "http_err_count", 2) + monkeypatch.setitem(_base._USAGE, "http_err_last", -1) + _base._record_net_error(2001) + hook.emit_metrics({"review_failed": True, "vulns_found": 0, "review_ms": 18000, + "api_error": -1}) + keys = list(self._row(capsys)) + at = keys.index("net_err") + assert keys[at - 1] == "http_err_count" and keys[at + 1] == "review_failed" + + def test_kept_on_a_row_over_the_key_cap(self, capsys): + _base._record_net_error(2001) + hook.emit_metrics({f"k{i}": i for i in range(25)}) + assert "net_err" in list(self._row(capsys))[:20] + + def test_absent_without_a_failure(self, capsys): + hook.emit_metrics({"vulns_found": 0, "review_ms": 10}) + assert "net_err" not in self._row(capsys) + + +def _free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def test_stop_review_that_cannot_connect(workspace, hook_env): + """Through the real hook: nothing listens at the endpoint.""" + _, repo = workspace + env = dict(hook_env) + env["ANTHROPIC_BASE_URL"] = f"http://127.0.0.1:{_free_port()}" + env["SECURITY_REVIEW_MODEL"] = "claude-sonnet-4-6" # one model: one round of retries + run_hook(ups_payload(repo), env) + (repo / "app.py").write_text(VULN_PY) + _, out, _ = run_hook(stop_payload(repo), env) + m = metrics_of(out) + assert m["api_error"] == -1 and m["review_failed"] is True + assert m["net_err"] % 100 == llm.NET_ERR_CONNECT + assert "net_err" in list(m)[:20] + + +def test_stop_review_that_ran_has_no_class(workspace, hook_env): + _, repo = workspace + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + _, out, _ = run_hook(stop_payload(repo), hook_env) + m = metrics_of(out) + assert "review_ms" in m and "net_err" not in m diff --git a/plugins/security-guidance/tests/test_no_credentials_notice.py b/plugins/security-guidance/tests/test_no_credentials_notice.py new file mode 100644 index 00000000..a828e2ed --- /dev/null +++ b/plugins/security-guidance/tests/test_no_credentials_notice.py @@ -0,0 +1,215 @@ +"""Without credentials the LLM reviews are skipped; the user is told.""" +import json +import os +import time + +import pytest + +from conftest import ( + bash_payload, commit_file, edit_payload, git, metrics_of, run_hook, + stop_payload, ups_payload, VULN_PY, +) +from test_gateway_sign_in import NOT_SIGNED_IN, SIGNED_IN, calls_of, make_cli + +# Anything in the developer's shell that changes whether a review would run. +_AMBIENT = ( + "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", + # Makes every session share one state file. + "CLAUDE_CODE_REMOTE_SESSION_ID", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "DISABLE_TELEMETRY", "DO_NOT_TRACK", + "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX", "CLAUDE_CODE_USE_FOUNDRY", + "CLAUDE_CODE_USE_MANTLE", "CLAUDE_CODE_USE_ANTHROPIC_AWS", + "SECURITY_GUIDANCE_DISABLE", "ENABLE_SECURITY_REMINDER", + "ENABLE_CODE_SECURITY_REVIEW", "ENABLE_STOP_REVIEW", "ENABLE_COMMIT_REVIEW", + "ENABLE_PATTERN_RULES", +) + + +def _env(hook_env, creds=False, **extra): + env = {k: v for k, v in hook_env.items() if k not in _AMBIENT} + if creds: + env["ANTHROPIC_API_KEY"] = "test-key" + env.update(extra) + return env + + +def _edit_output(repo, env, name="app.py", content=VULN_PY, session_id="s1"): + (repo / name).write_text(content) + rc, out, _ = run_hook( + edit_payload(repo, repo / name, new_string=content, session_id=session_id), env) + assert rc == 0 + lines = [l for l in out.splitlines() if l.startswith("{")] + assert len(lines) <= 1 + return json.loads(lines[0]) if lines else {} + + +def _notice(repo, env, **kw): + return _edit_output(repo, env, **kw).get("systemMessage") + + +def _marker(env): + return os.path.join(env["SECURITY_WARNINGS_STATE_DIR"], ".no_credentials_notice") + + +def test_edit_without_credentials_tells_the_user_once(workspace, hook_env): + _, repo = workspace + env = _env(hook_env) + notice = _notice(repo, env) + assert "is not running" in notice + assert "No Anthropic credentials were passed" in notice + assert "only the pattern warnings on edits are active" in notice + assert "ENABLE_CODE_SECURITY_REVIEW=0" in notice + assert _notice(repo, env) is None + assert _notice(repo, env, session_id="s2") is None + + +def test_notice_comes_back_after_a_week(workspace, hook_env): + _, repo = workspace + env = _env(hook_env) + assert _notice(repo, env) + six_days, eight_days = time.time() - 6 * 86400, time.time() - 8 * 86400 + os.utime(_marker(env), (six_days, six_days)) + assert _notice(repo, env) is None + os.utime(_marker(env), (eight_days, eight_days)) + assert _notice(repo, env) + assert _notice(repo, env) is None + + +def test_notice_names_every_setting_that_withholds_credentials(workspace, hook_env): + _, repo = workspace + notice = _notice(repo, _env(hook_env, DISABLE_TELEMETRY="1", DO_NOT_TRACK="true")) + assert "while DISABLE_TELEMETRY or DO_NOT_TRACK is set" in notice + + +@pytest.mark.parametrize("var", ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "DO_NOT_TRACK"]) +def test_notice_names_the_setting(workspace, hook_env, var): + _, repo = workspace + assert f"while {var} is set" in _notice(repo, _env(hook_env, **{var: "1"})) + + +def test_do_not_track_off_is_not_blamed(workspace, hook_env): + _, repo = workspace + notice = _notice(repo, _env(hook_env, DO_NOT_TRACK="0")) + assert "DO_NOT_TRACK" not in notice and "No Anthropic credentials" in notice + + +def test_notice_does_not_promise_pattern_warnings_that_are_off(workspace, hook_env): + _, repo = workspace + notice = _notice(repo, _env(hook_env, ENABLE_PATTERN_RULES="0")) + assert "is not running" in notice and "pattern warnings" not in notice + + +def test_notice_rides_along_with_a_pattern_warning(workspace, hook_env): + _, repo = workspace + risky = "import os\nos.system('ls ' + name)\n" + with_creds = _edit_output( + repo, _env(hook_env, creds=True), name="a.py", content=risky) + assert with_creds["metrics"]["pattern_hits"] == 1 + assert "systemMessage" not in with_creds + out = _edit_output(repo, _env(hook_env), name="b.py", content=risky) + assert "is not running" in out["systemMessage"] + assert out["metrics"] == with_creds["metrics"] + assert (out["hookSpecificOutput"]["additionalContext"] + == with_creds["hookSpecificOutput"]["additionalContext"]) + + +def test_no_notice_with_credentials(workspace, hook_env): + _, repo = workspace + assert _notice(repo, _env(hook_env, creds=True)) is None + assert _notice(repo, _env(hook_env, ANTHROPIC_AUTH_TOKEN="t")) is None + assert _notice(repo, _env(hook_env, CLAUDE_CODE_USE_BEDROCK="1")) is None + assert not os.path.exists(_marker(hook_env)) + + +def test_no_notice_when_the_reviews_are_turned_off(workspace, hook_env): + _, repo = workspace + assert _notice(repo, _env(hook_env, ENABLE_CODE_SECURITY_REVIEW="0")) is None + assert _notice(repo, _env(hook_env, SECURITY_GUIDANCE_DISABLE="1")) is None + assert _notice(repo, _env( + hook_env, ENABLE_STOP_REVIEW="0", SECURITY_GUIDANCE_COMMIT_REVIEW="off", + SG_PUSH_SWEEP="off")) is None + assert not os.path.exists(_marker(hook_env)) + assert _notice(repo, _env(hook_env, ENABLE_STOP_REVIEW="0")) + + +def test_no_notice_outside_a_git_repo(workspace, hook_env, tmp_path): + _, repo = workspace + plain = tmp_path / "plain" + plain.mkdir() + env = _env(hook_env) + assert _notice(plain, env, name="notes.py") is None + assert not os.path.exists(_marker(env)) + assert _notice(repo, env) + + +def test_no_notice_when_the_state_dir_is_unwritable(workspace, hook_env, tmp_path): + _, repo = workspace + blocker = tmp_path / "not-a-dir" + blocker.write_text("") + env = _env(hook_env, SECURITY_WARNINGS_STATE_DIR=str(blocker / "state")) + assert _notice(repo, env) is None + assert _notice(repo, env) is None + + +def test_skipped_reviews_report_the_missing_credentials(workspace, hook_env, stub_api): + _, repo = workspace + env = _env(hook_env) + run_hook(ups_payload(repo), env) + _edit_output(repo, env) + rc, out, _ = run_hook(stop_payload(repo), env) + m = metrics_of(out) + assert rc == 0 and m["skip_reason"] == 3 and m["no_creds"] is True + + _, commit_out = commit_file(repo, "app.py", VULN_PY + "# again\n") + rc, out, _ = run_hook( + bash_payload(repo, "git commit -m change", stdout=commit_out), env) + m = metrics_of(out) + assert rc == 0 and m["skip_reason"] == 22 and m["no_creds"] is True + + rc, out, _ = run_hook(bash_payload(repo, "git push origin main"), env) + m = metrics_of(out) + assert rc == 0 and m["push_sweep"] and m["skip_reason"] == 22 and m["no_creds"] is True + assert stub_api.calls == [] + + off = _env(hook_env, ENABLE_CODE_SECURITY_REVIEW="0") + for payload in (stop_payload(repo, session_id="s3"), + bash_payload(repo, "git push origin main", session_id="s3")): + rc, out, _ = run_hook(payload, off) + assert "no_creds" not in metrics_of(out) + + +def test_no_notice_in_a_signed_in_gateway_session(workspace, hook_env, tmp_path): + # Claude Code passes no credential there, but the reviews run through + # the user's claude, so they are running. + _, repo = workspace + cli, calls = make_cli(tmp_path, SIGNED_IN) + env = _env(hook_env, SG_AGENTIC_CLI_PATH=str(cli)) + assert _notice(repo, env) is None + # The check counts as this week's: a new session's edit does not ask. + assert _notice(repo, env, session_id="s2") is None + assert os.path.exists(_marker(env)) + assert [c.split(" | ")[0] for c in calls_of(calls)] == ["auth status --json"] + # A week later the edit asks again. + eight_days = time.time() - 8 * 86400 + os.utime(_marker(env), (eight_days, eight_days)) + assert _notice(repo, env, session_id="s3") is None + assert len(calls_of(calls)) == 2 + + +def test_edit_outside_a_git_repo_does_not_ask_claude(workspace, hook_env, tmp_path): + cli, calls = make_cli(tmp_path, SIGNED_IN) + plain = tmp_path / "plain" + plain.mkdir() + env = _env(hook_env, SG_AGENTIC_CLI_PATH=str(cli)) + assert _notice(plain, env, name="notes.py") is None + assert calls_of(calls) == [] + + +def test_notice_when_the_gateway_session_is_not_signed_in(workspace, hook_env, tmp_path): + _, repo = workspace + cli, calls = make_cli(tmp_path, NOT_SIGNED_IN) + env = _env(hook_env, SG_AGENTIC_CLI_PATH=str(cli)) + assert "is not running" in _notice(repo, env) + # Within the week no notice is due, so claude is not asked again. + assert _notice(repo, env, session_id="s2") is None + assert len(calls_of(calls)) == 1 diff --git a/plugins/security-guidance/tests/test_notebook_edit_patterns.py b/plugins/security-guidance/tests/test_notebook_edit_patterns.py new file mode 100644 index 00000000..df4d37a9 --- /dev/null +++ b/plugins/security-guidance/tests/test_notebook_edit_patterns.py @@ -0,0 +1,193 @@ +"""NotebookEdit cell sources go through the same pattern checks as file edits.""" +import json +import subprocess +import sys + +import pytest + +import extensibility +import security_reminder_hook as hook + +from conftest import HOOKS_DIR, metrics_of, run_hook + +CELL = ( + "import pickle, os\n" + "data = pickle.loads(open('model.pkl', 'rb').read())\n" + "os.system(f'convert {user_input}')\n" +) +CELL_RULES = ["os_system_injection", "pickle_deserialization"] + + +@pytest.fixture +def env(hook_env, tmp_path): + # Keep the developer's own security-patterns files out of the counts. + home = tmp_path / "home" + home.mkdir() + hook_env.update(HOME=str(home), USERPROFILE=str(home)) + hook_env.pop("CLAUDE_CONFIG_DIR", None) + return hook_env + + +def notebook_payload(cwd, notebook, new_source=CELL, tool_response=None, **tool_input): + return { + "session_id": "s1", + "hook_event_name": "PostToolUse", + "tool_name": "NotebookEdit", + "tool_input": {"notebook_path": str(notebook), "new_source": new_source, **tool_input}, + "tool_response": tool_response if tool_response is not None else {}, + "cwd": str(cwd), + } + + +def warning_of(stdout): + for line in stdout.splitlines(): + if line.strip().startswith("{"): + out = json.loads(line).get("hookSpecificOutput") + if out: + return out["additionalContext"] + return "" + + +@pytest.mark.parametrize("tool_input", [ + {"cell_id": "c1"}, + {"cell_id": "c1", "edit_mode": "replace", "cell_type": "code"}, + {"cell_id": "c1", "edit_mode": "insert", "cell_type": "code"}, + {"edit_mode": "insert", "cell_type": "code"}, + {"cell_id": "c1", "cell_type": "markdown"}, +]) +def test_cell_source_is_checked(tmp_path, env, tool_input): + rc, out, _ = run_hook( + notebook_payload(tmp_path, tmp_path / "analysis.ipynb", **tool_input), env) + assert rc == 0 + warning = warning_of(out) + assert "pickle" in warning + assert "os.system" in warning + assert metrics_of(out)["pattern_hits"] == 2 + + +def test_same_cell_matches_a_python_file_edit(tmp_path, env): + _, nb_out, _ = run_hook( + notebook_payload(tmp_path, tmp_path / "analysis.ipynb", cell_id="c1"), env) + py = notebook_payload(tmp_path, tmp_path / "analysis.py") + py.update(tool_name="Edit", tool_input={ + "file_path": str(tmp_path / "analysis.py"), "old_string": "", "new_string": CELL}) + _, py_out, _ = run_hook(py, env) + assert metrics_of(nb_out)["rule_mask"] == metrics_of(py_out)["rule_mask"] + + +@pytest.mark.parametrize("tool_input, tool_response", [ + ({"cell_id": "c1", "edit_mode": "delete"}, {}), + ({"cell_id": "c1", "edit_mode": "delete", "cell_type": "code"}, {"error": ""}), + ({"cell_id": "c1"}, {"error": "File has been modified since read"}), + ({"cell_id": "c1"}, {"new_source": CELL, "error": "Edit rejected"}), +]) +def test_deleted_cell_and_failed_edit_are_not_checked(tmp_path, env, tool_input, tool_response): + rc, out, _ = run_hook( + notebook_payload(tmp_path, tmp_path / "analysis.ipynb", + tool_response=tool_response, **tool_input), env) + assert rc == 0 + assert metrics_of(out) is None + assert warning_of(out) == "" + + +def test_successful_edit_with_an_empty_error_is_checked(tmp_path, env): + # What Claude Code sends for an edit that worked. + _, out, _ = run_hook( + notebook_payload(tmp_path, tmp_path / "analysis.ipynb", cell_id="c1", + tool_response={"error": "", "cell_type": "code", "old_source": CELL}), + env) + assert metrics_of(out)["pattern_hits"] == 2 + + +@pytest.mark.parametrize("tool_input, tool_response", [ + ({"cell_id": "c1", "new_source": None}, {}), + ({"cell_id": "c1", "new_source": ["import pickle"]}, {}), + ({"cell_id": None}, {}), + ({"cell_id": "c1"}, "Updated cell c1"), + ({"cell_id": "c1"}, None), +]) +def test_unexpected_shapes_do_not_break_the_hook(tmp_path, env, tool_input, tool_response): + payload = notebook_payload(tmp_path, tmp_path / "analysis.ipynb", **tool_input) + payload["tool_response"] = tool_response + rc, _, err = run_hook(payload, env) + assert rc == 0 + assert "Traceback" not in err + + +def test_clean_cell_is_quiet(tmp_path, env): + rc, out, _ = run_hook( + notebook_payload(tmp_path, tmp_path / "analysis.ipynb", + new_source="import json\ndata = json.load(open('m.json'))\n", + cell_id="c1"), env) + assert rc == 0 + assert metrics_of(out) is None + + +def saved_notebook(code, bom=""): + # NotebookEdit saves a cell's source as one string; Jupyter saves a list of lines. + return bom + json.dumps({ + "cells": [ + {"cell_type": "markdown", "id": "m", "metadata": {}, "source": "# Notes\n"}, + {"cell_type": "code", "id": "c", "metadata": {}, "source": code, "outputs": []}, + ], + "metadata": {}, "nbformat": 4, "nbformat_minor": 5, + }, indent=1) + + +@pytest.fixture +def rules(monkeypatch): + monkeypatch.setattr(extensibility, "_user_patterns", []) + return lambda path, content: sorted(r for r, _ in hook.check_patterns(str(path), content)) + + +@pytest.mark.parametrize("bom", ["", ""]) +def test_saved_notebook_is_checked_by_its_cells_as_written(rules, bom): + saved = saved_notebook(CELL, bom) + assert "\\nos.system(" in saved + assert rules("analysis.ipynb", saved) == CELL_RULES + assert rules("analysis.ipynb", saved_notebook(CELL.splitlines(True), bom)) == CELL_RULES + assert rules("analysis.ipynb", saved_notebook("x = 1\n", bom)) == [] + + +def test_saved_notebook_outputs_are_not_checked(rules): + saved = json.loads(saved_notebook("x = 1\n")) + saved["cells"][1]["outputs"] = [{"output_type": "stream", "name": "stdout", "text": [CELL]}] + assert rules("analysis.ipynb", json.dumps(saved)) == [] + saved["cells"][0]["source"] = CELL.splitlines(True) + assert rules("analysis.ipynb", json.dumps(saved)) == CELL_RULES + + +@pytest.mark.parametrize("content", [ + "{not json", "{}", '{"cells": 5}', '{"cells": [5]}', '{"cells": [{"source": 5}]}', "[]", +]) +def test_content_that_is_not_a_saved_notebook_is_checked_as_is(rules, content): + assert rules("analysis.ipynb", content) == [] + assert rules("analysis.ipynb", content + "\nimport os\nos.system(cmd)\n") == ["os_system_injection"] + + +def test_other_files_are_not_read_as_notebooks(rules): + assert rules("analysis.py", saved_notebook("import os\nos.system(cmd)\n")) == [] + + +SWEEP = ( + "import sys, json; sys.path.insert(0, sys.argv[1]);" + "import security_reminder_hook as h;" + "print(json.dumps(h.sweep_pending_warnings('s1')[:2]))" +) + + +def sweep(env): + r = subprocess.run([sys.executable, "-c", SWEEP, str(HOOKS_DIR)], + capture_output=True, text=True, env=env) + return json.loads(r.stdout.strip().splitlines()[-1]) + + +def test_notebook_warning_is_tallied_from_the_saved_cells(tmp_path, env): + nb = tmp_path / "analysis.ipynb" + nb.write_text(saved_notebook(CELL)) + run_hook(notebook_payload(tmp_path, nb, cell_id="c"), env) + assert sweep(env) == [0, 2] + + run_hook(notebook_payload(tmp_path, nb, cell_id="c"), env) + nb.write_text(saved_notebook("x = 1\n")) + assert sweep(env) == [2, 0] diff --git a/plugins/security-guidance/tests/test_pattern_fired_metrics.py b/plugins/security-guidance/tests/test_pattern_fired_metrics.py new file mode 100644 index 00000000..c1e31300 --- /dev/null +++ b/plugins/security-guidance/tests/test_pattern_fired_metrics.py @@ -0,0 +1,90 @@ +import json + +from conftest import ( + STUB_VULN, edit_payload, metrics_of, run_hook, stop_payload, ups_payload, +) + +from patterns import RuleId + +EVAL = 1 << RuleId.EVAL_INJECTION +SHELL = 1 << RuleId.PYTHON_SUBPROCESS_SHELL + + +def _edit(repo, env, name, content): + (repo / name).write_text(content) + run_hook(edit_payload(repo, repo / name, content), env) + + +def _turn(repo, env, edits, after=None): + run_hook(ups_payload(repo), env) + for name, content in edits.items(): + _edit(repo, env, name, content) + for name, content in (after or {}).items(): + (repo / name).write_text(content) + rc, so, se = run_hook(stop_payload(repo), env) + return metrics_of(so) + + +class TestFiredRulesReachTheStopRow: + def test_clean_review(self, env, gateway, workspace): + _, repo = workspace + m = _turn(repo, env, {"calc.js": "const v = eval(src)\n"}) + assert m["vulns_found"] == 0 and "review_ms" in m + assert m["warn_fired_mask"] == EVAL + assert m["warn_unresolved_mask"] == EVAL + assert list(m).index("warn_fired_mask") < 20 + + def test_fired_mask_follows_the_unresolved_mask(self, env, gateway, workspace): + """A row cut at the key cap loses the fired mask before the + unresolved one.""" + _, repo = workspace + gateway.vulns = [STUB_VULN] + m = _turn(repo, env, {"calc.js": "const v = eval(src)\n"}) + keys = list(m) + assert keys.index("warn_unresolved_mask") < keys.index("warn_fired_mask") + assert m["warn_fixed"] == 0 and m["warn_unresolved"] == 1 + + def test_review_with_findings(self, env, gateway, workspace): + _, repo = workspace + gateway.vulns = [STUB_VULN] + m = _turn(repo, env, {"calc.js": "const v = eval(src)\n"}) + assert m["vulns_found"] == 1 + assert m["warn_fired_mask"] == EVAL and m["warn_unresolved_mask"] == EVAL + + def test_review_skipped(self, env, gateway, workspace): + _, repo = workspace + m = _turn(repo, {**env, "ENABLE_STOP_REVIEW": "0"}, + {"calc.js": "const v = eval(src)\n"}) + assert m["skipped"] is True + assert m["warn_fired_mask"] == EVAL and m["warn_unresolved_mask"] == EVAL + + def test_fixed_rule_is_fired_but_not_unresolved(self, env, gateway, workspace): + _, repo = workspace + m = _turn( + repo, env, + {"calc.js": "const v = eval(src)\n", + "run.py": "import subprocess\nsubprocess.run(cmd, shell=True)\n"}, + after={"calc.js": "const v = JSON.parse(src)\n"}, + ) + assert m["warn_fired_mask"] == EVAL | SHELL + assert m["warn_unresolved_mask"] == SHELL + + def test_no_warning_no_keys(self, env, gateway, workspace): + _, repo = workspace + m = _turn(repo, env, {"ok.py": "def ok():\n return 1\n"}) + assert "warn_fired_mask" not in m and "warn_unresolved_mask" not in m + + def test_masks_are_reported_once(self, env, gateway, workspace): + _, repo = workspace + _turn(repo, env, {"calc.js": "const v = eval(src)\n"}) + m = _turn(repo, env, {"ok.py": "def ok():\n return 2\n"}) + assert "warn_fired_mask" not in m and "warn_unresolved_mask" not in m + + def test_user_rule_is_not_counted_as_a_builtin_of_the_same_name(self, env, gateway, workspace): + _, repo = workspace + (repo / ".claude").mkdir() + (repo / ".claude" / "security-patterns.json").write_text(json.dumps({"patterns": [ + {"rule_name": "eval_injection", "regex": "FORBIDDEN_CALL", "reminder": "no"}]})) + m = _turn(repo, {**env, "ENABLE_STOP_REVIEW": "0"}, {"a.py": "FORBIDDEN_CALL()\n"}) + assert "warn_fired_mask" not in m + assert m["warn_fixed"] == 0 and m["warn_unresolved"] == 1 diff --git a/plugins/security-guidance/tests/test_patterns_file_shape.py b/plugins/security-guidance/tests/test_patterns_file_shape.py new file mode 100644 index 00000000..a63494b6 --- /dev/null +++ b/plugins/security-guidance/tests/test_patterns_file_shape.py @@ -0,0 +1,116 @@ +"""A security-patterns file that parses to something other than a mapping is +skipped on its own; rules from the other locations still load.""" +import json + +import pytest + +import extensibility + +from conftest import run_hook + +VALID = {"patterns": [{"rule_name": "demo_rule", "regex": "AKIA[0-9A-Z]{16}", + "reminder": "DEMO_RULE_REMINDER"}]} + + +@pytest.fixture +def project(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.setenv("USERPROFILE", str(home)) + monkeypatch.delenv("CLAUDE_CONFIG_DIR", raising=False) + monkeypatch.setattr(extensibility, "_user_patterns", []) + monkeypatch.setattr(extensibility, "_guidance_block", "") + proj = tmp_path / "proj" + (proj / ".claude").mkdir(parents=True) + (proj / ".claude" / "security-patterns.json").write_text(json.dumps(VALID)) + return proj + + +def loaded(proj): + extensibility.load_for_session(str(proj)) + return [r["ruleName"] for r in extensibility.user_patterns()] + + +def rule(name="x", **kw): + return dict(VALID["patterns"][0], rule_name=name, **kw) + + +WRONG_SHAPE_JSON = [ + json.dumps([rule()]), + "5", + '"just a string"', + '{"patterns": 5}', + '{"patterns": "demo"}', + '{"patterns": {"rule_name": "x"}}', + '{"patterns": {}}', + '{"patterns": null}', + "{not json", + json.dumps({"patterns": [rule(regex="a{99999999999}")]}), +] + + +@pytest.mark.parametrize("body", WRONG_SHAPE_JSON) +def test_wrong_shape_local_json_keeps_the_project_rules(project, body): + (project / ".claude" / "security-patterns.local.json").write_text(body) + assert loaded(project) == ["user:demo_rule"] + + +@pytest.mark.parametrize("body", [ + "just a bare scalar\n", + "- rule_name: x\n regex: a\n reminder: r\n", + "patterns: 5\n", + "a: [1\n", + "when: 2001-13-45\n", +]) +def test_wrong_shape_local_yaml_keeps_the_project_rules(project, body): + pytest.importorskip("yaml") + (project / ".claude" / "security-patterns.local.yaml").write_text(body) + assert loaded(project) == ["user:demo_rule"] + + +def test_file_that_is_not_utf8_keeps_the_project_rules(project): + body = json.dumps({"patterns": [rule("local_rule")]}).encode("utf-16") + (project / ".claude" / "security-patterns.local.json").write_bytes(body) + assert loaded(project) == ["user:demo_rule"] + + +@pytest.mark.parametrize("body", ["- a\n- b\n", "patterns: 5\n"]) +def test_unusable_yaml_does_not_hide_the_json_next_to_it(project, body): + pytest.importorskip("yaml") + (project / ".claude" / "security-patterns.yaml").write_text(body) + assert loaded(project) == ["user:demo_rule"] + + +def test_one_bad_rule_does_not_drop_the_rest_of_its_file(project): + local = {"patterns": [rule("bad", regex="a{99999999999}"), rule("local_rule")]} + (project / ".claude" / "security-patterns.local.json").write_text(json.dumps(local)) + assert loaded(project) == ["user:demo_rule", "user:local_rule"] + + +def test_rules_from_a_valid_local_file_still_load(project): + local = {"patterns": [rule("local_rule")]} + (project / ".claude" / "security-patterns.local.json").write_text(json.dumps(local)) + assert loaded(project) == ["user:demo_rule", "user:local_rule"] + + +def test_wrong_shape_file_is_named_in_the_debug_log(project, monkeypatch): + lines = [] + monkeypatch.setattr(extensibility, "debug_log", lines.append) + (project / ".claude" / "security-patterns.local.json").write_text("[1, 2]") + assert loaded(project) == ["user:demo_rule"] + assert any("security-patterns.local.json" in l and "got list" in l for l in lines) + + +def test_edit_hook_still_warns_with_a_wrong_shape_file_present(project, hook_env): + (project / ".claude" / "security-patterns.local.json").write_text("[1, 2]") + env = dict(hook_env, HOME=str(project.parent / "home"), CLAUDE_PROJECT_DIR=str(project)) + env.pop("CLAUDE_CONFIG_DIR", None) + payload = { + "session_id": "s1", "hook_event_name": "PostToolUse", "tool_name": "Write", + "tool_input": {"file_path": str(project / "cfg.py"), + "content": "KEY = 'AKIAABCDEFGHIJKLMNOP'\n"}, + "cwd": str(project), "tool_response": {}, + } + _rc, stdout, stderr = run_hook(payload, env) + assert "DEMO_RULE_REMINDER" in stdout + stderr diff --git a/plugins/security-guidance/tests/test_portable_output.py b/plugins/security-guidance/tests/test_portable_output.py new file mode 100644 index 00000000..cad2052c --- /dev/null +++ b/plugins/security-guidance/tests/test_portable_output.py @@ -0,0 +1,181 @@ +import json +import os +import subprocess +import sys + +import pytest + +from conftest import ( + HOOKS_DIR, STUB_VULN, VULN_PY, edit_payload, metrics_of, run_hook, stop_payload, ups_payload, +) + +# What a host with a closed hook output schema accepts. +STOP_FIELDS = {"continue", "decision", "reason", "stopReason", "suppressOutput", "systemMessage"} +POST_TOOL_USE_FIELDS = STOP_FIELDS | {"hookSpecificOutput"} +HOST_MARKERS = ("CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT", "PLUGIN_ROOT", + "SECURITY_GUIDANCE_PORTABLE_OUTPUT") + + +def _env(hook_env, **extra): + env = {k: v for k, v in hook_env.items() if k not in HOST_MARKERS} + env.update(extra) + return env + + +def _objects(stdout): + lines = [l for l in stdout.splitlines() if l.strip()] + return [json.loads(l) for l in lines] + + +def _review_with_finding(workspace, env, stub_api): + ws, repo = workspace + stub_api.vulns = [STUB_VULN] + run_hook(ups_payload(repo), env) + (repo / "app.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py", VULN_PY), env) + return run_hook(stop_payload(repo), env) + + +class TestOtherHost: + def test_stop_skip_path_prints_only_standard_fields(self, workspace, hook_env): + ws, repo = workspace + rc, so, se = run_hook(stop_payload(repo), _env(hook_env, PLUGIN_ROOT=str(ws))) + assert rc == 0 + assert _objects(so) == [{}] + + def test_stop_finding_keeps_the_block_and_the_guidance(self, workspace, hook_env, stub_api): + rc, so, se = _review_with_finding(workspace, _env(hook_env, PLUGIN_ROOT="/p"), stub_api) + (out,) = _objects(so) + assert set(out) <= STOP_FIELDS, out + assert out["decision"] == "block" + assert "command_injection" in out["reason"] + assert "command_injection" in se + + def test_pattern_warning_keeps_the_guidance(self, workspace, hook_env): + ws, repo = workspace + env = _env(hook_env, PLUGIN_ROOT="/p") + (repo / "app.py").write_text(VULN_PY) + rc, so, se = run_hook(edit_payload(repo, repo / "app.py", VULN_PY), env) + assert rc == 0 + (out,) = _objects(so) + assert set(out) <= POST_TOOL_USE_FIELDS, out + assert out["hookSpecificOutput"]["additionalContext"] + + def test_kill_switch_path(self, workspace, hook_env): + ws, repo = workspace + env = _env(hook_env, PLUGIN_ROOT="/p", SECURITY_GUIDANCE_DISABLE="1") + rc, so, se = run_hook(stop_payload(repo), env) + assert rc == 0 + assert all(set(o) <= STOP_FIELDS for o in _objects(so)) + + + def test_mode_is_not_changed_by_the_hook_process_itself(self, hook_env): + code = ("import os, _base\n" + "os.environ['CLAUDE_CODE_ENTRYPOINT'] = 'sdk-py'\n" + "_base.write_hook_output({'metrics': {'x': 1}, 'reason': 'r'})\n") + r = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, + cwd=HOOKS_DIR, env=_env(hook_env, PLUGIN_ROOT="/p"), timeout=60) + assert _objects(r.stdout) == [{"reason": "r"}], r.stderr + + +class TestClaudeCodeUnchanged: + @pytest.mark.parametrize("extra", [ + {}, + {"PLUGIN_ROOT": "/p", "CLAUDECODE": "1"}, + {"PLUGIN_ROOT": "/p", "CLAUDE_CODE_ENTRYPOINT": "cli"}, + {"PLUGIN_ROOT": "/p", "SECURITY_GUIDANCE_PORTABLE_OUTPUT": "0"}, + ]) + def test_metrics_and_summary_are_still_sent(self, workspace, hook_env, stub_api, extra): + rc, so, se = _review_with_finding(workspace, _env(hook_env, **extra), stub_api) + (out,) = _objects(so) + assert out["metrics"]["vulns_found"] == 1 + assert out["rewakeSummary"] + assert out["decision"] == "block" + + def test_forced_on_under_claude_code(self, workspace, hook_env): + ws, repo = workspace + env = _env(hook_env, CLAUDECODE="1", SECURITY_GUIDANCE_PORTABLE_OUTPUT="1") + rc, so, se = run_hook(stop_payload(repo), env) + assert metrics_of(so) is None and _objects(so) == [{}] + + +# SessionStart runs ensure_agent_sdk.py. The wrapper picks the branch without +# installing anything: whether the SDK looks importable, and whether the +# interpreter looks older than the SDK supports. +SESSION_START_FIELDS = STOP_FIELDS | {"hookSpecificOutput"} +_BOOTSTRAP = """ +import importlib.util, os, runpy, sys +script, sdk, old_py = sys.argv[1:4] +sys.path.insert(0, os.path.dirname(script)) +real = importlib.util.find_spec +def find_spec(name, *a, **k): + if name == "claude_agent_sdk": + return object() if sdk == "present" else None + return real(name, *a, **k) +importlib.util.find_spec = find_spec +class V(tuple): + major, minor = property(lambda s: s[0]), property(lambda s: s[1]) +sys.version_info = V(((3, 9, 6) if old_py == "old" else (3, 12, 0)) + ("final", 0)) +sys.argv = [script] +runpy.run_path(script, run_name="__main__") +""" + + +def _session_start(tmp_path, branch, **extra): + state = tmp_path / "state" + state.mkdir(exist_ok=True) + sdk, old_py = "absent", "new" + if branch == "noop": + sdk = "present" + elif branch == "concurrent_build": + (state / "agent-sdk-venv.building").touch() + elif branch == "failure": + (tmp_path / "file").write_text("") + state = tmp_path / "file" / "state" + elif branch == "notice": + old_py = "old" + env = {k: v for k, v in os.environ.items() if k not in HOST_MARKERS} + env.update(SECURITY_WARNINGS_STATE_DIR=str(state), HOME=str(tmp_path), + PYTHONDONTWRITEBYTECODE="1", **extra) + r = subprocess.run( + [sys.executable, "-c", _BOOTSTRAP, str(HOOKS_DIR / "ensure_agent_sdk.py"), sdk, old_py], + input="{}", capture_output=True, text=True, env=env, timeout=60, + ) + return r.returncode, r.stdout, r.stderr + + +SESSION_START_BRANCHES = ["noop", "concurrent_build", "failure", "notice"] + + +class TestSessionStartOtherHost: + @pytest.mark.parametrize("branch", SESSION_START_BRANCHES) + def test_one_document_with_only_standard_fields(self, tmp_path, branch): + rc, so, se = _session_start(tmp_path, branch, PLUGIN_ROOT="/p") + assert rc == 0 and se == "" + out = json.loads(so) # the whole stream is one JSON document + assert set(out) <= SESSION_START_FIELDS, out + assert (out != {}) == (branch == "notice") + + def test_notice_is_kept(self, tmp_path): + rc, so, se = _session_start(tmp_path, "notice", PLUGIN_ROOT="/p") + out = json.loads(so) + assert "Python" in out["systemMessage"] + assert out["hookSpecificOutput"] == { + "hookEventName": "SessionStart", "additionalContext": out["systemMessage"]} + + +class TestSessionStartClaudeCodeUnchanged: + @pytest.mark.parametrize("branch,outcome", [ + ("noop", 0), ("concurrent_build", 5), ("failure", 3), ("notice", 6)]) + @pytest.mark.parametrize("extra", [ + {}, + {"PLUGIN_ROOT": "/p", "CLAUDECODE": "1"}, + {"PLUGIN_ROOT": "/p", "SECURITY_GUIDANCE_PORTABLE_OUTPUT": "0"}, + ]) + def test_async_line_then_metrics(self, tmp_path, branch, outcome, extra): + rc, so, se = _session_start(tmp_path, branch, **extra) + assert rc == 0 + first, second = _objects(so) + assert first == {"async": True, "asyncTimeout": 180000} + assert second["metrics"]["sdk_bootstrap"] == outcome + assert ("systemMessage" in second) == (branch == "notice") diff --git a/plugins/security-guidance/tests/test_project_config_files.py b/plugins/security-guidance/tests/test_project_config_files.py new file mode 100644 index 00000000..5cef98c0 --- /dev/null +++ b/plugins/security-guidance/tests/test_project_config_files.py @@ -0,0 +1,117 @@ +"""Project guidance and pattern files are read only as real files in the +project, and the guidance block ends where the plugin ends it.""" +import os +import re +import sys + +import pytest + +import extensibility + + +GUIDANCE = extensibility.GUIDANCE_BASENAME +RULE = '{"patterns": [{"rule_name": "r", "reminder": "from the file", "substrings": ["zz"]}]}' + + +@pytest.fixture +def project(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path / "home")) + monkeypatch.setenv("USERPROFILE", str(tmp_path / "home")) + monkeypatch.delenv("CLAUDE_CONFIG_DIR", raising=False) + monkeypatch.setattr(extensibility, "_guidance_block", "") + monkeypatch.setattr(extensibility, "_user_patterns", []) + repo = tmp_path / "repo" + (repo / ".claude").mkdir(parents=True) + return repo + + +def _link(link, target, is_dir=False): + try: + os.symlink(target, link, target_is_directory=is_dir) + except (OSError, NotImplementedError) as e: + pytest.skip(f"cannot make a link here: {e}") + + +def _guidance(cwd): + extensibility.load_for_session(str(cwd)) + return extensibility.guidance_block() + + +def test_guidance_file_in_the_project_is_read(project): + (project / ".claude" / GUIDANCE).write_text("use the vault\n") + assert "use the vault" in _guidance(project) + + +def test_guidance_link_inside_the_repository_is_skipped(project): + (project / ".env").write_text("TOKEN=abc\n") + _link(project / ".claude" / GUIDANCE, project / ".env") + assert _guidance(project) == "" + + +def test_guidance_link_outside_is_skipped(project, tmp_path): + (tmp_path / "outside.md").write_text("outside text\n") + _link(project / ".claude" / GUIDANCE, tmp_path / "outside.md") + assert _guidance(project) == "" + + +def test_config_folder_link_is_skipped(project, tmp_path): + other = tmp_path / "other" + other.mkdir() + (other / GUIDANCE).write_text("outside text\n") + (project / ".claude").rmdir() + _link(project / ".claude", other, is_dir=True) + assert _guidance(project) == "" + + +def test_patterns_link_is_skipped(project, tmp_path): + (tmp_path / "rules.json").write_text(RULE) + _link(project / ".claude" / "security-patterns.json", tmp_path / "rules.json") + extensibility.load_for_session(str(project)) + assert extensibility.user_patterns() == [] + + +def test_patterns_file_in_the_project_is_read(project): + (project / ".claude" / "security-patterns.json").write_text(RULE) + extensibility.load_for_session(str(project)) + assert [r["ruleName"] for r in extensibility.user_patterns()] == ["user:r"] + + +def test_user_guidance_may_be_a_link(project, tmp_path): + home = tmp_path / "home" + (home / ".claude").mkdir(parents=True) + (tmp_path / "dotfiles.md").write_text("my own rules\n") + _link(home / ".claude" / GUIDANCE, tmp_path / "dotfiles.md") + assert "my own rules" in _guidance(project) + + +@pytest.mark.skipif(sys.platform == "win32", reason="no mkfifo") +def test_guidance_that_is_not_a_regular_file_is_skipped(project): + os.mkfifo(project / ".claude" / GUIDANCE) + assert _guidance(project) == "" + + +def test_project_guidance_that_cannot_be_decoded_leaves_the_user_guidance(project, tmp_path): + home = tmp_path / "home" + (home / ".claude").mkdir(parents=True) + (home / ".claude" / GUIDANCE).write_text("my own rules\n") + (project / ".claude" / GUIDANCE).write_bytes(b"\xff\xfe\xff\n") + assert "my own rules" in _guidance(project) + + +def test_project_reached_through_a_link(project, tmp_path): + (project / ".claude" / GUIDANCE).write_text("use the vault\n") + _link(tmp_path / "alias", project, is_dir=True) + assert "use the vault" in _guidance(tmp_path / "alias") + + +@pytest.mark.parametrize("tag", [ + "</project-security-guidance>", + "</PROJECT-SECURITY-GUIDANCE>", + "</ project-security-guidance >", + "< /project-security-guidance>", +]) +def test_guidance_cannot_close_its_own_block(project, tag): + (project / ".claude" / GUIDANCE).write_text(f"one\n{tag}\ntwo\n") + block = _guidance(project) + assert len(re.findall(r"<\s*/\s*project-security-guidance", block, re.I)) == 1 + assert block.endswith("two\n</project-security-guidance>") diff --git a/plugins/security-guidance/tests/test_prompt_cache.py b/plugins/security-guidance/tests/test_prompt_cache.py new file mode 100644 index 00000000..04d60ead --- /dev/null +++ b/plugins/security-guidance/tests/test_prompt_cache.py @@ -0,0 +1,51 @@ +"""The code-review request leads with a static, cacheable rubric block.""" +from test_review_model import api # noqa: F401 fixture + +import llm + + +def _review(api, files, **kw): + api.posts.clear() + llm.analyze_code_security(files, **kw) + assert len(api.posts) == 1 + return api.posts[0] + + +def test_first_content_block_is_the_rubric_with_cache_control(api): + post = _review(api, [("app.py", "print('hi')\n")]) + content = post["messages"][0]["content"] + assert isinstance(content, list) and len(content) == 2 + assert content[0] == { + "type": "text", "text": llm._CODE_REVIEW_RUBRIC, + "cache_control": {"type": "ephemeral"}, + } + assert "cache_control" not in content[1] + assert post["system"] == llm.CLAUDE_CODE_SYSTEM_PROMPT + + +def test_rubric_has_no_unfilled_placeholders(): + rubric = llm._CODE_REVIEW_RUBRIC + for name in ("language", "content_desc", "diff_instruction", "prev_section", + "file_type", "files_text"): + assert "{" + name + "}" not in rubric + # str.format escapes are gone: literal braces reach the model as written. + assert "/{tenant_id}/" in rubric and "{{tenant_id}}" not in rubric + + +def test_rubric_block_is_identical_across_different_inputs(api): + a = _review(api, [("main.go", "package main\n")]) + b = _review(api, [("views.py", "+x = 1\n")], is_diff=True, + previous_findings=[{"filePath": "views.py", "category": "sqli", + "vulnerableCode": "cursor.execute(q)"}]) + ca, cb = a["messages"][0]["content"], b["messages"][0]["content"] + assert ca[0]["text"] == cb[0]["text"] + assert ca[1]["text"] != cb[1]["text"] + assert "Go code" in ca[1]["text"] and "main.go" in ca[1]["text"] + assert "Python diff" in cb[1]["text"] and "unified diff" in cb[1]["text"] + assert "PREVIOUS FINDINGS" in cb[1]["text"] and "=== DIFF: views.py ===" in cb[1]["text"] + + +def test_plain_prompt_without_prefix_stays_a_string(api): + api.posts.clear() + llm._call_claude("review this", {"type": "object"}) + assert api.posts[0]["messages"][0]["content"] == "review this" diff --git a/plugins/security-guidance/tests/test_prompt_untracked_snapshot.py b/plugins/security-guidance/tests/test_prompt_untracked_snapshot.py new file mode 100644 index 00000000..be0fc1ee --- /dev/null +++ b/plugins/security-guidance/tests/test_prompt_untracked_snapshot.py @@ -0,0 +1,169 @@ +import json +import os +import shutil + +import pytest + +from conftest import ( + VULN_PY, edit_payload, make_repo, metrics_of, run_hook, stop_payload, ups_payload, +) + +OFF_SWITCHES = [ + pytest.param({"ENABLE_STOP_REVIEW": "0"}, id="stop-review-off"), + pytest.param({"ENABLE_CODE_SECURITY_REVIEW": "0"}, id="all-reviews-off"), +] +OLD = "import os\nos.system(cmd)\n" + + +@pytest.fixture +def on(hook_env, tmp_path): + """hook_env with the Stop review on, and a `git` first on PATH that + records each call before running it.""" + shim = tmp_path / "bin" + shim.mkdir() + log = tmp_path / "git.log" + log.write_text("") + (shim / "git").write_text( + f'#!/bin/sh\nprintf \'%s\\n\' "$*" >> "{log}"\nexec "{shutil.which("git")}" "$@"\n') + (shim / "git").chmod(0o755) + env = {k: v for k, v in hook_env.items() if k != "CLAUDE_CODE_REMOTE_SESSION_ID"} + env.update(HOME=str(tmp_path), PATH=f"{shim}{os.pathsep}{hook_env['PATH']}", + GIT_CALLS=str(log)) + return env + + +def _listed_untracked(env): + with open(env["GIT_CALLS"]) as f: + return any("ls-files --others" in line for line in f) + + +def _state(env, session_id="s1"): + path = os.path.join(env["SECURITY_WARNINGS_STATE_DIR"], + f"security_warnings_state_{session_id}.json") + with open(path) as f: + return json.load(f) + + +def _repo(tmp_path): + repo = make_repo(tmp_path / "repo", {"app.py": OLD}) + (repo / "notes.txt").write_text("scratch\n") + return repo + + +def _write_payload(cwd, file_path, content, session_id="s1"): + return { + "session_id": session_id, + "hook_event_name": "PostToolUse", + "tool_name": "Write", + "tool_input": {"file_path": str(file_path), "content": content}, + "tool_response": {}, + "cwd": str(cwd), + } + + +def _reviewed(stub_api): + n = len(stub_api.calls) + del stub_api.calls[:] + return n > 0 + + +pytestmark = pytest.mark.skipif(os.name == "nt", reason="the git recorder is a sh script") + + +class TestPrompt: + def test_review_on_takes_the_snapshot(self, tmp_path, on): + repo = _repo(tmp_path) + rc, _, _ = run_hook(ups_payload(repo), on) + assert rc == 0 + assert _listed_untracked(on) + assert list(_state(on)["untracked_at_baseline"]) == ["notes.txt"] + + @pytest.mark.parametrize("off", OFF_SWITCHES) + def test_review_off_does_not_list_the_worktree(self, tmp_path, on, off): + repo = _repo(tmp_path) + rc, _, _ = run_hook(ups_payload(repo), {**on, **off}) + assert rc == 0 + assert not _listed_untracked(on) + assert "untracked_at_baseline" not in _state(on) + + @pytest.mark.parametrize("off", OFF_SWITCHES) + def test_review_off_write_filter_still_knows_what_was_there(self, tmp_path, on, off): + """The stash baseline is still captured: a full-file Write that keeps + a risky call from before the prompt is not reported as new.""" + repo = _repo(tmp_path) + env = {**on, **off, "CLAUDE_PROJECT_DIR": str(repo)} + (repo / "app.py").write_text(OLD + "# edited before the prompt\n") + run_hook(ups_payload(repo), env) + + _rc, out, _ = run_hook(_write_payload(repo, repo / "app.py", OLD + "print('x')\n"), env) + assert "additionalContext" not in out + _rc, out, _ = run_hook(_write_payload( + repo, repo / "app.py", OLD + "import pickle\npickle.loads(data)\n"), env) + assert "pickle" in out and "os.system" not in out + + def test_review_off_uses_the_repo_hint_without_listing(self, tmp_path, on): + repo = _repo(tmp_path) + outside = tmp_path / "not-a-repo" + outside.mkdir() + (repo / "app.py").write_text(VULN_PY) + # The hint a commit review that resolved the repo leaves behind. + with open(os.path.join(on["SECURITY_WARNINGS_STATE_DIR"], + "security_warnings_state_s1.json"), "w") as f: + json.dump({"shown_warnings": [], "repo_root_hint": str(repo)}, f) + rc, _, _ = run_hook(ups_payload(outside), {**on, "ENABLE_STOP_REVIEW": "0"}) + assert rc == 0 + assert not _listed_untracked(on) + assert _state(on).get("baseline_sha") + + def test_review_off_drops_the_snapshot_of_an_older_baseline(self, tmp_path, on): + repo = _repo(tmp_path) + run_hook(ups_payload(repo), on) + assert "untracked_at_baseline" in _state(on) + run_hook(ups_payload(repo), {**on, "ENABLE_STOP_REVIEW": "0"}) + assert "untracked_at_baseline" not in _state(on) + + +class TestTurn: + def test_review_on_reviews_only_new_untracked_files(self, tmp_path, on, stub_api): + repo = _repo(tmp_path) + (repo / "old_vuln.py").write_text(VULN_PY) + run_hook(ups_payload(repo), on) + _rc, out, _ = run_hook(stop_payload(repo), on) + assert metrics_of(out)["skip_reason"] == 9 # empty review set + assert not _reviewed(stub_api) + + run_hook(ups_payload(repo), on) + (repo / "new_vuln.py").write_text(VULN_PY) + _rc, out, _ = run_hook(stop_payload(repo), on) + assert metrics_of(out)["files_reviewed"] == 1 + assert _reviewed(stub_api) + + @pytest.mark.parametrize("off", OFF_SWITCHES) + def test_review_switched_back_on_at_the_next_prompt(self, tmp_path, on, stub_api, off): + """Old untracked files are not reviewed as this turn's work.""" + repo = _repo(tmp_path) + (repo / "old_vuln.py").write_text(VULN_PY) + run_hook(ups_payload(repo), {**on, **off}) + run_hook(stop_payload(repo), {**on, **off}) + + run_hook(ups_payload(repo), on) + _rc, out, _ = run_hook(stop_payload(repo), on) + assert metrics_of(out)["skip_reason"] == 9 + assert not _reviewed(stub_api) + + def test_review_switched_back_on_after_an_interrupted_turn(self, tmp_path, on, stub_api): + """The interrupted turn's baseline is kept, so its edit is reviewed, + and the old untracked file beside it is not.""" + off = {**on, "ENABLE_STOP_REVIEW": "0"} + repo = _repo(tmp_path) + (repo / "old_vuln.py").write_text(VULN_PY) + run_hook(ups_payload(repo), off) + (repo / "app.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py"), off) # no Stop: interrupted + baseline = _state(on)["baseline_sha"] + + run_hook(ups_payload(repo), on) + assert _state(on)["baseline_sha"] == baseline + _rc, out, _ = run_hook(stop_payload(repo), on) + assert metrics_of(out)["files_reviewed"] == 1 + assert _reviewed(stub_api) diff --git a/plugins/security-guidance/tests/test_repeated_diff_count.py b/plugins/security-guidance/tests/test_repeated_diff_count.py new file mode 100644 index 00000000..ede4fdcb --- /dev/null +++ b/plugins/security-guidance/tests/test_repeated_diff_count.py @@ -0,0 +1,268 @@ +"""Reviews report whether their diff was reviewed before; nothing is skipped.""" +from conftest import bash_payload, commit_file, git, make_repo, metrics_of, run_hook + +import diffstate + +import pytest + +DIFF = [("handlers.py", "@@ -0,0 +1,2 @@\n+def h(x):\n+ return x")] + + +@pytest.fixture(autouse=True) +def _claude_code_output(monkeypatch): + # diffstate reads the output mode from the test runner's environment. + monkeypatch.setattr(diffstate, "portable_output", lambda: False) + + +def _reviews(stub_api): + return [c for c in stub_api.calls if c.endswith("/v1/messages")] + + +def _records(repo): + p = repo / ".git" / "sg-reviewed-diffs" + return p.read_text().splitlines() if p.exists() else [] + + +def _recommit(repo, msg): + git(repo, "reset", "-q", "--soft", "HEAD~1") + out = git(repo, "commit", "-m", msg) + return git(repo, "rev-parse", "HEAD").strip(), out + + +def _seen(repo, diff=DIFF): + return diffstate._diff_review_started(str(repo), diff) + + +def _finished(repo, diff=DIFF): + token, _ = _seen(repo, diff) + diffstate._diff_review_done(token) + + +def _hunk(start, context, added): + return f"@@ -{start},2 +{start},3 @@ def f():\n {context}\n+{added}\n pass" + + +def test_first_review_of_a_diff_is_not_seen(tmp_path): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + token, m = _seen(repo) + assert m == {"diff_seen": diffstate.DIFF_SEEN_NO} + assert token and len(_records(repo)) == 1 + + +def test_unfinished_then_finished_on_same_then_other_tree(tmp_path): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + token, _ = _seen(repo) + assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_UNFINISHED} + diffstate._diff_review_done(token) + assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_SAME_TREE} + commit_file(repo, "other.py", "x = 1\n") + assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_OTHER_TREE} + + +def test_tree_is_the_reviewed_commits_not_where_head_went(tmp_path): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + sha, _ = commit_file(repo, "handlers.py", "def h(x):\n return x\n") + token, _ = diffstate._diff_review_started(str(repo), DIFF, tip=sha) + diffstate._diff_review_done(token) + git(repo, "reset", "-q", "--hard", "HEAD~1") + assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_OTHER_TREE} + assert diffstate._diff_review_started(str(repo), DIFF, tip=sha)[1] == { + "diff_seen": diffstate.DIFF_SEEN_SAME_TREE} + + +def test_a_different_change_path_or_place_is_not_seen(tmp_path): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + one = [("auth.py", _hunk(3, "def login():", "return True"))] + _finished(repo, one) + for other in ([("auth.py", _hunk(3, "def login():", "return False"))], + [("other.py", one[0][1])], + # The same added line next to different code. + [("auth.py", _hunk(3, "def is_admin():", "return True"))], + one + [("b.py", "@@ -0,0 +1 @@\n+1")]): + assert _seen(repo, other)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO} + + +def test_a_moved_hunk_keeps_its_key(tmp_path): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + _finished(repo, [("auth.py", _hunk(3, "def login():", "return True"))]) + moved = "@@ -40,2 +44,3 @@ class A:\n def login():\n+return True\n pass" + assert _seen(repo, [("auth.py", moved)])[1] == {"diff_seen": diffstate.DIFF_SEEN_SAME_TREE} + + +def test_real_diffs_of_two_commits_match_their_net_diff(tmp_path): + from gitutil import parse_diff_into_files + body = "".join(f"line {i}\n" for i in range(60)) + repo = make_repo(tmp_path / "repo", {"app.py": body}) + base = git(repo, "rev-parse", "HEAD").strip() + commit_file(repo, "app.py", body.replace("line 5\n", "line 5\nx = 1\n")) + _finished(repo, parse_diff_into_files(git(repo, "show", "-p", "HEAD"))) + commit_file(repo, "app.py", body.replace("line 5\n", "line 5\nx = 1\n") + .replace("line 50\n", "line 50\ny = 2\n")) + _finished(repo, parse_diff_into_files(git(repo, "show", "-p", "HEAD"))) + net = parse_diff_into_files(git(repo, "diff", base, "HEAD")) + assert len(diffstate._hunk_keys(net)) == 2 + # Both hunks had a finished review; the first one on an older tree. + assert _seen(repo, net)[1] == {"diff_seen": diffstate.DIFF_SEEN_OTHER_TREE} + + +def test_hunks_past_the_key_cap_are_not_dropped(tmp_path, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + monkeypatch.setattr(diffstate, "_SEEN_DIFFS_MAX_KEYS", 3) + hunks = [_hunk(10 * i, f"def f{i}():", f"x = {i}") for i in range(6)] + _finished(repo, [("app.py", "\n".join(hunks[:5]))]) + assert len(_records(repo)[0].split("\t")[3].split(" ")) == 3 + assert _seen(repo, [("app.py", "\n".join(hunks))])[1] == {"diff_seen": diffstate.DIFF_SEEN_NO} + assert _seen(repo, [("app.py", "\n".join(hunks[:5]))])[1] == { + "diff_seen": diffstate.DIFF_SEEN_SAME_TREE} + + +def test_removed_sql_comment_lines_tell_diffs_apart(tmp_path): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + _finished(repo, [("q.sql", "@@ -1,2 +1 @@\n--- drop table users;\n select 1;")]) + other = [("q.sql", "@@ -1,2 +1 @@\n--- drop table orders;\n select 1;")] + assert _seen(repo, other)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO} + + +def test_commits_reviewed_one_by_one_then_as_one_net_diff(tmp_path): + # Two commit reviews of the same file, then a push review of their net diff. + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + h1, h2 = _hunk(3, "def a():", "x = 1"), _hunk(90, "def z():", "y = 2") + _finished(repo, [("app.py", h1)]) + net = [("app.py", h1 + "\n" + h2), ("b.py", "@@ -0,0 +1 @@\n+b = 2")] + assert _seen(repo, net)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO} + _finished(repo, [("app.py", h2)]) + _finished(repo, [net[1]]) + assert _seen(repo, net)[1] == {"diff_seen": diffstate.DIFF_SEEN_SAME_TREE} + + +def test_old_records_do_not_count(tmp_path, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + _finished(repo) + monkeypatch.setattr(diffstate, "_SEEN_DIFFS_TTL_S", -1) + assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO} + + +@pytest.mark.parametrize("flock", [True, False]) +def test_record_is_cut_back_to_its_newest_rows(tmp_path, monkeypatch, flock): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + if not flock: + def no_flock(fd, exclusive): + raise ImportError("fcntl") + monkeypatch.setattr(diffstate, "_flock_nb", no_flock) + _finished(repo) + row = len(_records(repo)[0]) + 1 + monkeypatch.setattr(diffstate, "_SEEN_DIFFS_TRIM_BYTES", 6 * row) + for i in range(12): + _finished(repo, [("f.py", f"@@ -0,0 +1 @@\n+{i}")]) + rows = _records(repo) + assert 3 <= len(rows) <= 6 + assert rows[-1].split("\t")[2:] == ["1", diffstate._hunk_keys([("f.py", "@@ -0,0 +1 @@\n+11")])[0]] + assert _seen(repo, [("f.py", "@@ -0,0 +1 @@\n+11")])[1]["diff_seen"] == diffstate.DIFF_SEEN_SAME_TREE + assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO} + + +def test_damaged_record_is_read_around_and_shrinks(tmp_path, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + p = repo / ".git" / "sg-reviewed-diffs" + p.write_bytes(b"\xff\xfe garbage\nshort\tline\n" + b"9" * 5000 + b"\tt\t1\tk\n" + + b"x" * 5000) + monkeypatch.setattr(diffstate, "_SEEN_DIFFS_TRIM_BYTES", 2000) + token, m = _seen(repo) + assert m == {"diff_seen": diffstate.DIFF_SEEN_NO} + diffstate._diff_review_done(token) + assert p.stat().st_size < 2000 + assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_SAME_TREE} + + +def test_a_lock_another_process_keeps_is_not_waited_on(tmp_path, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + token, _ = _seen(repo) + monkeypatch.setattr(diffstate, "_flock_nb", lambda fd, exclusive: False) + monkeypatch.setattr(diffstate, "_SEEN_DIFFS_LOCK_TRIES", 2) + monkeypatch.setattr(diffstate, "_SEEN_DIFFS_TRIM_BYTES", 0) + # A start that cannot read the record under the lock does not guess. + assert _seen(repo) == (None, {"diff_seen": diffstate.DIFF_SEEN_ERROR}) + # A verdict is still noted, and the record is not cut back without the lock. + diffstate._diff_review_done(token) + assert [r.split("\t")[2] for r in _records(repo)] == ["0", "1"] + + +def test_failures_are_reported_and_never_raise(tmp_path): + assert _seen(tmp_path / "nope") == (None, {"diff_seen": diffstate.DIFF_SEEN_ERROR}) + diffstate._diff_review_done(None) + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + (repo / ".git" / "sg-reviewed-diffs").mkdir() + assert _seen(repo) == (None, {"diff_seen": diffstate.DIFF_SEEN_ERROR}) + diffstate._diff_review_done((str(repo / ".git" / "sg-reviewed-diffs"), ["k"], "t")) + + +def test_a_record_that_cannot_be_written_is_an_error_not_unseen(tmp_path, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + def full_disk(f, locked, token, done): + raise OSError(28, "No space left on device") + monkeypatch.setattr(diffstate, "_append_diff_row", full_disk) + assert _seen(repo) == (None, {"diff_seen": diffstate.DIFF_SEEN_ERROR}) + + +def test_nothing_is_recorded_in_portable_output_mode(tmp_path, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + monkeypatch.setattr(diffstate, "portable_output", lambda: True) + assert _seen(repo) == (None, {}) + assert _records(repo) == [] + + +def test_recommit_of_the_same_change_is_reviewed_again_and_counted(tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + sha, out = commit_file(repo, "handlers.py", "def h(x):\n return x\n") + rc, so, _ = run_hook(bash_payload(repo, "git commit -m change", out), hook_env) + first = metrics_of(so) + assert rc == 0 and first["diff_seen"] == diffstate.DIFF_SEEN_NO + + sha2, out2 = _recommit(repo, "same change again") + assert sha2 != sha + rc, so, _ = run_hook(bash_payload(repo, "git commit -m 'same change again'", out2), hook_env) + second = metrics_of(so) + assert rc == 0 and second["diff_seen"] == diffstate.DIFF_SEEN_SAME_TREE + # Last in the row, so a row over the key cap loses this and nothing older. + assert list(second)[-1] == "diff_seen" + # Count only: both commits were reviewed and recorded. + assert len(_reviews(stub_api)) == 2 + assert {sha, sha2} <= diffstate._load_reviewed_shas(str(repo)) + + +def test_push_of_a_rebased_reviewed_commit_is_counted(tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + remote = tmp_path / "remote.git" + git(tmp_path, "init", "-q", "--bare", str(remote)) + git(repo, "remote", "add", "origin", str(remote)) + git(repo, "push", "-q", "-u", "origin", "main") + git(repo, "checkout", "-q", "-b", "feat") + sha, out = commit_file(repo, "handlers.py", "def h(x):\n return x\n") + run_hook(bash_payload(repo, "git commit -m change", out), hook_env) + + git(repo, "checkout", "-q", "main") + commit_file(repo, "notes.md", "notes\n") + git(repo, "push", "-q", "origin", "main") + git(repo, "checkout", "-q", "feat") + git(repo, "rebase", "-q", "main") + git(repo, "push", "-q", "-u", "origin", "feat") + push_out = f"To {remote}\n * [new branch] feat -> feat\n" + rc, so, _ = run_hook(bash_payload(repo, "git push -u origin feat", push_out), hook_env) + m = metrics_of(so) + assert rc == 0 and m["push_sweep"] is True and m["unreviewed"] == 1 + assert m["diff_seen"] == diffstate.DIFF_SEEN_OTHER_TREE and list(m)[-1] == "diff_seen" + assert len(_reviews(stub_api)) == 2 + + +def test_failed_review_leaves_the_diff_unfinished(tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + sha, out = commit_file(repo, "handlers.py", "def h(x):\n return x\n") + stub_api.status = 500 + rc, so, _ = run_hook(bash_payload(repo, "git commit -m change", out), hook_env) + failed = metrics_of(so) + assert failed["review_failed"] is True and failed["diff_seen"] == diffstate.DIFF_SEEN_NO + + stub_api.status = 200 + sha2, out2 = _recommit(repo, "same change again") + rc, so, _ = run_hook(bash_payload(repo, "git commit -m 'same change again'", out2), hook_env) + assert metrics_of(so)["diff_seen"] == diffstate.DIFF_SEEN_UNFINISHED diff --git a/plugins/security-guidance/tests/test_repo_resolution.py b/plugins/security-guidance/tests/test_repo_resolution.py index f6b8d352..30c94a7c 100644 --- a/plugins/security-guidance/tests/test_repo_resolution.py +++ b/plugins/security-guidance/tests/test_repo_resolution.py @@ -191,6 +191,10 @@ class TestRegexes: "git -c core.editor=true -C sub commit --amend", "git --git-dir=x/.git --work-tree=x commit -m x", "git --git-dir x/.git commit -m x", + "git --no-pager commit -m x", + "git -p --no-optional-locks -c user.name=x --bare commit -m x", + 'git -c user.name="First Last" -c user.email=a@b.c commit -S -m x', + "git --git-dir='a b/.git' --namespace ns commit -m x", "cd sub && git commit -m x", "gt create -m x", ]) @@ -199,16 +203,29 @@ class TestRegexes: @pytest.mark.parametrize("cmd", [ "git push", "git -C sub push origin main", 'git -C "a b" push', - "git --work-tree x push -u origin HEAD", "gt submit", + "git --work-tree x push -u origin HEAD", "git --no-pager push", + 'git -c http.extraHeader="X-A: b" push', "git --config-env http.extraheader=TOK push", "gt submit", ]) def test_push_re(self, cmd): assert hook._GIT_PUSH_RE.search(cmd) - @pytest.mark.parametrize("cmd", ["git status", "git log --oneline", "echo commit"]) + @pytest.mark.parametrize("cmd", ["git status", "git log --oneline", "echo commit", + "git commit-graph write --reachable", "git commit-tree HEAD^{tree}", + "git log --grep commit", "git stash push", + "git log --oneline commit", "git --no-pager log --grep commit", + "git merge --no-commit topic", "git --git-dir commit status", + "git --namespace commit log", "git --help commit", + 'git -c user.name="commit status']) def test_commit_re_negative(self, cmd): assert not hook._GIT_COMMIT_RE.search(cmd) - @pytest.mark.parametrize("unit", ['-c "a"', "-c 'a'", "--no-a=b", "--a=b=c=d", "--git-dir x"]) + @pytest.mark.parametrize("cmd", ["git stash push", "git log --grep push", "git commit-graph write", + "git --no-pager stash push", "git --no-pager log push"]) + def test_push_re_negative(self, cmd): + assert not hook._GIT_PUSH_RE.search(cmd) + + @pytest.mark.parametrize("unit", ['-c "a"', "-c 'a'", "--no-a=b", "--a=b=c=d", "--git-dir x", + "--no-pager", "-p", '-c a="b c"', "--namespace x"]) @pytest.mark.parametrize("regex,verb", [("_GIT_COMMIT_RE", "commit"), ("_GIT_PUSH_RE", "push")]) def test_global_option_prefix_is_linear(self, regex, verb, unit): cre = getattr(hook, regex) @@ -222,15 +239,25 @@ class TestRegexes: class TestHooksJson: def test_matchers_and_events(self): cfg = json.loads((HOOKS_DIR / "hooks.json").read_text()) - assert "SubagentStop" in cfg["hooks"] - assert cfg["hooks"]["SubagentStop"][0]["hooks"][0]["command"] == \ - cfg["hooks"]["Stop"][0]["hooks"][0]["command"] + # Stop already reviews what subagents changed during the turn. + assert "SubagentStop" not in cfg["hooks"] + stop = cfg["hooks"]["Stop"][0]["hooks"][0] + assert stop["command"].endswith('security_reminder_hook.py"') + assert stop["asyncRewake"] is True bash = [g for g in cfg["hooks"]["PostToolUse"] if g.get("matcher") == "Bash"][0] - ifs = {h.get("if") for h in bash["hooks"]} - assert {"Bash(git commit:*)", "Bash(git push:*)", "Bash(git -C * commit *)", - "Bash(git -C * push*)", "Bash(gt create:*)", "Bash(gt modify:*)", - "Bash(gt submit:*)"} <= ifs - + # One entry per kind, not per spelling: when Claude Code cannot parse + # a command it runs every entry, so each extra entry is one more process. + assert [h.get("if") for h in bash["hooks"]] == [ + "Bash(git*commit*)", "Bash(git*push*)", "Bash(gt *)"] + assert all(h["asyncRewake"] is True and h["rewakeMessage"] for h in bash["hooks"]) + + @pytest.mark.parametrize("cmd", ["git stash push", "git commit-graph write --reachable", + "git log --grep commit", "gt log"]) + def test_other_commands_the_if_patterns_let_through_do_nothing(self, workspace, hook_env, stub_api, cmd): + ws, repo = workspace + commit_file(repo, "app.py", VULN_PY) + rc, so, se = run_hook(bash_payload(repo, cmd, ""), hook_env) + assert rc == 0 and so.strip() == "" and not stub_api.calls def _read_state(env): diff --git a/plugins/security-guidance/tests/test_review_model.py b/plugins/security-guidance/tests/test_review_model.py new file mode 100644 index 00000000..e290f35e --- /dev/null +++ b/plugins/security-guidance/tests/test_review_model.py @@ -0,0 +1,404 @@ +"""The default review model follows the newest Opus instead of a pinned release.""" +import http.server +import json +import sys +import threading +import time +import types + +import pytest + +from conftest import ( + VULN_PY, edit_payload, metrics_of, run_hook, stop_payload, ups_payload, +) + +import llm + +SCHEMA = {"type": "object", "properties": {"hasVulnerabilities": {"type": "boolean"}}} +PINNED = "claude-opus-5-5" + + +class _Api(http.server.BaseHTTPRequestHandler): + def do_GET(self): + self.server.gets.append((self.path, dict(self.headers))) + if self.server.models_status != 200: + return self._send(self.server.models_status, {"type": "error"}) + self._send(200, self.server.models_body) + + def do_POST(self): + n = int(self.headers.get("Content-Length") or 0) + body = json.loads(self.rfile.read(n)) + self.server.posts.append(body) + self._send(200, { + "id": "msg_stub", "type": "message", "role": "assistant", + "model": body["model"], "stop_reason": "end_turn", + "content": [{"type": "text", "text": json.dumps( + {"hasVulnerabilities": False, "vulnerabilities": []})}], + "usage": {"input_tokens": 1, "output_tokens": 1}, + }) + + def _send(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *a): + pass + + +def _models(*ids): + return {"data": [{"type": "model", "id": i} for i in ids], "has_more": False} + + +@pytest.fixture +def api(tmp_path, monkeypatch): + srv = http.server.HTTPServer(("127.0.0.1", 0), _Api) + srv.gets, srv.posts = [], [] + srv.models_status = 200 + srv.models_body = _models( + "claude-fable-5-1", "claude-opus-5-5", "claude-sonnet-5-5", "claude-opus-5", + "claude-opus-4-8", "claude-opus-4-7", "claude-opus-4-5-20251101", + ) + threading.Thread(target=srv.serve_forever, daemon=True).start() + for var in ("SECURITY_REVIEW_MODEL", "SG_AGENTIC_MODEL", "SG_DUAL_OR", + "CLAUDE_CODE_EXECPATH", "SG_AGENTIC_CLI_PATH", *llm._PROVIDER_ENV_VARS): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("ANTHROPIC_BASE_URL", f"http://127.0.0.1:{srv.server_port}") + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(tmp_path / "state")) + monkeypatch.setenv("NO_PROXY", "*") + monkeypatch.setenv("no_proxy", "*") + monkeypatch.setattr(llm, "ANTHROPIC_API_KEY", "test-key") + monkeypatch.setattr(llm, "ANTHROPIC_AUTH_TOKEN", "") + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", True) + monkeypatch.setattr(llm, "_auth_prefer_token", False) + monkeypatch.setattr(llm, "_review_model_memo", None, raising=False) + try: + yield srv + finally: + srv.shutdown() + + +def _new_process(monkeypatch): + """Each hook fire is a fresh process: only the on-disk cache carries over.""" + monkeypatch.setattr(llm, "_review_model_memo", None, raising=False) + + +def _review(): + return llm._call_claude_dual_or( + "review this", SCHEMA, + bool_key="hasVulnerabilities", list_key="vulnerabilities", + ) + + +class TestNewestOpus: + def test_highest_version_wins_whatever_the_order(self): + ids = ["claude-opus-4-7", "claude-opus-5-5", "claude-opus-5", "claude-opus-4-8"] + assert llm._newest_opus(ids) == "claude-opus-5-5" + assert llm._newest_opus(reversed(ids)) == "claude-opus-5-5" + + def test_minor_version_compares_as_a_number(self): + assert llm._newest_opus(["claude-opus-5-9", "claude-opus-5-10"]) == "claude-opus-5-10" + + def test_snapshot_date_is_not_a_version(self): + ids = ["claude-opus-4-20250514", "claude-opus-4-1-20250805", "claude-opus-4-5-20251101"] + assert llm._newest_opus(ids) == "claude-opus-4-5-20251101" + + def test_other_families_and_malformed_ids_are_ignored(self): + ids = [ + "claude-fable-5-1", "claude-sonnet-5-5", "claude-haiku-4-5", + "claude-opus-9-preview", "bedrock/claude-opus-9", "claude-opus-9 --flag", + "us.anthropic.claude-opus-9", None, 7, "claude-opus-4-7", + ] + assert llm._newest_opus(ids) == "claude-opus-4-7" + + def test_no_opus_listed(self): + assert llm._newest_opus(["claude-sonnet-5-5", "gpt-x"]) is None + + +class TestReviewRequestModel: + def test_review_uses_the_newest_opus_the_endpoint_lists(self, api): + assert _review() == {"hasVulnerabilities": False, "vulnerabilities": []} + assert [p["model"] for p in api.posts] == ["claude-opus-5-5"] + + def test_a_release_newer_than_the_pinned_default_is_picked_up(self, api): + api.models_body = _models("claude-opus-6", "claude-opus-5-5") + _review() + assert [p["model"] for p in api.posts] == ["claude-opus-6"] + + def test_gateway_listing_only_an_older_opus_gets_that_one(self, api): + api.models_body = _models("claude-opus-4-8", "claude-sonnet-5") + _review() + assert [p["model"] for p in api.posts] == ["claude-opus-4-8"] + + def test_truncated_listing_does_not_pick_an_older_opus(self, api): + api.models_body = {**_models("org-model-2", "org-model-1", "claude-opus-4-7"), + "has_more": True} + _review() + assert [p["model"] for p in api.posts] == [PINNED] + + def test_truncated_listing_still_picks_a_newer_opus(self, api): + api.models_body = {**_models("claude-opus-6", "org-model-1"), "has_more": True} + _review() + assert [p["model"] for p in api.posts] == ["claude-opus-6"] + + def test_lookup_asks_for_one_modest_page(self, api): + _review() + assert api.gets[0][0] == "/v1/models?limit=100" + + def test_lookup_authenticates_like_the_review_call(self, api): + _review() + path, headers = api.gets[0] + assert path.startswith("/v1/models") + lowered = {k.lower(): v for k, v in headers.items()} + assert lowered["x-api-key"] == "test-key" + assert lowered["anthropic-version"] == "2023-06-01" + + def test_oauth_only_session_sends_the_bearer_token(self, api, monkeypatch): + monkeypatch.setattr(llm, "ANTHROPIC_API_KEY", "") + monkeypatch.setattr(llm, "ANTHROPIC_AUTH_TOKEN", "oauth-token") + _review() + lowered = {k.lower(): v for k, v in api.gets[0][1].items()} + assert lowered["authorization"] == "Bearer oauth-token" + assert "oauth-2025-04-20" in lowered["anthropic-beta"] + assert "x-api-key" not in lowered + + def test_explicit_model_is_used_as_is_without_a_lookup(self, api, monkeypatch): + monkeypatch.setenv("SECURITY_REVIEW_MODEL", "claude-sonnet-5") + _review() + assert [p["model"] for p in api.posts] == ["claude-sonnet-5"] + assert api.gets == [] + + +class TestLookupFailure: + @pytest.mark.parametrize("status", [401, 403, 404, 500]) + def test_falls_back_to_the_pinned_default(self, api, status): + api.models_status = status + _review() + assert [p["model"] for p in api.posts] == [PINNED] + + @pytest.mark.parametrize("body", [ + {"data": []}, {"data": "nope"}, {"object": "list"}, [], "text", + {"data": [{"id": "gpt-x"}, {"name": "claude-opus-9"}, "claude-opus-9"]}, + ]) + def test_unusable_listing_falls_back_to_the_pinned_default(self, api, body): + api.models_body = body + assert llm.default_review_model() == PINNED + + def test_unreachable_endpoint_falls_back_to_the_pinned_default(self, api, monkeypatch): + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://127.0.0.1:1") + assert llm.default_review_model() == PINNED + + def test_no_credentials_means_no_lookup(self, api, monkeypatch): + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", False) + assert llm.default_review_model() == PINNED + assert api.gets == [] + + def test_failure_is_not_retried_on_every_hook_fire(self, api, monkeypatch): + api.models_status = 500 + assert llm.default_review_model() == PINNED + _new_process(monkeypatch) + assert llm.default_review_model() == PINNED + assert len(api.gets) == 1 + + def test_failure_is_retried_after_the_retry_window(self, api, monkeypatch): + api.models_status = 500 + assert llm.default_review_model() == PINNED + api.models_status = 200 + later = time.time() + llm._REVIEW_MODEL_RETRY_SECONDS + 1 + monkeypatch.setattr(llm.time, "time", lambda: later) + _new_process(monkeypatch) + assert llm.default_review_model() == "claude-opus-5-5" + + +class TestCache: + def test_one_lookup_serves_later_hook_fires(self, api, monkeypatch): + assert llm.default_review_model() == "claude-opus-5-5" + _new_process(monkeypatch) + api.models_body = _models("claude-opus-6") + assert llm.default_review_model() == "claude-opus-5-5" + assert len(api.gets) == 1 + + def test_one_lookup_per_process(self, api): + llm.default_review_model() + llm.default_review_model() + assert len(api.gets) == 1 + + def test_new_release_is_seen_once_the_cache_expires(self, api, monkeypatch): + assert llm.default_review_model() == "claude-opus-5-5" + api.models_body = _models("claude-opus-6", "claude-opus-5-5") + later = time.time() + llm._REVIEW_MODEL_TTL_SECONDS + 1 + monkeypatch.setattr(llm.time, "time", lambda: later) + _new_process(monkeypatch) + assert llm.default_review_model() == "claude-opus-6" + + def test_cache_from_another_endpoint_is_not_reused(self, api, monkeypatch): + assert llm.default_review_model() == "claude-opus-5-5" + other = http.server.HTTPServer(("127.0.0.1", 0), _Api) + other.gets, other.posts = [], [] + other.models_status = 200 + other.models_body = _models("claude-opus-4-8") + threading.Thread(target=other.serve_forever, daemon=True).start() + try: + monkeypatch.setenv("ANTHROPIC_BASE_URL", f"http://127.0.0.1:{other.server_port}") + _new_process(monkeypatch) + assert llm.default_review_model() == "claude-opus-4-8" + finally: + other.shutdown() + + @pytest.mark.parametrize("model", [ + "claude-opus-9 --dangerously-skip-permissions", "../../etc/passwd", + "claude-fable-5-1", 7, ["claude-opus-9"], + ]) + def test_tampered_cache_entry_is_ignored(self, api, monkeypatch, tmp_path, model): + assert llm.default_review_model() == "claude-opus-5-5" + cache = tmp_path / "state" / llm._REVIEW_MODEL_CACHE_FILE + entry = json.loads(cache.read_text()) + entry["model"] = model + cache.write_text(json.dumps(entry)) + _new_process(monkeypatch) + assert llm.default_review_model() == "claude-opus-5-5" + assert len(api.gets) == 2 + + @pytest.mark.parametrize("content", ["", "not json", "[]", '{"at": "soon"}', '{"at": 1e999}']) + def test_corrupt_cache_file_is_ignored(self, api, monkeypatch, tmp_path, content): + (tmp_path / "state").mkdir() + (tmp_path / "state" / llm._REVIEW_MODEL_CACHE_FILE).write_text(content) + assert llm.default_review_model() == "claude-opus-5-5" + + def test_future_dated_cache_entry_is_ignored(self, api, monkeypatch, tmp_path): + assert llm.default_review_model() == "claude-opus-5-5" + cache = tmp_path / "state" / llm._REVIEW_MODEL_CACHE_FILE + entry = json.loads(cache.read_text()) + entry["at"] = time.time() + 10 * 365 * 24 * 3600 + cache.write_text(json.dumps(entry)) + api.models_body = _models("claude-opus-6") + _new_process(monkeypatch) + assert llm.default_review_model() == "claude-opus-6" + + def test_unwritable_state_dir_still_resolves(self, api, monkeypatch, tmp_path): + blocker = tmp_path / "file" + blocker.write_text("x") + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(blocker / "state")) + assert llm.default_review_model() == "claude-opus-5-5" + + def test_endpoint_url_is_not_written_to_disk(self, api, monkeypatch, tmp_path): + llm.default_review_model() + cache = (tmp_path / "state" / llm._REVIEW_MODEL_CACHE_FILE).read_text() + assert "127.0.0.1" not in cache + + +@pytest.fixture +def fake_sdk(monkeypatch): + """Stand-in for claude_agent_sdk that records the options of each spawn.""" + spawned = [] + + class ClaudeAgentOptions: + def __init__(self, **kwargs): + self.kwargs = kwargs + spawned.append(kwargs) + + class AssistantMessage: + pass + + class ResultMessage: + subtype = "success" + usage = {} + total_cost_usd = 0.0 + structured_output = {"findings": [], "hasVulnerabilities": False, + "vulnerabilities": []} + + async def query(prompt, options): + yield ResultMessage() + + sdk = types.ModuleType("claude_agent_sdk") + sdk.ClaudeAgentOptions = ClaudeAgentOptions + sdk.AssistantMessage = AssistantMessage + sdk.ResultMessage = ResultMessage + sdk.query = query + monkeypatch.setitem(sys.modules, "claude_agent_sdk", sdk) + return spawned + + +class TestSdkPaths: + """Reviews that run through the Agent SDK name the CLI's own `opus` alias, + so the CLI that drives the model is the one that picks it.""" + + def test_agentic_review_defaults_to_the_cli_alias(self, api, fake_sdk, tmp_path): + llm.agentic_review(str(tmp_path), [("a.py", "+x = 1\n")], ["a.py"]) + assert fake_sdk[0]["model"] == "opus" + assert fake_sdk[0]["fallback_model"] is None + assert api.gets == [] + + def test_agentic_review_override_falls_back_to_the_alias(self, api, fake_sdk, + tmp_path, monkeypatch): + monkeypatch.setenv("SG_AGENTIC_MODEL", "claude-sonnet-5") + llm.agentic_review(str(tmp_path), [("a.py", "+x = 1\n")], ["a.py"]) + assert fake_sdk[0]["model"] == "claude-sonnet-5" + assert fake_sdk[0]["fallback_model"] == "opus" + + @pytest.mark.parametrize("provider_var", [ + "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX", "CLAUDE_CODE_USE_FOUNDRY", + ]) + def test_3p_review_defaults_to_the_cli_alias(self, api, fake_sdk, monkeypatch, + provider_var): + monkeypatch.setenv(provider_var, "1") + _review() + assert fake_sdk[0]["model"] == "opus" + assert fake_sdk[0]["fallback_model"] is None + assert api.gets == [] and api.posts == [] + + def test_3p_review_keeps_an_explicit_provider_id(self, api, fake_sdk, monkeypatch): + monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "1") + monkeypatch.setenv("SECURITY_REVIEW_MODEL", "us.anthropic.claude-opus-4-7") + _review() + assert fake_sdk[0]["model"] == "us.anthropic.claude-opus-4-7" + + +class TestStopHookProcess: + """The hook as Claude Code runs it: one process per fire, config from env.""" + + def _env(self, api, hook_env): + env = dict(hook_env) + env["ANTHROPIC_BASE_URL"] = f"http://127.0.0.1:{api.server_port}" + env.pop("SECURITY_REVIEW_MODEL", None) + return env + + def _edit_then_stop(self, repo, env, session_id): + run_hook(ups_payload(repo, session_id=session_id), env) + (repo / "app.py").write_text(VULN_PY + f"# {session_id}\n") + run_hook(edit_payload(repo, repo / "app.py", VULN_PY, session_id=session_id), env) + rc, so, se = run_hook(stop_payload(repo, session_id=session_id), env) + m = metrics_of(so) + assert m.get("skip_reason") is None and m["files_reviewed"] == 1, (m, se) + + def test_stop_review_runs_on_the_newest_opus(self, api, hook_env, workspace): + _, repo = workspace + env = self._env(api, hook_env) + self._edit_then_stop(repo, env, "s1") + assert [p["model"] for p in api.posts] == ["claude-opus-5-5"] + self._edit_then_stop(repo, env, "s2") + assert [p["model"] for p in api.posts] == ["claude-opus-5-5"] * 2 + assert len(api.gets) == 1 + + def test_stop_review_honors_the_override(self, api, hook_env, workspace): + _, repo = workspace + env = self._env(api, hook_env) + env["SECURITY_REVIEW_MODEL"] = "claude-sonnet-5" + self._edit_then_stop(repo, env, "s1") + assert [p["model"] for p in api.posts] == ["claude-sonnet-5"] + assert api.gets == [] + + +class TestPricing: + def test_current_opus_is_not_priced_as_the_unknown_model_default(self): + import _base + with _base._USAGE_LOCK: + before = _base._USAGE["cost"] + _base._record_usage({"input_tokens": 1_000_000, "output_tokens": 1_000_000}, + "claude-opus-5-5") + with _base._USAGE_LOCK: + after = _base._USAGE["cost"] + assert after - before == pytest.approx(24.0) diff --git a/plugins/security-guidance/tests/test_review_outcomes.py b/plugins/security-guidance/tests/test_review_outcomes.py new file mode 100644 index 00000000..806e9bdd --- /dev/null +++ b/plugins/security-guidance/tests/test_review_outcomes.py @@ -0,0 +1,318 @@ +"""A review that got no verdict is never reported as clean.""" +import json +import threading +import time + +import pytest + +from conftest import ( + STUB_VULN, VULN_PY, bash_payload, commit_file, edit_payload, git, + metrics_of, run_hook, stop_payload, ups_payload, +) + +import llm +import security_reminder_hook as hook + + +def _reviewed(repo): + p = repo / ".git" / "sg-reviewed-shas" + return p.read_text() if p.exists() else "" + + +def _rewake_body(stdout, stderr): + """What Claude Code shows Claude for an exit-2 asyncRewake hook: stderr, + else stdout.""" + return stderr or stdout + + +def _commit(repo, env, content=VULN_PY, session_id="s1"): + sha, out = commit_file(repo, "app.py", content) + rc, so, se = run_hook(bash_payload(repo, "git commit -m change", out, + session_id=session_id), env) + return sha, rc, so, se + + +class TestFailedReviewIsNotClean: + def test_commit_left_unreviewed_and_user_told(self, env, gateway, workspace): + _, repo = workspace + gateway.status = 401 + sha, rc, so, se = _commit(repo, env) + m = metrics_of(so) + assert m["review_failed"] is True and m["api_error"] == 401 and m["vulns_found"] == 0 + assert sha not in _reviewed(repo) + assert rc == 2 + body = _rewake_body(so, se) + assert ("The security review of this commit did not complete: " + "the API rejected the plugin's credentials (HTTP 401).") in body + assert "ANTHROPIC_CUSTOM_HEADERS" in body # ANTHROPIC_BASE_URL is set + assert "If it is pushed later from Claude Code, the push review will check it." in body + assert "log.txt" in body + summary = json.loads(so.strip().splitlines()[-1])["rewakeSummary"] + assert summary.startswith("Commit security review did not complete: " + "the API rejected the plugin's credentials (HTTP 401). Log: ") + + def test_commit_pushed_by_the_same_command(self, env, gateway, workspace): + """The push hook does not run for `git commit && git push`, so no later + push review covers this commit.""" + _, repo = workspace + gateway.status = 401 + sha, out = commit_file(repo, "app.py", VULN_PY) + out += f"To /srv/remote.git\n 1111111..{sha[:7]} main -> main\n" + rc, so, se = run_hook(bash_payload(repo, "git commit -m change && git push", out), env) + assert rc == 2 + assert "The same command pushed it, so a later push will not re-check it." in se + assert "If it is pushed later" not in se + + @pytest.mark.parametrize("command", [ + 'git commit -m "retry git push on 403"', # push only in the message + "git commit -m change && git push", # push rejected or output hidden + ]) + def test_push_in_the_command_without_push_output(self, env, gateway, workspace, command): + _, repo = workspace + gateway.status = 401 + sha, out = commit_file(repo, "app.py", VULN_PY) + rc, so, se = run_hook(bash_payload(repo, command, out), env) + assert rc == 2 + assert "The same command pushed it" not in se + assert ("If the same command pushed it, a later push will not re-check it; " + "otherwise a later push from Claude Code will.") in se + + def test_told_once_per_session_for_each_kind_of_failure(self, env, gateway, workspace): + _, repo = workspace + gateway.status = 401 + _commit(repo, env) + sha2, rc, so, se = _commit(repo, env, VULN_PY + "# 2\n") + assert rc == 0 and se == "" + assert metrics_of(so)["review_failed"] is True + assert sha2 not in _reviewed(repo) + gateway.status = 503 # a different kind of failure is still told + sha3, rc, so, se = _commit(repo, env, VULN_PY + "# 3\n") + assert rc == 2 and "busy or unavailable (HTTP 503)" in se + assert sha3 not in _reviewed(repo) + + def test_unreadable_reply_is_a_failed_review(self, env, gateway, workspace): + _, repo = workspace + gateway.reply = "I can't help with that." + sha, rc, so, se = _commit(repo, env) + assert metrics_of(so)["review_failed"] is True + assert sha not in _reviewed(repo) + + def test_reply_without_the_verdict_flag_is_a_failed_review(self, env, gateway, workspace): + _, repo = workspace + gateway.reply = json.dumps({"findings": []}) + sha, rc, so, se = _commit(repo, env) + assert metrics_of(so)["review_failed"] is True + assert sha not in _reviewed(repo) + + def test_unexpected_error_in_the_call_is_a_failed_review(self, monkeypatch): + def boom(*a, **k): + raise ConnectionResetError("dropped") + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", True) + monkeypatch.setattr(llm, "_call_claude_dual_or", boom) + assert llm.analyze_code_security([("a.py", "+x = 1\n")], is_diff=True) == (None, []) + assert llm.last_review_failed() + + def test_pushed_commit_is_told_after_an_unpushed_one(self, env, gateway, workspace): + _, repo = workspace + gateway.status = 401 + _commit(repo, env) + sha, out = commit_file(repo, "app.py", VULN_PY + "# 2\n") + out += "To github.com:o/r.git\n 1111111..2222222 main -> main\n" + rc, so, se = run_hook(bash_payload(repo, "git commit -am x && git push", out), env) + assert rc == 2 and "The same command pushed it" in se + + def test_clean_review_still_marks_the_commit(self, env, gateway, workspace): + _, repo = workspace + sha, rc, so, se = _commit(repo, env) + m = metrics_of(so) + assert rc == 0 and "review_failed" not in m and m["vulns_found"] == 0 + assert sha in _reviewed(repo) + + def test_push_sweep_leaves_the_range_unreviewed(self, env, gateway, workspace, tmp_path): + """Also told after a commit review failed the same way: a failed push + review is not re-checked later, so it has its own notice.""" + ws, repo = workspace + remote = tmp_path / "remote.git" + git(ws, "init", "-q", "--bare", str(remote)) + git(repo, "remote", "add", "origin", str(remote)) + git(repo, "push", "-q", "-u", "origin", "main") + base = git(repo, "rev-parse", "HEAD").strip() + gateway.status = 401 + sha, rc, so, se = _commit(repo, env) + assert rc == 2 + git(repo, "push", "-q", "origin", "main") + push_stdout = f"To {remote}\n {base[:7]}..{sha[:7]} main -> main\n" + rc, so, se = run_hook(bash_payload(repo, "git push origin main", push_stdout), env) + m = metrics_of(so) + assert m["push_sweep"] is True and m["review_failed"] is True, m + assert sha not in _reviewed(repo) + assert rc == 2 + assert "The security review of this push did not complete" in se + assert ("It was for the pushed commits not already reviewed when they were " + "committed, and a later push will not re-check them.") in se + + def test_subagent_review_failure_keeps_the_diff_unreviewed(self, env, gateway, workspace): + """A SubagentStop review that finds nothing records its diff as + reviewed; one that got no verdict must not, or it is never retried.""" + _, repo = workspace + gateway.reply = "not json" + run_hook(ups_payload(repo), env) + (repo / "app.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py", VULN_PY), env) + rc, so, se = run_hook(stop_payload(repo, event="SubagentStop"), env) + assert rc == 0, (so, se) + gateway.reply, gateway.vulns = None, [STUB_VULN] + rc, so, se = run_hook(stop_payload(repo, event="SubagentStop"), env) + assert rc == 2 and "command_injection" in se, (so, se) + + @pytest.fixture + def race(self, monkeypatch): + """Runs the race with the fallback starting at once; agentic_review + and the single-shot review are stand-ins.""" + calls = [] + monkeypatch.setenv("SG_AGENTIC_RACE_DELAY_S", "0") + monkeypatch.delenv("SG_AGENTIC_NO_RACE", raising=False) + + def failing_single_shot(*a, **k): + calls.append("single_shot") + llm._review_status.failed = True + return None, [] + + monkeypatch.setattr(hook, "analyze_code_security", failing_single_shot) + + def run(agentic): + monkeypatch.setattr(hook, "agentic_review", agentic) + llm._review_status.failed = False + result = hook._agentic_review_with_race("/nonexistent", [("a.py", "+x")], ["a.py"], []) + assert llm.last_review_failed() is False # the flag stays on its own thread + return result + run.calls = calls + return run + + def test_review_ends_when_neither_side_returns(self, race, monkeypatch): + """A request that never answers holds both sides; the deadline holds anyway.""" + monkeypatch.setattr(hook, "_RACE_MAX_WAIT_S", 0.1) + monkeypatch.setattr(hook, "_RACE_END_S", 0.2) + release = threading.Event() + monkeypatch.setattr(hook, "analyze_code_security", lambda *a, **k: release.wait(5)) + + def stuck_agentic(*a, **k): + release.wait(5) + started = time.monotonic() + try: + guidance, vulns, m = race(stuck_agentic) + finally: + release.set() + assert time.monotonic() - started < 2 + assert (guidance, vulns) == (None, []) + assert m["review_failed"] is True + assert m["api_error"] == hook._RACE_TIMED_OUT + + def test_failed_fallback_waits_for_agentic(self, race): + """Direct calls can fail where the agentic reviewer's CLI works (it + may hold a credential the direct calls lack); its verdict must count.""" + def slow_agentic(*a, **k): + time.sleep(1) + return None, [], {"agentic": True} + + g, v, m = race(slow_agentic) + assert m["race_winner"] == 1 and not m.get("review_failed") + + def test_both_failing_reports_the_fallback_once(self, race): + def failing_agentic(*a, **k): + time.sleep(1) + return None, [], {"agentic_fallback": "investigate:ProcessError"} + + g, v, m = race(failing_agentic) + assert m["race_winner"] == 2 and m["review_failed"] is True + assert "agentic_fallback" not in m # the caller must not rerun it + assert race.calls == ["single_shot"] + + def test_no_fallback_after_agentic_finished(self, race, monkeypatch): + """Once agentic has answered, the delayed fallback never starts, even + after the caller has taken agentic's result off the queue.""" + monkeypatch.setenv("SG_AGENTIC_RACE_DELAY_S", "1") + + def quick_failing_agentic(*a, **k): + return None, [], {"agentic_fallback": "investigate:ProcessError"} + + g, v, m = race(quick_failing_agentic) + assert m["race_winner"] == 1 + time.sleep(1.5) + assert race.calls == [] + + def test_agentic_failing_while_the_fallback_overruns(self, race, monkeypatch): + """Past the deadline the race reports no verdict instead of letting the + caller start yet another review.""" + monkeypatch.setattr(hook, "_RACE_MAX_WAIT_S", 2) + + def slow_single_shot(*a, **k): + race.calls.append("single_shot") + time.sleep(10) + return None, [] + + monkeypatch.setattr(hook, "analyze_code_security", slow_single_shot) + + def failing_agentic(*a, **k): + time.sleep(0.5) + return None, [], {"agentic_fallback": "investigate:ProcessError"} + + t0 = time.monotonic() + g, v, m = race(failing_agentic) + assert time.monotonic() - t0 < 5 + assert m["review_failed"] is True and m["api_error"] == hook._RACE_TIMED_OUT + assert "agentic_fallback" not in m + assert hook._review_failure_reason(m["api_error"])[1] == "the review did not finish in time" + + def test_failed_fallback_stops_waiting_at_the_deadline(self, race, monkeypatch): + monkeypatch.setattr(hook, "_RACE_MAX_WAIT_S", 1) + + def hung_agentic(*a, **k): + time.sleep(30) + + g, v, m = race(hung_agentic) + assert m["race_winner"] == 2 and m["review_failed"] is True + + +class TestFailureReason: + @pytest.mark.parametrize("status,failure,reason", [ + (401, "auth", "the API rejected the plugin's credentials (HTTP 401)"), + (403, "auth", "the API refused access (HTTP 403)"), + (529, "busy", "the API was busy or unavailable (HTTP 529)"), + (-1, "network", "the model could not be reached (network error or timeout)"), + (404, "request", "the API refused the request (HTTP 404)"), + (None, "reply", "the model's reply could not be read"), + ]) + def test_first_party(self, monkeypatch, status, failure, reason): + for var in llm._PROVIDER_ENV_VARS: + monkeypatch.delenv(var, raising=False) + assert hook._review_failure_reason(status)[:2] == (failure, reason) + + def test_cloud_provider(self, monkeypatch): + monkeypatch.setenv("CLAUDE_CODE_USE_BEDROCK", "1") + for status in (-1, None): + assert hook._review_failure_reason(status)[:2] == ( + "provider", "the review through the cloud provider got no answer") + + def test_causes_name_only_settings_in_use(self, monkeypatch): + for var in llm._PROVIDER_ENV_VARS: + monkeypatch.delenv(var, raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("SECURITY_REVIEW_MODEL", raising=False) + assert "ANTHROPIC_CUSTOM_HEADERS" not in hook._review_failure_reason(401)[2] + assert hook._review_failure_reason(404)[2] == "" + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://gateway") + monkeypatch.setenv("SECURITY_REVIEW_MODEL", "claude-x") + assert "ANTHROPIC_CUSTOM_HEADERS" in hook._review_failure_reason(401)[2] + assert "SECURITY_REVIEW_MODEL (claude-x)" in hook._review_failure_reason(404)[2] + + +class TestAgenticReviewerOn: + def test_failed_review_is_not_clean(self, agentic_env, gateway, workspace): + _, repo = workspace + gateway.status = 401 + sha, rc, so, se = _commit(repo, agentic_env) + m = metrics_of(so) + assert m["review_failed"] is True and m["agentic_fallback"] == 2, m + assert sha not in _reviewed(repo) + assert rc == 2 and "HTTP 401" in se diff --git a/plugins/security-guidance/tests/test_review_session_persistence.py b/plugins/security-guidance/tests/test_review_session_persistence.py new file mode 100644 index 00000000..0643bea1 --- /dev/null +++ b/plugins/security-guidance/tests/test_review_session_persistence.py @@ -0,0 +1,134 @@ +"""Review runs through the Agent SDK are not saved as Claude Code session +transcripts unless SECURITY_GUIDANCE_PERSIST_SESSIONS is set.""" +import subprocess +import sys + +import pytest + +from conftest import ( + HOOKS_DIR, STUB_VULN, VULN_PY, bash_payload, commit_file, run_hook, + write_cli, +) + +import llm + +FLAG = "--no-session-persistence" +SINGLE_TURN = ( + "import sys; sys.path.insert(0, sys.argv[1]); import llm, review_api; " + "llm._call_claude_via_sdk('review', review_api.FINDINGS_SCHEMA)" +) + + +def _stub_cli(tmp_path, reject_flag=False): + """An inner CLI that records each launch's arguments on one line and + exits 1. With reject_flag it first refuses the flag the way a Claude Code + older than the flag does.""" + log = tmp_path / "inner-cli-launches" + reject = ( + f"if {FLAG!r} in sys.argv:\n" + f" sys.stderr.write(\"error: unknown option '{FLAG}'\\n\")\n" + " sys.exit(1)\n" + ) if reject_flag else "" + cli = write_cli(tmp_path, "claude", ( + "import sys\n" + # utf-8: the reviewer's prompt has characters cp1252 cannot write. + f"with open({str(log)!r}, 'a', encoding='utf-8') as f:\n" + " f.write(' '.join(sys.argv[1:]) + '\\n')\n" + f"{reject}" + "sys.exit(1)\n")) + return str(cli), log + + +def _reviews(log): + """Argument lines of the review launches (the SDK also probes `-v`).""" + return [l for l in log.read_text(encoding="utf-8").splitlines() + if "--input-format" in l] + + +@pytest.fixture +def sdk_env(env): + pytest.importorskip("claude_agent_sdk") + e = {**env, "SG_AGENTIC_COMMIT_REVIEW": "1"} + e.pop("SECURITY_GUIDANCE_PERSIST_SESSIONS", None) + return e + + +def _commit(repo, env): + _, out = commit_file(repo, "app.py", VULN_PY) + return run_hook(bash_payload(repo, "git commit -m change", out), env) + + +def _single_turn(env): + subprocess.run([sys.executable, "-c", SINGLE_TURN, str(HOOKS_DIR)], + env=env, capture_output=True, text=True, timeout=120) + + +class TestSessionArgs: + @pytest.mark.parametrize("value", [None, "", "0", "off"]) + def test_not_saved_by_default(self, monkeypatch, value): + monkeypatch.setattr(llm, "_send_session_flag", True) + if value is None: + monkeypatch.delenv("SECURITY_GUIDANCE_PERSIST_SESSIONS", raising=False) + else: + monkeypatch.setenv("SECURITY_GUIDANCE_PERSIST_SESSIONS", value) + assert llm._reviewer_session_args() == {"no-session-persistence": None} + + @pytest.mark.parametrize("value", ["1", " on ", "TRUE", "yes"]) + def test_opt_in_keeps_sessions(self, monkeypatch, value): + monkeypatch.setattr(llm, "_send_session_flag", True) + monkeypatch.setenv("SECURITY_GUIDANCE_PERSIST_SESSIONS", value) + assert llm._reviewer_session_args() == {} + + def test_only_a_rejection_of_this_flag_counts(self): + assert llm._refused_flags(["error: unknown option '--other'"]) == [] + assert llm._refused_flags([f"error: unknown option '{FLAG}'"]) == [FLAG] + + def test_left_out_once_refused(self, monkeypatch): + monkeypatch.setattr(llm, "_send_session_flag", False) + monkeypatch.delenv("SECURITY_GUIDANCE_PERSIST_SESSIONS", raising=False) + assert llm._reviewer_session_args() == {} + + +class TestInnerCliArguments: + def test_commit_reviewer_gets_the_flag(self, sdk_env, gateway, workspace, tmp_path): + cli, log = _stub_cli(tmp_path) + gateway.vulns = [STUB_VULN] + _commit(workspace[1], {**sdk_env, "SG_AGENTIC_CLI_PATH": cli}) + reviews = _reviews(log) + assert reviews and all(FLAG in r for r in reviews) + + def test_commit_reviewer_opt_in_drops_the_flag(self, sdk_env, gateway, workspace, tmp_path): + cli, log = _stub_cli(tmp_path) + gateway.vulns = [STUB_VULN] + _commit(workspace[1], {**sdk_env, "SG_AGENTIC_CLI_PATH": cli, + "SECURITY_GUIDANCE_PERSIST_SESSIONS": "1"}) + reviews = _reviews(log) + assert reviews and not any(FLAG in r for r in reviews) + + def test_single_turn_review_gets_the_flag(self, sdk_env, tmp_path): + cli, log = _stub_cli(tmp_path) + _single_turn({**sdk_env, "SG_AGENTIC_CLI_PATH": cli}) + reviews = _reviews(log) + assert reviews and all(FLAG in r for r in reviews) + + def test_single_turn_review_opt_in_drops_the_flag(self, sdk_env, tmp_path): + cli, log = _stub_cli(tmp_path) + _single_turn({**sdk_env, "SG_AGENTIC_CLI_PATH": cli, + "SECURITY_GUIDANCE_PERSIST_SESSIONS": "1"}) + reviews = _reviews(log) + assert reviews and not any(FLAG in r for r in reviews) + + +class TestInnerCliOlderThanTheFlag: + def test_commit_reviewer_runs_again_without_it(self, sdk_env, gateway, workspace, tmp_path): + cli, log = _stub_cli(tmp_path, reject_flag=True) + gateway.vulns = [STUB_VULN] + _commit(workspace[1], {**sdk_env, "SG_AGENTIC_CLI_PATH": cli}) + reviews = _reviews(log) + assert [FLAG in r for r in reviews] == [True, False] + + def test_single_turn_review_runs_again_without_it(self, sdk_env, tmp_path): + cli, log = _stub_cli(tmp_path, reject_flag=True) + _single_turn({**sdk_env, "SG_AGENTIC_CLI_PATH": cli}) + reviews = _reviews(log) + assert [FLAG in r for r in reviews] == [True, False] diff --git a/plugins/security-guidance/tests/test_review_usage_log.py b/plugins/security-guidance/tests/test_review_usage_log.py new file mode 100644 index 00000000..bca87d51 --- /dev/null +++ b/plugins/security-guidance/tests/test_review_usage_log.py @@ -0,0 +1,116 @@ +"""Each LLM review leaves one usage line in the plugin's debug log.""" +import re + +from conftest import ( + VULN_PY, bash_payload, commit_file, edit_payload, git, metrics_of, run_hook, + stop_payload, ups_payload, +) + +USAGE = re.compile( + r"Review usage: review=(\w+) session=(\S+) model=(\S+) calls=(\d+) " + r"in=(\d+) out=(\d+) cache_read=(\d+) cache_write=(\d+) est_cost_usd=(\d+\.\d{4}~?)$" +) + + +def _usage_lines(env): + log = env["SECURITY_GUIDANCE_DEBUG_LOG"] + try: + text = open(log).read() + except FileNotFoundError: + return [] + return [m.groups() for m in map(USAGE.search, text.splitlines()) if m] + + +def _env(hook_env, tmp_path): + return {**hook_env, "SECURITY_GUIDANCE_DEBUG_LOG": str(tmp_path / "log.txt"), + "SECURITY_REVIEW_MODEL": "claude-opus-5-5"} + + +def test_stop_review_logs_its_usage_with_the_session(workspace, hook_env, tmp_path): + _, repo = workspace + env = _env(hook_env, tmp_path) + run_hook(ups_payload(repo, session_id="sess-1"), env) + (repo / "app.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py", VULN_PY, session_id="sess-1"), env) + assert _usage_lines(env) == [] + rc, so, _ = run_hook(stop_payload(repo, session_id="sess-1"), env) + + m = metrics_of(so) + assert m["api_calls"] == 1 + # "stub" is not in the price table, so the estimate is marked. + assert _usage_lines(env) == [ + ("stop", "sess-1", "stub", "1", "1", "1", "0", "0", f"{m['cost_usd']:.4f}~"), + ] + + +def test_commit_review_logs_review_commit(workspace, hook_env, tmp_path): + _, repo = workspace + env = _env(hook_env, tmp_path) + _, out = commit_file(repo, "app.py", VULN_PY) + run_hook(bash_payload(repo, "git commit -m change", out, session_id="sess 2/x"), env) + + lines = _usage_lines(env) + assert len(lines) == 1 + assert lines[0][:2] == ("commit", "sess_2_x") and lines[0][3] == "1" + + +def test_no_line_when_no_review_ran(workspace, hook_env, tmp_path): + _, repo = workspace + env = _env(hook_env, tmp_path) + run_hook(ups_payload(repo), env) + run_hook(stop_payload(repo), env) + assert _usage_lines(env) == [] + + +def test_failed_request_logs_no_usage(workspace, hook_env, stub_api, tmp_path): + _, repo = workspace + env = _env(hook_env, tmp_path) + stub_api.status = 400 + _, out = commit_file(repo, "app.py", VULN_PY) + run_hook(bash_payload(repo, "git commit -m change", out), env) + assert _usage_lines(env) == [] + + +def test_push_review_logs_review_push(workspace, hook_env, tmp_path): + _, repo = workspace + env = _env(hook_env, tmp_path) + remote = tmp_path / "remote.git" + git(tmp_path, "init", "-q", "--bare", str(remote)) + git(repo, "remote", "add", "origin", str(remote)) + git(repo, "push", "-q", "-u", "origin", "main") + base = git(repo, "rev-parse", "HEAD").strip() + sha, _ = commit_file(repo, "app.py", VULN_PY) + git(repo, "push", "-q", "origin", "main") + out = f"To {remote}\n {base[:7]}..{sha[:7]} main -> main\n" + run_hook(bash_payload(repo, "git push origin main", out), env) + + lines = _usage_lines(env) + assert len(lines) == 1 and lines[0][0] == "push" + + +def test_line_carries_the_same_numbers_as_the_metrics(tmp_path, monkeypatch): + import _base + logged = [] + monkeypatch.setattr(_base, "debug_log", logged.append) + monkeypatch.setattr(_base, "_USAGE", { + "in": 0, "out": 0, "cr": 0, "cw": 0, "cost": 0.0, "n": 0, + "http_err_last": 0, "http_err_count": 0, "models": [], "default_priced": False, + }) + monkeypatch.setattr(_base, "_USAGE_CONTEXT", {"session_id": "abc", "review": "stop"}) + _base._log_usage(_base._usage_metrics()) + assert logged == [] + + _base._record_usage({"input_tokens": 31994, "output_tokens": 3250, + "cache_read_input_tokens": 7, "cache_creation_input_tokens": 9}, + "claude-opus-5-5") + _base._record_usage({"input_tokens": 1, "output_tokens": 1}, "opus", cost_usd=0.5) + _base._record_usage({"input_tokens": 1, "output_tokens": 1}, + "bad model\nReview usage: forged") + usage = _base._usage_metrics() + _base._log_usage(usage) + assert logged == [ + "Review usage: review=stop session=abc " + "model=claude-opus-5-5,opus,bad_model_Review_usage:_forged calls=3 " + f"in=31996 out=3252 cache_read=7 cache_write=9 est_cost_usd={usage['cost_usd']:.4f}~" + ] + assert usage["cost_usd"] > 0.69 diff --git a/plugins/security-guidance/tests/test_reviewer_context_metrics.py b/plugins/security-guidance/tests/test_reviewer_context_metrics.py new file mode 100644 index 00000000..99e94dc7 --- /dev/null +++ b/plugins/security-guidance/tests/test_reviewer_context_metrics.py @@ -0,0 +1,264 @@ +"""The agentic reviewer reports, as counts only, how its Read/Grep/Glob calls +went, whether the directory it ran in was still there, and how many turns and +how much model time the review took.""" +import json +import os +import shutil +import sys +import types + +import pytest + +import llm +import security_reminder_hook as hook + + +class _ToolUse: + def __init__(self, id, name): + self.id, self.name, self.input = id, name, {} + + +class _ToolResult: + def __init__(self, tool_use_id, is_error=None): + self.tool_use_id, self.is_error, self.content = tool_use_id, is_error, "" + + +@pytest.fixture +def sdk(monkeypatch, tmp_path): + """Stand-in for claude_agent_sdk whose reviewer replays `script.messages` + and then returns no findings. Like the SDK, it cannot start in a + directory that does not exist.""" + script = types.SimpleNamespace(messages=[], before_result=lambda: None, result={}) + + class ClaudeAgentOptions: + def __init__(self, **kwargs): + self.kwargs = kwargs + + class AssistantMessage: + def __init__(self, *content): + self.content = list(content) + + class UserMessage: + def __init__(self, content): + self.content = content + + class ResultMessage: + subtype, usage, total_cost_usd = "success", {}, 0.0 + structured_output = {"findings": []} + + async def query(prompt, options): + if not os.path.isdir(options.kwargs["cwd"]): + raise RuntimeError("Working directory does not exist") + for m in script.messages: + yield m + script.before_result() + result = ResultMessage() + for name, value in script.result.items(): + setattr(result, name, value) + yield result + + mod = types.ModuleType("claude_agent_sdk") + mod.ClaudeAgentOptions, mod.ResultMessage, mod.query = ClaudeAgentOptions, ResultMessage, query + mod.AssistantMessage = AssistantMessage + monkeypatch.setitem(sys.modules, "claude_agent_sdk", mod) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", "") + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(tmp_path / "state")) + for var in ("SG_AGENTIC_CONTEXT_DIR", "SG_AGENTIC_DEBUG_DIR", "SG_AGENTIC_MODEL"): + monkeypatch.delenv(var, raising=False) + script.assistant, script.user = AssistantMessage, UserMessage + return script + + +def _review(repo): + return llm.agentic_review(str(repo), [("a.py", "+x = 1\n")], ["a.py"])[2] + + +class TestAgenticReview: + def test_clean_review_counts_calls_and_no_errors(self, sdk, tmp_path): + sdk.messages = [sdk.assistant(_ToolUse("t1", "Read")), + sdk.user([_ToolResult("t1", is_error=False)])] + m = _review(tmp_path) + assert m["rv_tool_calls"] == 1 + assert "rv_tool_err" not in m and "rv_cwd_gone" not in m + + def test_errored_file_tool_results_are_counted(self, sdk, tmp_path): + sdk.messages = [ + sdk.assistant(_ToolUse("t1", "Read")), + sdk.user([_ToolResult("t1", is_error=True)]), + sdk.assistant(_ToolUse("t2", "Glob"), _ToolUse("t3", "Grep")), + sdk.user([_ToolResult("t2", is_error=True), _ToolResult("t3", is_error=False)]), + sdk.user("plain text"), + ] + m = _review(tmp_path) + assert (m["rv_tool_calls"], m["rv_tool_err"]) == (3, 2) + + def test_other_tools_are_not_counted(self, sdk, tmp_path): + # A rejected structured-output attempt or a denied tool is not a + # sign of missing context. + sdk.messages = [ + sdk.assistant(_ToolUse("t1", "StructuredOutput"), _ToolUse("t2", "Bash")), + sdk.user([_ToolResult("t1", is_error=True), _ToolResult("t2", is_error=True)]), + ] + m = _review(tmp_path) + assert "rv_tool_calls" not in m and "rv_tool_err" not in m + + def test_directory_missing_before_the_review(self, sdk, tmp_path): + m = _review(tmp_path / "removed") + assert m["agentic_fallback"].startswith("investigate:") + assert m["rv_cwd_gone"] == 1 + + def test_directory_removed_during_the_review(self, sdk, tmp_path): + repo = tmp_path / "wt" + repo.mkdir() + sdk.messages = [sdk.assistant(_ToolUse("t1", "Read")), + sdk.user([_ToolResult("t1", is_error=True)])] + sdk.before_result = lambda: shutil.rmtree(repo) + m = _review(repo) + assert (m["rv_cwd_gone"], m["rv_tool_err"]) == (2, 1) + + def test_counts_survive_a_failed_investigate(self, sdk, tmp_path, monkeypatch): + async def query(prompt, options): + yield sdk.assistant(_ToolUse("t1", "Read")) + yield sdk.user([_ToolResult("t1", is_error=True)]) + raise RuntimeError("inner CLI exited") + + monkeypatch.setattr(sys.modules["claude_agent_sdk"], "query", query) + m = _review(tmp_path) + assert m["agentic_fallback"].startswith("investigate:") + assert (m["rv_tool_calls"], m["rv_tool_err"]) == (1, 1) + assert "rv_turns" not in m and "rv_api_ms" not in m + + def test_turns_and_model_time_come_from_the_result_message(self, sdk, tmp_path): + # One assistant message per content block: three messages, two turns. + sdk.messages = [sdk.assistant(_ToolUse("t1", "Read")), sdk.assistant(_ToolUse("t2", "Grep")), + sdk.user([_ToolResult("t1"), _ToolResult("t2")]), sdk.assistant()] + sdk.result = {"num_turns": 2, "duration_api_ms": 4100, "duration_ms": 9000} + m = _review(tmp_path) + assert (m["rv_turns"], m["rv_api_ms"]) == (2, 4100) + + def test_turns_and_model_time_are_summed_over_stages(self, sdk, tmp_path, monkeypatch): + finding = {"severity": "high", "filePath": "a.py", "category": "xss", "vulnerableCode": "x = 1"} + monkeypatch.setattr(sys.modules["claude_agent_sdk"].ResultMessage, "structured_output", + {"findings": [finding], "verdicts": []}) + sdk.result = {"num_turns": 3, "duration_api_ms": 1000} + m = _review(tmp_path) + # Investigate, the second investigate pass, and refute. + assert (m["rv_turns"], m["rv_api_ms"]) == (9, 3000) + + def test_stage_that_ran_out_of_turns_is_counted(self, sdk, tmp_path): + sdk.result = {"num_turns": 18, "duration_api_ms": 60000, + "subtype": "error_max_turns", "structured_output": None} + m = _review(tmp_path) + assert m["agentic_fallback"] == "investigate:error_max_turns" + assert (m["rv_turns"], m["rv_api_ms"]) == (18, 60000) + + @pytest.mark.parametrize("result", [{}, {"num_turns": None, "duration_api_ms": None}, + {"num_turns": "3", "duration_api_ms": True}, + {"num_turns": -1, "duration_api_ms": 0}, + {"num_turns": float("nan"), "duration_api_ms": float("inf")}]) + def test_missing_or_unusable_result_fields_leave_the_keys_out(self, sdk, tmp_path, result): + sdk.result = result + m = _review(tmp_path) + assert "agentic_fallback" not in m + assert "rv_turns" not in m and "rv_api_ms" not in m + + +class TestEmittedRow: + @pytest.fixture(autouse=True) + def _clean_tail(self, monkeypatch): + monkeypatch.setattr(hook, "_TAIL_METRICS", {}) + monkeypatch.setattr(hook, "_REVIEWER_EFFORT_METRICS", {}) + + def _row(self, capsys): + return json.loads(capsys.readouterr().out)["metrics"] + + def test_context_keys_come_last_in_priority_order(self, capsys): + hook._note_reviewer_context( + {"rv_tool_calls": 4, "rv_tool_err": 2, "rv_cwd_gone": 2, "investigate_turns": 5}) + hook.emit_metrics({"vulns_found": 0, "review_ms": 10}) + keys = list(self._row(capsys)) + assert keys[-5:] == ["vulns_found", "review_ms", "rv_cwd_gone", "rv_tool_err", "rv_tool_calls"] + + def test_effort_is_one_key_after_every_other_key(self, capsys, monkeypatch): + monkeypatch.setattr(hook, "_LAUNCHER_METRICS", {"launcher_probe_ms": 7000}) + hook._note_reviewer_context({"rv_api_ms": 9100, "rv_turns": 5, "rv_tool_calls": 4}) + hook.emit_metrics({"vulns_found": 0, "review_ms": 9500}) + row = self._row(capsys) + assert list(row)[-3:] == ["rv_tool_calls", "launcher_probe_ms", "rv_effort"] + assert divmod(row["rv_effort"], 10_000_000) == (5, 9100) + assert "rv_turns" not in row and "rv_api_ms" not in row + + @pytest.mark.parametrize("metrics, packed", [ + ({"rv_turns": 3}, (3, 0)), + ({"rv_api_ms": 250}, (0, 250)), + ({"rv_turns": 54, "rv_api_ms": 123_456_789}, (54, 9_999_999)), + ]) + def test_effort_packing(self, capsys, metrics, packed): + hook._note_reviewer_context(metrics) + hook.emit_metrics({"vulns_found": 0}) + assert divmod(self._row(capsys)["rv_effort"], 10_000_000) == packed + + def test_no_effort_key_without_turns_or_model_time(self, capsys): + hook._note_reviewer_context({"rv_tool_calls": 2, "agentic_fallback": "import:ImportError"}) + hook.emit_metrics({"vulns_found": 0}) + assert "rv_effort" not in self._row(capsys) + + def test_effort_does_not_displace_a_key_at_the_cap(self, capsys, monkeypatch): + monkeypatch.setattr(hook, "_LAUNCHER_METRICS", {"launcher_probe_ms": 7000}) + hook._note_reviewer_context({"rv_api_ms": 9100, "rv_turns": 5}) + hook.emit_metrics({f"k{i}": i for i in range(40)}) + keys = list(self._row(capsys)) + assert keys[-2:] == ["launcher_probe_ms", "rv_effort"] + + def test_effort_fits_under_the_cap_on_a_clean_commit_review_row(self, capsys, monkeypatch): + """The keys of a clean agentic commit review row, with tool calls.""" + monkeypatch.setattr(hook, "_usage_metrics", lambda: { + "tok_in": 1, "tok_out": 1, "tok_cache_r": 0, "tok_cache_w": 0, "cost_usd": 0.1, "api_calls": 1}) + hook._note_reviewer_context({"rv_tool_calls": 2, "rv_turns": 3, "rv_api_ms": 9100}) + hook.emit_metrics({ + "vulns_found": 0, "commit_review": True, "commit_review_on": True, + "sha_via_reflog": True, "reflog_shas_n": 1, "rate_count": 1, "agentic": True, + "candidates": 0, "race_winner": 1, "files_reviewed": 1, "review_ms": 9500, + }) + keys = list(self._row(capsys)) + assert keys[-2:] == ["rv_tool_calls", "rv_effort"] and len(keys) <= 20 + + def test_zero_or_absent_keys_are_left_out(self, capsys): + hook._note_reviewer_context({"rv_tool_calls": 3, "rv_tool_err": 0}) + hook.emit_metrics({"vulns_found": 0}) + row = self._row(capsys) + assert row["rv_tool_calls"] == 3 + assert "rv_tool_err" not in row and "rv_cwd_gone" not in row + + def test_failed_review_row_keeps_vulns_found_ahead_of_context_keys(self, capsys): + hook._note_reviewer_context({"rv_cwd_gone": 1}) + with pytest.raises(SystemExit): + hook._exit_review_failed("Commit", "s1", {"files_reviewed": 1}, 500) + keys = list(self._row(capsys)) + assert keys.index("vulns_found") < keys.index("rv_cwd_gone") == len(keys) - 1 + + +class TestRace: + def test_context_kept_when_the_fallback_result_replaces_a_failed_agentic_run(self, monkeypatch): + import time + + def agentic(*a): + time.sleep(0.3) # fails after the fallback has started + return None, [], {"agentic_fallback": "investigate:RuntimeError", + "rv_cwd_gone": 2, "rv_tool_err": 4, "rv_tool_calls": 5, + "rv_turns": 18, "rv_api_ms": 60000} + + monkeypatch.setenv("SG_AGENTIC_RACE_DELAY_S", "0") + monkeypatch.delenv("SG_AGENTIC_NO_RACE", raising=False) + monkeypatch.setattr(hook, "agentic_review", agentic) + + def single_shot(*a, **k): + time.sleep(0.6) + return None, [] + + monkeypatch.setattr(hook, "analyze_code_security", single_shot) + monkeypatch.setattr(hook.llm, "last_review_failed", lambda: False) + _, _, m = hook._agentic_review_with_race("/nonexistent", [("a.py", "+x")], ["a.py"], []) + assert m["race_winner"] == 2 and m["agentic"] is False + assert (m["rv_cwd_gone"], m["rv_tool_err"], m["rv_tool_calls"]) == (2, 4, 5) + assert (m["rv_turns"], m["rv_api_ms"]) == (18, 60000) diff --git a/plugins/security-guidance/tests/test_reviewer_tools.py b/plugins/security-guidance/tests/test_reviewer_tools.py new file mode 100644 index 00000000..555ef0ad --- /dev/null +++ b/plugins/security-guidance/tests/test_reviewer_tools.py @@ -0,0 +1,272 @@ +"""The inner Claude Code the plugin starts for a review gets only the tools +the review needs.""" +import json +import sys +import threading +import types + +import pytest + +import llm + +READ_ONLY = {"tools": "Read,Grep,Glob", "strict-mcp-config": None} +NO_TOOLS = {"tools": "", "strict-mcp-config": None} +# Once an inner CLI has refused --strict-mcp-config. +READ_ONLY_NO_MCP = {"tools": "Read,Grep,Glob", "disallowedTools": "mcp__*"} +NO_TOOLS_NO_MCP = {"tools": "", "disallowedTools": "mcp__*"} +# Every review start also asks for its session not to be saved. +NOT_SAVED = {"no-session-persistence": None} + + +@pytest.fixture(autouse=True) +def strict_mcp_flag_sent(monkeypatch): + monkeypatch.setattr(llm, "_send_strict_mcp", True) + monkeypatch.setattr(llm, "_send_session_flag", True) + + +def test_tool_flags(): + assert llm._only_tools_args(("Read", "Grep", "Glob")) == READ_ONLY + assert llm._only_tools_args() == NO_TOOLS + llm._send_strict_mcp = False + assert llm._only_tools_args(("Read", "Grep", "Glob")) == READ_ONLY_NO_MCP + assert llm._only_tools_args() == NO_TOOLS_NO_MCP + + +def test_flags_reach_the_command_line_of_the_installed_sdk(): + sdk = pytest.importorskip("claude_agent_sdk") + transport = pytest.importorskip( + "claude_agent_sdk._internal.transport.subprocess_cli") + opts = sdk.ClaudeAgentOptions( + cli_path="claude", allowed_tools=["Read", "Grep", "Glob"], + extra_args=llm._only_tools_args(llm._REVIEWER_TOOLS)) + cmd = transport.SubprocessCLITransport(prompt="x", options=opts)._build_command() + assert cmd[cmd.index("--tools") + 1] == "Read,Grep,Glob" + assert cmd.count("--tools") == 1 and "--strict-mcp-config" in cmd + assert "--disallowedTools" not in cmd + + llm._send_strict_mcp = False + opts = sdk.ClaudeAgentOptions( + cli_path="claude", allowed_tools=["Read", "Grep", "Glob"], + extra_args=llm._only_tools_args(llm._REVIEWER_TOOLS)) + cmd = transport.SubprocessCLITransport(prompt="x", options=opts)._build_command() + assert cmd[cmd.index("--tools") + 1] == "Read,Grep,Glob" + assert cmd[cmd.index("--disallowedTools") + 1] == "mcp__*" + assert cmd.count("--disallowedTools") == 1 and "--strict-mcp-config" not in cmd + + +@pytest.fixture +def sdk(monkeypatch, tmp_path): + """Stand-in for claude_agent_sdk that records each spawn's options. The + first agentic stage reports one finding, later stages keep it.""" + spawned = [] + + class ClaudeAgentOptions: + def __init__(self, **kwargs): + self.kwargs = kwargs + + class ResultMessage: + subtype, usage, total_cost_usd = "success", {}, 0.0 + + def __init__(self, structured_output): + self.structured_output = structured_output + + async def query(prompt, options): + spawned.append(options.kwargs) + if len(spawned) == 1: + yield ResultMessage({"hasVulnerabilities": False, "vulnerabilities": [], + "findings": [{ + "filePath": "a.py", "category": "c", "vulnerableCode": "x = 1", + "explanation": "e", "fix": "f", "severity": "high"}]}) + else: + yield ResultMessage({"findings": [], "survived": [0]}) + + mod = types.ModuleType("claude_agent_sdk") + mod.ClaudeAgentOptions, mod.ResultMessage, mod.query = ClaudeAgentOptions, ResultMessage, query + mod.AssistantMessage = type("AssistantMessage", (), {}) + monkeypatch.setitem(sys.modules, "claude_agent_sdk", mod) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", "") + monkeypatch.delenv("SECURITY_GUIDANCE_PERSIST_SESSIONS", raising=False) + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(tmp_path / "state")) + for var in ("CONTEXT_DIR", "DEBUG_DIR", "MODEL", "ITER2", "FILTER", "NO_ADJUDICATE", + "DIFF_INTERSECT", "EXCLUDE_MEDIUM", "MAX_TURNS"): + monkeypatch.delenv("SG_AGENTIC_" + var, raising=False) + return spawned + + +def test_every_agentic_stage_gets_only_the_read_tools(sdk, tmp_path): + _, vulns, _ = llm.agentic_review(str(tmp_path), [("a.py", "+x = 1\n")], ["a.py"]) + assert len(vulns) == 1 + # investigate, second investigate pass, refute + assert len(sdk) == 3 + for opts in sdk: + assert opts["extra_args"] == {**READ_ONLY, **NOT_SAVED} + assert opts["allowed_tools"] == ["Read", "Grep", "Glob"] + assert "permission_mode" not in opts + + +def test_single_turn_sdk_review_gets_no_tools(sdk): + schema = {"type": "object", "properties": {}} + assert llm._call_claude_via_sdk("review this", schema) is not None + assert len(sdk) == 1 + assert sdk[0]["extra_args"] == {**NO_TOOLS, **NOT_SAVED} + assert sdk[0]["allowed_tools"] == [] + + +# What Claude Code prints, before exiting 1, on a machine whose administrator +# delivers the MCP servers. +REFUSAL = "You cannot use --strict-mcp-config when an enterprise MCP config is present" + + +@pytest.fixture +def managed_mcp(sdk, monkeypatch): + """The stand-in SDK's inner CLI now refuses --strict-mcp-config. Gives + the options of the refused starts and of the ones that ran.""" + answer = sys.modules["claude_agent_sdk"].query + refused = [] + + async def query(prompt, options): + if "strict-mcp-config" in options.kwargs["extra_args"]: + refused.append(options.kwargs) + options.kwargs["stderr"](REFUSAL) + raise RuntimeError("Command failed with exit code 1") + async for msg in answer(prompt, options): + yield msg + + monkeypatch.setattr(sys.modules["claude_agent_sdk"], "query", query) + return refused, sdk + + +def test_agentic_review_runs_where_the_strict_mcp_flag_is_refused(managed_mcp, tmp_path): + _, vulns, metrics = llm.agentic_review(str(tmp_path), [("a.py", "+x = 1\n")], ["a.py"]) + assert len(vulns) == 1 + # The refused start's line is not what child_stderr is there to count. + assert "agentic_fallback" not in metrics and "child_stderr" not in metrics + refused, ran = managed_mcp + assert [o["extra_args"] for o in refused] == [{**READ_ONLY, **NOT_SAVED}] + # investigate, second investigate pass, refute + assert [o["extra_args"] for o in ran] == [{**READ_ONLY_NO_MCP, **NOT_SAVED}] * 3 + for opts in ran: + assert opts["allowed_tools"] == ["Read", "Grep", "Glob"] + + +def test_single_turn_sdk_review_runs_where_the_strict_mcp_flag_is_refused(managed_mcp): + schema = {"type": "object", "properties": {}} + assert llm._call_claude_via_sdk("review this", schema) is not None + refused, ran = managed_mcp + assert [o["extra_args"] for o in refused] == [{**NO_TOOLS, **NOT_SAVED}] + assert [o["extra_args"] for o in ran] == [{**NO_TOOLS_NO_MCP, **NOT_SAVED}] + # The refusal is remembered for the rest of the hook process. + assert llm._call_claude_via_sdk("review this", schema) is not None + assert len(refused) == 1 and ran[-1]["extra_args"] == {**NO_TOOLS_NO_MCP, **NOT_SAVED} + + +def test_other_start_failures_are_not_retried(sdk, monkeypatch, tmp_path): + async def query(prompt, options): + sdk.append(options.kwargs) + options.kwargs["stderr"]("Fallback model cannot be the same as the main model") + raise RuntimeError("Command failed with exit code 1") + yield + + monkeypatch.setattr(sys.modules["claude_agent_sdk"], "query", query) + guidance, vulns, metrics = llm.agentic_review(str(tmp_path), [("a.py", "+x = 1\n")], ["a.py"]) + assert guidance is None and vulns == [] + assert metrics["agentic_fallback"] == "investigate:RuntimeError" + assert llm._call_claude_via_sdk("review this", {"type": "object", "properties": {}}) is None + assert [o["extra_args"] for o in sdk] == [{**READ_ONLY, **NOT_SAVED}, {**NO_TOOLS, **NOT_SAVED}] + + +def test_reviews_started_together_are_each_started_again(managed_mcp): + refused, ran = managed_mcp + both_refused = threading.Barrier(2, timeout=10) + refuse = sys.modules["claude_agent_sdk"].query + + async def query(prompt, options): + if "strict-mcp-config" in options.kwargs["extra_args"]: + both_refused.wait() + async for msg in refuse(prompt, options): + yield msg + + sys.modules["claude_agent_sdk"].query = query + schema = {"type": "object", "properties": {}} + results = [] + threads = [threading.Thread( + target=lambda: results.append(llm._call_claude_via_sdk("review this", schema))) + for _ in range(2)] + for t in threads: + t.start() + for t in threads: + t.join(20) + assert len(results) == 2 and None not in results + assert len(refused) == 2 and [o["extra_args"] for o in ran] == [{**NO_TOOLS_NO_MCP, **NOT_SAVED}] * 2 + + +def test_review_rows_say_when_the_flag_was_left_out(managed_mcp, capsys): + import security_reminder_hook as hook + + def row(): + hook.emit_metrics({"vulns_found": 0, "review_ms": 10}) + return json.loads(capsys.readouterr().out)["metrics"] + + assert "strict_mcp_off" not in row() + assert llm._call_claude_via_sdk("review this", {"type": "object", "properties": {}}) is not None + assert row()["strict_mcp_off"] == 1 + + +# What a Claude Code older than --no-session-persistence prints, before exiting 1. +UNKNOWN = "error: unknown option '--no-session-persistence'" + + +@pytest.fixture +def old_cli(sdk, monkeypatch): + """The stand-in SDK's inner CLI now refuses --no-session-persistence + and, once that is gone, --strict-mcp-config: one flag per start, as + Claude Code does. Gives the options of the refused starts and of the + ones that ran.""" + answer = sys.modules["claude_agent_sdk"].query + refused = [] + + async def query(prompt, options): + args = options.kwargs["extra_args"] + line = (UNKNOWN if "no-session-persistence" in args + else REFUSAL if "strict-mcp-config" in args else None) + if line: + refused.append(options.kwargs) + options.kwargs["stderr"](line) + raise RuntimeError("Command failed with exit code 1") + async for msg in answer(prompt, options): + yield msg + + monkeypatch.setattr(sys.modules["claude_agent_sdk"], "query", query) + return refused, sdk + + +def test_agentic_review_runs_where_both_flags_are_refused(old_cli, tmp_path): + _, vulns, metrics = llm.agentic_review(str(tmp_path), [("a.py", "+x = 1\n")], ["a.py"]) + assert len(vulns) == 1 + assert "agentic_fallback" not in metrics and "child_stderr" not in metrics + refused, ran = old_cli + assert [o["extra_args"] for o in refused] == [{**READ_ONLY, **NOT_SAVED}, READ_ONLY] + assert [o["extra_args"] for o in ran] == [READ_ONLY_NO_MCP] * 3 + + +def test_single_turn_sdk_review_runs_where_both_flags_are_refused(old_cli): + schema = {"type": "object", "properties": {}} + assert llm._call_claude_via_sdk("review this", schema) is not None + refused, ran = old_cli + assert [o["extra_args"] for o in refused] == [{**NO_TOOLS, **NOT_SAVED}, NO_TOOLS] + assert [o["extra_args"] for o in ran] == [NO_TOOLS_NO_MCP] + # Both refusals are remembered for the rest of the hook process. + assert llm._call_claude_via_sdk("review this", schema) is not None + assert len(refused) == 2 and ran[-1]["extra_args"] == NO_TOOLS_NO_MCP + + +def test_a_flag_refused_again_after_it_was_left_out_is_not_retried(sdk, monkeypatch): + async def query(prompt, options): + sdk.append(options.kwargs) + options.kwargs["stderr"](UNKNOWN) + raise RuntimeError("Command failed with exit code 1") + yield + + monkeypatch.setattr(sys.modules["claude_agent_sdk"], "query", query) + assert llm._call_claude_via_sdk("review this", {"type": "object", "properties": {}}) is None + assert [o["extra_args"] for o in sdk] == [{**NO_TOOLS, **NOT_SAVED}, NO_TOOLS] diff --git a/plugins/security-guidance/tests/test_sdk_bootstrap_switches.py b/plugins/security-guidance/tests/test_sdk_bootstrap_switches.py new file mode 100644 index 00000000..36f2742b --- /dev/null +++ b/plugins/security-guidance/tests/test_sdk_bootstrap_switches.py @@ -0,0 +1,183 @@ +"""The agent SDK is not installed for reviews that are switched off. + +ensure_agent_sdk.py (SessionStart) and the PostToolUse rescue spawn built a +venv of a few hundred MB whatever the environment said, including under +SECURITY_GUIDANCE_DISABLE=1, and the rescue started ensure_agent_sdk.py again +every five minutes on machines whose SDK lives in that venv. See #5331. +""" +import importlib.util +import subprocess +import sys + +import pytest + +from conftest import HOOKS_DIR +import _base +import ensure_agent_sdk as boot +import llm +import security_reminder_hook as hook + +OFF = [ + {"SECURITY_GUIDANCE_DISABLE": "1"}, + {"SECURITY_GUIDANCE_DISABLE": "true"}, + {"ENABLE_SECURITY_REMINDER": "0"}, + {"ENABLE_CODE_SECURITY_REVIEW": "0"}, + {"ENABLE_STOP_REVIEW": "0", "ENABLE_COMMIT_REVIEW": "0"}, +] +STILL_ON = [{}, {"ENABLE_STOP_REVIEW": "0"}, {"ENABLE_COMMIT_REVIEW": "0"}, + {"SECURITY_GUIDANCE_DISABLE": "0"}] +SWITCHES = ("SECURITY_GUIDANCE_DISABLE", "ENABLE_SECURITY_REMINDER", + "ENABLE_CODE_SECURITY_REVIEW", "ENABLE_STOP_REVIEW", "ENABLE_COMMIT_REVIEW") + + +@pytest.fixture +def state(tmp_path, monkeypatch): + """Empty state dir, no SDK in the interpreter, switches unset.""" + for name in SWITCHES: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(tmp_path / "state")) + monkeypatch.setattr(boot, "_sdk_on_syspath", lambda: False) + return tmp_path / "state" + + +@pytest.fixture +def builds(monkeypatch): + """Records the venv / pip commands instead of running them.""" + calls = [] + + def run(cmd, **kwargs): + calls.append([str(c) for c in cmd]) + return subprocess.CompletedProcess(cmd, 0, b"", b"") + monkeypatch.setattr(boot.subprocess, "run", run) + return calls + + +@pytest.fixture +def spawns(state, monkeypatch): + """Records the rescue's detached starts of ensure_agent_sdk.py.""" + calls = [] + monkeypatch.setattr(importlib.util, "find_spec", lambda name: None) + monkeypatch.setattr(hook, "_SDK_BOOTSTRAP_THROTTLE", str(state / ".sdk_bootstrap_spawned")) + monkeypatch.setattr(hook.subprocess, "Popen", lambda cmd, **kw: calls.append(cmd)) + return calls + + +def _install(root, finished=True): + pkg = root / "claude_agent_sdk" + pkg.mkdir(parents=True) + (pkg / "__init__.py").write_text("") + if finished: + (root / "claude_agent_sdk-0.2.0.dist-info").mkdir() + + +@pytest.mark.parametrize("env", OFF) +def test_session_start_builds_nothing_when_reviews_are_off(state, builds, monkeypatch, env): + for k, v in env.items(): + monkeypatch.setenv(k, v) + assert boot.main() == (boot.SKIP_REVIEWS_OFF, "", "") + assert builds == [] and not state.exists() + + +@pytest.mark.parametrize("env", STILL_ON) +def test_session_start_still_builds_when_a_review_can_run(state, builds, monkeypatch, env): + for k, v in env.items(): + monkeypatch.setenv(k, v) + assert boot.main()[0] == boot.BUILT + assert [c[1:3] for c in builds] == [["-m", "venv"], ["-m", "pip"]] + + +@pytest.mark.parametrize("env", OFF[2:]) +def test_rescue_does_not_start_the_bootstrap_when_reviews_are_off(spawns, monkeypatch, env): + for k, v in env.items(): + monkeypatch.setenv(k, v) + hook._maybe_bootstrap_agent_sdk_async() + assert spawns == [] + + +def test_rescue_starts_the_bootstrap_once_when_nothing_is_installed(spawns): + hook._maybe_bootstrap_agent_sdk_async() + hook._maybe_bootstrap_agent_sdk_async() + assert len(spawns) == 1 and spawns[0][1].endswith("ensure_agent_sdk.py") + + +@pytest.mark.parametrize("where", [ + "agent-sdk-venv/lib/python3.12/site-packages", + "agent-sdk-venv/Lib/site-packages", + "agent-sdk-libs", +]) +def test_rescue_leaves_an_existing_install_alone(spawns, state, where): + _install(state / where) + hook._maybe_bootstrap_agent_sdk_async() + assert spawns == [] and not (state / ".sdk_bootstrap_spawned").exists() + + +def test_rescue_still_runs_for_an_install_that_was_cut_short(spawns, state): + _install(state / "agent-sdk-venv/lib/python3.12/site-packages", finished=False) + hook._maybe_bootstrap_agent_sdk_async() + assert len(spawns) == 1 + + +def test_reviewer_and_bootstrap_agree_on_where_the_sdk_is_installed(state, monkeypatch): + for where in ("agent-sdk-venv/lib/python3.12/site-packages", "agent-sdk-libs"): + _install(state / where) + monkeypatch.setattr(sys, "path", list(sys.path)) + assert llm._inject_agent_sdk_venv_into_syspath(str(state)) + assert {str(state / "agent-sdk-venv/lib/python3.12/site-packages"), + str(state / "agent-sdk-libs")} <= set(sys.path) + + +def test_switches_are_read_the_same_way_as_the_hook_reads_them(monkeypatch): + for env in OFF + STILL_ON: + for name in SWITCHES: + monkeypatch.delenv(name, raising=False) + for k, v in env.items(): + monkeypatch.setenv(k, v) + out = subprocess.run( + [sys.executable, "-c", + "import security_reminder_hook as h; " + "print(h.SECURITY_GUIDANCE_DISABLED or not h.ENABLE_CODE_SECURITY_REVIEW" + " or not (h.ENABLE_STOP_REVIEW or h.ENABLE_COMMIT_REVIEW))"], + cwd=HOOKS_DIR, capture_output=True, text=True).stdout.strip() + assert out == str(_base.llm_reviews_switched_off()), env + + +def test_disabled_plugin_says_nothing_on_a_python_below_3_10(state, builds, monkeypatch): + monkeypatch.setenv("SECURITY_GUIDANCE_DISABLE", "1") + monkeypatch.setattr(boot.sys, "version_info", (3, 9, 6, "final", 0)) + assert boot.main()[0] == boot.SKIP_REVIEWS_OFF + + +def test_install_the_session_cannot_use_is_counted(state, tmp_path, monkeypatch): + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", False) + monkeypatch.setattr(boot, "_near_git_repo", lambda: False) + assert boot._install_context_metrics(boot.BUILT, state) == {"sdk_no_creds": 1, "sdk_no_git": 1} + assert boot._install_context_metrics(boot.BUILD_FAILED, state) == {} + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", True) + monkeypatch.setattr(boot, "_near_git_repo", lambda: True) + assert boot._install_context_metrics(boot.NOOP_VENV, state) == {} + + +def test_session_directory_in_or_holding_a_repository(tmp_path, monkeypatch): + monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path)) + if boot._near_git_repo(): + pytest.skip("the temp dir is itself inside a git work tree") + (tmp_path / "ws" / "repo" / ".git").mkdir(parents=True) + (tmp_path / "ws" / "repo" / "src").mkdir() + for inside in ("ws", "ws/repo", "ws/repo/src"): + monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(tmp_path / inside)) + assert boot._near_git_repo(), inside + + +@pytest.mark.parametrize("cfg, counted", [ + ("home = /usr/bin\nversion = 3.99.1\n", {"sdk_venv_py": 399}), + ("home = /usr/bin\nversion_info = 3.99.1.final.0\n", {"sdk_venv_py": 399}), + (f"version = {sys.version_info[0]}.{sys.version_info[1]}.0\n", {}), + ("home = /usr/bin\n", {}), +]) +def test_venv_built_by_another_python_is_counted(state, monkeypatch, cfg, counted): + monkeypatch.setattr(boot, "_near_git_repo", lambda: True) + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", True) + (state / "agent-sdk-venv").mkdir(parents=True) + (state / "agent-sdk-venv" / "pyvenv.cfg").write_text(cfg) + assert boot._install_context_metrics(boot.NOOP_VENV, state) == counted + assert boot._install_context_metrics(boot.BUILT, state) == {} diff --git a/plugins/security-guidance/tests/test_sdk_emptied_install.py b/plugins/security-guidance/tests/test_sdk_emptied_install.py new file mode 100644 index 00000000..415e8d99 --- /dev/null +++ b/plugins/security-guidance/tests/test_sdk_emptied_install.py @@ -0,0 +1,101 @@ +"""An SDK install whose package directory lost its files is rebuilt. + +`import claude_agent_sdk` succeeds on an empty directory (Python imports it +as a namespace package), so the bootstrap reported a healthy venv, the +reviewer's `from claude_agent_sdk import ...` failed on every commit, and +nothing repaired it until the venv was deleted by hand. See #6612. +""" +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest + +import _base +import ensure_agent_sdk as boot +from test_sdk_bootstrap_switches import SWITCHES + + +def _fill(pkg): + pkg.mkdir(parents=True, exist_ok=True) + (pkg / "__init__.py").write_text("AssistantMessage = object\n") + (pkg / "__pycache__").mkdir(exist_ok=True) + + +def _empty(pkg): + for f in pkg.rglob("*"): + if f.is_file(): + f.unlink() + + +@pytest.fixture +def venv(tmp_path, monkeypatch): + """A real venv (no pip, so it is quick) holding a stand-in SDK package; + the build commands are recorded instead of run, the probes run for real.""" + for name in SWITCHES + ("PYTHONPATH",): + monkeypatch.delenv(name, raising=False) + state = tmp_path / "state" + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(state)) + monkeypatch.setattr(boot, "_sdk_on_syspath", lambda: False) + monkeypatch.setattr(boot, "_near_git_repo", lambda: True) + subprocess.run([sys.executable, "-m", "venv", "--without-pip", + str(state / "agent-sdk-venv")], check=True) + pkg = Path(_base.agent_sdk_site_dirs(str(state))[0]) / "claude_agent_sdk" + _fill(pkg) + real_run = subprocess.run + builds = [] + + def run(cmd, **kwargs): + cmd = [str(c) for c in cmd] + if "-m" in cmd: + builds.append(cmd) + return subprocess.CompletedProcess(cmd, 0, b"", b"") + return real_run(cmd, **kwargs) + monkeypatch.setattr(boot.subprocess, "run", run) + return state, pkg, builds + + +def test_complete_install_is_left_alone(venv): + state, _, builds = venv + outcome = boot.main()[0] + assert outcome == boot.NOOP_VENV and builds == [] + assert "sdk_venv_emptied" not in boot._install_context_metrics(outcome, state) + + +def test_emptied_install_is_rebuilt_and_counted(venv): + state, pkg, builds = venv + _empty(pkg) + outcome = boot.main()[0] + assert outcome == boot.BUILT + assert builds[0][1:4] == ["-m", "venv", "--clear"] + assert any("pip" in c and "claude-agent-sdk" in c for c in builds) + assert boot._install_context_metrics(outcome, state)["sdk_venv_emptied"] == 1 + # The count belongs to the run that found it, not to later ones. + _fill(pkg) + outcome = boot.main()[0] + assert outcome == boot.NOOP_VENV + assert "sdk_venv_emptied" not in boot._install_context_metrics(outcome, state) + + +def test_venv_that_never_held_the_sdk_is_not_counted_as_emptied(venv): + state, pkg, builds = venv + _empty(pkg) + (pkg / "__pycache__").rmdir() + pkg.rmdir() + outcome = boot.main()[0] + assert outcome == boot.BUILT and builds + assert "sdk_venv_emptied" not in boot._install_context_metrics(outcome, state) + + +# An SDK installed for the interpreter running the tests is found behind the +# emptied directory, which hides what this test looks at. +@pytest.mark.skipif(importlib.util.find_spec("claude_agent_sdk") is not None, + reason="claude_agent_sdk is installed for this interpreter") +def test_emptied_fallback_install_is_not_reused(tmp_path, monkeypatch): + monkeypatch.delenv("PYTHONPATH", raising=False) + pkg = boot._target_dir(tmp_path) / "claude_agent_sdk" + _fill(pkg) + assert boot._target_sdk_importable(tmp_path) is True + _empty(pkg) + assert boot._target_sdk_importable(tmp_path) is False diff --git a/plugins/security-guidance/tests/test_secret_named_metric.py b/plugins/security-guidance/tests/test_secret_named_metric.py new file mode 100644 index 00000000..14d71923 --- /dev/null +++ b/plugins/security-guidance/tests/test_secret_named_metric.py @@ -0,0 +1,94 @@ +"""The Stop review counts the data files it sends whose name says they hold +secrets. Measurement only: the files are reviewed as before.""" +import pytest + +from conftest import ( + STUB_VULN, VULN_PY, make_repo, metrics_of, run_hook, stop_payload, ups_payload, +) + +import gitutil + + +@pytest.mark.parametrize("path", [ + "secrets.yml", + "config/secrets.yaml", + "secrets.prod.json", + ".secrets.toml", + "Secrets.YML", + "credentials.json", + "client_secret.json", + "keys/service-account.json", + "service_account.json", + "terraform/secrets.tfvars", + "client-secret.json", + "client_secret_1234-abc.apps.googleusercontent.com.json", + "serviceAccountKey.json", +]) +def test_secret_named_data_files(path): + assert gitutil._is_secret_named(path) + + +@pytest.mark.parametrize("path", [ + "secrets.py", + "src/credentials.ts", + "secrets_manager.yaml", + "app-secrets.json", + "docs/secrets.md", + "secrets", + "config.yaml", + "package.json", + "secrets/config.yaml", + "secrets.example.yml", + "credentials.sample.json", + "secrets.schema.json", + "secrets.enc.yaml", + "secrets.sops.yaml", + ".json", +]) +def test_other_files_are_not_counted(path): + assert not gitutil._is_secret_named(path) + + +def _review(tmp_path, hook_env, files): + repo = make_repo(tmp_path / "repo") + run_hook(ups_payload(repo), hook_env) + for name, body in files.items(): + (repo / name).write_text(body) + return run_hook(stop_payload(repo), hook_env) + + +def test_stop_hook_counts_secret_named_files_and_still_reviews_them(tmp_path, hook_env, stub_api): + _rc, stdout, _err = _review(tmp_path, hook_env, { + "secrets.yml": "db:\n password: example\n", + "credentials.json": '{"token": "example"}\n', + "app.py": "print(1)\n", + }) + m = metrics_of(stdout) + assert m.get("secret_named_reviewed") == 2 + assert m.get("files_reviewed") == 3 + assert "/v1/messages" in stub_api.calls + + +def test_count_is_on_the_findings_row_too(tmp_path, hook_env, stub_api): + stub_api.vulns = [STUB_VULN] + rc, stdout, _err = _review(tmp_path, hook_env, { + "secrets.yml": "db:\n password: example\n", + "app.py": VULN_PY, + }) + assert rc == 2 + assert metrics_of(stdout).get("secret_named_reviewed") == 1 + + +def test_no_key_when_no_such_file_is_reviewed(tmp_path, hook_env, stub_api): + _rc, stdout, _err = _review(tmp_path, hook_env, {"app.py": "print(1)\n"}) + m = metrics_of(stdout) + assert m.get("files_reviewed") == 1 + assert "secret_named_reviewed" not in m + + +def test_count_is_on_the_failed_review_row(tmp_path, hook_env, stub_api): + stub_api.status = 500 + _rc, stdout, _err = _review(tmp_path, hook_env, {"secrets.yml": "a: b\n"}) + m = metrics_of(stdout) + assert m.get("review_failed") is True + assert m.get("secret_named_reviewed") == 1 diff --git a/plugins/security-guidance/tests/test_secret_renames.py b/plugins/security-guidance/tests/test_secret_renames.py new file mode 100644 index 00000000..7d4e38fb --- /dev/null +++ b/plugins/security-guidance/tests/test_secret_renames.py @@ -0,0 +1,149 @@ +"""A tracked file renamed away from a secret-shaped name stays out of the +Stop review, however the rename reached git.""" +import errno +import os +import subprocess + +import pytest + +from conftest import ( + GIT_ENV, bash_payload, commit_file, git, make_repo, metrics_of, run_hook, + stop_payload, ups_payload, +) + +import diffstate +import gitutil + +SECRET_BODY = "".join(f"KEY_{i}=SECRET_VALUE_{i}\n" for i in range(30)) +EDIT = "import os\nos.system(input())\n" + + +def _repo(tmp_path): + return make_repo(tmp_path / "repo", {"deploy_rsa": SECRET_BODY, "app.py": "x = 1\n"}) + + +def _sent(stub_api): + return "".join(stub_api.bodies) + + +def _rename(repo, how): + if how == "moved": + os.rename(repo / "deploy_rsa", repo / "settings.py") + return + git(repo, "mv", "deploy_rsa", "settings.py") + if how == "committed": + git(repo, "commit", "-qm", "rename") + elif how == "amended": + git(repo, "commit", "-q", "--amend", "-m", "init, renamed") + # Edited after the commit, so the file is still part of the turn. + with open(repo / "settings.py", "a") as f: + f.write("KEY_NEW=SECRET_VALUE_NEW\n") + + +@pytest.mark.parametrize("how", ["moved", "staged", "committed", "amended"]) +def test_stop_review_leaves_the_renamed_file_out(tmp_path, hook_env, stub_api, how): + repo = _repo(tmp_path) + run_hook(ups_payload(repo), hook_env) + _rename(repo, how) + (repo / "app.py").write_text(EDIT) + _rc, stdout, _err = run_hook(stop_payload(repo), hook_env) + m = metrics_of(stdout) + assert m.get("skip_reason") is None, m + assert m.get("files_reviewed") == 1 + # A plain mv is not a rename to git: the old name is listed as deleted. + assert m.get("secret_renamed") == (None if how == "moved" else 1) + assert "os.system" in _sent(stub_api) + assert "SECRET_VALUE" not in _sent(stub_api) + + +def test_stop_review_with_nothing_but_the_rename_sends_nothing(tmp_path, hook_env, stub_api): + repo = _repo(tmp_path) + run_hook(ups_payload(repo), hook_env) + git(repo, "mv", "deploy_rsa", "settings.py") + _rc, stdout, _err = run_hook(stop_payload(repo), hook_env) + assert metrics_of(stdout).get("skip_reason") == 7 + assert stub_api.bodies == [] + + +def test_commit_review_leaves_the_renamed_file_out(tmp_path, hook_env, stub_api): + repo = _repo(tmp_path) + git(repo, "mv", "deploy_rsa", "settings.py") + _sha, out = commit_file(repo, "app.py", EDIT) + run_hook(bash_payload(repo, "git commit -m change", stdout=out), hook_env) + assert "os.system" in _sent(stub_api) + assert "SECRET_VALUE" not in _sent(stub_api) + + +@pytest.fixture +def git_env(monkeypatch): + """The suite's git settings for calls made in this process.""" + for k, v in GIT_ENV.items(): + monkeypatch.setenv(k, v) + monkeypatch.setattr(gitutil, "_secret_rename_origins", {}) + monkeypatch.setattr(diffstate, "_secret_rename_origins", gitutil._secret_rename_origins) + + +def test_listings_are_unchanged_and_note_the_secret_origin(tmp_path, git_env): + repo = make_repo(tmp_path / "repo", { + "deploy_rsa": SECRET_BODY, "old name.py": SECRET_BODY + "x = 1\n", + "gone.py": "x = 1\n", "edit.py": "x = 1\n"}) + git(repo, "mv", "deploy_rsa", "settings.py") + git(repo, "mv", "old name.py", "new name.py") + git(repo, "rm", "-q", "gone.py") + (repo / "edit.py").write_text("x = 2\n") + names = {"settings.py", "new name.py", "gone.py", "edit.py"} + assert gitutil._git_name_only(str(repo), "HEAD") == names + assert gitutil._secret_rename_origins == {"settings.py": "deploy_rsa"} + gitutil._secret_rename_origins.clear() + assert gitutil._git_status_porcelain(str(repo)) == (names, set()) + assert gitutil._secret_rename_origins == {"settings.py": "deploy_rsa"} + + +def test_review_set_gains_only_the_origin_of_a_reviewed_file(tmp_path, git_env): + repo = make_repo(tmp_path / "repo", { + "deploy_rsa": SECRET_BODY, "old.py": SECRET_BODY + "x = 1\n"}) + git(repo, "mv", "deploy_rsa", "settings.py") + git(repo, "mv", "old.py", "new.py") + paths, _base, root, _untracked, metrics = diffstate.compute_v2_review_set(str(repo), None, None, {}) + assert sorted(os.path.relpath(p, root) for p in paths) == ["deploy_rsa", "new.py", "settings.py"] + assert metrics["dirty_now_count"] == 2 and metrics["review_set_count"] == 3 + assert metrics["secret_renamed"] == 1 + + +def test_origin_still_pairs_beside_a_nested_repo_and_a_split_path_list( + tmp_path, git_env, monkeypatch): + repo = make_repo(tmp_path / "repo", {"deploy_rsa": SECRET_BODY, "app.py": "x = 1\n"}) + make_repo(repo / "nested", {"inner.py": "y = 1\n"}) + git(repo, "mv", "deploy_rsa", "settings.py") + (repo / "app.py").write_text(EDIT) + (repo / "a_much_longer_module_name.py").write_text("y = 1\n") + (repo / "another_quite_long_module_name.py").write_text("y = 2\n") + for i in range(20): + (repo / f"note{i:02d}.txt").write_text("x\n") + paths, base, root, untracked, metrics = diffstate.compute_v2_review_set(str(repo), None, None, {}) + names = [os.path.relpath(p, root) for p in paths] + assert "deploy_rsa" in names and "settings.py" in names + assert not any(n.startswith("nested") for n in names) + assert metrics["secret_renamed"] == 1 + + # The OS refuses the one command that holds every path, so the paths go + # out in several: the old name has to be in the one with the new name. + real_popen = subprocess.Popen + limit = sum(len(a) + 1 for a in gitutil.GIT_CMD) + 220 + diffs = [] + + def popen(cmd, *args, **kwargs): + if sum(len(a) + 1 for a in cmd) > limit: + raise OSError(errno.E2BIG, "Argument list too long") + if "diff" in cmd: + diffs.append(cmd) + return real_popen(cmd, *args, **kwargs) + + monkeypatch.setattr(subprocess, "Popen", popen) + monkeypatch.setattr(gitutil, "_MAX_PATH_ARGS_BYTES", 80) + diff = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + assert len(diffs) > 1 and all("deploy_rsa" in c for c in diffs) + reviewed = gitutil.parse_diff_into_files(diff) + assert sorted(f for f, _ in reviewed) == [ + "a_much_longer_module_name.py", "another_quite_long_module_name.py", "app.py"] + assert "SECRET_VALUE" not in "".join(d for _, d in reviewed) diff --git a/plugins/security-guidance/tests/test_secret_shaped_paths.py b/plugins/security-guidance/tests/test_secret_shaped_paths.py new file mode 100644 index 00000000..b20beef3 --- /dev/null +++ b/plugins/security-guidance/tests/test_secret_shaped_paths.py @@ -0,0 +1,115 @@ +import os +import subprocess + +import pytest + +from conftest import ( + GIT_ENV, git, make_repo, metrics_of, run_hook, stop_payload, ups_payload, +) + +import gitutil + + +@pytest.mark.parametrize("path", [ + "dot_config/private_example/encrypted_private_token", + "encrypted_private_dot_netrc", + "home/encrypted_config.yaml", + "id_ed25519", + ".ssh/id_rsa", + "private_dot_ssh/private_id_ecdsa", + "deploy_rsa", + "keys/github_ed25519", + ".env.production.json", + ".env.local.yaml", + "config/dot_env.toml", + "private_dot_netrc", + "dot_aws/credentials", + "Deploy_RSA", + ".credentials.json", + ".claude/.credentials.json", + "private_dot_claude/private_dot_credentials.json", + ".claude.json", + "dot_claude.json", +]) +def test_secret_shaped_names_are_not_reviewable(path): + assert gitutil._is_secret_shaped(path) + assert not gitutil._is_reviewable_source(path) + + +@pytest.mark.parametrize("path", [ + "src/private_api.py", + "src/id_generator.py", + "lib/encrypted_store.py", + "crypto/rsa.py", + "auth/sign_rsa.go", + "app/credentials.py", + "scripts/run_migrations.sh", + "dot_product.py", + "config/environment.json", + "bin/deploy", + "Dockerfile.dev", + "executable_backup.sh", + "config/credentials.json", + "src/claude.json", +]) +def test_ordinary_source_stays_reviewable(path): + assert not gitutil._is_secret_shaped(path) + assert gitutil._is_reviewable_source(path) + + +def _staged_diff(repo): + r = subprocess.run( + [*gitutil.GIT_CMD, "diff", "--cached", "--no-color", "-M"], cwd=repo, + capture_output=True, env={**os.environ, **GIT_ENV}, + ) + assert r.returncode == 0, r.stderr + return r.stdout.decode("utf-8", errors="replace") + + +def test_secret_content_is_left_out_of_the_parsed_diff(tmp_path): + repo = make_repo(tmp_path / "repo") + (repo / "encrypted_private_token").write_text("CIPHERTEXT_BODY\n") + (repo / "id_ed25519").write_text("PRIVATE_KEY_BODY\n") + (repo / ".env.production.json").write_text('{"API_KEY": "ENV_SECRET_BODY"}\n') + (repo / "app.py").write_text("import os\nos.system(input())\n") + git(repo, "add", "-A") + diff = _staged_diff(repo) + parsed = gitutil.parse_diff_into_files(diff) + assert [p for p, _ in parsed] == ["app.py"] + assert "_BODY" not in "".join(c for _, c in parsed) + assert gitutil.extract_file_paths_from_diff(diff) == ["app.py"] + + +def test_rename_away_from_a_secret_name_is_left_out(tmp_path): + body = "".join(f"KEY_{i}=SECRET_VALUE_{i}\n" for i in range(30)) + repo = make_repo(tmp_path / "repo", {"deploy_rsa": body}) + git(repo, "mv", "deploy_rsa", "notes.sh") + (repo / "notes.sh").write_text(body + "KEY_NEW=SECRET_VALUE_NEW\n") + git(repo, "add", "-A") + diff = _staged_diff(repo) + assert "rename from deploy_rsa" in diff + assert gitutil.parse_diff_into_files(diff) == [] + assert gitutil.extract_file_paths_from_diff(diff) == [] + + +def test_stop_hook_reports_how_many_files_it_left_out(tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo") + run_hook(ups_payload(repo), hook_env) + (repo / "id_ed25519").write_text("-----BEGIN OPENSSH PRIVATE KEY-----\n") + (repo / ".env.local").write_text("TOKEN=abc\n") + _rc, stdout, _err = run_hook(stop_payload(repo), hook_env) + assert metrics_of(stdout).get("secret_shaped_skipped") == 2 + assert stub_api.calls == [] + + +@pytest.mark.parametrize("name", [".credentials.json", ".claude.json"]) +def test_claude_code_files_rewritten_during_the_turn_are_not_sent(tmp_path, hook_env, stub_api, name): + repo = make_repo(tmp_path / "repo") + (repo / name).write_text('{"accessToken": "old"}\n') + run_hook(ups_payload(repo), hook_env) + (repo / name).write_text('{"accessToken": "new-and-longer"}\n') + later = (repo / name).stat().st_mtime + 5 + os.utime(repo / name, (later, later)) + _rc, stdout, _err = run_hook(stop_payload(repo), hook_env) + assert metrics_of(stdout).get("secret_shaped_skipped") == 1 + assert stub_api.calls == [] diff --git a/plugins/security-guidance/tests/test_slow_prompt_metrics.py b/plugins/security-guidance/tests/test_slow_prompt_metrics.py new file mode 100644 index 00000000..827b4642 --- /dev/null +++ b/plugins/security-guidance/tests/test_slow_prompt_metrics.py @@ -0,0 +1,175 @@ +import json +import os +import re +import shutil +import subprocess + +import pytest + +import gitutil +from conftest import git, make_repo, metrics_of, run_hook, ups_payload + +SLOW = 1.2 +SNAPSHOT = '*"add -u"*' +MOUNTS = ( + "25 30 8:32 / / rw,relatime - ext4 /dev/sdc rw\n" + "60 25 0:50 / /mnt/c rw,noatime - 9p C:\\134 rw,aname=drivers\n" + "61 25 0:51 / /mnt/my\\040disk rw,noatime - 9p D:\\134 rw\n" + "62 60 8:40 / /mnt/c/data rw - ext4 /dev/sdd rw\n" + "63 25 0:52 / /mnt/a\\134b rw - drvfs E:\\134 rw\n" +) + +# The stalling git is a shell script, and on Windows a clean prompt can itself +# take a second. +pytestmark = pytest.mark.skipif(os.name == "nt", reason="needs a shell-script git") + + +@pytest.fixture +def repo(tmp_path): + return make_repo(tmp_path / "repo", {"app.py": "print('hi')\n", "lib.py": "x = 1\n"}) + + +def _slow(hook_env, tmp_path, match): + """hook_env with a `git` first on PATH that stalls on commands matching + `match` (a shell case pattern).""" + shim = tmp_path / "bin" + shim.mkdir(exist_ok=True) + (shim / "git").write_text( + f'#!/bin/sh\ncase "$*" in {match}) sleep {SLOW};; esac\n' + f'exec "{shutil.which("git")}" "$@"\n') + (shim / "git").chmod(0o755) + return {**hook_env, "PATH": f"{shim}{os.pathsep}{hook_env['PATH']}"} + + +def test_slow_capture_reports_where_the_time_went(repo, hook_env, tmp_path): + rc, out, _ = run_hook(ups_payload(str(repo)), _slow(hook_env, tmp_path, SNAPSHOT)) + m = metrics_of(out) + assert rc == 0 + assert m["ups_ms"] >= SLOW * 1000 + assert m["ups_baseline_ms"] >= SLOW * 1000 + assert m["ups_untracked_ms"] < m["ups_baseline_ms"] + assert m["ups_index_entries"] == 2 + assert not {"ups_no_baseline", "ups_wsl_drive", "ups_baseline_fail"} & set(m) + + +def test_slow_untracked_listing_is_told_apart(repo, hook_env, tmp_path): + env = _slow(hook_env, tmp_path, '*"ls-files"*"--others"*') + m = metrics_of(run_hook(ups_payload(str(repo)), env)[1]) + assert m["ups_untracked_ms"] >= SLOW * 1000 + assert m["ups_baseline_ms"] < m["ups_untracked_ms"] + + +def test_slow_prompt_with_stop_review_off_lists_nothing(repo, hook_env, tmp_path): + env = {**_slow(hook_env, tmp_path, SNAPSHOT), "ENABLE_STOP_REVIEW": "0"} + m = metrics_of(run_hook(ups_payload(str(repo)), env)[1]) + assert m["ups_baseline_ms"] >= SLOW * 1000 + assert m["ups_untracked_ms"] == 0 + + +def test_slow_prompt_without_a_baseline_says_so(hook_env, tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + subprocess.run(["git", "init", "-q", str(empty)], check=True) + env = _slow(hook_env, tmp_path, '*"rev-parse HEAD"*') + m = metrics_of(run_hook(ups_payload(str(empty)), env)[1]) + assert m["ups_no_baseline"] == 1 and "ups_index_entries" not in m + + +def test_git_that_stalls_past_its_timeout_still_reports(repo, hook_env, tmp_path): + """A hung git reads as "not a repository"; the row must still go out.""" + shim = tmp_path / "bin" + shim.mkdir() + (shim / "git").write_text('#!/bin/sh\nexec sleep 6\n') + (shim / "git").chmod(0o755) + env = {**hook_env, "PATH": f"{shim}{os.pathsep}{hook_env['PATH']}"} + m = metrics_of(run_hook(ups_payload(str(repo)), env)[1]) + assert m["ups_no_baseline"] == 1 and m["ups_ms"] >= 5000 + + +def test_failed_snapshot_is_flagged(repo, hook_env, tmp_path): + shim = tmp_path / "bin" + shim.mkdir() + (shim / "git").write_text( + f'#!/bin/sh\ncase "$*" in *"write-tree"*) sleep {SLOW}; exit 1;; esac\n' + f'exec "{shutil.which("git")}" "$@"\n') + (shim / "git").chmod(0o755) + env = {**hook_env, "PATH": f"{shim}{os.pathsep}{hook_env['PATH']}"} + m = metrics_of(run_hook(ups_payload(str(repo)), env)[1]) + assert m["ups_baseline_fail"] == 1 + + +def test_fast_prompt_prints_nothing(repo, hook_env): + rc, out, _ = run_hook(ups_payload(str(repo)), hook_env) + assert rc == 0 and out.strip() == "" + + +def test_slow_launcher_is_reported_on_a_fast_prompt(repo, hook_env): + env = {**hook_env, "SG_LAUNCHER_PROBE_MS": "4350", "SG_LAUNCHER_PROBE_MAX_MS": "4000"} + m = metrics_of(run_hook(ups_payload(str(repo)), env)[1]) + assert m["launcher_probe_ms"] == 4350 and m["ups_ms"] < 1000 + + +def test_other_hosts_get_no_output(repo, hook_env, tmp_path): + env = {k: v for k, v in _slow(hook_env, tmp_path, SNAPSHOT).items() + if k not in ("CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT", "PLUGIN_ROOT")} + env["SECURITY_GUIDANCE_PORTABLE_OUTPUT"] = "1" + assert run_hook(ups_payload(str(repo)), env)[1].strip() == "" + + +def test_row_holds_no_path_and_valid_keys(repo, hook_env, tmp_path): + _, out, _ = run_hook(ups_payload(str(repo)), _slow(hook_env, tmp_path, SNAPSHOT)) + assert str(repo) not in out and str(tmp_path) not in out + metrics = json.loads(out)["metrics"] + assert all(re.fullmatch(r"[a-z][a-z0-9_]{0,39}", k) for k in metrics) + assert all(isinstance(v, (int, float)) for v in metrics.values()) + + +def test_slow_capture_saves_the_same_baseline(repo, hook_env, tmp_path): + (repo / "app.py").write_text("print('changed')\n") + (repo / "new.txt").write_text("untracked\n") + states = [] + for name, env in (("fast", hook_env), ("slow", _slow(hook_env, tmp_path, SNAPSHOT))): + d = tmp_path / name + d.mkdir() + env = {k: v for k, v in env.items() if k != "CLAUDE_CODE_REMOTE_SESSION_ID"} + env["SECURITY_WARNINGS_STATE_DIR"] = str(d) + run_hook(ups_payload(str(repo)), env) + with open(d / "security_warnings_state_s1.json") as f: + states.append(json.load(f)) + fast, slow = states + assert git(repo, "rev-parse", fast["baseline_sha"] + "^{tree}") == \ + git(repo, "rev-parse", slow["baseline_sha"] + "^{tree}") + assert fast["baseline_sha"] != fast["head_at_capture"] + assert "new.txt" in fast["untracked_at_baseline"] + assert fast["untracked_at_baseline"] == slow["untracked_at_baseline"] + + +@pytest.mark.parametrize("path,expected", [ + ("/mnt/c/code/app", True), + ("/mnt/c", True), + ("/mnt/my disk/app", True), + ("/mnt/a\\b/app", True), + ("/mnt/cool/app", False), + ("/home/me/app", False), + ("/mnt/c/data/app", False), +]) +def test_windows_drive_mount(path, expected): + assert gitutil.windows_drive_mount(path, MOUNTS) is expected + + +@pytest.mark.parametrize("junk", ["junk\n", "1 2 - \n", "1 2 3 / /mnt/c rw - \n"]) +def test_windows_drive_mount_skips_malformed_lines(junk): + assert gitutil.windows_drive_mount("/mnt/c/x", junk + MOUNTS) is True + + +def test_wsl_drive_reaches_the_row(monkeypatch): + import security_reminder_hook as hook + monkeypatch.setattr(hook, "on_windows_drive", lambda cwd: True) + assert hook._slow_prompt_metrics("/x", 2.0, 1.5, 0.1, True, 0)["ups_wsl_drive"] == 1 + + +def test_index_entry_count(repo, tmp_path): + assert gitutil.index_entry_count(str(repo / ".git" / "index")) == 2 + assert gitutil.index_entry_count(str(tmp_path / "missing")) is None + (tmp_path / "bad").write_bytes(b"not an index") + assert gitutil.index_entry_count(str(tmp_path / "bad")) is None diff --git a/plugins/security-guidance/tests/test_state_dir_mode.py b/plugins/security-guidance/tests/test_state_dir_mode.py new file mode 100644 index 00000000..d55afaf9 --- /dev/null +++ b/plugins/security-guidance/tests/test_state_dir_mode.py @@ -0,0 +1,107 @@ +"""The state dir holds the reviewer's venv, review logs and session state. It +is created, and kept, with access for its user alone, whatever the umask.""" +import os +import stat +import sys + +import pytest + +import _base +import session_state + +pytestmark = pytest.mark.skipif(sys.platform == "win32", reason="POSIX modes") + + +@pytest.fixture +def loose_umask(): + old = os.umask(0o002) + yield + os.umask(old) + + +def _mode(p): + return stat.S_IMODE(os.stat(p).st_mode) + + +def test_new_state_dir_is_owner_only(tmp_path, monkeypatch, loose_umask): + d = tmp_path / "state" + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(d)) + session_state.save_state("s1", {"k": 1}) + assert _mode(d) == 0o700 + + +def test_existing_default_state_dir_is_narrowed(tmp_path, monkeypatch, loose_umask): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.delenv("SECURITY_WARNINGS_STATE_DIR", raising=False) + monkeypatch.delenv("CLAUDE_CONFIG_DIR", raising=False) + d = tmp_path / ".claude" / "security" + d.mkdir(parents=True) + os.chmod(d, 0o775) + _base.make_state_dir() + assert _mode(d) == 0o700 + + +def test_existing_folder_named_by_the_user_is_left_alone(tmp_path, monkeypatch): + d = tmp_path / "team" + d.mkdir() + os.chmod(d, 0o2775) + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(d)) + _base.make_state_dir() + assert _mode(d) == 0o2775 + + +def test_a_link_at_the_default_state_dir_is_left_alone(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.delenv("SECURITY_WARNINGS_STATE_DIR", raising=False) + monkeypatch.delenv("CLAUDE_CONFIG_DIR", raising=False) + shared = tmp_path / "shared" + shared.mkdir() + os.chmod(shared, 0o775) + (tmp_path / ".claude").mkdir() + os.symlink(shared, tmp_path / ".claude" / "security") + _base.make_state_dir() + assert _mode(shared) == 0o775 + + +def test_other_folders_are_left_alone(tmp_path, monkeypatch): + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.delenv("SECURITY_WARNINGS_STATE_DIR", raising=False) + logs = tmp_path / "logs" + logs.mkdir() + os.chmod(logs, 0o755) + _base.make_state_dir(str(logs)) + assert _mode(logs) == 0o755 + + +def test_debug_log_in_the_working_directory_leaves_the_state_dir_alone(tmp_path, monkeypatch): + d = tmp_path / "state" + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(d)) + with pytest.raises(OSError): + _base.make_state_dir("") + assert not d.exists() + + +@pytest.mark.parametrize("nofollow", [True, False]) +def test_in_place_save_replaces_a_symlink(tmp_path, monkeypatch, nofollow): + d = tmp_path / "state" + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(d)) + monkeypatch.setattr(session_state, "_RENAME_INTO_PLACE", False) + if not nofollow: # the Windows path: no O_NOFOLLOW + monkeypatch.delattr(os, "O_NOFOLLOW") + _base.make_state_dir() + target = tmp_path / "elsewhere.txt" + target.write_text("keep\n") + state_file = session_state.get_state_file("s1") + os.symlink(target, state_file) + session_state.save_state("s1", {"k": 1}) + assert target.read_text() == "keep\n" + assert not os.path.islink(state_file) + assert session_state.load_state("s1").get("k") == 1 + + +def test_in_place_save_creates_an_owner_only_file(tmp_path, monkeypatch, loose_umask): + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(tmp_path / "state")) + monkeypatch.setattr(session_state, "_RENAME_INTO_PLACE", False) + session_state.save_state("s1", {"k": 1}) + assert _mode(session_state.get_state_file("s1")) == 0o600 + assert session_state.load_state("s1").get("k") == 1 diff --git a/plugins/security-guidance/tests/test_state_file.py b/plugins/security-guidance/tests/test_state_file.py new file mode 100644 index 00000000..0d3a56b0 --- /dev/null +++ b/plugins/security-guidance/tests/test_state_file.py @@ -0,0 +1,197 @@ +import json +import os +import signal +import subprocess +import sys +import time + +import pytest + +from conftest import HOOKS_DIR, make_repo, metrics_of, run_hook, stop_payload, ups_payload + +import session_state as ss + + +@pytest.fixture +def state_dir(tmp_path, monkeypatch): + d = tmp_path / "state" + d.mkdir() + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(d)) + monkeypatch.delenv("CLAUDE_CODE_REMOTE_SESSION_ID", raising=False) + # Keep the legacy lock-file sweep and the debug log out of the real home. + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setattr(ss, "debug_log", lambda _msg: None) + return d + + +def _seed(session_id="s1"): + ss.with_locked_state(session_id, lambda st: st.update( + baseline_sha="a" * 40, touched_paths=["x.py"], + untracked_at_baseline={f"dir/file_{i:05d}.py": i for i in range(1500)})) + + +def _on_disk(state_dir, session_id="s1"): + return json.loads((state_dir / f"security_warnings_state_{session_id}.json").read_text()) + + +def _leftovers(state_dir): + return sorted(p.name for p in state_dir.iterdir() if p.name.endswith(".tmp")) + + +_MARK_ONE = ( + "import sys; sys.path.insert(0, sys.argv[1]); import session_state as s; " + "s.with_locked_state('s1', lambda st: st['shown_warnings'].append(sys.argv[2]))" +) + +# A save that dies half-way through writing, as a hook killed on timeout does. +_KILLED_MID_SAVE = """ +import json, os, signal, sys +sys.path.insert(0, sys.argv[1]) +import session_state as s +def dump(obj, f, **kw): + data = json.dumps(obj) + f.write(data[:len(data) // 2]); f.flush() + os.kill(os.getpid(), signal.SIGKILL) +s.json.dump = dump +s.with_locked_state('s1', lambda st: st['touched_paths'].append('y.py')) +""" + + +@pytest.mark.skipif(os.name == "nt", reason="rename into place is not used on Windows") +class TestSaveIsAtomic: + def test_a_save_killed_part_way_keeps_the_previous_state(self, state_dir): + _seed() + r = subprocess.run([sys.executable, "-c", _KILLED_MID_SAVE, str(HOOKS_DIR)], + env=dict(os.environ)) + assert r.returncode == -signal.SIGKILL + state = ss.load_state("s1") + assert state["baseline_sha"] == "a" * 40 + assert state["touched_paths"] == ["x.py"] + assert len(state["untracked_at_baseline"]) == 1500 + assert "state_resets" not in state + + def test_no_temporary_file_is_left_behind(self, state_dir): + _seed() + ss.with_locked_state("s1", lambda st: st["touched_paths"].append("y.py")) + assert _on_disk(state_dir)["touched_paths"] == ["x.py", "y.py"] + assert _leftovers(state_dir) == [] + + def test_state_that_cannot_be_written_leaves_the_file_alone(self, state_dir): + _seed() + with pytest.raises(TypeError): + ss.save_state("s1", {"shown_warnings": [], "bad": object()}) + assert _on_disk(state_dir)["baseline_sha"] == "a" * 40 + assert _leftovers(state_dir) == [] + + def test_writes_in_place_where_the_directory_refuses_a_rename(self, state_dir, monkeypatch): + _seed() + + def refuse(_src, _dst): + raise PermissionError("rename refused") + monkeypatch.setattr(ss.os, "replace", refuse) + ss.with_locked_state("s1", lambda st: st["touched_paths"].append("y.py")) + assert _on_disk(state_dir)["touched_paths"] == ["x.py", "y.py"] + assert _leftovers(state_dir) == [] + + def test_a_full_disk_leaves_the_previous_state(self, state_dir, monkeypatch): + _seed() + + def full(_obj, f, **_kw): + f.write('{"shown_warn') + raise OSError(28, "No space left on device") + monkeypatch.setattr(ss.json, "dump", full) + ss.save_state("s1", {"shown_warnings": []}) + assert _on_disk(state_dir)["baseline_sha"] == "a" * 40 + assert _leftovers(state_dir) == [] + + def test_in_place_where_rename_is_not_used(self, state_dir, monkeypatch): + monkeypatch.setattr(ss, "_RENAME_INTO_PLACE", False) + monkeypatch.setattr(ss.os, "replace", lambda *_a: pytest.fail("renamed")) + _seed() + assert _on_disk(state_dir)["touched_paths"] == ["x.py"] + + +@pytest.mark.skipif(ss.fcntl is None, reason="needs file locking") +def test_concurrent_writers_all_land(state_dir): + """Eight hooks marking one warning each at the same moment keep all eight.""" + for _round in range(5): + (state_dir / "security_warnings_state_s1.json").unlink(missing_ok=True) + procs = [subprocess.Popen([sys.executable, "-c", _MARK_ONE, str(HOOKS_DIR), f"k{i}"], + env=dict(os.environ)) for i in range(8)] + assert [p.wait() for p in procs] == [0] * 8 + assert sorted(_on_disk(state_dir)["shown_warnings"]) == [f"k{i}" for i in range(8)] + assert _leftovers(state_dir) == [] + + +class TestStateLossIsCounted: + def test_missing_file_is_a_fresh_state(self, state_dir): + assert ss.load_state("s1") == {"shown_warnings": []} + + @pytest.mark.parametrize("content", [ + b"", b'{"baseline_sha": "abc", "touched_pa', b"42", b'{"baseline_sha": "\xff\xfe"}']) + def test_unreadable_file_is_counted(self, state_dir, content): + (state_dir / "security_warnings_state_s1.json").write_bytes(content) + assert ss.load_state("s1") == {"shown_warnings": [], "state_resets": 1} + + def test_overwriting_another_hooks_save_is_counted_without_locking( + self, state_dir, monkeypatch): + monkeypatch.setattr(ss, "fcntl", None) + _seed() + assert "state_overwrites" not in _on_disk(state_dir) + + def slow_hook(state): + # Another hook saves while this one is between its read and write. + time.sleep(0.01) + ss.save_state("s1", {**state, "touched_paths": ["x.py", "other.py"]}) + state["shown_warnings"].append("k") + ss.with_locked_state("s1", slow_hook) + after = _on_disk(state_dir) + assert after["state_overwrites"] == 1 + assert after["touched_paths"] == ["x.py"] # the other hook's save was lost + + def test_a_concurrent_save_of_the_same_state_is_not_counted(self, state_dir, monkeypatch): + monkeypatch.setattr(ss, "fcntl", None) + _seed() + + def reader(state): + ss.save_state("s1", dict(state)) # another hook that only read + return state["baseline_sha"] + assert ss.with_locked_state("s1", reader) == "a" * 40 + assert "state_overwrites" not in _on_disk(state_dir) + + def test_stop_reports_the_counts_once(self, tmp_path, hook_env): + repo = make_repo(tmp_path / "repo") + hook_env = {k: v for k, v in hook_env.items() if k != "CLAUDE_CODE_REMOTE_SESSION_ID"} + state_file = os.path.join(hook_env["SECURITY_WARNINGS_STATE_DIR"], + "security_warnings_state_s1.json") + run_hook(ups_payload(repo), hook_env) + _rc, stdout, _err = run_hook(stop_payload(repo), hook_env) + m = metrics_of(stdout) + assert "state_resets" not in m and "state_overwrites" not in m + + with open(state_file, "w") as f: + f.write('{"baseline_sha": "abc", "touched_pa') + run_hook(ups_payload(repo), hook_env) + with open(state_file) as f: + state = json.load(f) + state["state_overwrites"] = 3 + with open(state_file, "w") as f: + json.dump(state, f) + _rc, stdout, _err = run_hook(stop_payload(repo), hook_env) + m = metrics_of(stdout) + assert (m["state_resets"], m["state_overwrites"]) == (1, 3) + + _rc, stdout, _err = run_hook(stop_payload(repo), hook_env) + m = metrics_of(stdout) + assert "state_resets" not in m and "state_overwrites" not in m + + +def test_cleanup_removes_old_temporary_files(state_dir): + old = state_dir / "security_warnings_state_s1.json.a1b2c3.tmp" + new = state_dir / "security_warnings_state_s2.json.a1b2c3.tmp" + old.write_text("{}") + new.write_text("{}") + long_ago = time.time() - 31 * 24 * 60 * 60 + os.utime(old, (long_ago, long_ago)) + ss.cleanup_old_state_files() + assert not old.exists() and new.exists() diff --git a/plugins/security-guidance/tests/test_stop_baseline_checkout.py b/plugins/security-guidance/tests/test_stop_baseline_checkout.py new file mode 100644 index 00000000..97c20ab3 --- /dev/null +++ b/plugins/security-guidance/tests/test_stop_baseline_checkout.py @@ -0,0 +1,282 @@ +"""The turn baseline belongs to the checkout the prompt hook captured it in. +A Stop review in another checkout (the session entered a worktree, or edited +another repo) does not use it: commits count from that checkout's own HEAD at +the time of the prompt, files are diffed against HEAD, and uncommitted files +last written before the prompt are left out. Stop rows carry `bl_origin`, +saying which case the row was.""" +import json +import os +import shutil +import time + +import pytest + +from conftest import ( + STUB_VULN, VULN_PY, edit_payload, git, make_repo, metrics_of, run_hook, + stop_payload, ups_payload, +) + + +@pytest.fixture +def checkouts(tmp_path, monkeypatch): + """a: a repo's primary checkout. b: a linked worktree of it whose branch + already holds a committed file that a's branch lacks. Both were last + committed to an hour ago.""" + monkeypatch.setenv("GIT_COMMITTER_DATE", f"@{int(time.time()) - 3600} +0000") + a = make_repo(tmp_path / "ws" / "a", {"app.py": "print('hi')\n"}) + b = tmp_path / "ws" / "b" + git(a, "worktree", "add", "-q", "-b", "feature", str(b)) + (b / "old.py").write_text(VULN_PY) + git(b, "add", "-A") + git(b, "commit", "-q", "-m", "earlier work") + monkeypatch.delenv("GIT_COMMITTER_DATE") + return a, b + + +def _stop(cwd, hook_env): + return metrics_of(run_hook(stop_payload(cwd), hook_env)[1]) + + +def _state(hook_env): + d = hook_env["SECURITY_WARNINGS_STATE_DIR"] + for e in os.scandir(d): + if e.name.endswith(".json"): + with open(e.path) as f: + state = json.load(f) + if "baseline_sha" in state: + return state + raise AssertionError(os.listdir(d)) + + +def test_baseline_from_the_reviewed_checkout(hook_env, checkouts): + a, _ = checkouts + run_hook(ups_payload(a), hook_env) + (a / "app.py").write_text(VULN_PY) + m = _stop(a, hook_env) + assert m["files_reviewed"] == 1 and m["bl_origin"] == 0 + + +def test_sent_on_a_skipped_review_too(hook_env, checkouts): + a, _ = checkouts + run_hook(ups_payload(a), hook_env) + m = _stop(a, hook_env) + assert m["skipped"] is True and m["bl_origin"] == 0 + + +def test_prompt_in_a_subdirectory_is_the_same_checkout(hook_env, checkouts): + a, _ = checkouts + (a / "pkg").mkdir() + run_hook(ups_payload(a / "pkg"), hook_env) + (a / "app.py").write_text(VULN_PY) + assert _stop(a, hook_env)["bl_origin"] == 0 + + +def test_turn_that_only_enters_another_worktree_reviews_nothing( + hook_env, checkouts, stub_api): + a, b = checkouts + run_hook(ups_payload(a), hook_env) + m = _stop(b, hook_env) + # b's earlier commit is not a commit made this turn. + assert m["skipped"] is True and m["bl_origin"] == 1 + assert not stub_api.calls + + +def test_edit_after_entering_another_worktree_is_reviewed( + hook_env, checkouts, stub_api): + a, b = checkouts + run_hook(ups_payload(a), hook_env) + (b / "new.py").write_text(VULN_PY) + run_hook(edit_payload(b, b / "new.py"), hook_env) + m = _stop(b, hook_env) + assert m["files_reviewed"] == 1 and m["bl_origin"] == 1 + assert stub_api.calls.count("/v1/messages") == 1 + + +def test_commit_made_after_entering_another_worktree_is_reviewed( + hook_env, checkouts): + a, b = checkouts + run_hook(ups_payload(a), hook_env) + (b / "new.py").write_text(VULN_PY) + git(b, "add", "-A") + git(b, "commit", "-q", "-m", "this turn") + m = _stop(b, hook_env) + # new.py, not old.py from the earlier commit. + assert m["files_reviewed"] == 1 and m["bl_origin"] == 1 + + +def test_commit_in_a_worktree_created_this_turn_is_reviewed(hook_env, checkouts): + a, _ = checkouts + run_hook(ups_payload(a), hook_env) + c = a.parent / "c" + git(a, "worktree", "add", "-q", "-b", "fresh", str(c)) + (c / "new.py").write_text(VULN_PY) + git(c, "add", "-A") + git(c, "commit", "-q", "-m", "this turn") + (c / "more.py").write_text(VULN_PY) + m = _stop(c, hook_env) + assert m["files_reviewed"] == 2 and m["bl_origin"] == 1 + + +def test_uncommitted_work_already_in_the_other_worktree_is_left_out( + hook_env, checkouts, stub_api): + a, b = checkouts + (b / "scratch.py").write_text(VULN_PY) # untracked + (b / "old.py").write_text(VULN_PY + "# todo\n") # tracked, edited + time.sleep(2.2) # past the slack allowed around the prompt time + run_hook(ups_payload(a), hook_env) + m = _stop(b, hook_env) + assert m["skipped"] is True and m["bl_origin"] == 1 + assert not stub_api.calls + # Both are reviewed once the turn writes to them. + (b / "scratch.py").write_text(VULN_PY + "x = 1\n") + (b / "old.py").write_text(VULN_PY + "# done\n") + m = _stop(b, hook_env) + assert m["files_reviewed"] == 2 and m["bl_origin"] == 1 + + +def test_baseline_kept_from_an_interrupted_turn_in_another_worktree( + hook_env, checkouts, stub_api): + a, b = checkouts + run_hook(ups_payload(a), hook_env) + run_hook(edit_payload(a, a / "app.py"), hook_env) + run_hook(ups_payload(b), hook_env) + m = _stop(b, hook_env) + assert m["skipped"] is True and m["bl_origin"] == 21 + assert not stub_api.calls + + +def test_baseline_kept_from_an_interrupted_turn_in_the_same_checkout(hook_env, checkouts): + a, _ = checkouts + run_hook(ups_payload(a), hook_env) + run_hook(edit_payload(a, a / "app.py"), hook_env) + run_hook(ups_payload(a), hook_env) + (a / "app.py").write_text(VULN_PY) + assert _stop(a, hook_env)["bl_origin"] == 20 + + +def test_interrupted_turn_then_a_prompt_that_cannot_capture(hook_env, checkouts): + a, _ = checkouts + run_hook(ups_payload(a), hook_env) + run_hook(edit_payload(a, a / "app.py"), hook_env) + run_hook(ups_payload(a), hook_env) + run_hook(ups_payload(a.parent), hook_env) + (a / "app.py").write_text(VULN_PY) + assert _stop(a, hook_env)["bl_origin"] == 10 + + +def test_baseline_from_another_repository(hook_env, checkouts, tmp_path): + a, _ = checkouts + other = make_repo(tmp_path / "ws" / "other", {"lib.py": "x = 1\n"}) + run_hook(ups_payload(a), hook_env) + (other / "lib.py").write_text(VULN_PY) + m = _stop(other, hook_env) + assert m["files_reviewed"] == 1 and m["bl_origin"] == 2 + + +def test_baseline_from_a_checkout_that_is_gone(hook_env, checkouts): + a, b = checkouts + run_hook(ups_payload(b), hook_env) + git(a, "worktree", "remove", "--force", str(b)) + shutil.rmtree(b, ignore_errors=True) + (a / "app.py").write_text(VULN_PY) + m = _stop(a, hook_env) + assert m["files_reviewed"] == 1 and m["bl_origin"] == 3 + + +def test_prompt_in_a_folder_that_is_not_a_repo_keeps_the_untracked_snapshot( + hook_env, checkouts): + a, _ = checkouts + ws = a.parent + (a / "notes.py").write_text("n = 0\n") # untracked since before the session + run_hook(ups_payload(a), hook_env) + (a / "first.py").write_text("x = 1\n") + assert _stop(a, hook_env)["bl_origin"] == 0 + run_hook(ups_payload(ws), hook_env) + (a / "second.py").write_text("y = 2\n") + run_hook(edit_payload(ws, a / "second.py"), hook_env) + m = _stop(ws, hook_env) + # The earlier baseline is kept, so the first turn's file is reviewed + # again. notes.py is not: the snapshot that goes with it is kept too. + assert m["files_reviewed"] == 2 and m["bl_origin"] == 10 + # The next prompt in the repo captures a fresh baseline there. + run_hook(ups_payload(a), hook_env) + (a / "third.py").write_text("z = 3\n") + m = _stop(a, hook_env) + assert m["files_reviewed"] == 1 and m["bl_origin"] == 0 + + +def test_baseline_taken_after_a_finding_belongs_to_the_reviewed_checkout( + hook_env, checkouts, stub_api): + a, b = checkouts + run_hook(ups_payload(a), hook_env) + (b / "new.py").write_text(VULN_PY) + stub_api.vulns = [dict(STUB_VULN, filePath="new.py")] + assert _stop(b, hook_env)["bl_origin"] == 1 + st = _state(hook_env) + assert os.path.samefile(st["baseline_root"], b) + # b's HEAD at the prompt, not a's. + assert st["head_at_capture"] == git(b, "rev-parse", "HEAD").strip() + # Same turn, next Stop: only what changed since the finding. + stub_api.vulns = [] + (b / "newer.py").write_text("v = 1\n") + m = _stop(b, hook_env) + assert m["files_reviewed"] == 1 and m["bl_origin"] == 0 + + +def test_baseline_taken_after_a_finding_keeps_the_turns_first_commit( + hook_env, checkouts, stub_api): + a, _ = checkouts + head = git(a, "rev-parse", "HEAD").strip() + run_hook(ups_payload(a), hook_env) + (a / "app.py").write_text(VULN_PY) + git(a, "commit", "-q", "-am", "this turn") + stub_api.vulns = [dict(STUB_VULN, filePath="app.py")] + assert _stop(a, hook_env)["bl_origin"] == 0 + assert _state(hook_env)["head_at_capture"] == head + # A fix folded into the commit is still inside the turn's range. + (a / "app.py").write_text(VULN_PY + "x = 1\n") + git(a, "commit", "-q", "-a", "--amend", "-m", "this turn") + assert _stop(a, hook_env)["files_reviewed"] == 1 + + +def test_first_baseline_taken_after_a_finding_records_its_checkout( + hook_env, checkouts, stub_api): + a, b = checkouts + ws = a.parent + run_hook(ups_payload(ws), hook_env) + (a / "new.py").write_text(VULN_PY) + run_hook(edit_payload(ws, a / "new.py"), hook_env) + stub_api.vulns = [dict(STUB_VULN, filePath="new.py")] + assert _stop(ws, hook_env)["bl_origin"] == 4 + assert os.path.samefile(_state(hook_env)["baseline_root"], a) + stub_api.vulns = [] + assert _stop(b, hook_env)["bl_origin"] == 1 + + +def test_no_baseline_at_all(hook_env, checkouts): + a, _ = checkouts + ws = a.parent + run_hook(ups_payload(ws), hook_env) + (a / "new.py").write_text("v = 1\n") + run_hook(edit_payload(ws, a / "new.py"), hook_env) + m = _stop(ws, hook_env) + assert m["files_reviewed"] == 1 and m["bl_origin"] == 4 + + +def test_prompt_subdirectory_removed_during_the_turn(hook_env, checkouts): + a, _ = checkouts + (a / "pkg").mkdir() + run_hook(ups_payload(a / "pkg"), hook_env) + (a / "pkg").rmdir() + (a / "app.py").write_text(VULN_PY) + assert _stop(a, hook_env)["bl_origin"] == 0 + + +def test_nested_worktree_removed_during_the_turn(hook_env, checkouts): + a, _ = checkouts + nested = a / ".claude" / "worktrees" / "wt" + git(a, "worktree", "add", "-q", "-b", "nested", str(nested)) + run_hook(ups_payload(nested), hook_env) + git(a, "worktree", "remove", "--force", str(nested)) + (a / "app.py").write_text(VULN_PY) + assert _stop(a, hook_env)["bl_origin"] == 3 diff --git a/plugins/security-guidance/tests/test_stop_diff_memory.py b/plugins/security-guidance/tests/test_stop_diff_memory.py new file mode 100644 index 00000000..31a524f9 --- /dev/null +++ b/plugins/security-guidance/tests/test_stop_diff_memory.py @@ -0,0 +1,349 @@ +"""The Stop review reads `git diff` output without holding all of it: files it +would drop come back as a header line, and one file's diff is cut at a cap.""" +import random +import subprocess +import sys +import tempfile +import time +import tracemalloc + +import pytest + +from conftest import ( + GIT_ENV, VULN_PY, make_repo, metrics_of, run_hook, stop_payload, + ups_payload, +) + +import gitutil + + +@pytest.fixture(autouse=True) +def _fresh_read_state(): + yield + gitutil._unparsed_diff_headers.clear() + gitutil._diff_read.update(bytes=0, cut_files=0, cut_bytes=0, timeouts=0) + + +def _rewrite(path, removed, added): + return (f"diff --git a/{path} b/{path}\n" + "index 1111111..2222222 100644\n" + f"--- a/{path}\n+++ b/{path}\n" + f"@@ -1,{len(removed)} +1,{len(added)} @@\n" + + "".join(f"-{ln}\n" for ln in removed) + + "".join(f"+{ln}\n" for ln in added)) + + +@pytest.fixture +def git_env(tmp_path, monkeypatch): + d = tmp_path / "tmproot" + d.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(d)) + for k, v in GIT_ENV.items(): + monkeypatch.setenv(k, v) + + +def _section(path, body_lines, old=None): + return (f"diff --git a/{old or path} b/{path}\n" + "index 1111111..2222222 100644\n" + f"--- a/{old or path}\n+++ b/{path}\n" + f"@@ -1,2 +1,{len(body_lines)} @@\n" + + "".join(f"+{ln}\n" for ln in body_lines)) + + +def _keep(raw, chunk_sizes): + keeper = gitutil._DiffKeeper() + pos = 0 + while pos < len(raw): + n = next(chunk_sizes) + keeper.feed(raw[pos:pos + n]) + pos += n + return keeper, keeper.result() + + +def _sizes(*cycle): + while True: + yield from cycle + + +MIXED = ( + _section("app.py", ["import os", "os.system(cmd)"]) + + _section("notes.txt", ["plain text", "diff --git a/fake.py b/fake.py is only text here"]) + + _section("web/café.js", ["eval(x)"]) + + 'diff --git "a/we\\"ird.py" "b/we\\"ird.py"\n--- "a/we\\"ird.py"\n+++ "b/we\\"ird.py"\n@@ -1 +1 @@\n+x = 1\n' + + "diff --git nonsense header\n@@ -1 +1 @@\n+y\n" + + _section("new_name.py", ["z = 2"], old="old_name.txt") + + _section("data.csv", ["1,2,3"] * 50) + + _section("last.sh", ["rm -rf $1"]) +).encode() + + +class TestDiffKeeper: + @pytest.mark.parametrize("seed", range(40)) + def test_same_files_as_reading_it_whole_at_any_chunking(self, seed): + rng = random.Random(seed) + sizes = iter(lambda: rng.choice([1, 2, 3, 7, 11, 12, 13, 64, 500]), None) + gitutil._unparsed_diff_headers.clear() + want = gitutil.parse_diff_into_files(MIXED.decode()) + want_unparsed = set(gitutil._unparsed_diff_headers) + gitutil._unparsed_diff_headers.clear() + keeper, kept = _keep(MIXED, sizes) + assert gitutil.parse_diff_into_files(kept.decode()) == want + assert gitutil._unparsed_diff_headers == want_unparsed + assert [p for p, _ in want] == ["app.py", "web/café.js", 'we"ird.py', "new_name.py", "last.sh"] + assert keeper.seen == len(MIXED) + assert keeper.cut_files == 0 + + def test_dropped_file_keeps_its_header_line_only(self): + _, kept = _keep(MIXED, _sizes(4096)) + text = kept.decode() + assert "diff --git a/notes.txt b/notes.txt\n" in text + assert "plain text" not in text and "1,2,3" not in text + assert len(kept) < len(MIXED) + + def test_untouched_when_every_file_is_reviewable_and_small(self): + raw = (_section("a.py", ["x"]) + _section("b.js", ["y"])).encode() + assert _keep(raw, _sizes(5))[1] == raw + + @pytest.mark.parametrize("chunk", [1, 17, 4096]) + def test_large_file_is_cut_and_the_next_file_survives(self, monkeypatch, chunk): + monkeypatch.setattr(gitutil, "MAX_FILE_DIFF_BYTES", 300) + big = _section("big.json", ["k" * 37] * 100) + raw = (big + _section("after.py", ["import os"])).encode() + keeper, kept = _keep(raw, _sizes(chunk)) + files = dict(gitutil.parse_diff_into_files(kept.decode())) + assert set(files) == {"big.json", "after.py"} + assert files["after.py"] == dict(gitutil.parse_diff_into_files(raw.decode()))["after.py"] + # The start of the section, then as much again of its added lines. + assert 300 < len(files["big.json"]) <= 2 * 300 + len(gitutil._CUT_MARK) + 2 + assert keeper.cut_files == 1 + assert keeper.cut_bytes == len(big) - 2 * 300 + + def test_cut_file_says_so_on_its_last_line(self, monkeypatch): + monkeypatch.setattr(gitutil, "MAX_FILE_DIFF_BYTES", 300) + raw = (_section("big.json", ["k" * 37] * 100) + _section("after.py", ["x"])).encode() + _, kept = _keep(raw, _sizes(64)) + files = dict(gitutil.parse_diff_into_files(kept.decode())) + assert files["big.json"].endswith(gitutil._CUT_MARK.decode()) + assert "cut by security-guidance" not in files["after.py"] + assert kept.count(gitutil._CUT_MARK) == 1 + + @pytest.mark.parametrize("chunk", [1, 23, 4096]) + def test_rewrite_past_the_cap_keeps_its_added_lines(self, monkeypatch, chunk): + monkeypatch.setattr(gitutil, "MAX_FILE_DIFF_BYTES", 400) + raw = (_rewrite("views.py", ["old line %03d" % i for i in range(200)], + ["os.system(cmd)"] + ["new line %03d" % i for i in range(200)]) + + _section("after.py", ["x"])).encode() + keeper, kept = _keep(raw, _sizes(chunk)) + body = dict(gitutil.parse_diff_into_files(kept.decode()))["views.py"] + assert "+os.system(cmd)\n" in body + assert "-old line 199" not in body + past = body[400:] + assert all(ln.startswith(("+", "\\")) for ln in past.splitlines()[1:]) + assert len(body) <= 2 * 400 + len(gitutil._CUT_MARK) + 2 + assert keeper.cut_files == 1 + + def test_dropped_large_file_is_not_counted_as_cut(self, monkeypatch): + monkeypatch.setattr(gitutil, "MAX_FILE_DIFF_BYTES", 300) + raw = (_section("big.log", ["k" * 37] * 100) + _section("after.py", ["x"])).encode() + keeper, kept = _keep(raw, _sizes(64)) + assert keeper.cut_files == 0 and keeper.cut_bytes == 0 + assert [p for p, _ in gitutil.parse_diff_into_files(kept.decode())] == ["after.py"] + + def test_renamed_from_a_secret_shaped_name_keeps_its_header_only(self): + raw = (_section("config.py", ["TOKEN = 'abc'"], old=".env") + + _section("after.py", ["x"])).encode() + want = gitutil.parse_diff_into_files(raw.decode()) + _, kept = _keep(raw, _sizes(64)) + assert gitutil.parse_diff_into_files(kept.decode()) == want + assert [p for p, _ in want] == ["after.py"] + assert b"TOKEN" not in kept + + def test_past_the_total_later_files_keep_less_and_less(self, monkeypatch): + monkeypatch.setattr(gitutil, "MAX_FILE_DIFF_BYTES", 2000) + monkeypatch.setattr(gitutil, "MAX_KEPT_DIFF_BYTES", 3000) + monkeypatch.setattr(gitutil, "_LATE_FILE_DIFF_BYTES", 200) + monkeypatch.setattr(gitutil, "_LAST_FILE_DIFF_BYTES", 120) + raw = "".join(_section(f"f{i}.py", ["v" * 30] * 400) for i in range(40)).encode() + keeper, kept = _keep(raw, _sizes(333)) + files = gitutil.parse_diff_into_files(kept.decode()) + assert [p for p, _ in files] == [f"f{i}.py" for i in range(40)] + sizes = [len(body) for _, body in files] + mark = len(gitutil._CUT_MARK) + 2 + assert sizes[0] > 2000 and sizes[0] <= 4000 + mark + assert 200 < sizes[5] <= 400 + mark + assert sizes[-1] <= 120 + mark + assert sizes[0] > sizes[5] > sizes[-1] + assert keeper.cut_files == 40 + # Everything not kept is counted; a cut may add a newline or two. + from_git = len(kept) - 40 * len(gitutil._CUT_MARK) + assert from_git - 80 <= len(raw) - keeper.cut_bytes <= from_git + + def test_output_cut_inside_a_line_ends_with_a_newline(self, monkeypatch): + monkeypatch.setattr(gitutil, "MAX_FILE_DIFF_BYTES", 300) + raw = _rewrite("big.json", ["k" * 37] * 100, []).encode() + keeper, kept = _keep(raw, _sizes(64)) + assert kept == raw[:300] + b"\n" + gitutil._CUT_MARK + assert keeper.cut_bytes == len(raw) - 300 + both = kept + _section("next.py", ["x"]).encode() + assert [p for p, _ in gitutil.parse_diff_into_files(both.decode())] == ["big.json", "next.py"] + + def test_empty_output(self): + keeper, kept = _keep(b"", _sizes(1)) + assert kept == b"" and keeper.seen == 0 + + +class TestRunDiffBounded: + def test_returns_what_run_would(self): + code = ("import sys; sys.stdout.buffer.write(b'diff --git a/a.py b/a.py\\n@@ -1 +1 @@\\n+x\\n'); " + "sys.stderr.write('warn'); sys.exit(3)") + r = gitutil._run_diff_bounded([sys.executable, "-c", code], 30) + assert r.returncode == 3 + assert r.stdout == b"diff --git a/a.py b/a.py\n@@ -1 +1 @@\n+x\n" + assert r.stderr == b"warn" + + def test_timeout_kills_and_raises(self): + code = "import sys, time; sys.stdout.write('x' * 100000); sys.stdout.flush(); time.sleep(60)" + before = gitutil._diff_read["timeouts"] + with pytest.raises(subprocess.TimeoutExpired): + gitutil._run_diff_bounded([sys.executable, "-c", code], 0.5) + assert gitutil._diff_read["timeouts"] == before + 1 + + def test_timeout_holds_while_another_process_keeps_the_pipe_open(self): + # What a `git diff` that started a fetch looks like: the program is + # gone, something it started still holds its stdout. + code = ("import subprocess, sys; " + "subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(8)']); " + "sys.stdout.write('x'); sys.stdout.flush()") + before = gitutil._diff_read["timeouts"] + t0 = time.monotonic() + with pytest.raises(subprocess.TimeoutExpired): + gitutil._run_diff_bounded([sys.executable, "-c", code], 1) + assert time.monotonic() - t0 < 5 + assert gitutil._diff_read["timeouts"] == before + 1 + + def test_a_timed_out_read_still_reports_its_size(self): + code = "import sys, time; sys.stdout.write('x' * 300000); sys.stdout.flush(); time.sleep(60)" + with pytest.raises(subprocess.TimeoutExpired): + gitutil._run_diff_bounded([sys.executable, "-c", code], 1) + assert gitutil._diff_read["bytes"] >= 200000 + + def test_a_read_that_breaks_is_an_error_not_a_short_diff(self, monkeypatch): + def broken(self, data): + raise MemoryError("no room") + monkeypatch.setattr(gitutil._DiffKeeper, "feed", broken) + code = "import sys; sys.stdout.write('diff --git a/a.py b/a.py\\n' * 50000)" + t0 = time.monotonic() + with pytest.raises(OSError): + gitutil._run_diff_bounded([sys.executable, "-c", code], 30) + assert time.monotonic() - t0 < 10 + + def test_no_thread_to_read_with_is_an_error(self, monkeypatch): + def refuse(self): + raise RuntimeError("can't start new thread") + monkeypatch.setattr(gitutil.threading.Thread, "start", refuse) + with pytest.raises(OSError): + gitutil._run_diff_bounded([sys.executable, "-c", "print('x')"], 30) + + def test_missing_program_raises_like_run(self): + with pytest.raises(FileNotFoundError): + gitutil._run_diff_bounded(["sg-no-such-program"], 5) + + +class TestGetGitDiff: + def test_large_data_file_is_not_held(self, tmp_path, git_env): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + (repo / "app.py").write_text(VULN_PY) + line = "generated text line that is about eighty bytes long once the number is on it %07d\n" + (repo / "gen.txt").write_text("".join(line % i for i in range(250000))) + paths = [str(repo / "app.py"), str(repo / "gen.txt")] + tracemalloc.start() + try: + diff = gitutil.get_git_diff(str(repo), "HEAD", paths=paths, untracked_paths=["gen.txt"]) + peak = tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + assert [p for p, _ in gitutil.parse_diff_into_files(diff)] == ["app.py"] + assert "diff --git a/gen.txt b/gen.txt\n" in diff and "generated text" not in diff + assert gitutil._diff_read["bytes"] > 20 * 1024 * 1024 + assert gitutil._diff_read["cut_files"] == 0 + # Reading it whole took several times the 20 MB of output. + assert peak < 8 * 1024 * 1024 + + def test_timeout_is_none(self, tmp_path, git_env, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + + def slow(cmd, timeout, **kwargs): + raise subprocess.TimeoutExpired(cmd, timeout) + + monkeypatch.setattr(gitutil, "_run_diff_bounded", slow) + assert gitutil.get_git_diff(str(repo), "HEAD", paths=[str(repo / "app.py")], + untracked_paths=[]) is None + + @pytest.mark.parametrize("value, want", [ + ("", 4 * 1024 * 1024), ("junk", 4 * 1024 * 1024), ("0", 4 * 1024 * 1024), + ("-5", 4 * 1024 * 1024), ("1000", 256 * 1024), ("1000000", 1000000)]) + def test_cap_setting_cannot_break_the_read(self, monkeypatch, value, want): + monkeypatch.setenv("SG_MAX_FILE_DIFF_BYTES", value) + assert gitutil._file_diff_cap() == want + + def test_rewritten_large_file_still_shows_what_is_new(self, tmp_path, git_env): + body = "".join(f"def handler_{i}(request):\n return render(request, 'page_{i}.html')\n" + for i in range(18000)) + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "views.py").write_bytes(body.replace("\n", "\r\n").encode()) + subprocess.run(["git", "add", "-A"], cwd=repo, check=True) + subprocess.run(["git", "commit", "-q", "-m", "views"], cwd=repo, check=True) + assert (repo / "views.py").stat().st_size > 1200000 + (repo / "views.py").write_bytes(("import os\nos.system(request.GET['cmd'])\n" + body).encode()) + diff = gitutil.get_git_diff(str(repo), "HEAD", paths=[str(repo / "views.py")], untracked_paths=[]) + assert gitutil._diff_read["cut_files"] == 0 + files = gitutil.filter_preexisting_from_diff( + gitutil.parse_diff_into_files(diff), str(repo), "HEAD") + assert "+os.system(request.GET['cmd'])" in files[0][1][:2000] + + def test_rewrite_past_the_cap_still_shows_what_is_new(self, tmp_path, git_env, monkeypatch): + monkeypatch.setattr(gitutil, "MAX_FILE_DIFF_BYTES", 200000) + body = "".join(f"def handler_{i}(request):\n return render(request, 'page_{i}.html')\n" + for i in range(18000)) + repo = make_repo(tmp_path / "repo", {"views.py": body.replace("\n", "\r\n")}) + (repo / "views.py").write_bytes(("import os\nos.system(request.GET['cmd'])\n" + body).encode()) + diff = gitutil.get_git_diff(str(repo), "HEAD", paths=[str(repo / "views.py")], untracked_paths=[]) + assert gitutil._diff_read["cut_files"] == 1 + assert len(diff) < 2 * 200000 + 1000 + files = gitutil.filter_preexisting_from_diff( + gitutil.parse_diff_into_files(diff), str(repo), "HEAD") + assert "+os.system(request.GET['cmd'])" in files[0][1][:2000] + + def test_failed_diff_is_still_none(self, tmp_path, git_env): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + assert gitutil.get_git_diff(str(repo), "0" * 40, paths=[str(repo / "app.py")], + untracked_paths=[]) is None + + +class TestStopHook: + def test_reports_what_it_read_and_reviews_the_source(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + (repo / "gen.txt").write_text("some generated text\n" * 150000) + (repo / "big.json").write_text('"k": "some generated value",\n' * 20000) + env = {**hook_env, "SG_MAX_FILE_DIFF_BYTES": "1000"} + rc, so, se = run_hook(stop_payload(repo), env) + m = metrics_of(so) + assert not m.get("skipped"), m + assert m["files_reviewed"] == 2 + assert m["diff_mb"] == 3 + assert m["diff_cut_files"] == 1 + # 256 KiB from the start and as much again of added lines were read; + # the prompt took 80,000 bytes of that. + assert m["diff_truncated"] > 20000 * 29 - 80000 - 1000 + assert "diff_timeout" not in m + assert stub_api.calls + + def test_small_turn_reports_nothing_new(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"}) + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + m = metrics_of(run_hook(stop_payload(repo), hook_env)[1]) + assert m["files_reviewed"] == 1 + assert not {"diff_mb", "diff_cut_files", "diff_timeout"} & set(m) diff --git a/plugins/security-guidance/tests/test_stop_review_paths.py b/plugins/security-guidance/tests/test_stop_review_paths.py new file mode 100644 index 00000000..b6f9793c --- /dev/null +++ b/plugins/security-guidance/tests/test_stop_review_paths.py @@ -0,0 +1,319 @@ +"""The Stop review still runs when the changed paths of a turn do not fit in +one git command.""" +import errno +import os +import subprocess +import tempfile +import types + +import pytest + +from conftest import ( + GIT_ENV, VULN_PY, make_repo, metrics_of, run_hook, stop_payload, + ups_payload, +) + +import diffstate +import gitutil + +GIT_CMD_CHARS = sum(len(a) + 1 for a in gitutil.GIT_CMD) + + +@pytest.fixture +def git_env(tmp_path, monkeypatch): + """Git identity for in-process calls, and a temp dir of our own for the + index copies.""" + d = tmp_path / "tmproot" + d.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(d)) + monkeypatch.setattr(gitutil, "_path_list_too_long", set()) + for k, v in GIT_ENV.items(): + monkeypatch.setenv(k, v) + + +def _more_than_one_command_holds(repo, exts): + """3,000 untracked files, cycling through `exts`, whose names add up to + more than a single command can carry on any platform (the largest limit + is about 2 MB, on Linux).""" + deep = repo / ("d" * 180) / ("e" * 180) / ("f" * 180) / ("g" * 180) + deep.mkdir(parents=True) + for i in range(3000): + (deep / f"file_{i:05d}{exts[i % len(exts)]}").write_text("x\n") + + +def _unusual_argument_limit(): + """The long-path tests are sized for the usual limits: 1 MB (macOS) to + 2 MB (Linux with an 8 MB stack).""" + try: + return not 1024 * 1024 <= os.sysconf("SC_ARG_MAX") <= 2 * 1024 * 1024 + 200000 + except (AttributeError, ValueError, OSError): + return False + + +long_paths = pytest.mark.skipif( + os.name == "nt" or _unusual_argument_limit(), + reason="needs paths longer than Windows allows by default, and the usual " + "1 to 2 MB limit on one command's arguments") + + +def _refuse_long_commands(monkeypatch, limit, error): + """Make starting a command fail like the OS does when its arguments + exceed `limit` characters. Returns the list of commands run. Popen is + where subprocess.run and the bounded diff reader both start theirs.""" + real_popen = subprocess.Popen + ran = [] + + def popen(cmd, *args, **kwargs): + if sum(len(a) + 1 for a in cmd) > limit: + raise error + ran.append(cmd) + return real_popen(cmd, *args, **kwargs) + + monkeypatch.setattr(subprocess, "Popen", popen) + return ran + + +class TestPathBatches: + def test_every_path_once_and_each_batch_bounded(self): + paths = [f"dir/file_{i}.py" for i in range(100000)] + batches = list(gitutil._path_batches(paths)) + assert [p for b in batches for p in b] == paths + assert len(batches) > 1 + assert all(sum(len(p) for p in b) <= gitutil._MAX_PATH_ARGS_BYTES + for b in batches) + + def test_budget_is_in_bytes(self, monkeypatch): + monkeypatch.setattr(gitutil, "_MAX_PATH_ARGS_BYTES", 100) + assert len(list(gitutil._path_batches(["\u6f22" * 20 + ".py"] * 3))) == 3 + + def test_one_overlong_path_still_goes_out(self): + long = "x" * (gitutil._MAX_PATH_ARGS_BYTES + 10) + assert list(gitutil._path_batches(["a", long, "b"])) == [["a"], [long], ["b"]] + + def test_no_paths_no_batches(self): + assert list(gitutil._path_batches([])) == [] + + +class TestGetGitDiff: + def _review_set(self, tmp_path): + repo = make_repo(tmp_path / "repo", {f"t{i}.py": "x = 1\n" for i in range(6)}) + for i in range(6): + (repo / f"t{i}.py").write_text("x = 2\n") + (repo / f"new{i}.py").write_text("y = 1\n") + paths, base, root, untracked, _ = diffstate.compute_v2_review_set(str(repo), None, None, {}) + assert len(paths) == 12 and len(untracked) == 6 + return root, base, paths, untracked + + def test_one_command_when_the_paths_fit(self, tmp_path, git_env, monkeypatch): + root, base, paths, untracked = self._review_set(tmp_path) + ran = _refuse_long_commands(monkeypatch, 10**9, OSError(errno.E2BIG, "too long")) + diff = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + assert len(gitutil.parse_diff_into_files(diff)) == 12 + assert len([c for c in ran if "diff" in c]) == 1 + assert len([c for c in ran if "add" in c]) == 1 + + @pytest.mark.parametrize("error", [ + OSError(errno.E2BIG, "Argument list too long"), + # What Python raises on Windows for ERROR_FILENAME_EXCED_RANGE. + type("WinTooLong", (FileNotFoundError,), {"winerror": 206})(errno.ENOENT, "too long"), + ], ids=["posix", "windows"]) + def test_batches_when_one_command_is_refused(self, tmp_path, git_env, monkeypatch, error): + root, base, paths, untracked = self._review_set(tmp_path) + whole = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + monkeypatch.setattr(gitutil, "_MAX_PATH_ARGS_BYTES", 60) + limit = GIT_CMD_CHARS + 100 # the six-path add fits, the twelve-path diff does not + ran = _refuse_long_commands(monkeypatch, limit, error) + batched = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + assert sorted(gitutil.parse_diff_into_files(batched)) == \ + sorted(gitutil.parse_diff_into_files(whole)) + assert len([c for c in ran if "diff" in c]) > 1 + # The batches are read through the bounded reader, which counts them. + assert gitutil._diff_read["bytes"] > 0 + + def test_only_reviewable_paths_after_a_refusal(self, tmp_path, git_env, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("x = 2\n") + for i in range(20): + (repo / f"note{i:02d}.txt").write_text("x\n") + paths, base, root, untracked, _ = diffstate.compute_v2_review_set(str(repo), None, None, {}) + whole = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + assert whole.count("diff --git ") == 21 + _refuse_long_commands(monkeypatch, GIT_CMD_CHARS + 100, OSError(errno.E2BIG, "too long")) + cut = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + assert cut.count("diff --git ") == 1 + assert gitutil.parse_diff_into_files(cut) == gitutil.parse_diff_into_files(whole) + + def test_rename_away_from_a_secret_shaped_name_stays_out_after_a_refusal( + self, tmp_path, git_env, monkeypatch): + secret = "".join(f"KEY_{i}=value-{i}\n" for i in range(20)) + repo = make_repo(tmp_path / "repo", {".env": secret, "app.py": "x = 1\n"}) + os.rename(repo / ".env", repo / "settings.py") + (repo / "app.py").write_text("x = 2\n") + (repo / "a_much_longer_module_name.py").write_text("y = 1\n") + for i in range(20): + (repo / f"note{i:02d}.txt").write_text("x\n") + paths, base, root, untracked, _ = diffstate.compute_v2_review_set(str(repo), None, None, {}) + whole = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + reviewed = ["a_much_longer_module_name.py", "app.py"] + assert sorted(f for f, _ in gitutil.parse_diff_into_files(whole)) == reviewed + # Room for .env beside each batch, and not for the three .py together. + monkeypatch.setattr(gitutil, "_MAX_PATH_ARGS_BYTES", 60) + ran = _refuse_long_commands(monkeypatch, GIT_CMD_CHARS + 120, OSError(errno.E2BIG, "too long")) + cut = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + diffs = [c for c in ran if "diff" in c] + assert len(diffs) > 1 and all(".env" in c for c in diffs) + assert sorted(f for f, _ in gitutil.parse_diff_into_files(cut)) == reviewed + assert "KEY_1=" not in "".join(d for _, d in gitutil.parse_diff_into_files(cut)) + + def test_gives_up_when_the_secret_shaped_paths_alone_are_too_many( + self, tmp_path, git_env, monkeypatch): + repo = make_repo(tmp_path / "repo", { + "app.py": "x = 1\n", **{f"dir{i:02d}/.env": "K=v\n" for i in range(20)}}) + (repo / "app.py").write_text("x = 2\n") + for i in range(20): + os.remove(repo / f"dir{i:02d}" / ".env") + paths, base, root, untracked, _ = diffstate.compute_v2_review_set(str(repo), None, None, {}) + monkeypatch.setattr(gitutil, "_MAX_PATH_ARGS_BYTES", 200) + ran = _refuse_long_commands(monkeypatch, GIT_CMD_CHARS + 100, OSError(errno.E2BIG, "too long")) + assert gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) is None + assert [c for c in ran if "diff" in c] == [] + assert "gave_up" in gitutil._path_list_too_long + + def test_secret_shaped_files_still_in_the_work_tree_do_not_ride_along( + self, tmp_path, git_env, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("x = 2\n") + for i in range(20): + (repo / f"dir{i:02d}").mkdir() + (repo / f"dir{i:02d}" / ".env").write_text("K=v\n") + paths, base, root, untracked, _ = diffstate.compute_v2_review_set(str(repo), None, None, {}) + monkeypatch.setattr(gitutil, "_MAX_PATH_ARGS_BYTES", 200) + ran = _refuse_long_commands(monkeypatch, GIT_CMD_CHARS + 100, OSError(errno.E2BIG, "too long")) + cut = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + assert [f for f, _ in gitutil.parse_diff_into_files(cut)] == ["app.py"] + assert not any(".env" in a for c in ran for a in c) + assert not gitutil._path_list_too_long + + def test_batches_shrink_to_a_lower_system_limit(self, monkeypatch): + monkeypatch.setattr(os, "sysconf", lambda name: 100000, raising=False) + budget = gitutil._path_args_budget({"A": "b" * 9997}) + # Half of what the environment leaves, or the platform figure where + # that is lower (Windows). + assert budget == min(45000, gitutil._MAX_PATH_ARGS_BYTES) + paths = [f"dir/file_{i}.py" for i in range(10000)] + assert all(sum(len(p) + 9 for p in b) <= budget + for b in gitutil._path_batches(paths, budget)) + # Never above the platform figure, never uselessly small. + monkeypatch.setattr(os, "sysconf", lambda name: 10**9, raising=False) + assert gitutil._path_args_budget({}) == gitutil._MAX_PATH_ARGS_BYTES + monkeypatch.setattr(os, "sysconf", lambda name: 100, raising=False) + assert gitutil._path_args_budget({}) == 4096 + + def test_gives_up_past_the_batch_limit(self, tmp_path, git_env, monkeypatch): + root, base, paths, untracked = self._review_set(tmp_path) + monkeypatch.setattr(gitutil, "_MAX_PATH_ARGS_BYTES", 60) + monkeypatch.setattr(gitutil, "_MAX_PATH_BATCHES", 2) + ran = _refuse_long_commands(monkeypatch, GIT_CMD_CHARS + 100, OSError(errno.E2BIG, "too long")) + assert gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) is None + assert [c for c in ran if "diff" in c] == [] + + def test_other_start_failures_are_not_retried(self, tmp_path, git_env, monkeypatch): + root, base, paths, untracked = self._review_set(tmp_path) + ran = _refuse_long_commands(monkeypatch, 0, FileNotFoundError(errno.ENOENT, "git")) + assert gitutil.get_git_diff(root, base, paths=paths, untracked_paths=[]) is None + assert ran == [] + + def test_batches_share_one_time_limit(self, tmp_path, git_env, monkeypatch): + repo = make_repo(tmp_path / "repo") + clock = iter([0, 0, 31]) + monkeypatch.setattr(gitutil, "time", types.SimpleNamespace(monotonic=lambda: next(clock))) + monkeypatch.setattr(gitutil, "_MAX_PATH_ARGS_BYTES", 30) + ran = _refuse_long_commands(monkeypatch, GIT_CMD_CHARS + 60, OSError(errno.E2BIG, "too long")) + with pytest.raises(subprocess.TimeoutExpired): + gitutil._run_git_on_review_paths( + ["status"], ["a" * 40, "b" * 40, "c" * 40], timeout=30, cwd=str(repo)) + assert len(ran) == 1 + + @long_paths + def test_more_paths_than_one_command_holds(self, tmp_path, git_env): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("x = 2\n") + _more_than_one_command_holds(repo, [".py", ".txt"]) + paths, base, root, untracked, _ = diffstate.compute_v2_review_set(str(repo), None, None, {}) + assert sum(len(p) for p in untracked) > 2 * 1024 * 1024 + diff = gitutil.get_git_diff(root, base, paths=paths, untracked_paths=untracked) + assert diff is not None + # app.py and the 1,500 .py files, from more than one command. + assert len(gitutil.parse_diff_into_files(diff)) == 1501 + assert diff.count("diff --git ") == 1501 + + +class TestWholeHook: + @long_paths + def test_edit_reviewed_beside_more_paths_than_one_command_holds(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + _more_than_one_command_holds(repo, [".txt"]) + rc, so, se = run_hook(stop_payload(repo), hook_env) + m = metrics_of(so) + assert m.get("skip_reason") is None, m + assert m.get("review_set_count") == 3001 + assert stub_api.calls + + def test_review_set_past_the_cap_skips_without_a_diff(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + (repo / "new.py").write_text(VULN_PY) + rc, so, se = run_hook(stop_payload(repo), {**hook_env, "MAX_REVIEW_SET_PATHS": "1"}) + m = metrics_of(so) + assert m.get("skip_reason") == 8, m + # The skip taken after a diff reports how many files the diff held. + assert "diff_files_count" not in m + assert not stub_api.calls + + def test_cap_counts_only_reviewable_paths(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + for i in range(20): + (repo / f"note{i:02d}.txt").write_text("x\n") + rc, so, se = run_hook(stop_payload(repo), {**hook_env, "MAX_REVIEW_SET_PATHS": "5"}) + m = metrics_of(so) + assert m.get("skip_reason") is None, m + assert m.get("review_set_count") == 21 + assert stub_api.calls + + @long_paths + def test_giving_up_is_reported_as_too_many_paths(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + # 3,000 reviewable names of about 750 bytes: more than four commands. + _more_than_one_command_holds(repo, [".py"]) + rc, so, se = run_hook(stop_payload(repo), hook_env) + m = metrics_of(so) + assert m.get("skip_reason") == 8, m + assert not stub_api.calls + + @long_paths + def test_nothing_reviewable_is_reported_as_no_source_files(self, tmp_path, hook_env, stub_api): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + run_hook(ups_payload(repo), hook_env) + _more_than_one_command_holds(repo, [".txt"]) + rc, so, se = run_hook(stop_payload(repo), hook_env) + m = metrics_of(so) + assert m.get("skip_reason") == 7, m + assert not stub_api.calls + + def test_cap_reads_paths_from_the_repo_root(self, tmp_path, hook_env, stub_api): + # A directory above the repo named like a skipped one (build/) must + # not make every path look unreviewable. + repo = make_repo(tmp_path / "build" / "repo", {"app.py": "x = 1\n"}) + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + (repo / "new.py").write_text(VULN_PY) + rc, so, se = run_hook(stop_payload(repo), {**hook_env, "MAX_REVIEW_SET_PATHS": "1"}) + assert metrics_of(so).get("skip_reason") == 8 + assert not stub_api.calls diff --git a/plugins/security-guidance/tests/test_stop_scope_metrics.py b/plugins/security-guidance/tests/test_stop_scope_metrics.py new file mode 100644 index 00000000..7ae6dc64 --- /dev/null +++ b/plugins/security-guidance/tests/test_stop_scope_metrics.py @@ -0,0 +1,181 @@ +"""Stop rows carry one count-only number, `review_scope`, saying how the +reviewed files relate to what the session's edit tools wrote and whether the +turn-start snapshot was a real one.""" +import os +import shutil + +import pytest + +from conftest import ( + STUB_VULN, VULN_PY, edit_payload, metrics_of, run_hook, stop_payload, + ups_payload, +) + +import diffstate + +needs_sh = pytest.mark.skipif(os.name == "nt", reason="uses a /bin/sh git shim") + + +def _stop(repo, env): + return metrics_of(run_hook(stop_payload(repo), env)[1]) + + +def _turn(repo, env, edits=(), during=None): + run_hook(ups_payload(repo), env) + for path in edits: + run_hook(edit_payload(repo, path), env) + if during: + during() + return _stop(repo, env) + + +def _no_write_tree_env(tmp_path, hook_env): + """An env whose `git write-tree` fails, like a snapshot that could not be taken.""" + bindir = tmp_path / "bin" + bindir.mkdir() + shim = bindir / "git" + shim.write_text( + '#!/bin/sh\nfor a in "$@"; do [ "$a" = write-tree ] && exit 1; done\n' + f'exec "{shutil.which("git")}" "$@"\n') + shim.chmod(0o755) + return {**hook_env, "PATH": f"{bindir}{os.pathsep}{hook_env['PATH']}"} + + +class TestScope: + def test_edits_outside_repo_review_of_a_file_they_did_not_write(self, hook_env, workspace): + ws, repo = workspace + m = _turn(repo, hook_env, edits=[ws / "notes.md"], + during=lambda: (repo / "other.py").write_text(VULN_PY)) + assert m["files_reviewed"] == 1 and m["review_scope"] == 30001 + + def test_edit_inside_repo(self, hook_env, workspace): + _, repo = workspace + m = _turn(repo, hook_env, edits=[repo / "app.py"], + during=lambda: (repo / "app.py").write_text(VULN_PY)) + assert m["files_reviewed"] == 1 and m["review_scope"] == 10000 + + def test_relative_edit_path_is_read_against_the_session_cwd(self, hook_env, workspace): + _, repo = workspace + m = _turn(repo, hook_env, edits=["app.py"], + during=lambda: (repo / "app.py").write_text(VULN_PY)) + assert m["files_reviewed"] == 1 and m["review_scope"] == 10000 + + def test_mixed_edits_and_a_file_changed_by_other_means(self, hook_env, workspace): + ws, repo = workspace + + def during(): + (repo / "app.py").write_text(VULN_PY) + (repo / "gen.py").write_text(VULN_PY) + m = _turn(repo, hook_env, edits=[ws / "notes.md", repo / "app.py"], during=during) + assert m["files_reviewed"] == 2 and m["review_scope"] == 20001 + + def test_no_edit_tool_paths(self, hook_env, workspace): + _, repo = workspace + m = _turn(repo, hook_env, during=lambda: (repo / "app.py").write_text(VULN_PY)) + assert m["files_reviewed"] == 1 and m["review_scope"] == 1 + + def test_outside_only_turn_with_nothing_to_review(self, hook_env, workspace): + ws, repo = workspace + m = _turn(repo, hook_env, edits=[ws / "notes.md"]) + assert m["skipped"] is True and m["review_scope"] == 30000 + + def test_skipped_review_counts_over_the_review_set(self, hook_env, workspace): + _, repo = workspace + m = _turn(repo, hook_env, during=lambda: (repo / "notes.txt").write_text("x\n")) + assert m["skipped"] is True and m["review_scope"] == 1 + + def test_second_review_of_the_same_turn_still_knows_the_edits(self, hook_env, workspace): + _, repo = workspace + m = _turn(repo, hook_env, edits=[repo / "app.py"], + during=lambda: (repo / "app.py").write_text(VULN_PY)) + assert m["review_scope"] == 10000 + m = _stop(repo, hook_env) # a rewake: no new prompt, same baseline + assert m["files_reviewed"] == 1 and m["review_scope"] == 10000 + # The next prompt starts a new count. + m = _turn(repo, hook_env, during=lambda: (repo / "gen.py").write_text(VULN_PY)) + assert m["files_reviewed"] == 1 and m["review_scope"] == 1 + + def test_no_key_outside_a_repo(self, hook_env, tmp_path): + plain = tmp_path / "plain" + plain.mkdir() + m = _turn(plain, hook_env, edits=[plain / "a.py"], + during=lambda: (plain / "a.py").write_text(VULN_PY)) + assert m["skipped"] is True and "review_scope" not in m + + def test_unreadable_path_does_not_stop_the_review(self, hook_env, workspace): + _, repo = workspace + m = _turn(repo, hook_env, edits=[str(repo / "a\ud800b.py"), repo / "app.py"], + during=lambda: (repo / "app.py").write_text(VULN_PY)) + assert m["files_reviewed"] == 1 + if os.name == "nt": + # Windows paths can hold a lone surrogate: it is one more file + # in the repo. + assert m["review_scope"] == 10000 + else: + assert "review_scope" not in m + + +class TestSnapshot: + def test_no_prompt_hook_means_no_snapshot(self, hook_env, workspace): + _, repo = workspace + (repo / "app.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py"), hook_env) + m = _stop(repo, hook_env) + assert m["files_reviewed"] == 1 and m["review_scope"] == 310000 + + @needs_sh + def test_failed_snapshot_is_counted(self, hook_env, workspace, tmp_path): + ws, repo = workspace + (repo / "app.py").write_text(VULN_PY) # uncommitted work from before the turn + m = _turn(repo, _no_write_tree_env(tmp_path, hook_env), edits=[ws / "notes.md"]) + assert m["files_reviewed"] == 1 and m["review_scope"] == 130001 + # The next prompt takes a real snapshot and the flag is gone. + m = _turn(repo, hook_env, edits=[repo / "app.py"], + during=lambda: (repo / "app.py").write_text(VULN_PY + "# more\n")) + assert m["files_reviewed"] == 1 and m["review_scope"] == 10000 + + @needs_sh + def test_flag_follows_the_snapshot_taken_after_a_finding(self, hook_env, stub_api, + workspace, tmp_path): + _, repo = workspace + broken = _no_write_tree_env(tmp_path, hook_env) + stub_api.vulns = [STUB_VULN] + # Good snapshot at the prompt; the one taken after the finding fails. + run_hook(ups_payload(repo), hook_env) + (repo / "app.py").write_text(VULN_PY) + run_hook(edit_payload(repo, repo / "app.py"), hook_env) + m = _stop(repo, broken) + assert m["vulns_found"] == 1 and m["review_scope"] == 10000 + (repo / "app.py").write_text(VULN_PY + "# fix\n") + run_hook(edit_payload(repo, repo / "app.py"), hook_env) + m = _stop(repo, hook_env) + assert m["review_scope"] == 110000 + # That Stop took a real one again. + (repo / "app.py").write_text(VULN_PY + "# fix 2\n") + run_hook(edit_payload(repo, repo / "app.py"), hook_env) + m = _stop(repo, hook_env) + assert m["review_scope"] == 10000 + + +class TestCode: + def _code(self, root, cwd, touched, files, fail=0): + scope, inside = diffstate.touched_scope(root, cwd, touched) + return diffstate.review_scope_code(root, scope, inside, files, fail) + + def test_repo_at_the_filesystem_root(self): + root = os.path.abspath(os.sep) + assert self._code(root, root, [os.path.join(root, "x.py")], ["x.py", "y.py"]) == 10001 + + def test_names_that_only_start_like_a_parent_dir(self, tmp_path): + root = str(tmp_path) + assert self._code(root, root, [str(tmp_path / "..hidden.py")], ["..hidden.py"], 2) == 210000 + + def test_untouched_count_is_capped(self, tmp_path): + files = [f"f{i}.py" for i in range(12000)] + assert self._code(str(tmp_path), "", [], files) == 9999 + + def test_absolute_and_relative_file_names_count_the_same(self, tmp_path): + root = str(tmp_path) + touched = [str(tmp_path / "a.py")] + assert self._code(root, root, touched, ["a.py", "b.py"]) == 10001 + assert self._code(root, root, touched, [str(tmp_path / "a.py"), str(tmp_path / "b.py")]) == 10001 diff --git a/plugins/security-guidance/tests/test_temp_index_cleanup.py b/plugins/security-guidance/tests/test_temp_index_cleanup.py new file mode 100644 index 00000000..9b103a71 --- /dev/null +++ b/plugins/security-guidance/tests/test_temp_index_cleanup.py @@ -0,0 +1,467 @@ +import os +import stat +import tempfile +import time + +import pytest + +from conftest import GIT_ENV, make_repo + +import gitutil + +posix_only = pytest.mark.skipif(not hasattr(os, "getuid"), reason="needs a uid") + + +@pytest.fixture +def tmproot(tmp_path, monkeypatch): + """A temp dir of our own, so nothing here reads or sweeps the real one.""" + d = tmp_path / "tmproot" + d.mkdir() + monkeypatch.setattr(tempfile, "tempdir", str(d)) + for k, v in GIT_ENV.items(): + monkeypatch.setenv(k, v) + return d + + +def _leftovers(root): + """Everything left under `root` but the slot files, which stay.""" + return sorted( + os.path.join(dirpath, f) + for dirpath, _, files in os.walk(root) for f in files + if not (f.startswith("slot") and f.endswith(".lock")) + ) + + +def _write(path): + with open(path, "w") as f: + f.write("x") + return path + + +def _advance_clock(monkeypatch, seconds): + """ctime cannot be set, so files are aged by moving the clock instead.""" + real = time.time + monkeypatch.setattr(time, "time", lambda: real() + seconds) + + +class TestTempIndex: + @posix_only + def test_copy_lives_in_private_dir_and_is_removed(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo") + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + idx = env["GIT_INDEX_FILE"] + private = tmproot / f"claude-security-guidance-{os.getuid()}" + assert os.path.dirname(idx) == str(private) + assert stat.S_IMODE(os.lstat(private).st_mode) == 0o700 + assert os.path.isfile(idx) + assert _leftovers(tmproot) == [] + + def test_cleanup_removes_what_a_killed_git_leaves(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo") + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + for suffix in (".lock", ".stash.12345", ".stash.12345.lock"): + _write(env["GIT_INDEX_FILE"] + suffix) + assert _leftovers(tmproot) == [] + + def test_cleanup_when_body_raises(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo") + with pytest.raises(RuntimeError): + with gitutil._temp_index(str(repo), untracked_paths=[]): + raise RuntimeError("boom") + assert _leftovers(tmproot) == [] + + def test_live_copy_keeps_index_mtime_and_survives_a_sweep(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo") + old = int(time.time()) - 7200 + os.utime(repo / ".git" / "index", (old, old)) + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + assert int(os.stat(env["GIT_INDEX_FILE"]).st_mtime) == old + gitutil._hook_tmpdir() + assert os.path.isfile(env["GIT_INDEX_FILE"]) + + +class TestSweep: + @posix_only + def test_reclaims_stale_files_in_both_dirs(self, tmproot, monkeypatch): + private = gitutil._hook_tmpdir() + stale = [ + _write(os.path.join(d, name)) + for d in (private, str(tmproot)) + for name in ("security_hook_idx_dead", "security_hook_idx_dead.lock") + ] + _advance_clock(monkeypatch, 3600) + keep = [ + _write(os.path.join(private, "security_hook_idx_live")), + _write(os.path.join(str(tmproot), "security_hook_idx_live")), + ] + for p in keep: + os.utime(p, (time.time(), time.time())) + keep.append(_write(os.path.join(str(tmproot), "unrelated"))) + gitutil._hook_tmpdir() + assert [p for p in stale if os.path.exists(p)] == [] + assert [p for p in keep if not os.path.exists(p)] == [] + + def test_backdated_mtime_alone_is_not_stale(self, tmproot): + p = _write(str(tmproot / "security_hook_idx_copy2")) + old = time.time() - 7200 + os.utime(p, (old, old)) + gitutil._hook_tmpdir() + assert os.path.exists(p) + + @posix_only + def test_only_our_own_regular_files(self, tmproot, monkeypatch): + target = _write(str(tmproot / "target")) + os.symlink(target, tmproot / "security_hook_idx_link") + (tmproot / "security_hook_idx_dir").mkdir() + mine = _write(str(tmproot / "security_hook_idx_mine")) + _advance_clock(monkeypatch, 3600) + gitutil._sweep_stale_indexes(str(tmproot), os.getuid() + 1) + assert os.path.exists(mine) + gitutil._sweep_stale_indexes(str(tmproot), os.getuid()) + assert not os.path.exists(mine) + assert os.path.lexists(tmproot / "security_hook_idx_link") + assert os.path.exists(target) + assert (tmproot / "security_hook_idx_dir").is_dir() + + def test_large_backlog_is_drained_in_slices(self, tmproot, monkeypatch): + for i in range(3): + _write(str(tmproot / f"security_hook_idx_{i}")) + _advance_clock(monkeypatch, 3600) + monkeypatch.setattr(gitutil, "_SWEEP_BUDGET_S", -1) + gitutil._sweep_stale_indexes(str(tmproot), None) + assert len(_leftovers(tmproot)) == 2 + + +class TestPrivateDir: + @posix_only + def test_refuses_symlink_or_file_at_the_predictable_path(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo") + predictable = tmproot / f"claude-security-guidance-{os.getuid()}" + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + os.symlink(elsewhere, predictable) + assert gitutil._hook_tmpdir() is None + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + assert os.path.dirname(env["GIT_INDEX_FILE"]) == str(tmproot) + assert list(elsewhere.iterdir()) == [] + predictable.unlink() + predictable.write_text("") + assert gitutil._hook_tmpdir() is None + + @posix_only + def test_refuses_a_directory_owned_by_someone_else(self, tmproot, monkeypatch): + uid = os.getuid() + (tmproot / f"claude-security-guidance-{uid + 1}").mkdir() + monkeypatch.setattr(os, "getuid", lambda: uid + 1) + assert gitutil._hook_tmpdir() is None + + def test_without_getuid_uses_and_sweeps_the_bare_temp_dir( + self, tmp_path, tmproot, monkeypatch): + repo = make_repo(tmp_path / "repo") + monkeypatch.delattr(os, "getuid", raising=False) + stale = _write(str(tmproot / "security_hook_idx_dead")) + _advance_clock(monkeypatch, 3600) + assert gitutil._hook_tmpdir() is None + assert not os.path.exists(stale) + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + assert os.path.dirname(env["GIT_INDEX_FILE"]) == str(tmproot) + assert _leftovers(tmproot) == [] + + +def _dead_pid(): + import subprocess + import sys + p = subprocess.Popen([sys.executable, "-c", "pass"]) + p.wait() + return p.pid + + +class TestOwnerGone: + @posix_only + def test_copy_name_carries_the_hook_pid(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo") + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + name = os.path.basename(env["GIT_INDEX_FILE"]) + assert name.startswith(f"security_hook_idx_pid{os.getpid()}_") + assert not gitutil._owner_is_gone(name) + assert not gitutil._owner_is_gone(name + ".lock") + + @posix_only + def test_fresh_copies_of_a_dead_hook_go_at_once(self, tmproot): + private = gitutil._hook_tmpdir() + dead = f"security_hook_idx_pid{_dead_pid()}_abcdefgh" + gone = [_write(os.path.join(d, dead + suffix)) + for d in (private, str(tmproot)) + for suffix in ("", ".lock", ".stash.77")] + keep = [ + _write(os.path.join(private, f"security_hook_idx_pid{os.getpid()}_abcdefgh")), + # Earlier releases: 8 random characters, which may look like a pid. + _write(os.path.join(private, "security_hook_idx_pid1_abc")), + _write(os.path.join(private, "security_hook_idx_12345678")), + _write(os.path.join(str(tmproot), f"unrelated_pid{_dead_pid()}_abcdefgh")), + ] + gitutil._hook_tmpdir() + assert [p for p in gone if os.path.exists(p)] == [] + assert [p for p in keep if not os.path.exists(p)] == [] + + def test_no_process_is_probed_without_getuid(self, tmproot, monkeypatch): + """On Windows os.kill ends the process it is given.""" + monkeypatch.delattr(os, "getuid", raising=False) + + def no_kill(*a): + raise AssertionError("probed a process") + + monkeypatch.setattr(os, "kill", no_kill) + p = _write(str(tmproot / "security_hook_idx_pid999999_abcdefgh")) + gitutil._hook_tmpdir() + assert os.path.exists(p) + + +@pytest.mark.skipif(os.name == "nt", reason="uses signals and a /bin/sh git shim") +class TestHookProcessKilled: + def _start_slow_hook(self, tmp_path, hook_env, repo): + """Start a UserPromptSubmit hook whose snapshot `git add -u` hangs, and + return once its index copy exists.""" + import shutil + import subprocess + import sys + import json + from conftest import HOOK_SCRIPT, ups_payload + real_git = shutil.which("git") + bindir = tmp_path / "bin" + bindir.mkdir(exist_ok=True) + shim = bindir / "git" + shim.write_text( + "#!/bin/sh\n" + 'case "$*" in *"add -u"*) [ -n "$SLOW_GIT" ] && exec sleep 60;; esac\n' + f'exec "{real_git}" "$@"\n' + ) + shim.chmod(0o755) + tmproot = tmp_path / "hooktmp" + tmproot.mkdir(exist_ok=True) + env = dict(hook_env) + env["PATH"] = f"{bindir}{os.pathsep}{env['PATH']}" + env["TMPDIR"] = str(tmproot) + env["SLOW_GIT"] = "1" + proc = subprocess.Popen( + [sys.executable, str(HOOK_SCRIPT)], stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, env=env) + proc.stdin.write(json.dumps(ups_payload(repo)).encode()) + proc.stdin.close() + deadline = time.time() + 30 + while time.time() < deadline and not self._copies(tmproot): + assert proc.poll() is None, "hook exited before copying the index" + time.sleep(0.05) + assert self._copies(tmproot) + return proc, env, tmproot + + def _copies(self, root): + return [p for p in _leftovers(root) + if os.path.basename(p).startswith("security_hook_idx_")] + + def test_terminated_hook_removes_its_copy(self, tmp_path, hook_env, stub_api): + import signal + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("x = 2\n") + proc, env, tmproot = self._start_slow_hook(tmp_path, hook_env, repo) + proc.send_signal(signal.SIGTERM) + assert proc.wait(timeout=30) == 128 + signal.SIGTERM + assert self._copies(tmproot) == [] + + def test_next_run_reclaims_the_copy_of_a_killed_hook(self, tmp_path, hook_env, stub_api): + from conftest import run_hook, ups_payload + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("x = 2\n") + proc, env, tmproot = self._start_slow_hook(tmp_path, hook_env, repo) + proc.kill() + proc.wait(timeout=30) + assert self._copies(tmproot) + env.pop("SLOW_GIT") + run_hook(ups_payload(repo, session_id="s2"), env) + assert self._copies(tmproot) == [] + + +class TestHardLink: + @posix_only + def test_copy_is_a_link_and_git_cannot_change_the_index_through_it(self, tmp_path, tmproot): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "new.py").write_text("n = 1\n") + index = repo / ".git" / "index" + before = index.read_bytes() + with gitutil._temp_index(str(repo), untracked_paths=["new.py"]) as env: + copy = env["GIT_INDEX_FILE"] + # `add -N` wrote a new index over the link. + assert os.stat(copy).st_ino != os.stat(index).st_ino + assert open(copy, "rb").read() != before + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + assert os.stat(env["GIT_INDEX_FILE"]).st_ino == os.stat(index).st_ino + assert index.read_bytes() == before + assert _leftovers(tmproot) == [] + + def test_falls_back_to_a_copy(self, tmp_path, tmproot, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + index = repo / ".git" / "index" + old = int(time.time()) - 7200 + os.utime(index, (old, old)) + + def no_link(*a, **kw): + raise OSError(18, "Invalid cross-device link") + + monkeypatch.setattr(os, "link", no_link) + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + copy = env["GIT_INDEX_FILE"] + assert os.stat(copy).st_ino != os.stat(index).st_ino + assert open(copy, "rb").read() == index.read_bytes() + assert int(os.stat(copy).st_mtime) == old + assert _leftovers(tmproot) == [] + + +@posix_only +class TestCopySlots: + def test_only_so_many_copies_at_once(self, tmp_path, tmproot, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + monkeypatch.setattr(gitutil, "_MAX_LIVE_COPIES", 1) + monkeypatch.setattr(gitutil, "_SLOT_WAIT_S", 0.3) + with gitutil._temp_index(str(repo), untracked_paths=[]) as first: + assert first is not None + with gitutil._temp_index(str(repo), untracked_paths=[]) as second: + assert second is None + with gitutil._temp_index(str(repo), untracked_paths=[]) as again: + assert again is not None + assert _leftovers(tmproot) == [] + + def test_fewer_slots_when_the_disk_is_nearly_full(self, tmproot, monkeypatch): + import collections + import shutil + usage = collections.namedtuple("usage", "total used free") + gb = 1024 ** 3 + index = 350 * 1024 ** 2 + + def free(n): + monkeypatch.setattr(shutil, "disk_usage", lambda p: usage(250 * gb, 0, n)) + + free(200 * gb) + assert gitutil._copy_slots(str(tmproot), index) == gitutil._MAX_LIVE_COPIES + free(10 * gb) + assert gitutil._copy_slots(str(tmproot), index) == 2 + free(5 * gb) + assert gitutil._copy_slots(str(tmproot), index) == 1 + free(0) + assert gitutil._copy_slots(str(tmproot), index) == 1 + free(1 * gb) + assert gitutil._copy_slots(str(tmproot), 1000) == gitutil._MAX_LIVE_COPIES + + def test_waits_for_a_slot(self, tmp_path, tmproot, monkeypatch): + import threading + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + monkeypatch.setattr(gitutil, "_MAX_LIVE_COPIES", 1) + monkeypatch.setattr(gitutil, "_SLOT_WAIT_S", 10) + release = threading.Event() + holding = threading.Event() + + def holder(): + with gitutil._temp_index(str(repo), untracked_paths=[]): + holding.set() + release.wait(10) + + t = threading.Thread(target=holder) + t.start() + assert holding.wait(10) + threading.Timer(0.5, release.set).start() + with gitutil._temp_index(str(repo), untracked_paths=[]) as env: + assert env is not None + t.join() + + def test_slot_of_a_killed_hook_is_free(self, tmp_path, tmproot, monkeypatch): + import signal + import subprocess + import sys + monkeypatch.setattr(gitutil, "_MAX_LIVE_COPIES", 1) + monkeypatch.setattr(gitutil, "_SLOT_WAIT_S", 0.3) + private = gitutil._hook_tmpdir() + holder = subprocess.Popen( + [sys.executable, "-c", + "import fcntl, os, sys, time\n" + "fd = os.open(sys.argv[1], os.O_RDWR | os.O_CREAT, 0o600)\n" + "fcntl.flock(fd, fcntl.LOCK_EX)\n" + "print('held', flush=True)\n" + "time.sleep(60)\n", + os.path.join(private, "slot0.lock")], + stdout=subprocess.PIPE) + assert holder.stdout.readline().strip() == b"held" + with gitutil._copy_slot(private) as got: + assert not got + holder.send_signal(signal.SIGKILL) + holder.wait() + with gitutil._copy_slot(private) as got: + assert got + + def test_baseline_falls_back_to_head_without_a_slot(self, tmp_path, tmproot, monkeypatch): + import diffstate + from conftest import git + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + (repo / "app.py").write_text("x = 2\n") + head = git(repo, "rev-parse", "HEAD").strip() + monkeypatch.setattr(gitutil, "_MAX_LIVE_COPIES", 1) + monkeypatch.setattr(gitutil, "_SLOT_WAIT_S", 0.3) + private = gitutil._hook_tmpdir() + with gitutil._copy_slot(private) as got: + assert got + assert diffstate.capture_git_baseline(str(repo)) == head + assert diffstate.capture_git_baseline(str(repo)) != head + assert _leftovers(tmproot) == [] + + +class TestStashIndexesLeftInGitDir: + """What releases before 2.0.10 left next to the real index.""" + + def test_old_ones_go_and_nothing_else_does(self, tmp_path, tmproot, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + git_dir = repo / ".git" + index = (git_dir / "index").read_bytes() + old = [_write(str(git_dir / n)) for n in ("index.stash.123", "index.stash.99999")] + not_ours = [_write(str(git_dir / n)) for n in ( + "index.lock", "index.stash.12.bak", "index.stash.abc", "index.stash.", "myindex.stash.5")] + _advance_clock(monkeypatch, gitutil._TEMP_INDEX_STALE_S + 60) + fresh = _write(str(git_dir / "index.stash.456")) + now = time.time() # the moved clock + os.utime(fresh, (now, now)) + + with gitutil._temp_index(str(repo), untracked_paths=[]): + pass + + assert [p for p in old if os.path.exists(p)] == [] + assert [p for p in not_ours if not os.path.exists(p)] == [] + assert os.path.exists(fresh) + assert (git_dir / "index").read_bytes() == index + + def test_a_directory_or_link_by_that_name_stays(self, tmp_path, tmproot, monkeypatch): + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + git_dir = repo / ".git" + target = _write(str(tmp_path / "elsewhere")) + os.mkdir(git_dir / "index.stash.1") + os.symlink(target, git_dir / "index.stash.2") + _advance_clock(monkeypatch, gitutil._TEMP_INDEX_STALE_S + 60) + + with gitutil._temp_index(str(repo), untracked_paths=[]): + pass + + assert os.path.isdir(git_dir / "index.stash.1") + assert os.path.islink(git_dir / "index.stash.2") + assert os.path.exists(target) + + def test_linked_worktree_has_its_own(self, tmp_path, tmproot, monkeypatch): + from conftest import git + repo = make_repo(tmp_path / "repo", {"app.py": "x = 1\n"}) + wt = tmp_path / "wt" + git(repo, "worktree", "add", "-q", str(wt), "-b", "side") + wt_git_dir = repo / ".git" / "worktrees" / "wt" + mine = _write(str(wt_git_dir / "index.stash.7")) + main = _write(str(repo / ".git" / "index.stash.7")) + _advance_clock(monkeypatch, gitutil._TEMP_INDEX_STALE_S + 60) + + with gitutil._temp_index(str(wt), untracked_paths=[]): + pass + + assert not os.path.exists(mine) + assert os.path.exists(main) diff --git a/plugins/security-guidance/tests/test_tls_store_unreadable_cert.py b/plugins/security-guidance/tests/test_tls_store_unreadable_cert.py new file mode 100644 index 00000000..c76aa8aa --- /dev/null +++ b/plugins/security-guidance/tests/test_tls_store_unreadable_cert.py @@ -0,0 +1,387 @@ +"""A Windows certificate store with an entry OpenSSL cannot read must not stop reviews. + +Older Python releases load each Windows store in one call, so one unreadable +certificate makes the default TLS context fail to build. The store and that +loading are stood in for here, so the cases run on every platform. +""" +import http.server +import json +import shutil +import ssl +import subprocess +import threading +import urllib.error +import urllib.request + +import pytest + +from conftest import _Stub + +import _base +import llm + +# A self-signed test CA (public certificate only). +UNRELATED_CA_PEM = """-----BEGIN CERTIFICATE----- +MIIBkjCCATmgAwIBAgIUbv89EXoDEbZBR6jKRVAU1V/0fGgwCgYIKoZIzj0EAwIw +HzEdMBsGA1UEAwwUc2cgdGVzdCB1bnJlbGF0ZWQgQ0EwHhcNMjYxMDIxMTYyMjQ1 +WhcNMzYxMDE4MTYyMjQ1WjAfMR0wGwYDVQQDDBRzZyB0ZXN0IHVucmVsYXRlZCBD +QTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABPNt0cYQuFTK2120/wnrAXDuqQUE +8Cu0MpykWGDKTpymrBpKT9tB3wtr6Z951brKADRTgA7sKsIRoo9WmwnJKdyjUzBR +MB0GA1UdDgQWBBQbn4fFArXUPihIY/CG7IxtVswc1DAfBgNVHSMEGDAWgBQbn4fF +ArXUPihIY/CG7IxtVswc1DAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0cA +MEQCIDuCfFRxNvoQhAXHiaA+Pgd7nuZ7hHVE16dERXHiVT6JAiACb2vks3dsL40u +UAQ7MEoy37JzJExADeVOBhztTqTkkw== +-----END CERTIFICATE----- +""" +UNRELATED_CA = ssl.PEM_cert_to_DER_cert(UNRELATED_CA_PEM) +# The same certificate with its serial number's tag overwritten. +UNREADABLE = UNRELATED_CA[:13] + b"\x04" + UNRELATED_CA[14:] + +SERVER_AUTH = ssl.Purpose.SERVER_AUTH.oid +CLIENT_AUTH = ssl.Purpose.CLIENT_AUTH.oid + +_real_create_default_context = ssl.create_default_context + + +class _Store: + """Stand-in for the Windows certificate stores and for a Python that + loads each of them in one call.""" + + def __init__(self): + self.stores = {"CA": [], "ROOT": []} + self.enumerations = 0 + self.default_builds = 0 + + def enum_certificates(self, name): + self.enumerations += 1 + return list(self.stores.get(name, [])) + + def create_default_context(self, purpose=ssl.Purpose.SERVER_AUTH, *, + cafile=None, capath=None, cadata=None): + if cafile or capath or cadata: + return _real_create_default_context( + purpose, cafile=cafile, capath=capath, cadata=cadata) + self.default_builds += 1 + # Never the machine's own trust: only what the test puts in the store. + context = _real_create_default_context(purpose, cadata=UNRELATED_CA) + for name in ("CA", "ROOT"): + certs = b"".join( + cert for cert, encoding, trust in self.stores[name] + if encoding == "x509_asn" and (trust is True or purpose.oid in trust)) + if certs: + context.load_verify_locations(cadata=certs) + return context + + +@pytest.fixture +def store(monkeypatch): + s = _Store() + monkeypatch.setattr(ssl, "enum_certificates", s.enum_certificates, raising=False) + monkeypatch.setattr(ssl, "create_default_context", s.create_default_context) + monkeypatch.setattr(ssl, "_create_default_https_context", s.create_default_context) + # urllib keeps one opener per process, and on newer Pythons it holds the + # context it was built with; a hook process starts without one. + monkeypatch.setattr(urllib.request, "_opener", None) + # The store is checked once per process. + monkeypatch.setattr(llm, "_store_tls_context", llm._STORE_TLS_UNSET) + monkeypatch.setattr(llm, "_store_certs_skipped", 0) + for var in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy", + "ALL_PROXY", "all_proxy", "SSL_CERT_FILE", "SSL_CERT_DIR"): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("NO_PROXY", "*") + monkeypatch.setenv("no_proxy", "*") + return s + + +def _entry(cert, trust=True, encoding="x509_asn"): + return (cert, encoding, trust) + + +def _subjects(context): + """The test certificates in a context (it also holds the machine's default file).""" + names = {dict(x[0] for x in c["subject"]).get("commonName", "") + for c in context.get_ca_certs()} + return {n for n in names if n.startswith("sg test ")} + + +# ── certificates and an HTTPS endpoint for the handshake cases ─────────── + +def _openssl(*args, cwd): + r = subprocess.run(["openssl", *args], cwd=cwd, capture_output=True, text=True) + assert r.returncode == 0, r.stderr + + +@pytest.fixture(scope="module") +def pki(tmp_path_factory): + if not shutil.which("openssl"): + pytest.skip("needs the openssl command to make test certificates") + d = tmp_path_factory.mktemp("pki") + leaf = ("basicConstraints=CA:FALSE\nextendedKeyUsage=serverAuth\n" + "subjectKeyIdentifier=hash\nauthorityKeyIdentifier=keyid\n") + (d / "leaf.cnf").write_text("subjectAltName=DNS:localhost,IP:127.0.0.1\n" + leaf) + (d / "other-host.cnf").write_text("subjectAltName=DNS:other.invalid\n" + leaf) + # The subject goes in the config file: LibreSSL ignores -subj when the config has prompt=no. + def req_config(common_name): + (d / "req.cnf").write_text( + "[req]\ndistinguished_name=dn\nx509_extensions=v3\nprompt=no\n" + f"[dn]\nCN={common_name}\n" + "[v3]\nbasicConstraints=critical,CA:TRUE\nkeyUsage=critical,keyCertSign,cRLSign\n" + "subjectKeyIdentifier=hash\n") + out = {} + # server certificate -> (the CA that signs it, its extensions) + servers = {"trusted": ("trusted", "leaf.cnf"), "untrusted": ("untrusted", "leaf.cnf"), + "other-host": ("trusted", "other-host.cnf")} + for ca in ("trusted", "untrusted"): + req_config(f"sg test {ca} CA") + _openssl("req", "-x509", "-newkey", "rsa:2048", "-nodes", "-config", "req.cnf", + "-keyout", f"{ca}-ca.key", "-out", f"{ca}-ca.pem", "-days", "2", cwd=d) + req_config("sg test server") + for name, (ca, extensions) in servers.items(): + _openssl("req", "-newkey", "rsa:2048", "-nodes", "-config", "req.cnf", + "-keyout", f"{name}.key", "-out", f"{name}.csr", cwd=d) + _openssl("x509", "-req", "-in", f"{name}.csr", "-CA", f"{ca}-ca.pem", + "-CAkey", f"{ca}-ca.key", "-CAcreateserial", "-out", f"{name}.pem", + "-days", "2", "-extfile", extensions, cwd=d) + out[name] = { + "ca": ssl.PEM_cert_to_DER_cert((d / f"{ca}-ca.pem").read_text()), + "cert": str(d / f"{name}.pem"), "key": str(d / f"{name}.key"), + } + return out + + +@pytest.fixture +def https_api(pki, store, tmp_path, monkeypatch): + servers = [] + + def start(signed_by): + srv = http.server.HTTPServer(("127.0.0.1", 0), _Stub) + srv.calls, srv.status, srv.delay, srv.vulns = [], 200, 0, [] + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(pki[signed_by]["cert"], pki[signed_by]["key"]) + srv.socket = context.wrap_socket(srv.socket, server_side=True) + threading.Thread(target=srv.serve_forever, daemon=True).start() + servers.append(srv) + monkeypatch.setenv("ANTHROPIC_BASE_URL", f"https://localhost:{srv.server_port}") + return srv + + for var in ("SG_AGENTIC_MODEL", "SG_DUAL_OR", "CLAUDE_CODE_EXECPATH", + "SG_AGENTIC_CLI_PATH", "CLAUDE_CODE_REMOTE", *llm._PROVIDER_ENV_VARS): + monkeypatch.delenv(var, raising=False) + monkeypatch.setenv("SECURITY_REVIEW_MODEL", "claude-opus-5-5") + monkeypatch.setenv("SECURITY_WARNINGS_STATE_DIR", str(tmp_path / "state")) + monkeypatch.setattr(llm, "ANTHROPIC_API_KEY", "test-key") + monkeypatch.setattr(llm, "ANTHROPIC_AUTH_TOKEN", "") + monkeypatch.setattr(llm, "HAS_API_CREDENTIALS", True) + monkeypatch.setattr(llm, "_auth_prefer_token", False) + monkeypatch.setattr(llm, "_review_model_memo", None, raising=False) + monkeypatch.setattr(llm, "_last_call_claude_http_error", None) + monkeypatch.setattr(_base, "_USAGE", {**_base._USAGE, "models": []}) + # No waiting between the attempts of a call that keeps failing. + monkeypatch.setattr("time.sleep", lambda s: None) + try: + yield start + finally: + for srv in servers: + srv.shutdown() + srv.server_close() + + +def _review(): + return llm.analyze_code_security([("app.py", "+print('hi')\n")], is_diff=True) + + +# ── the stand-in behaves like the reported machines ────────────────────── + +class TestStandIn: + def test_the_unreadable_certificate_is_unreadable(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with pytest.raises(ssl.SSLError): + context.load_verify_locations(cadata=UNREADABLE) + + @pytest.mark.parametrize("entries", [ + [UNRELATED_CA, UNREADABLE], [UNREADABLE, UNRELATED_CA], + ], ids=["after a readable one", "first in the store"]) + def test_one_unreadable_certificate_breaks_the_default_context(self, store, entries): + store.stores["ROOT"] = [_entry(c) for c in entries] + with pytest.raises(ssl.SSLError): + ssl.create_default_context() + + def test_a_readable_store_builds(self, store): + store.stores["ROOT"] = [_entry(UNRELATED_CA)] + ssl.create_default_context() + + +# ── the context built without the unreadable entries ───────────────────── + +def _context(): + return llm._context_without_unreadable_store_certs() + + +class TestContext: + def test_none_when_the_default_context_builds(self, store): + store.stores["ROOT"] = [_entry(UNRELATED_CA)] + assert _context() == (None, 0) + assert store.enumerations == 0 + + def test_none_off_windows(self, store, monkeypatch): + store.stores["ROOT"] = [_entry(UNREADABLE), _entry(UNRELATED_CA)] + monkeypatch.delattr(ssl, "enum_certificates") + assert _context() == (None, 0) + assert store.default_builds == 0 + + def test_none_when_no_certificate_is_readable(self, store): + store.stores["ROOT"] = [_entry(UNREADABLE)] + assert _context() == (None, 1) + + def test_still_verifies_the_chain_and_the_host_name(self, store): + store.stores["ROOT"] = [_entry(UNREADABLE), _entry(UNRELATED_CA)] + context, _ = _context() + assert context.verify_mode == ssl.CERT_REQUIRED + assert context.check_hostname is True + default = _real_create_default_context(cadata=UNRELATED_CA) + assert context.verify_flags == default.verify_flags + assert context.options == default.options + assert context.minimum_version == default.minimum_version + + def test_readable_certificates_from_both_stores_are_kept(self, store, pki): + store.stores["CA"] = [_entry(UNREADABLE), _entry(pki["trusted"]["ca"])] + store.stores["ROOT"] = [_entry(UNRELATED_CA), _entry(UNREADABLE), _entry(UNREADABLE)] + context, skipped = _context() + assert _subjects(context) == {"sg test trusted CA", "sg test unrelated CA"} + assert skipped == 3 + + def test_only_certificates_trusted_for_servers_are_loaded(self, store, pki): + store.stores["ROOT"] = [ + _entry(UNREADABLE), + _entry(UNRELATED_CA, trust={SERVER_AUTH}), + _entry(pki["trusted"]["ca"], trust={CLIENT_AUTH}), + _entry(pki["untrusted"]["ca"], encoding="pkcs_7_asn"), + _entry(UNREADABLE, trust={CLIENT_AUTH}), + ] + context, skipped = _context() + assert _subjects(context) == {"sg test unrelated CA"} + assert skipped == 1 + + def test_a_store_that_cannot_be_listed_is_left_out(self, store, monkeypatch): + store.stores["ROOT"] = [_entry(UNREADABLE), _entry(UNRELATED_CA)] + listed = store.enum_certificates + + def enum_certificates(name): + if name == "CA": + raise PermissionError("denied") + return listed(name) + + monkeypatch.setattr(ssl, "enum_certificates", enum_certificates) + context, skipped = _context() + assert _subjects(context) == {"sg test unrelated CA"} + assert skipped == 1 + + +# ── the plugin's HTTPS calls ───────────────────────────────────────────── + +@pytest.fixture +def urlopen_calls(monkeypatch): + calls = [] + monkeypatch.setattr(urllib.request, "urlopen", + lambda request, **kw: calls.append(kw) or "response") + return calls + + +def _head(): + return urllib.request.Request(llm._anthropic_base_url() + "/", method="HEAD") + + +class TestUrlopen: + def test_default_context_is_used_when_it_builds(self, store, urlopen_calls): + store.stores["ROOT"] = [_entry(UNRELATED_CA)] + assert llm._urlopen("request", 7) == "response" + assert llm._urlopen("request", 7) == "response" + assert urlopen_calls == [{"timeout": 7}] * 2 + assert (store.default_builds, store.enumerations) == (1, 0) + assert llm._store_certs_skipped == 0 + + def test_nothing_is_checked_off_windows(self, store, urlopen_calls, monkeypatch): + monkeypatch.delattr(ssl, "enum_certificates") + assert llm._urlopen("request", 7) == "response" + assert urlopen_calls == [{"timeout": 7}] + assert store.default_builds == 0 + + def test_each_request_is_sent_once(self, store, urlopen_calls): + store.stores["ROOT"] = [_entry(UNREADABLE), _entry(UNRELATED_CA)] + assert llm._urlopen("request", 7) == "response" + assert len(urlopen_calls) == 1 + assert urlopen_calls[0]["timeout"] == 7 + assert isinstance(urlopen_calls[0]["context"], ssl.SSLContext) + + def test_a_store_check_that_fails_leaves_the_default_route( + self, store, urlopen_calls, monkeypatch): + def enum_certificates(name): + raise OSError("store unavailable") + + store.stores["ROOT"] = [_entry(UNREADABLE)] + monkeypatch.setattr(ssl, "enum_certificates", enum_certificates) + assert llm._urlopen("request", 7) == "response" + assert urlopen_calls == [{"timeout": 7}] + + def test_review_runs_with_an_unreadable_certificate_in_the_store( + self, store, pki, https_api): + srv = https_api("trusted") + store.stores["CA"] = [_entry(UNRELATED_CA), _entry(UNREADABLE)] + store.stores["ROOT"] = [_entry(UNREADABLE), _entry(pki["trusted"]["ca"])] + _review() + assert not llm.last_review_failed() + assert len(srv.calls) == 1 + assert llm._store_certs_skipped == 2 + + def test_the_store_is_read_once_per_process(self, store, pki, https_api): + srv = https_api("trusted") + store.stores["ROOT"] = [_entry(UNREADABLE), _entry(pki["trusted"]["ca"])] + _review() + builds, enumerations = store.default_builds, store.enumerations + _review() + assert len(srv.calls) == 2 + assert (store.default_builds, store.enumerations) == (builds, enumerations) == (1, 2) + + def test_a_server_the_store_does_not_trust_is_still_refused( + self, store, pki, https_api): + srv = https_api("untrusted") + store.stores["ROOT"] = [_entry(UNREADABLE), _entry(pki["trusted"]["ca"])] + with pytest.raises(urllib.error.URLError) as raised: + llm._urlopen(_head(), 5) + assert isinstance(raised.value.reason, ssl.SSLCertVerificationError) + _review() + assert llm.last_review_failed() + assert srv.calls == [] + + def test_a_host_name_the_certificate_does_not_cover_is_still_refused( + self, store, pki, https_api): + srv = https_api("other-host") + store.stores["ROOT"] = [_entry(UNREADABLE), _entry(pki["trusted"]["ca"])] + with pytest.raises(urllib.error.URLError) as raised: + llm._urlopen(_head(), 5) + assert isinstance(raised.value.reason, ssl.SSLCertVerificationError) + assert "localhost" in str(raised.value.reason) + _review() + assert llm.last_review_failed() + assert srv.calls == [] + + def test_review_runs_on_a_readable_store(self, store, pki, https_api): + srv = https_api("trusted") + store.stores["ROOT"] = [_entry(pki["trusted"]["ca"])] + _review() + assert not llm.last_review_failed() + assert len(srv.calls) == 1 + assert llm._store_certs_skipped == 0 + + +def test_review_rows_count_the_skipped_store_entries(monkeypatch, capsys): + import security_reminder_hook as hook + + def row(): + hook.emit_metrics({"vulns_found": 0, "review_ms": 10}) + return json.loads(capsys.readouterr().out)["metrics"] + + monkeypatch.setattr(llm, "_store_certs_skipped", 0) + assert "tls_store_skipped" not in row() + monkeypatch.setattr(llm, "_store_certs_skipped", 7) + assert row()["tls_store_skipped"] == 7 diff --git a/plugins/security-guidance/tests/test_unmatched_bash_metric.py b/plugins/security-guidance/tests/test_unmatched_bash_metric.py new file mode 100644 index 00000000..8933bbd1 --- /dev/null +++ b/plugins/security-guidance/tests/test_unmatched_bash_metric.py @@ -0,0 +1,162 @@ +"""Bash hook starts that match no commit or push are counted; nothing changes.""" +import itertools +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from conftest import HOOK_SCRIPT, STUB_VULN, VULN_PY, bash_payload, commit_file, metrics_of, run_hook + +import security_reminder_hook as hook + +WEIGHT = hook._UNMATCHED_BASH_SAMPLE +SHELL_PATTERN = "|".join(f"*{a}*{b}*" for a, b in hook._SHELL_PRECHECK_PAIRS) + + +def _call_id(sampled): + for n in itertools.count(): + tool_use_id = f"toolu_{n}" + if hook._unmatched_bash_sampled({"tool_use_id": tool_use_id}) is sampled: + return tool_use_id + + +def _raw(command, stdout="ok"): + return json.dumps(bash_payload("/w", command, stdout=stdout)) + + +def test_unmatched_call_reports_what_a_text_check_would_do(workspace, hook_env): + ws, _ = workspace + loop = bash_payload("/w", "for f in a b; do echo $f; done", tool_use_id=_call_id(True)) + rc, so, se = run_hook(loop, hook_env) + m = metrics_of(so) + assert rc == 0 and se == "" + assert m["bash_no_match"] == WEIGHT and m["precheck_skip"] == WEIGHT and m["pv"] + + # Text mentions pass a text check and still match nothing. + for command, stdout in (("git log --grep commit", ""), ("ls", "run git push next")): + p = bash_payload("/w", command, stdout, tool_use_id=_call_id(True)) + rc, so, se = run_hook(p, hook_env) + m = metrics_of(so) + assert rc == 0 and m["bash_no_match"] == WEIGHT and m["precheck_skip"] == 0 + + +def test_unsampled_call_stays_silent(workspace, hook_env): + loop = "for f in a b; do echo $f; done" + assert run_hook(bash_payload("/w", loop, tool_use_id=_call_id(False)), hook_env) == (0, "", "") + assert run_hook(bash_payload("/w", loop), hook_env) == (0, "", "") # no call id + + +def test_other_hosts_get_no_row(workspace, hook_env): + markers = ("CLAUDECODE", "CLAUDE_CODE_ENTRYPOINT", "SECURITY_GUIDANCE_PORTABLE_OUTPUT") + env = {k: v for k, v in hook_env.items() if k not in markers} + env["PLUGIN_ROOT"] = "/p" + p = bash_payload("/w", "for f in a b; do echo $f; done", tool_use_id=_call_id(True)) + assert run_hook(p, env) == (0, "", "") + + +def test_one_call_in_ten_is_sampled(): + picked = sum(hook._unmatched_bash_sampled({"tool_use_id": f"toolu_{n}"}) for n in range(20000)) + assert 20000 / WEIGHT * 0.85 < picked < 20000 / WEIGHT * 1.15 + assert not hook._unmatched_bash_sampled({}) and not hook._unmatched_bash_sampled({"tool_use_id": 7}) + + +def test_pairs_cover_every_verb_in_the_regexes(): + # A verb added to either regex has to be added to _SHELL_PRECHECK_PAIRS + # and to _commands() below. + assert hook._GIT_COMMIT_RE.pattern.endswith(r"\s+commit\b(?!-)|\bgt\s+(?:create|modify)\b") + assert hook._GIT_PUSH_RE.pattern.endswith(r"\s+push\b|\bgt\s+submit\b)") + + +def _commands(): + """Commands the commit and push regexes match, in the spellings they allow.""" + opts = ["", " -C sub", " -c user.name=x", " -c 'a.b=c d'", " --git-dir=.git", " --no-pager", + " -P", " --namespace=n", " --work-tree .", " -C a -c b=c --no-pager"] + seps = [" ", "\t", " ", "\n"] + heads = ["", "cd x && ", "FOO=1 ", "(", "for i in 1; do ", "echo \"$(", "/usr/bin/", "x=`"] + for head, opt, sep, verb in itertools.product(heads, opts, seps, ["commit", "push"]): + yield f"{head}git{opt}{sep}{verb} -m 'é \"q\" \\ done'" + for head, sep, verb in itertools.product(heads, seps, ["create", "modify", "submit"]): + yield f"{head}gt{sep}{verb} --all" + + +def test_text_check_never_drops_a_reviewed_command(): + matched = [c for c in _commands() if hook._GIT_COMMIT_RE.search(c) or hook._GIT_PUSH_RE.search(c)] + assert len(matched) > 600 + for command in matched: + for ensure_ascii in (True, False): + raw = json.dumps(bash_payload("/w", command), ensure_ascii=ensure_ascii) + assert hook._shell_precheck_passes(raw), command + + +CASES = [ + ("ls -la | head", False), + ("echo commit the git way", False), # wrong order + ("grep -c gt app.py", False), + ("git status && echo 'ready to commit'", True), + ("gt log; echo submit", True), + ("git\tpush", True), + ("digit; echo pushed", True), +] + + +def test_text_check_on_known_strings(): + for command, passes in CASES: + assert hook._shell_precheck_passes(_raw(command)) is passes, command + + +def _working_bash(): + """A bash that runs a script. On Windows that is the one shipped with + git: the `bash` on PATH there can be the WSL launcher, which is not one.""" + found = [] + git = shutil.which("git") + if sys.platform == "win32" and git: + root = Path(git).resolve().parent.parent + found += [root / "bin" / "bash.exe", root / "usr" / "bin" / "bash.exe"] + if shutil.which("bash"): + found.append(Path(shutil.which("bash"))) + for bash in found: + try: + r = subprocess.run([str(bash), "-c", "echo ok"], capture_output=True, timeout=30) + except (OSError, subprocess.SubprocessError): + continue + if r.returncode == 0 and r.stdout.strip() == b"ok": + return str(bash) + return None + + +def test_text_check_agrees_with_the_shell_pattern(): + bash = _working_bash() + if not bash: + pytest.skip("needs a working bash") + script = f'input=$(cat); case "$input" in {SHELL_PATTERN}) echo pass ;; *) echo skip ;; esac' + for command, _ in CASES: + raw = _raw(command) + r = subprocess.run([bash, "-c", script], input=raw.encode(), capture_output=True, timeout=30) + assert r.stdout.strip() in (b"pass", b"skip"), r.stderr + assert (r.stdout.strip() == b"pass") is hook._shell_precheck_passes(raw), command + + +def test_reviewed_command_a_text_check_would_drop_is_flagged(workspace, hook_env, stub_api): + ws, repo = workspace + stub_api.vulns = [STUB_VULN] + _, out = commit_file(repo, "app.py", VULN_PY) + raw = json.dumps(bash_payload(repo, "git commit -m change", out)) + # Same command, with one letter written as a JSON escape. + raw = raw.replace('"git commit -m change"', '"git \\u0063ommit -m change"') + assert json.loads(raw)["tool_input"]["command"] == "git commit -m change" + if hook._shell_precheck_passes(raw): + pytest.skip("the temporary path itself passes the text check") + r = subprocess.run([sys.executable, str(HOOK_SCRIPT)], input=raw, capture_output=True, + text=True, env=hook_env, timeout=120) + m = metrics_of(r.stdout) + assert m["precheck_miss"] == 1 and m.get("commit_review") is True + assert list(m).index("precheck_miss") < 3 # ahead of the row's own keys + + # The ordinary spelling carries no flag. + _, out = commit_file(repo, "b.py", VULN_PY) + rc, so, se = run_hook(bash_payload(repo, "git commit -m change", out), hook_env) + m = metrics_of(so) + assert m.get("commit_review") is True and "precheck_miss" not in m diff --git a/plugins/security-guidance/tests/test_untracked_cap.py b/plugins/security-guidance/tests/test_untracked_cap.py new file mode 100644 index 00000000..00fd4015 --- /dev/null +++ b/plugins/security-guidance/tests/test_untracked_cap.py @@ -0,0 +1,85 @@ +import os +import time + +from conftest import make_repo, metrics_of, run_hook, stop_payload, ups_payload + +import diffstate + +SRC = "import os\nos.system(input())\n" + + +def _review(repo, snapshot): + paths, _base, root, _untracked, metrics = diffstate.compute_v2_review_set( + str(repo), None, None, snapshot) + return sorted(os.path.relpath(p, root) for p in paths), metrics + + +class TestCappedSnapshot: + def _repo(self, tmp_path, monkeypatch, n=30, cap=10): + monkeypatch.setattr(diffstate, "UNTRACKED_BASELINE_CAP", cap) + monkeypatch.setattr(diffstate, "_CAPPED_BASELINE_SLACK_NS", 20_000_000) + repo = make_repo(tmp_path / "repo") + for i in range(n): + (repo / f"old_{i:03d}.py").write_text(SRC) + time.sleep(0.1) + return repo + + def test_snapshot_records_when_it_was_capped(self, tmp_path, monkeypatch): + repo = self._repo(tmp_path, monkeypatch) + before = time.time_ns() + snap = diffstate._list_untracked(str(repo)) + assert before <= snap[diffstate.UNTRACKED_CAPPED_AT_KEY] <= time.time_ns() + assert len(snap) == 10 + 1 + + def test_uncapped_snapshot_has_no_marker(self, tmp_path, monkeypatch): + repo = self._repo(tmp_path, monkeypatch, n=5) + assert diffstate.UNTRACKED_CAPPED_AT_KEY not in diffstate._list_untracked(str(repo)) + + def test_files_past_the_cap_are_not_reviewed_when_untouched(self, tmp_path, monkeypatch): + repo = self._repo(tmp_path, monkeypatch) + snap = diffstate._list_untracked(str(repo)) + review, metrics = _review(repo, snap) + assert review == [] + assert metrics["preexisting_untracked_excluded"] == 30 + + def test_new_and_edited_files_past_the_cap_are_reviewed(self, tmp_path, monkeypatch): + repo = self._repo(tmp_path, monkeypatch) + snap = diffstate._list_untracked(str(repo)) + time.sleep(0.1) + (repo / "zz_new.py").write_text(SRC) + (repo / "old_025.py").write_text(SRC + "eval(input())\n") + (repo / "old_003.py").write_text(SRC + "eval(input())\n") # inside the cap + review, _ = _review(repo, snap) + assert review == ["old_003.py", "old_025.py", "zz_new.py"] + + def test_file_moved_in_with_an_old_mtime_is_reviewed(self, tmp_path, monkeypatch): + repo = self._repo(tmp_path, monkeypatch) + snap = diffstate._list_untracked(str(repo)) + time.sleep(0.1) + new = repo / "zz_extracted.py" + new.write_text(SRC) + os.utime(new, (1_600_000_000, 1_600_000_000)) # as tar or cp -p would leave it + review, _ = _review(repo, snap) + assert review == ["zz_extracted.py"] + + +def test_stop_hook_does_not_re_review_files_past_the_cap(tmp_path, hook_env, stub_api): + """More untracked files than the snapshot holds: the same old files must + not be sent for review turn after turn (#6407).""" + repo = make_repo(tmp_path / "repo") + n_files = diffstate.UNTRACKED_BASELINE_CAP + 60 + for i in range(n_files): + (repo / f"old_{i:05d}.py").write_text(SRC) + time.sleep(2.2) # older than the snapshot by more than the timestamp slack + + for _turn in range(2): + run_hook(ups_payload(repo), hook_env) + _rc, stdout, _err = run_hook(stop_payload(repo), hook_env) + # Telemetry: a count above the cap is how this path shows up. + assert metrics_of(stdout)["preexisting_untracked_excluded"] == n_files + assert stub_api.calls == [] + + run_hook(ups_payload(repo), hook_env) + (repo / "zz_new.py").write_text(SRC) + run_hook(stop_payload(repo), hook_env) + assert len([c for c in stub_api.calls if "messages" in c]) == 1 diff --git a/plugins/security-guidance/tests/test_user_config_dir.py b/plugins/security-guidance/tests/test_user_config_dir.py new file mode 100644 index 00000000..7c8a1245 --- /dev/null +++ b/plugins/security-guidance/tests/test_user_config_dir.py @@ -0,0 +1,98 @@ +"""User-level claude-security-guidance.md and security-patterns files are +found in the directory CLAUDE_CONFIG_DIR names, as well as in ~/.claude.""" +import json +import os + +import pytest + +import extensibility + +MARKER = "MARKER_POLICY_LINE" +PATTERNS = {"patterns": [{"rule_name": "marker_rule", "substrings": ["x"], "reminder": "r"}]} + + +@pytest.fixture +def home(tmp_path, monkeypatch): + h = tmp_path / "home" + h.mkdir() + monkeypatch.setenv("HOME", str(h)) + monkeypatch.setenv("USERPROFILE", str(h)) + monkeypatch.delenv("CLAUDE_CONFIG_DIR", raising=False) + return h + + +def write_user_files(d, marker=MARKER, rule="marker_rule"): + d.mkdir(parents=True, exist_ok=True) + (d / "claude-security-guidance.md").write_text(f"- {marker}\n") + patterns = {"patterns": [dict(PATTERNS["patterns"][0], rule_name=rule)]} + (d / "security-patterns.json").write_text(json.dumps(patterns)) + + +def load(): + extensibility.load_for_session(None) + return extensibility.guidance_block(), [r["ruleName"] for r in extensibility.user_patterns()] + + +def test_files_in_configured_dir_are_loaded(home, monkeypatch): + cfg = home / ".claude-work" + write_user_files(cfg) + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(cfg)) + guidance, rules = load() + assert MARKER in guidance + assert rules == ["user:marker_rule"] + + +def test_default_dir_still_loaded_when_config_dir_is_set(home, monkeypatch): + write_user_files(home / ".claude") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(home / ".claude-work")) + guidance, rules = load() + assert MARKER in guidance + assert rules == ["user:marker_rule"] + + +def test_both_dirs_loaded_default_first(home, monkeypatch): + write_user_files(home / ".claude", marker="FROM_DEFAULT", rule="default_rule") + cfg = home / ".claude-work" + write_user_files(cfg, marker="FROM_CONFIGURED", rule="configured_rule") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(cfg)) + guidance, rules = load() + assert guidance.index("FROM_DEFAULT") < guidance.index("FROM_CONFIGURED") + assert rules == ["user:default_rule", "user:configured_rule"] + + +@pytest.mark.parametrize("spelling", ["{home}/.claude", "{home}/.claude/", "~/.claude", "{home}/link"]) +def test_config_dir_naming_the_default_dir_is_read_once(home, monkeypatch, spelling): + write_user_files(home / ".claude") + os.symlink(home / ".claude", home / "link") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", spelling.format(home=home)) + guidance, rules = load() + assert guidance.count(MARKER) == 1 + assert rules == ["user:marker_rule"] + + +@pytest.mark.parametrize("value", ["", " "]) +def test_empty_config_dir_is_not_set(home, monkeypatch, value): + write_user_files(home / ".claude") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", value) + assert extensibility._user_config_dirs() == [str(home / ".claude")] + guidance, rules = load() + assert guidance.count(MARKER) == 1 + + +def test_tilde_in_config_dir_is_expanded(home, monkeypatch): + write_user_files(home / ".claude-work") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", "~/.claude-work") + guidance, _ = load() + assert MARKER in guidance + + +def test_project_files_unchanged(home, monkeypatch, tmp_path): + cfg = home / ".claude-work" + write_user_files(cfg, marker="FROM_USER") + monkeypatch.setenv("CLAUDE_CONFIG_DIR", str(cfg)) + proj = tmp_path / "proj" + write_user_files(proj / ".claude", marker="FROM_PROJECT", rule="project_rule") + extensibility.load_for_session(str(proj)) + guidance = extensibility.guidance_block() + assert guidance.index("FROM_USER") < guidance.index("FROM_PROJECT") + assert [r["ruleName"] for r in extensibility.user_patterns()] == ["user:marker_rule", "user:project_rule"] diff --git a/plugins/security-guidance/tests/test_windows_cli_lookup.py b/plugins/security-guidance/tests/test_windows_cli_lookup.py new file mode 100644 index 00000000..3afea8e5 --- /dev/null +++ b/plugins/security-guidance/tests/test_windows_cli_lookup.py @@ -0,0 +1,279 @@ +"""Which Claude Code the Agent SDK is told to start on Windows.""" +import os +import subprocess +import sys +import types + +import pytest + +from conftest import HOOKS_DIR + +import _base +import llm + +VAR = "NoDefaultCurrentDirectoryInExePath" +windows_only = pytest.mark.skipif(os.name != "nt", reason="needs Windows path rules") + + +def put(folder, name="claude.exe"): + folder.mkdir(parents=True, exist_ok=True) + (folder / name).write_text("") + return str(folder / name) + + +def set_path(monkeypatch, *entries): + monkeypatch.setenv("PATH", os.pathsep.join(str(e) for e in entries)) + + +@pytest.fixture +def win(monkeypatch, tmp_path): + """llm as on Windows: an empty home and PATH, no CLI named in the + environment, and a working directory that has a claude.exe of its own.""" + ns = types.SimpleNamespace(home=tmp_path / "home", repo=tmp_path / "repo", tools=tmp_path / "tools") + for d in (ns.home, ns.repo, ns.tools): + d.mkdir() + put(ns.repo) + monkeypatch.setattr(llm, "_WINDOWS", True) + monkeypatch.setattr(llm, "_gateway_cli", None) + monkeypatch.setattr(_base, "DEBUG_LOG_FILE", str(tmp_path / "log.txt")) + monkeypatch.setenv("HOME", str(ns.home)) + monkeypatch.setenv("USERPROFILE", str(ns.home)) # "~" on Windows + for var in ("SG_AGENTIC_CLI_PATH", "CLAUDE_CODE_EXECPATH", "CLAUDE_PROJECT_DIR"): + monkeypatch.delenv(var, raising=False) + set_path(monkeypatch) + monkeypatch.chdir(ns.repo) + return ns + + +class TestUsersOwnCli: + def test_native_installers_folder_comes_first(self, win, monkeypatch): + put(win.tools) + set_path(monkeypatch, win.tools) + native = put(win.home / ".local" / "bin") + assert llm._windows_user_cli() == native + + @pytest.mark.parametrize("spelling", [ + pytest.param("{}", id="plain"), + pytest.param('"{}"', id="quoted"), + pytest.param("{}" + os.sep, id="trailing-separator"), + ]) + def test_found_on_path(self, win, monkeypatch, spelling): + put(win.tools) + set_path(monkeypatch, win.tools / "empty", spelling.format(win.tools)) + assert os.path.samefile(llm._windows_user_cli(), win.tools / "claude.exe") + + def test_batch_file_is_not_used(self, win, monkeypatch): + put(win.tools, "claude.cmd") + set_path(monkeypatch, win.tools) + assert llm._windows_user_cli() is None + + @pytest.mark.parametrize("entry", [ + pytest.param("", id="empty"), + pytest.param(".", id="dot"), + pytest.param("{repo}", id="full-path"), + pytest.param("{repo}" + os.sep, id="trailing-separator"), + pytest.param(os.path.join("..", "repo"), id="relative"), + ]) + def test_working_directory_is_not_searched(self, win, monkeypatch, entry): + set_path(monkeypatch, entry.format(repo=win.repo)) + assert llm._windows_user_cli() is None + + def test_search_continues_past_the_working_directory(self, win, monkeypatch): + exe = put(win.tools) + set_path(monkeypatch, ".", win.repo, win.tools) + assert llm._windows_user_cli() == exe + + def test_relative_entry_is_skipped(self, win, monkeypatch): + put(win.tools) + set_path(monkeypatch, os.path.join("..", "tools")) + assert llm._windows_user_cli() is None + + @pytest.mark.parametrize("home", ["~", "", "home"]) + def test_home_that_is_not_a_full_path_is_skipped(self, win, monkeypatch, home): + put(win.repo / home / ".local" / "bin") + monkeypatch.setattr(llm.os.path, "expanduser", lambda p: home) + assert llm._windows_user_cli() is None + + def test_project_root_is_not_searched_from_a_subfolder(self, win, monkeypatch): + (win.repo / "sub").mkdir() + monkeypatch.chdir(win.repo / "sub") + monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(win.repo).replace(os.sep, "/")) + set_path(monkeypatch, win.repo) + assert llm._windows_user_cli() is None + + def test_link_to_the_working_directory_is_not_searched(self, win, monkeypatch, tmp_path): + try: + if os.name == "nt": + import _winapi + _winapi.CreateJunction(str(win.repo), str(tmp_path / "link")) + else: + os.symlink(win.repo, tmp_path / "link", target_is_directory=True) + except OSError: + pytest.skip("cannot make a link here") + set_path(monkeypatch, tmp_path / "link") + assert llm._windows_user_cli() is None + + @pytest.mark.parametrize("project", [False, True]) + def test_folder_inside_the_project_is_not_searched(self, win, monkeypatch, project): + put(win.repo / ".venv" / "Scripts") + if project: + (win.repo / "sub").mkdir() + monkeypatch.chdir(win.repo / "sub") + monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(win.repo)) + set_path(monkeypatch, win.repo / ".venv" / "Scripts") + assert llm._windows_user_cli() is None + + def test_folder_that_only_shares_a_prefix_is_found(self, win, monkeypatch, tmp_path): + exe = put(tmp_path / "repo-tools") + set_path(monkeypatch, tmp_path / "repo-tools") + assert llm._windows_user_cli() == exe + + def test_session_above_the_home_folder_still_finds_the_native_install(self, win, monkeypatch): + native = put(win.home / ".local" / "bin") + monkeypatch.chdir(win.home.parent) + monkeypatch.setenv("CLAUDE_PROJECT_DIR", str(win.home.parent)) + assert llm._windows_user_cli() == native + + def test_tools_under_the_home_folder_are_found(self, win, monkeypatch): + exe = put(win.home / "AppData" / "tool") + monkeypatch.chdir(win.home) + set_path(monkeypatch, win.home / "AppData" / "tool") + assert llm._windows_user_cli() == exe + + @windows_only + def test_entries_without_a_drive_or_a_root_are_skipped(self, win, monkeypatch): + put(win.tools) + drive, rest = os.path.splitdrive(str(win.tools)) + set_path(monkeypatch, rest, rest.replace("\\", "/"), drive + os.path.relpath(str(win.tools))) + assert llm._windows_user_cli() is None + + @windows_only + def test_other_spellings_of_the_working_directory_are_skipped(self, win, monkeypatch): + import ctypes + short = ctypes.create_unicode_buffer(1024) + ctypes.windll.kernel32.GetShortPathNameW(str(win.repo), short, 1024) + repo = str(win.repo) + set_path(monkeypatch, repo.upper(), repo.replace("\\", "/"), "\\\\?\\" + repo, short.value or repo) + assert llm._windows_user_cli() is None + + +class TestFixedPaths: + @pytest.fixture + def installed_as_on_linux(self, win, monkeypatch): + monkeypatch.setattr(llm.os.path, "isfile", lambda p: p.endswith("/bin/claude")) + + def test_not_used_on_windows(self, installed_as_on_linux): + assert llm._user_cli_path() is None + + def test_used_elsewhere(self, installed_as_on_linux, monkeypatch): + monkeypatch.setattr(llm, "_WINDOWS", False) + assert llm._user_cli_path().endswith("/.local/bin/claude") + + +@pytest.fixture +def sdk(monkeypatch, tmp_path): + """Stand-in for claude_agent_sdk, installed in tmp_path, that records the + options of each spawn in .spawned.""" + class ResultMessage: + subtype, usage, total_cost_usd = "success", {}, 0.0 + structured_output = {"findings": []} + + async def query(prompt, options): + yield ResultMessage() + + mod = types.ModuleType("claude_agent_sdk") + mod.spawned = [] + mod.ClaudeAgentOptions = lambda **kwargs: mod.spawned.append(kwargs) + mod.AssistantMessage = type("AssistantMessage", (), {}) + mod.ResultMessage, mod.query = ResultMessage, query + mod.__file__ = str(tmp_path / "site" / "claude_agent_sdk" / "__init__.py") + mod.bundled = str(tmp_path / "site" / "claude_agent_sdk" / "_bundled" / "claude.exe") + monkeypatch.setitem(sys.modules, "claude_agent_sdk", mod) + monkeypatch.setenv("SECURITY_REVIEW_MODEL", "opus") + return mod + + +def bundle(tmp_path): + return put(tmp_path / "site" / "claude_agent_sdk" / "_bundled") + + +REVIEWS = [ + pytest.param(lambda: llm.agentic_review(os.getcwd(), [("a.py", "+x = 1\n")], ["a.py"]), id="agentic"), + pytest.param(lambda: llm._call_claude_via_sdk("review this", {"type": "object"}), id="cloud-provider"), +] + + +@pytest.mark.parametrize("review", REVIEWS) +class TestWhatTheSdkIsTold: + @staticmethod + def told(sdk): + assert sdk.spawned + return {o["cli_path"] for o in sdk.spawned} + + def test_bundled_cli_before_the_users(self, win, sdk, tmp_path, review): + put(win.home / ".local" / "bin") + bundle(tmp_path) + review() + assert self.told(sdk) == {sdk.bundled} + + def test_users_cli_when_nothing_is_bundled(self, win, sdk, monkeypatch, review): + tolerant = [] + monkeypatch.setattr(llm, "_tolerate_unknown_sdk_messages", lambda: tolerant.append(1)) + set_path(monkeypatch, ".", win.tools) + cli = put(win.tools) + review() + assert self.told(sdk) == {cli} and tolerant + + def test_full_path_is_named_when_no_cli_exists(self, win, sdk, review): + review() + assert self.told(sdk) == {sdk.bundled} + + def test_named_cli_is_used_as_named(self, win, sdk, tmp_path, monkeypatch, review): + bundle(tmp_path) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", "X:\\my\\claude.exe") + review() + assert self.told(sdk) == {"X:\\my\\claude.exe"} + + def test_empty_name_is_no_name(self, win, sdk, tmp_path, monkeypatch, review): + put(win.home / ".local" / "bin") + bundle(tmp_path) + monkeypatch.setenv("SG_AGENTIC_CLI_PATH", "") + review() + assert self.told(sdk) == {sdk.bundled} + + def test_sdk_chooses_off_windows(self, win, sdk, tmp_path, monkeypatch, review): + monkeypatch.setattr(llm, "_WINDOWS", False) + monkeypatch.setattr(llm, "_user_cli_path", lambda: None) + bundle(tmp_path) + review() + assert self.told(sdk) == {None} + + +def test_empty_name_is_no_name_off_windows(monkeypatch): + monkeypatch.setattr(llm, "_WINDOWS", False) + assert llm._cli_for_sdk("") is None + + +def test_sdk_module_without_a_file_still_gets_a_full_path(win, sdk): + del sdk.__file__ + assert os.path.isabs(llm._cli_for_sdk(None)) + + +class TestSearchOrderVariable: + @staticmethod + def after_import(os_name, **env): + code = (f"import os; os.name = {os_name!r}; import _base; " + f"print(os.environ.get({VAR!r}))") + base = {k: v for k, v in os.environ.items() if k.upper() != VAR.upper()} + return subprocess.run([sys.executable, "-c", code], cwd=HOOKS_DIR, env={**base, **env}, + capture_output=True, text=True, check=True).stdout.strip() + + def test_set_on_windows_when_absent(self): + assert self.after_import("nt") == "1" + + def test_existing_value_is_kept(self): + assert self.after_import("nt", **{VAR: "0"}) == "0" + + @pytest.mark.skipif(os.name == "nt", reason="checks that the variable is not set off Windows") + def test_not_set_elsewhere(self): + assert self.after_import(os.name) == "None" |
