diff options
Diffstat (limited to 'plugins/security-guidance/hooks')
| -rw-r--r-- | plugins/security-guidance/hooks/_base.py | 224 | ||||
| -rw-r--r-- | plugins/security-guidance/hooks/diffstate.py | 808 | ||||
| -rw-r--r-- | plugins/security-guidance/hooks/ensure_agent_sdk.py | 160 | ||||
| -rw-r--r-- | plugins/security-guidance/hooks/extensibility.py | 117 | ||||
| -rw-r--r-- | plugins/security-guidance/hooks/gitutil.py | 1450 | ||||
| -rw-r--r-- | plugins/security-guidance/hooks/hooks.json | 63 | ||||
| -rw-r--r-- | plugins/security-guidance/hooks/llm.py | 1432 | ||||
| -rwxr-xr-x | plugins/security-guidance/hooks/security_reminder_hook.py | 1302 | ||||
| -rw-r--r-- | plugins/security-guidance/hooks/session_state.py | 93 | ||||
| -rwxr-xr-x | plugins/security-guidance/hooks/sg-python.sh | 364 |
10 files changed, 5291 insertions, 722 deletions
diff --git a/plugins/security-guidance/hooks/_base.py b/plugins/security-guidance/hooks/_base.py index ce051753..d518b88e 100644 --- a/plugins/security-guidance/hooks/_base.py +++ b/plugins/security-guidance/hooks/_base.py @@ -5,11 +5,20 @@ This module exists so that ``patterns``/``session_state``/``gitutil`` can use ``debug_log`` without importing ``security_reminder_hook`` (which would be a circular import). It must stay free of any other intra-plugin imports. """ +import glob import json import os +import re +import stat import threading from datetime import datetime +if os.name == "nt": + # Unless this variable exists, Windows looks for a program that is started + # by name in the working directory before PATH. + os.environ.setdefault("NoDefaultCurrentDirectoryInExePath", "1") + + def state_dir(): """Return the absolute path of the plugin's state directory. @@ -32,11 +41,73 @@ def state_dir(): """ explicit = os.environ.get("SECURITY_WARNINGS_STATE_DIR") if explicit: - return os.path.expanduser(explicit) - cc_config = os.environ.get("CLAUDE_CONFIG_DIR") - if cc_config: - return os.path.expanduser(os.path.join(cc_config, "security")) - return os.path.expanduser("~/.claude/security") + d = explicit + elif os.environ.get("CLAUDE_CONFIG_DIR"): + d = os.path.join(os.environ["CLAUDE_CONFIG_DIR"], "security") + else: + d = "~/.claude/security" + return os.path.abspath(os.path.expanduser(d)) + + +def make_state_dir(d=None): + """Create the state dir (or `d`) with access for its user alone, and + narrow the default one to that if it is already there, so a umask such as + 002 does not leave the reviewer's venv, logs and session state open to the + rest of a shared group. A folder named by SECURITY_WARNINGS_STATE_DIR + that already exists is the user's to set. Raises OSError like os.makedirs.""" + if d is None: + d = state_dir() + os.makedirs(d, mode=0o700, exist_ok=True) + if (os.name != "nt" and not os.environ.get("SECURITY_WARNINGS_STATE_DIR", "").strip() + and os.path.abspath(d) == state_dir()): + try: + st = os.lstat(d) + if stat.S_ISDIR(st.st_mode) and st.st_uid == os.getuid() and st.st_mode & 0o077: + os.chmod(d, 0o700) + except OSError: + pass + return d + + +def llm_reviews_switched_off(): + """True when the environment turns off every review that calls a model: + the kill switch (SECURITY_GUIDANCE_DISABLE=1 / ENABLE_SECURITY_REMINDER=0), + ENABLE_CODE_SECURITY_REVIEW=0, or ENABLE_STOP_REVIEW=0 together with + ENABLE_COMMIT_REVIEW=0. Nothing loads the agent SDK then, so nothing + should install it. Same reading of each variable as + security_reminder_hook.py.""" + env = os.environ.get + if env("SECURITY_GUIDANCE_DISABLE", "").strip().lower() in ("1", "true", "yes", "on"): + return True + if env("ENABLE_SECURITY_REMINDER", "1") == "0": + return True + if env("ENABLE_CODE_SECURITY_REVIEW", "1") == "0": + return True + return env("ENABLE_STOP_REVIEW", "1") == "0" and env("ENABLE_COMMIT_REVIEW", "1") == "0" + + +def agent_sdk_site_dirs(state_dir): + """Directories under the state dir that ensure_agent_sdk.py installs the + agent SDK into and llm.py adds to sys.path: the venv's site-packages + (POSIX `lib/pythonX.Y/site-packages`, Windows `Lib/site-packages`) and + the flat `pip install --target` fallback.""" + venv_root = os.path.join(state_dir, "agent-sdk-venv") + return ( + glob.glob(os.path.join(venv_root, "lib", "python*", "site-packages")) + + glob.glob(os.path.join(venv_root, "Lib", "site-packages")) + + [os.path.join(state_dir, "agent-sdk-libs")] + ) + + +def agent_sdk_installed(state_dir): + """True iff one of those directories holds a finished install of the SDK: + its package and the metadata pip writes once the package is unpacked. A + file check only, no import.""" + return any( + os.path.isfile(os.path.join(d, "claude_agent_sdk", "__init__.py")) + and glob.glob(os.path.join(d, "claude_agent_sdk-*.dist-info")) + for d in agent_sdk_site_dirs(state_dir) + ) # Debug log file. Lives under the plugin state dir (default ~/.claude/security/) @@ -94,6 +165,47 @@ PROVENANCE_BANNER = ( ) +# Output that only Claude Code reads: the `metrics` and `rewakeSummary` fields, +# and the separate `{"async": ...}` line an async hook prints first. Other hosts +# that run this plugin's hooks can validate hook output against a closed schema +# and reject the whole output over a field or a second line they do not know, +# which shows up as a hook error. +_CLAUDE_CODE_OUTPUT_FIELDS = ("metrics", "rewakeSummary") + + +def _detect_portable_output(): + forced = os.environ.get("SECURITY_GUIDANCE_PORTABLE_OUTPUT", "").strip().lower() + if forced in ("1", "true", "yes", "on"): + return True + if forced in ("0", "false", "no", "off"): + return False + if os.environ.get("CLAUDECODE") or os.environ.get("CLAUDE_CODE_ENTRYPOINT"): + return False + return bool(os.environ.get("PLUGIN_ROOT")) + + +# Decided once, from the environment the host started the hook with: code the +# hook runs in-process later can set these variables itself. +_PORTABLE_OUTPUT = _detect_portable_output() + + +def portable_output(): + """True when stdout should carry only the standard hook output fields. + + SECURITY_GUIDANCE_PORTABLE_OUTPUT=1 or =0 forces it on or off. Otherwise it + is on only when the host passes PLUGIN_ROOT and does not identify itself as + Claude Code, so Claude Code and direct invocations are unchanged.""" + return _PORTABLE_OUTPUT + + +def write_hook_output(out): + """Print one hook output line. In portable mode the Claude Code only + fields are left out; what remains (possibly `{}`) is still valid output.""" + if portable_output(): + out = {k: v for k, v in out.items() if k not in _CLAUDE_CODE_OUTPUT_FIELDS} + print(json.dumps(out), flush=True) + + def _read_plugin_version_int(): """Encode plugin.json version "M.m.p" as M*10000 + m*100 + p so it fits the bool|number metrics constraint. Returns 0 if unreadable.""" @@ -127,6 +239,11 @@ _USAGE = { # reported it manually. "http_err_last": 0, # most recent HTTP error code this invocation "http_err_count": 0, # total HTTP errors (4xx + 5xx + network) + "net_err": 0, # why the last call got no HTTP response, 0 if it got one + # For the usage log line only: model ids as recorded (the requested alias + # on the Agent SDK path), and whether any call had no price-table entry. + "models": [], + "default_priced": False, } _USAGE_LOCK = threading.Lock() @@ -140,6 +257,9 @@ _PRICE_PER_MTOK = { "claude-sonnet-4-6": (3.0, 15.0), "claude-opus-4-6": (15.0, 75.0), "claude-opus-4-7": (5.0, 25.0), + "claude-opus-4-8": (5.0, 25.0), + "claude-opus-5": (5.0, 25.0), + "claude-opus-5-5": (4.0, 20.0), } _PRICE_DEFAULT = (3.0, 15.0) @@ -160,12 +280,15 @@ def _record_usage(usage, model, cost_usd=None): cw = int(u.get("cache_creation_input_tokens") or 0) except (TypeError, ValueError): return + default_priced = False if cost_usd is None: pin, pout = _PRICE_DEFAULT + default_priced = True m = (model or "").lower() for k, v in sorted(_PRICE_PER_MTOK.items(), key=lambda kv: -len(kv[0])): if m.startswith(k): pin, pout = v + default_priced = False break cost_usd = (i * pin + o * pout + cr * pin * 0.1 + cw * pin * 1.25) / 1_000_000 with _USAGE_LOCK: @@ -175,6 +298,10 @@ def _record_usage(usage, model, cost_usd=None): _USAGE["cw"] += cw _USAGE["cost"] += float(cost_usd or 0.0) _USAGE["n"] += 1 + _USAGE["default_priced"] = _USAGE["default_priced"] or default_priced + name = re.sub(r"[^A-Za-z0-9_.:/@-]", "_", str(model or ""))[:80] + if name and name not in _USAGE["models"]: + _USAGE["models"].append(name) def _record_http_error(status): @@ -201,6 +328,93 @@ def _record_http_error(status): _USAGE["http_err_count"] += 1 +def _record_net_error(code): + """Record why an LLM API call got no HTTP response: the packed code from + llm._net_error_code. `api_error` -1 cannot tell a certificate failure + from a refused connection or a timeout, and a failure the HTTP client + raises outside its own error type (a default TLS context that cannot be + built, a connection closed before the reply) has no `api_error` at all. + Emitted as `net_err` on the row of a hook whose last call failed this + way; a later call clears it.""" + try: + c = int(code) + except (TypeError, ValueError): + return + if c > 0: + with _USAGE_LOCK: + _USAGE["net_err"] = c + + +def _clear_net_error(): + with _USAGE_LOCK: + _USAGE["net_err"] = 0 + + +def _net_error_metric(): + with _USAGE_LOCK: + return _USAGE.get("net_err", 0) + + +# Set by main() and the review handlers: which review this invocation runs +# and for which Claude Code session, so the usage log line can be attributed. +_USAGE_CONTEXT = {"session_id": "", "review": "stop"} + + +def _log_usage(usage): + """Write this invocation's review usage to the debug log. + + `usage` is a `_usage_metrics()` snapshot, the same numbers that go out + with the hook's metrics. The raw-HTTP review leaves no Claude Code + transcript and no OpenTelemetry event; without this line nothing on the + machine records what it used. `est_cost_usd` is that same rollup: list + prices (the inner CLI's own figure on the Agent SDK path), `~` appended + when a model had no price-table entry and the default rate was used. + """ + if not usage.get("api_calls"): + return + with _USAGE_LOCK: + models = ",".join(_USAGE["models"]) or "-" + approx = "~" if _USAGE["default_priced"] else "" + debug_log( + f"Review usage: review={_USAGE_CONTEXT.get('review') or '-'} " + f"session={_USAGE_CONTEXT.get('session_id') or '-'} " + f"model={models} calls={usage['api_calls']} " + f"in={usage['tok_in']} out={usage['tok_out']} " + f"cache_read={usage['tok_cache_r']} cache_write={usage['tok_cache_w']} " + f"est_cost_usd={usage['cost_usd']:.4f}{approx}" + ) + + +def launcher_probe_metrics(): + """How long sg-python.sh waited for interpreters to answer its probe. + + Only when this launch's probes took a second or more together, so ordinary + rows carry no extra keys: `launcher_probe_ms` is all of them, + `launcher_probe_max_ms` the slowest one. Both are capped at an hour (the + launcher reads the wall clock, which can jump). Read once: the variables + are removed so child processes do not report this launch as their own. + """ + try: + slowest = int(os.environ.pop("SG_LAUNCHER_PROBE_MAX_MS", "")) + total = int(os.environ.pop("SG_LAUNCHER_PROBE_MS", "")) + except ValueError: + return {} + if total < 1000 or slowest > total: + return {} + cap = 3_600_000 + return {"launcher_probe_ms": min(total, cap), "launcher_probe_max_ms": min(slowest, cap)} + + +def launcher_direct_metric(): + """`launcher_direct` when sg-python.sh ran this hook with the interpreter + behind a packaged Windows alias instead of the alias: 1 from the path it + remembered (one process for the hook), 2 when it had just worked that + path out. Read once, like the probe timings. + """ + how = os.environ.pop("SG_LAUNCHER_DIRECT", "") + return {"launcher_direct": int(how)} if how in ("1", "2") else {} + + def _usage_metrics(): """Snapshot the accumulator as metric keys. Returns {} when no API calls AND no HTTP errors were made so skip-path emits don't burn key budget. diff --git a/plugins/security-guidance/hooks/diffstate.py b/plugins/security-guidance/hooks/diffstate.py index 8d0665dd..34655335 100644 --- a/plugins/security-guidance/hooks/diffstate.py +++ b/plugins/security-guidance/hooks/diffstate.py @@ -6,16 +6,22 @@ there so callers keep resolving bare names through the hook module's globals — tests that ``monkeypatch.setattr(hook, "<fn>", …)`` continue to work without retargeting. """ +import hashlib import os import subprocess +import sys +import time -from _base import debug_log, _PV +from _base import debug_log, portable_output, _PV +import gitutil from gitutil import ( GIT_CMD, _git_dir, _git_toplevel, _git_status_porcelain, - _git_rev_parse_head, _is_ancestor, _git_name_only, + _git_rev_parse_head, _is_ancestor, _git_name_only, _temp_index, + _secret_rename_origins, index_entry_count, ) from session_state import with_locked_state +from extensibility import _same_dir # ===================================================================== @@ -95,6 +101,10 @@ def consume_stop_state(session_id, clear=True): "touched_paths": list(state.get("touched_paths", [])), "baseline_sha": state.get("baseline_sha"), "head_at_capture": state.get("head_at_capture"), + # Where and when the prompt hook captured it; see baseline_origin. + "baseline_root": state.get("baseline_root"), + "prompt_at": state.get("prompt_at"), + "baseline_kept": state.get("baseline_kept") or 0, "untracked_at_baseline": ( dict(state["untracked_at_baseline"]) if isinstance(state.get("untracked_at_baseline"), dict) else {} @@ -103,20 +113,92 @@ def consume_stop_state(session_id, clear=True): "fire_count_expired": expired and state.get("stop_hook_fire_count", 0) > 0, "previous_findings": [] if findings_expired else list(state.get("previous_findings", [])), "reviewed_diff_hash": state.get("reviewed_diff_hash"), + # State lost or overwritten since the last Stop (session_state). + "state_resets": state.get("state_resets", 0), + "state_overwrites": state.get("state_overwrites", 0), + "baseline_fail": state.get("baseline_fail") or 0, } + # touched_paths is cleared per Stop, but one baseline can be + # reviewed by several Stops (rewakes). turn_touched keeps every path + # recorded since the baseline; only telemetry reads it. + turn = list(dict.fromkeys( + [p for p in state.get("turn_touched", []) if isinstance(p, str)] + + snap["touched_paths"]))[-200:] + snap["turn_touched"] = turn if clear: + state["turn_touched"] = turn state["touched_paths"] = [] state.pop("reviewed_diff_hash", None) + state.pop("state_resets", None) + state.pop("state_overwrites", None) return snap return with_locked_state(session_id, _snap) or { "touched_paths": [], "baseline_sha": None, "head_at_capture": None, - "untracked_at_baseline": {}, + "untracked_at_baseline": {}, "baseline_root": None, "prompt_at": None, + "baseline_kept": 0, "fire_count": 0, "fire_count_expired": False, "previous_findings": [], - "reviewed_diff_hash": None, + "reviewed_diff_hash": None, "state_resets": 0, "state_overwrites": 0, + "baseline_fail": 0, "turn_touched": [], } +# Path names compare without case where the filesystem usually does. +_FOLD_CASE = os.name == "nt" or sys.platform == "darwin" + + +def _rel_in_repo(real_root, path): + """`path` relative to the repo root (a real path), or None when it lies + outside the repo.""" + real = os.path.realpath(path) + try: + rel = os.path.relpath(real, real_root) + except ValueError: # another drive + return None + if rel == os.pardir or rel.startswith(os.pardir + os.sep): + return None + return rel.casefold() if _FOLD_CASE else rel + + +def touched_scope(repo_root, cwd, touched_paths): + """Where the paths the session's edit tools wrote since the baseline lie: + (scope, repo-relative paths inside the repo). scope: 0 none recorded, + 1 all inside the reviewed repo, 2 some outside, 3 all outside. Relative + paths are read against `cwd`. The list holds the newest 200 paths, so + this is approximate past that. Telemetry only.""" + root = os.path.realpath(repo_root) + inside, outside = set(), 0 + for p in touched_paths: + rel = _rel_in_repo(root, os.path.join(cwd or root, p)) + if rel is None: + outside += 1 + else: + inside.add(rel) + if not touched_paths: + return 0, inside + if not outside: + return 1, inside + return (2 if inside else 3), inside + + +def review_scope_code(repo_root, scope, inside, files, base_fail): + """One number, for telemetry only, saying how a Stop review relates to + what the session's edit tools wrote since the baseline: + + base_fail * 100000 + scope * 10000 + min(untouched, 9999) + + scope and inside: from touched_scope. untouched: how many of `files` + (repo-root-relative or absolute) no edit tool wrote. base_fail: 0 the + review diffed against a real turn-start snapshot, 1 and 2 see + last_baseline_fail, 3 there was no usable snapshot. Packed into one key + and sent even when 0, so a row without it was cut by the metrics key cap + or had no repo.""" + root = os.path.realpath(repo_root) + untouched = sum(1 for f in files + if _rel_in_repo(root, os.path.join(root, f)) not in inside) + return int(base_fail or 0) * 100000 + scope * 10000 + min(untouched, 9999) + + def record_reviewed_diff(session_id, diff_hash): def _save(state): state["reviewed_diff_hash"] = diff_hash @@ -147,6 +229,48 @@ def restore_unreviewed_stop_state(session_id, paths, baseline_sha): with_locked_state(session_id, _restore) +# state["baseline_kept"]: why a prompt left an earlier turn's baseline in +# place. Only telemetry reads it. +KEPT_NO_CAPTURE = 1 # it could not capture one (not a repo, or git failed) +KEPT_UNREVIEWED = 2 # the previous turn's edits were never reviewed + +# baseline_origin() values that mean "captured in another checkout". +FOREIGN_ORIGINS = (1, 2, 3) + + +def baseline_origin(repo_root, baseline_sha, baseline_root): + """Where the session's baseline was captured, relative to the checkout a + Stop review is about to read (`repo_root`): + + 0 that checkout (or the origin was not recorded) + 1 another worktree of the same repository + 2 another repository + 3 another folder, which is gone or cannot be read as a checkout + 4 there is no baseline + + `baseline_root` is the repo root the prompt hook stored beside the SHA.""" + if not baseline_sha: + return 4 + if not baseline_root or not isinstance(baseline_root, str): + return 0 + if _same_dir(baseline_root, repo_root): + return 0 + if not os.path.isdir(baseline_root): + return 3 + here, there = _git_dir(repo_root), _git_dir(baseline_root) + if not here or not there: + return 3 + return 1 if _same_dir(here, there) else 2 + + +def baseline_origin_metric(origin, kept): + """The `bl_origin` number on Stop rows: baseline_origin, plus 10 or 20 + (KEPT_*) when the baseline is an earlier turn's.""" + if origin == 4 or kept not in (KEPT_NO_CAPTURE, KEPT_UNREVIEWED): + return origin + return origin + 10 * kept + + def get_baseline_file_content(session_id, file_path, cwd): """Get the content of a file at the baseline SHA. Returns None if unavailable. @@ -165,65 +289,186 @@ def get_baseline_file_content(session_id, file_path, cwd): try: abs_path = os.path.abspath(file_path) cwd_abs = os.path.abspath(cwd) if cwd else os.getcwd() - try: - rel_path = os.path.relpath(abs_path, cwd_abs) - except ValueError: - return None - result = subprocess.run( - [*GIT_CMD, "show", f"{baseline_sha}:{rel_path}"], - cwd=cwd, capture_output=True, timeout=5 - ) - if result.returncode == 0: - return (result.stdout or b"").decode("utf-8", errors="replace") + attempts = [(abs_path, cwd_abs, cwd)] + # The tool's path and the session directory can name the same place + # through different symlinks (/tmp and /var on macOS). The relative + # path between the two spellings then leaves the repo and git finds + # nothing, so the lookup is tried again with both resolved. + real_path, real_cwd = os.path.realpath(abs_path), os.path.realpath(cwd_abs) + if (real_path, real_cwd) != (abs_path, cwd_abs): + attempts.append((real_path, real_cwd, real_cwd)) + for path, start, git_cwd in attempts: + try: + rel_path = os.path.relpath(path, start) + except ValueError: + continue + # `<rev>:<path>` is read from the repo root unless the path starts + # with `./`, and takes forward slashes only. Without both, the + # lookup fails for a session started below the repo root, and on + # Windows for any file that is not directly in the session + # directory. + rel_path = rel_path.replace(os.sep, "/") + result = subprocess.run( + [*GIT_CMD, "show", f"{baseline_sha}:./{rel_path}"], + cwd=git_cwd, capture_output=True, timeout=5 + ) + if result.returncode == 0: + return (result.stdout or b"").decode("utf-8", errors="replace") return None except (subprocess.TimeoutExpired, FileNotFoundError, OSError, ValueError): return None +# How the last capture_git_baseline call got its SHA: 0 a real snapshot, +# 1 the snapshot failed and HEAD was returned, 2 no index copy and HEAD was +# returned. With 1 or 2 the next Stop review cannot tell the turn's changes +# from uncommitted work that was already there. Stored with baseline_sha. +# Telemetry only. +last_baseline_fail = 0 +# Entries in the index the last capture copied, None if unknown. Telemetry only. +last_index_entries = None + + +# Who the snapshot commit is made by. Set on every commit-tree call, so the +# user's own name never appears on a commit they did not make and a repo with +# no identity configured still works. The commit object does land in +# `.git/objects`, but nothing refers to it, like the one `git stash create` +# used to make here: it is prunable from the moment it is written. +_SNAPSHOT_IDENT = { + "GIT_AUTHOR_NAME": "security-guidance", "GIT_AUTHOR_EMAIL": "security-guidance@localhost", + "GIT_COMMITTER_NAME": "security-guidance", "GIT_COMMITTER_EMAIL": "security-guidance@localhost", +} + +# The whole snapshot, not one command: `stash create` was capped at 15 s and a +# per-command cap would let a slow filesystem hold the prompt for far longer. +_SNAPSHOT_BUDGET_S = 15 + + +def _snapshot_commit(cwd, env, head_tree): + """A commit of every tracked change, on the index copy `env` names. + + Returns its SHA, "" when nothing tracked has changed (so the caller uses + HEAD, `head_tree` being HEAD's tree), or None when git refused or the + time budget ran out. + + No `--missing-ok`: git counts a partial clone's promisor blobs as + present, so write-tree neither fetches nor fails on one, and the + existence check it would turn off is worth keeping. + """ + deadline = time.monotonic() + _SNAPSHOT_BUDGET_S + + def run(args, extra=None): + left = deadline - time.monotonic() + if left <= 0: + raise subprocess.TimeoutExpired(args, _SNAPSHOT_BUDGET_S) + return subprocess.run( + [*GIT_CMD, *args], cwd=cwd, capture_output=True, timeout=left, + env={**env, **(extra or {})}, + ) + + # No pathspec: `add -u` has staged the whole work tree, whatever directory + # of it `cwd` is, since git 2.0. `-- :/` would say the same but fails + # outright under a user's GIT_LITERAL_PATHSPECS. + add = run(["add", "-u"]) + if add.returncode != 0: + debug_log(f"Snapshot: add -u failed: {_stderr_of(add)}") + return None + tree = run(["write-tree"]) + sha = (tree.stdout or b"").decode("utf-8", errors="replace").strip() + if tree.returncode != 0 or not sha: + debug_log(f"Snapshot: write-tree failed: {_stderr_of(tree)}") + return None + if sha == head_tree: + return "" + if gitutil.last_dropped_gitlinks: + # A nested repository's entry is gone from the index copy, so the + # tree cannot equal HEAD's even on a clean work tree. Ask whether + # anything but those entries differs, rather than writing a commit + # object on every prompt in a repo with submodules. + same = run(["diff-tree", "--quiet", "--ignore-submodules=all", head_tree, sha]) + if same.returncode == 0: + return "" + commit = run(["-c", "commit.gpgsign=false", "commit-tree", sha, "-p", "HEAD", + "-m", "security-guidance turn baseline"], _SNAPSHOT_IDENT) + out = (commit.stdout or b"").decode("utf-8", errors="replace").strip() + if commit.returncode != 0 or not out: + debug_log(f"Snapshot: commit-tree failed: {_stderr_of(commit)}") + return None + return out + + +def _stderr_of(result): + return (result.stderr or b"").decode("utf-8", errors="replace").strip()[:200] + + def capture_git_baseline(cwd): """ Capture a git ref representing the current working tree state. - Uses `git stash create` which creates a commit object for the current state - (HEAD + uncommitted changes) without modifying the stash list or working tree. + Builds a commit object for the current state (HEAD + uncommitted changes) + without touching the stash list or the working tree: `add -u` stages every + tracked change into a throwaway index copy, `write-tree` turns that into a + tree, and `commit-tree` hangs it off HEAD. Falls back to HEAD if the working tree is clean. Returns the SHA string, or None if not in a git repo or if the repo has no commits. - NOTE: `git stash create` does NOT capture untracked files. UPS pairs this + This was `git stash create`, which does the same thing in one command but + checks the work tree on a single thread, while `add -u` uses git's index + preload (one thread per 500 entries, up to 20). On a filesystem where each + file status is slow — a WSL repository on a Windows drive, a network mount + — that is the whole cost of the prompt: see #6854, where `stash create` + took 4 s on 1,424 files and never finished on 10,000. + + The commands run on a throwaway copy of the index (GIT_INDEX_FILE), so + their `index.lock` lands next to the copy, not in `.git`: they cannot + collide with a concurrent git command, and a timeout kill cannot leave a + lock that blocks the user's next one. GIT_OPTIONAL_LOCKS does not cover + these. Copying the real index without a lock is safe: git replaces it by + atomic rename. + + NOTE: the snapshot does NOT capture untracked files. UPS pairs this SHA with a `_list_untracked()` snapshot stored as `untracked_at_baseline`, and `compute_v2_review_set` subtracts that set so pre-existing untracked - files are not reviewed as Claude-authored. + files are not reviewed as Claude-authored. UPS takes no snapshot while the + Stop review, its only reader, is switched off. """ # stdout is a SHA so text=True is safe on stdout, but a non-ASCII - # filename in `git stash create`'s STDERR warning (e.g. a worktree + # filename in git's STDERR warning (e.g. a worktree # with `Ávila_report.txt` triggers a quotePath/locale warning) would # trip the stderr reader thread on Windows cp1252. Decode both streams # leniently for symmetry with _list_untracked. See #2056. + global last_baseline_fail, last_index_entries + last_baseline_fail = 0 + last_index_entries = None try: - # Check if HEAD exists (i.e., repo has at least one commit) + # HEAD's commit and its tree in one call: the commit is the answer + # for a clean tree, the tree is what the snapshot is compared with. + # A failure here means the repo has no commits — skip the review + # rather than create commits in the user's repo. head_check = subprocess.run( - [*GIT_CMD, "rev-parse", "HEAD"], + [*GIT_CMD, "rev-parse", "HEAD", "HEAD^{tree}"], cwd=cwd, capture_output=True, timeout=5 ) - if head_check.returncode != 0: - # No commits yet — skip review rather than creating commits in the user's repo + head_out = (head_check.stdout or b"").decode("utf-8", errors="replace").split() + if head_check.returncode != 0 or len(head_out) != 2: debug_log("No commits in repo, skipping baseline capture") return None - - result = subprocess.run( - [*GIT_CMD, "stash", "create"], - cwd=cwd, capture_output=True, timeout=15 - ) - sha = (result.stdout or b"").decode("utf-8", errors="replace").strip() - if sha: - return sha - - # Working tree is clean — stash create returns empty. Use HEAD. - result = subprocess.run( - [*GIT_CMD, "rev-parse", "HEAD"], - cwd=cwd, capture_output=True, timeout=5 - ) - sha = (result.stdout or b"").decode("utf-8", errors="replace").strip() - return sha if sha else None + head_sha, head_tree = head_out + + sha = "" + # untracked_paths=[]: a plain copy of the index, no `add -N` pass. + with _temp_index(cwd, untracked_paths=[]) as env: + # env is None when no usable index copy could be made: fall + # back to HEAD. + if env is not None: + last_index_entries = index_entry_count(env["GIT_INDEX_FILE"]) + sha = _snapshot_commit(cwd, env, head_tree) + if sha is None: + last_baseline_fail = 1 + sha = "" + else: + last_baseline_fail = 2 + # No snapshot: nothing tracked has changed, or it could not be taken. + return sha or head_sha except (subprocess.TimeoutExpired, FileNotFoundError, OSError, ValueError) as e: debug_log(f"Failed to capture git baseline: {e}") return None @@ -267,14 +512,19 @@ def capture_git_baseline(cwd): _REVIEWED_SHAS_BASENAME = "sg-reviewed-shas" _REVIEWED_SHAS_CAP = 500 +def _is_full_sha(sha): + return len(sha) == 40 and all(c in "0123456789abcdef" for c in sha) + + def _reviewed_shas_path(repo_root): gd = _git_dir(repo_root) return os.path.join(gd, _REVIEWED_SHAS_BASENAME) if gd else None -def _load_reviewed_shas(repo_root): +def _load_reviewed_shas(repo_root, _gd=None): """Set of full 40-hex shas previously reviewed in this clone.""" - p = _reviewed_shas_path(repo_root) + p = (os.path.join(_gd, _REVIEWED_SHAS_BASENAME) if _gd + else _reviewed_shas_path(repo_root)) if not p or not os.path.exists(p): return set() out = set() @@ -282,7 +532,7 @@ def _load_reviewed_shas(repo_root): with open(p, "r") as f: for line in f: sha = line.split("\t", 1)[0].strip() - if len(sha) == 40 and all(c in "0123456789abcdef" for c in sha): + if _is_full_sha(sha): out.add(sha) except OSError: pass @@ -334,11 +584,429 @@ def _append_reviewed_shas(repo_root, shas, vulns_found=0): pass +# ─── repeated-diff count ──────────────────────────────────────────────────── +# +# Count only. A change that comes back under a new commit (undo and re-commit, +# rebase before the first push) is unknown to the reviewed-shas record and is +# reviewed again. These helpers say how often that happens, by remembering a +# hash of each reviewed hunk next to the reviewed-shas file. Nothing here +# decides whether a review runs, and nothing here waits on another process +# for more than a moment. + +_SEEN_DIFFS_BASENAME = "sg-reviewed-diffs" +_SEEN_DIFFS_TTL_S = 7 * 24 * 3600 +# Past this size the record is cut back to its newest half. +_SEEN_DIFFS_TRIM_BYTES = 1024 * 1024 +_SEEN_DIFFS_MAX_KEYS = 500 +_SEEN_DIFFS_LOCK_TRIES = 10 + +# `diff_seen` values, weakest first. A review reports the weakest of its +# hunks: 3 means every hunk already had a finished review in this clone. +DIFF_SEEN_ERROR = -1 # the record could not be read or written +DIFF_SEEN_NO = 0 # some hunk was never reviewed +DIFF_SEEN_UNFINISHED = 1 # a review of it started and has no verdict (yet) +DIFF_SEEN_OTHER_TREE = 2 # finished, on different repo contents +DIFF_SEEN_SAME_TREE = 3 # finished, on the same tree + + +def _hunk_keys(diff_files): + """One hash per hunk: the path plus the hunk's lines, context included. + + The `@@` line is left out, so a hunk keeps its key when a rebase only + moves it; the same edit next to different code gets a different key. + """ + keys = {} + for path, content in diff_files: + hunks, cur = [], [] + for ln in content.split("\n"): + if ln.startswith("@@"): + hunks.append(cur) + cur = [] + else: + cur.append(ln) + hunks.append(cur) + for body in hunks: + # A file's last hunk ends with the diff's trailing blank line. + while body and not body[-1]: + body.pop() + if not body: + continue + h = hashlib.sha256(path.encode("utf-8", "replace")) + h.update(b"\0") + h.update("\n".join(body).encode("utf-8", "replace")) + keys[h.hexdigest()[:16]] = None + keys = list(keys) + if len(keys) > _SEEN_DIFFS_MAX_KEYS: + # Keep the record small: the hunks past the cap share one key, so + # they only match the same set of hunks again. + rest = hashlib.sha256(" ".join(keys[_SEEN_DIFFS_MAX_KEYS - 1:]).encode("ascii")) + keys = keys[:_SEEN_DIFFS_MAX_KEYS - 1] + [rest.hexdigest()[:16]] + return keys + + +def _seen_lock(fd): + """True: locked. False: another process kept it. None: no flock here.""" + for _ in range(_SEEN_DIFFS_LOCK_TRIES): + try: + if _flock_nb(fd, exclusive=True): + return True + except (ImportError, OSError): + return None + time.sleep(0.05) + return False + + +def _append_diff_row(f, locked, token, done): + """Append one record for the review to `f`, the record opened "a+b".""" + _, keys, tree = token + row = f"{int(time.time())}\t{tree}\t{int(done)}\t{' '.join(keys)}\n".encode("ascii") + size = f.seek(0, os.SEEK_END) + if size: + f.seek(size - 1) + if f.read(1) != b"\n": + row = b"\n" + row # a torn last record must not swallow this one + # Cutting the record back rewrites it: not while another process holds + # the lock. Without flock at all, bounded beats exact. + if locked is not False and size + len(row) > _SEEN_DIFFS_TRIM_BYTES: + f.seek(0) + keep, room = [], _SEEN_DIFFS_TRIM_BYTES // 2 + for ln in reversed(f.read().splitlines(keepends=True)): + room -= len(ln) + if room < 0: + break + keep.append(ln) + f.seek(0) + f.truncate() + row = b"".join(reversed(keep)) + row + f.write(row) + f.flush() + + +def _diff_review_started(repo_root, diff_files, tip="HEAD"): + """Note that a review of `diff_files`, a diff ending at `tip`, is starting. + + Returns (token, metrics). `metrics` is {"diff_seen": <DIFF_SEEN_* code>}: + whether this clone already reviewed every hunk of this diff. `token` goes + to `_diff_review_done` once the review has a verdict. Never raises. + """ + if portable_output() or not diff_files: + # No metrics leave the hook in portable mode, so nothing to count. + return None, {} + try: + gd = _git_dir(repo_root) + keys = _hunk_keys(diff_files) + if not gd or not keys: + raise ValueError("no git dir" if not gd else "no hunks") + token = (os.path.join(gd, _SEEN_DIFFS_BASENAME), keys, + "".join(_full_shas(repo_root, [tip + "^{tree}"])[:1])) + tree = token[2] + now = time.time() + best = dict.fromkeys(keys, DIFF_SEEN_NO) + # Read and note under one lock, so two reviews of one diff that start + # together cannot both miss each other. + with open(token[0], "a+b") as f: + locked = _seen_lock(f.fileno()) + if locked is False: + raise OSError("record is locked") + f.seek(0) + for ln in f.read().decode("ascii", "replace").splitlines(): + row = ln.split("\t") + if len(row) != 4 or not row[0].isdigit() or len(row[0]) > 12: + continue + if now - int(row[0]) > _SEEN_DIFFS_TTL_S: + continue + if row[2] != "1": + seen = DIFF_SEEN_UNFINISHED + elif tree and row[1] == tree: + seen = DIFF_SEEN_SAME_TREE + else: + seen = DIFF_SEEN_OTHER_TREE + for k in row[3].split(" "): + if best.get(k, seen) < seen: + best[k] = seen + _append_diff_row(f, locked, token, done=False) + return token, {"diff_seen": min(best.values())} + except Exception as e: + debug_log(f"Repeated-diff count: not counted ({type(e).__name__}: {e})") + return None, {"diff_seen": DIFF_SEEN_ERROR} + + +def _diff_review_done(token): + """Note that the review `token` came from got a verdict. Never raises.""" + if not token: + return + try: + with open(token[0], "a+b") as f: + _append_diff_row(f, _seen_lock(f.fileno()), token, done=True) + except Exception as e: + debug_log(f"Repeated-diff count: verdict not noted ({type(e).__name__}: {e})") + + +# ─── in-flight review markers ─────────────────────────────────────────────── +# +# A commit or push review records its shas only once it has a verdict, which +# is minutes after it starts. A review that starts in between (the push right +# after a commit, the next commit's reflog scan) would send the same commits +# through a second full review. So a review first takes an exclusive flock on +# `sg-inflight-<sha>`, next to the reviewed-shas file, and holds it until its +# process ends; other reviews leave a locked commit to it. The kernel drops +# the lock however the holder dies, so a killed review blocks nobody. Native +# Windows Python has no flock; there the same lock is taken with LockFileEx. +# Where files cannot be locked (some network filesystems) there are no +# markers and nothing changes. + +_INFLIGHT_PREFIX = "sg-inflight-" +# A marker file nobody holds (a killed review's) is removed once this old. +_INFLIGHT_GC_AGE_S = 3600 +_inflight_held = {} +# Windows refuses to delete a file that is open, so a marker is closed first +# there. Elsewhere it is deleted while still locked, so nobody can lock a +# file that is about to go. +_UNLINK_BEFORE_CLOSE = os.name != "nt" + + +def _full_shas(repo_root, shas): + """Resolve abbreviated commit refs to full 40-hex shas; drops unresolved.""" + out = [] + for s in shas: + try: + r = subprocess.run( + [*GIT_CMD, "rev-parse", "--verify", "-q", s], + cwd=repo_root, capture_output=True, timeout=5, + ) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + continue + if r.returncode == 0: + out.append(r.stdout.decode("utf-8", errors="replace").strip()) + return out + + +_win_lock_file_ex = None + + +def _win_lock_nb(fd, exclusive): + """`_flock_nb` for native Windows Python, with LockFileEx. + + Shared and exclusive as with flock, never waits, and Windows drops the + lock when the handle is closed or the holder's process ends. Windows + locks stop other processes reading the locked bytes, so the byte locked + is one far past the end of the (empty) marker: copying a repository while + a review runs still reads the marker. + """ + global _win_lock_file_ex + import errno + try: + import ctypes + import msvcrt + if _win_lock_file_ex is None: + from ctypes import wintypes + + class _Overlapped(ctypes.Structure): + _fields_ = [("Internal", ctypes.c_void_p), ("InternalHigh", ctypes.c_void_p), + ("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD), + ("hEvent", wintypes.HANDLE)] + + fn = ctypes.WinDLL("kernel32", use_last_error=True).LockFileEx + fn.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD, + wintypes.DWORD, ctypes.POINTER(_Overlapped)] + fn.restype = wintypes.BOOL + _win_lock_file_ex = (fn, _Overlapped) + except (ImportError, AttributeError, OSError) as e: + raise OSError(errno.ENOLCK, f"no file locking in this Python ({e})") + fn, _Overlapped = _win_lock_file_ex + fail_immediately, exclusive_lock = 0x1, 0x2 + where = _Overlapped(Offset=0, OffsetHigh=0x7FFFFFFF) + if fn(msvcrt.get_osfhandle(fd), fail_immediately | (exclusive_lock if exclusive else 0), + 0, 1, 0, ctypes.byref(where)): + return True + err = ctypes.get_last_error() + if err == 33: # ERROR_LOCK_VIOLATION: another handle holds it + return False + raise OSError(errno.ENOLCK, f"LockFileEx failed (WinError {err})") + + +def _flock_nb(fd, exclusive): + """True when the lock was taken, False when another process holds it. + Raises OSError when the file cannot be locked at all.""" + import errno + if os.name == "nt": + return _win_lock_nb(fd, exclusive) + try: + import fcntl + except ImportError as e: + raise OSError(errno.ENOLCK, f"no file locking in this Python ({e})") + try: + fcntl.flock(fd, (fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) | fcntl.LOCK_NB) + return True + except OSError as e: + if e.errno in (errno.EWOULDBLOCK, errno.EAGAIN): + return False + raise + + +def _again_if_refused(op, *args): + """`op(*args)`, tried a few times where another process having the file + open for a moment refuses it (Windows). Raises the last OSError.""" + import time as _time + tries = 1 if _UNLINK_BEFORE_CLOSE else 3 + for attempt in range(tries): + try: + return op(*args) + except PermissionError: + if attempt == tries - 1: + raise + _time.sleep(0.01) + + +def _drop_marker(p, fd, unlink=True): + """Close a marker this process has open and, with `unlink`, delete it, in + the order this platform needs. Never raises. A delete that stays refused + leaves an unlocked file, which is swept once old.""" + steps = [(_again_if_refused, os.unlink, p)] if unlink else [] + steps.insert(len(steps) if _UNLINK_BEFORE_CLOSE else 0, (os.close, fd)) + for step, *args in steps: + try: + step(*args) + except OSError: + pass + + +def _release_inflight(): + for p, fd in list(_inflight_held.items()): + _drop_marker(p, fd) + _inflight_held.clear() + + +def _sweep_inflight(gd): + """Remove old marker files no review holds.""" + import time as _time + cutoff = _time.time() - _INFLIGHT_GC_AGE_S + for name in os.listdir(gd): + p = os.path.join(gd, name) + if not name.startswith(_INFLIGHT_PREFIX) or p in _inflight_held: + continue + try: + if os.path.getmtime(p) > cutoff: + continue + fd = os.open(p, os.O_RDWR) + except OSError: + continue + try: + unheld = _flock_nb(fd, exclusive=True) + except OSError: + unheld = False + _drop_marker(p, fd, unlink=unheld) + + +def _mark_inflight(repo_root, shas): + """Claim `shas` (full 40-hex) for the review this process is about to + run, until the process ends. Best-effort; never raises. A commit another + live review holds stays with that review.""" + try: + gd = _git_dir(repo_root) + if not gd: + return + _sweep_inflight(gd) + for sha in shas: + p = os.path.join(gd, _INFLIGHT_PREFIX + sha) + if not _is_full_sha(sha) or p in _inflight_held: + continue + try: + fd = _again_if_refused(os.open, p, os.O_RDWR | os.O_CREAT, 0o600) + except OSError: + # One marker that cannot be opened does not cost the others + # their claim. + continue + try: + held = _flock_nb(fd, exclusive=True) + except OSError: + # Cannot lock here at all: leave no file that reads as held. + _drop_marker(p, fd) + return + if held and _UNLINK_BEFORE_CLOSE: + # The inode check: a holder unlinks its marker when it ends, + # and a lock on that unlinked file would claim nothing. Not + # where an open file cannot be unlinked in the first place. + try: + held = os.fstat(fd).st_ino == os.stat(p).st_ino + except OSError: + held = False + if not held: + os.close(fd) + continue + if not _inflight_held: + import atexit + atexit.register(_release_inflight) + _inflight_held[p] = fd + os.utime(p) + except Exception: + pass + + +def _load_inflight_shas(repo_root, shas, _gd=None): + """Which of `shas` another live review in this clone holds right now.""" + out = set() + gd = _gd or _git_dir(repo_root) + if not gd: + return out + for sha in shas: + p = os.path.join(gd, _INFLIGHT_PREFIX + sha) + if not _is_full_sha(sha) or p in _inflight_held: + continue + try: + fd = os.open(p, os.O_RDONLY) + except OSError: + continue + try: + # Shared, so two reviews asking at once do not look like a + # holder to each other. + if not _flock_nb(fd, exclusive=False): + out.add(sha) + except OSError: + pass + os.close(fd) + return out + + +def _reviewed_after_inflight(repo_root, push_range, max_wait_s, poll_s=1.0): + """The reviewed-sha set for a push of `push_range` (oldest first), read + once no other review is still running on the commit the push review + would start from, or after `max_wait_s`. Returns (reviewed, waited_s).""" + import time as _time + gd = _git_dir(repo_root) + start = _time.monotonic() + waited = False + reviewed = _load_reviewed_shas(repo_root, _gd=gd) + while gd: + # Only the first unreviewed commit matters: if nobody is reviewing + # it, the push review covers everything from there on anyway. + first = next((s for s in push_range if s not in reviewed), None) + if first is None: + break + # In-flight before reviewed: a review that ends between the two + # reads then shows up as reviewed rather than as neither. + busy = _load_inflight_shas(repo_root, [first], _gd=gd) + reviewed = _load_reviewed_shas(repo_root, _gd=gd) + if first in reviewed: + continue + if not busy or _time.monotonic() - start >= max_wait_s: + break + waited = True + _time.sleep(poll_s) + return reviewed, (_time.monotonic() - start) if waited else 0.0 + + # ===================================================================== # v2 review-set computation (Stop hook) # ===================================================================== UNTRACKED_BASELINE_CAP = 2000 +# Set in a snapshot that hit the cap, to the time the listing started. NUL +# cannot occur in a path, so the key never collides with a file. +UNTRACKED_CAPPED_AT_KEY = "\0capped_at_ns" +# Coarse filesystem timestamps (1 s on ext3, 2 s on FAT) can stamp a file +# created just after the snapshot as slightly older than it. +_CAPPED_BASELINE_SLACK_NS = 2_000_000_000 def _list_untracked(cwd): @@ -365,6 +1033,7 @@ def _list_untracked(cwd): the holdouts.""" try: repo = _git_toplevel(cwd) or cwd + started_ns = time.time_ns() # core.quotePath=false comes from GIT_CMD globally (see gitutil.py). r = subprocess.run( [*GIT_CMD, "ls-files", "--others", "--exclude-standard", "-z"], @@ -385,6 +1054,10 @@ def _list_untracked(cwd): out[p] = 0 if len(out) >= UNTRACKED_BASELINE_CAP: debug_log(f"_list_untracked: capped at {UNTRACKED_BASELINE_CAP}") + # The files past the cap are not recorded one by one. Record + # when the snapshot was taken so the Stop hook can still tell + # them from files created or edited during the turn. + out[UNTRACKED_CAPPED_AT_KEY] = started_ns break return out except (subprocess.TimeoutExpired, FileNotFoundError, OSError, ValueError) as e: @@ -393,21 +1066,26 @@ def _list_untracked(cwd): debug_log(f"_list_untracked error: {e}") return {} -def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_baseline=None): +def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_baseline=None, + since_ns=None): """v2 diff strategy: derive the review set from git state alone. review_set = (files dirty vs current HEAD, plus files committed this turn when HEAD advanced linearly) ∩ (files whose content differs from the - pre-turn stash baseline). The first term is immune to checkout/pull + pre-turn baseline). The first term is immune to checkout/pull ballooning; the second filters out the user's untouched pre-turn WIP. Falls back to dirty_now alone when no baseline is available. untracked_at_baseline: {repo-root-relative path: mtime_ns} captured at - UPS. `git stash create` doesn't include untracked files, so without this + UPS. The baseline snapshot doesn't include untracked files, so without this snapshot a pre-existing untracked file looks "new since baseline" forever. A file is excluded only if it was untracked at baseline AND its mtime is unchanged — an in-place edit during the turn is still reviewed. + since_ns: for a checkout the prompt hook took no baseline or snapshot of + (the turn moved into it), the time the turn began. A dirty or untracked + file last written before then was already that way and is left out. + Known limitation: a Bash-only turn that's interrupted before Stop fires leaves touched_paths empty, so the next UPS re-baselines past those edits. v1 never reviews Bash-only turns at all, so v2 is no worse there. @@ -430,16 +1108,36 @@ def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_basel if tracked_dirty is None: return [], "HEAD", repo, [], {"dirty_now_count": -1, "changed_since_count": -1, "review_set_count": 0} + capped_at_ns = untracked_at_baseline.get(UNTRACKED_CAPPED_AT_KEY) + def _unchanged_since_baseline(p): base_mtime = untracked_at_baseline.get(p) - if base_mtime is None: - return False try: - return os.stat(os.path.join(repo, p)).st_mtime_ns == base_mtime + st = os.stat(os.path.join(repo, p)) except OSError: return False + if base_mtime is not None: + return st.st_mtime_ns == base_mtime + if not capped_at_ns: + return False + # The snapshot stopped at the cap, so this file may simply be one it + # never reached. Without this, every pre-existing file past the cap + # looks new on every turn and the same files are reviewed again and + # again. A file created or edited during the turn has a later mtime + # or ctime than the snapshot; anything older was already there. + return max(st.st_mtime_ns, st.st_ctime_ns) < capped_at_ns - _CAPPED_BASELINE_SLACK_NS preexisting_unchanged = {p for p in untracked if _unchanged_since_baseline(p)} + if since_ns: + def _written_before_turn(p): + try: + st = os.stat(os.path.join(repo, p)) + except OSError: + return False + return max(st.st_mtime_ns, st.st_ctime_ns) < since_ns - _CAPPED_BASELINE_SLACK_NS + + preexisting_unchanged |= {p for p in untracked if _written_before_turn(p)} + tracked_dirty = {p for p in tracked_dirty if not _written_before_turn(p)} new_untracked = untracked - preexisting_unchanged dirty_now = tracked_dirty | new_untracked @@ -450,11 +1148,11 @@ def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_basel dirty_now |= _git_name_only(repo, f"{head_at_capture}..HEAD") or set() diff_base = head_at_capture - # changed_since: tracked files vs the stash baseline (no temp index — the - # stash never contained untracked files anyway), then union with + # changed_since: tracked files vs the turn baseline (no `add -N` — the + # baseline never contained untracked files anyway), then union with # currently-untracked. The previous `include_untracked=True` arm cost a # full `git add -N .` (slow in large repos) per call to surface - # untracked files in the diff output — but `git diff <stash>` already + # untracked files in the diff output — but `git diff <baseline>` already # lists them as "only in worktree" without that, and we have the explicit # set from status regardless. if baseline_sha: @@ -464,9 +1162,13 @@ def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_basel else: changed_since = None # changed_since is None on missing baseline OR on git error (e.g. the - # dangling stash SHA was pruned). Either way, don't intersect with ∅ — + # dangling baseline SHA was pruned). Either way, don't intersect with ∅ — # that would silently zero the review set. Fall back to dirty_now. review_set = (dirty_now & changed_since) if changed_since is not None else dirty_now + # Kept beside its new name so that the content diff pairs the two. + secret_origins = { + old for new, old in _secret_rename_origins.items() if new in review_set} + review_set = review_set | secret_origins review_paths = [os.path.join(repo, p) for p in sorted(review_set)] untracked_in_review = sorted(new_untracked & review_set) @@ -478,4 +1180,6 @@ def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_basel # Only emit when nonzero to stay under the 10-key telemetry cap. if preexisting_unchanged: metrics["preexisting_untracked_excluded"] = len(preexisting_unchanged) + if secret_origins: + metrics["secret_renamed"] = len(secret_origins) return review_paths, diff_base, repo, untracked_in_review, metrics diff --git a/plugins/security-guidance/hooks/ensure_agent_sdk.py b/plugins/security-guidance/hooks/ensure_agent_sdk.py index f5d016db..80c69c9c 100644 --- a/plugins/security-guidance/hooks/ensure_agent_sdk.py +++ b/plugins/security-guidance/hooks/ensure_agent_sdk.py @@ -27,7 +27,28 @@ from pathlib import Path # → ~/.claude/security. See _base.state_dir for resolution precedence. Re-aliased # here to match the existing local name (state_dir was already a local var in # main() and _maybe_emit_user_notice). -from _base import state_dir as _resolve_state_dir +from _base import ( + launcher_direct_metric, launcher_probe_metrics, portable_output, state_dir as _resolve_state_dir, + write_hook_output, +) +from _base import llm_reviews_switched_off, make_state_dir + + +def _spawn_cwd() -> str: + """Working directory for every Python this file starts. + + `python -c` and `python -m` put the working directory first on the import + path, and hooks run in the user's repository, so a repository file named + `claude_agent_sdk.py`, `venv.py` or `pip.py` would be imported in place + of the real module. The state dir is the plugin's own (the venv is built + there, so the interpreter can reach it); the plugin folder is the + fallback when it can't be created. + """ + d = _resolve_state_dir() + try: + return make_state_dir(d) + except OSError: + return str(Path(__file__).resolve().parent.parent) # Outcome codes for the sdk_bootstrap metric. Values are stable for telemetry. NOOP_SYSTEM = 0 # claude_agent_sdk already importable in system python @@ -55,6 +76,9 @@ SKIP_COOLDOWN = 9 # a recent build was signal-killed (memory pressure) — no # repro confirmed the dominant Linux BUILD_FAILED is a # SIGKILL/SIGSEGV of the memory-heavy venv+pip subprocess # (rc<0, empty streams). See #2154 follow-up. +SKIP_REVIEWS_OFF = 10 # the environment turns off every review that would + # load the SDK (kill switch included), so nothing is + # installed. See #5331. # How long to skip rebuilds after a signal kill. Retries at most once per # window so a machine whose memory frees up still recovers (just not every @@ -231,7 +255,7 @@ def _write_cooldown(state_dir) -> None: """Start/refresh the signal-kill cooldown so we stop re-attempting a build that keeps getting killed every session. Best-effort.""" try: - Path(state_dir).mkdir(parents=True, exist_ok=True) + make_state_dir(str(state_dir)) (Path(state_dir) / "agent-sdk-venv.cooldown").write_text( time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())) except OSError: @@ -311,7 +335,7 @@ def _probe_has_pip() -> bool: try: r = subprocess.run( [sys.executable, "-m", "pip", "--version"], - capture_output=True, timeout=10, + capture_output=True, timeout=10, cwd=_spawn_cwd(), ) return r.returncode == 0 except Exception: @@ -392,6 +416,20 @@ def _pip_err_from_stderr(stderr_b): return f"other:{tail}" if tail else "other" +# A package directory that has lost its files still imports, as a namespace +# package, so a bare `import claude_agent_sdk` calls an emptied install +# healthy and it is never rebuilt. A real install has a __file__; the probe +# exits with _PROBE_EMPTIED when there is none. See #6612. +_PROBE_EMPTIED = 3 +_SDK_IMPORT_PROBE = ( + "import sys, claude_agent_sdk; " + f"sys.exit(0 if getattr(claude_agent_sdk, '__file__', None) else {_PROBE_EMPTIED})" +) + +# Set by main() when the venv's probe exited with _PROBE_EMPTIED. +_venv_was_emptied = False + + def _target_dir(state_dir) -> Path: return Path(state_dir) / "agent-sdk-libs" @@ -407,9 +445,9 @@ def _target_sdk_importable(state_dir) -> bool: try: r = subprocess.run( [sys.executable, "-c", - "import sys; sys.path.insert(0, sys.argv[1]); import claude_agent_sdk", + "import sys; sys.path.insert(0, sys.argv[1]); " + _SDK_IMPORT_PROBE, str(target)], - capture_output=True, timeout=10, + capture_output=True, timeout=10, cwd=_spawn_cwd(), ) return r.returncode == 0 except Exception: @@ -434,7 +472,7 @@ def _build_via_target(state_dir) -> tuple[int, str, str]: "--target", str(target), "--upgrade", "--disable-pip-version-check", "--prefer-binary", "--no-cache-dir", "claude-agent-sdk"], - capture_output=True, timeout=120, check=True, + capture_output=True, timeout=120, check=True, cwd=_spawn_cwd(), ) return BUILT_TARGET, "", "" except subprocess.CalledProcessError as e: @@ -453,6 +491,79 @@ def _build_via_target(state_dir) -> tuple[int, str, str]: return BUILD_FAILED, "pip_target", f"exc:{type(e).__name__}" +def _near_git_repo() -> bool: + """Whether the session's directory is in a git work tree (a `.git` entry + here or above) or directly holds one (a workspace of repositories). The + reviews the SDK serves only run on a repository.""" + try: + top = d = os.path.abspath(os.environ.get("CLAUDE_PROJECT_DIR") or os.getcwd()) + while True: + if os.path.exists(os.path.join(d, ".git")): + return True + parent = os.path.dirname(d) + if parent == d: + break + d = parent + with os.scandir(top) as entries: + for i, entry in enumerate(entries): + if i >= 500: + return True # too large to say; not counted + if entry.is_dir() and os.path.exists(os.path.join(entry.path, ".git")): + return True + return False + except OSError: + return True + + +def _venv_python_version(venv) -> int: + """The Python the venv was built with, as major*100+minor (312), read + from its pyvenv.cfg; 0 if unknown.""" + try: + for line in (Path(venv) / "pyvenv.cfg").read_text(errors="replace").splitlines(): + key, _, value = line.partition("=") + if key.strip() in ("version", "version_info"): + major, minor = value.strip().split(".")[:2] + return int(major) * 100 + int(minor) + except (OSError, ValueError): + pass + return 0 + + +def _install_context_metrics(outcome, state_dir) -> dict: + """Counts that size open questions about the private SDK install; no + behaviour depends on them. Only when set, and the first three only on + rows where the install exists. + + sdk_no_creds / sdk_no_git: the session holds the install while the + reviews it serves cannot run, for lack of model credentials or of a git + repository. Should the build wait for those? See #5331. + + sdk_venv_py: the venv was built by another Python version than the one + running the hooks, which is the one that loads it; its compiled + dependencies do not load there. See #5706. + + sdk_venv_emptied: the venv's SDK package directory had lost its files + and this run went on to rebuild it, whatever the outcome. See #6612.""" + out = {} + if _venv_was_emptied: + out["sdk_venv_emptied"] = 1 + if outcome not in (NOOP_VENV, BUILT, BUILT_TARGET, NOOP_TARGET): + return out + try: + import llm + if not llm.HAS_API_CREDENTIALS: + out["sdk_no_creds"] = 1 + except Exception: + pass + if not _near_git_repo(): + out["sdk_no_git"] = 1 + if outcome == NOOP_VENV: + built_by = _venv_python_version(Path(state_dir) / "agent-sdk-venv") + if built_by and built_by != sys.version_info[0] * 100 + sys.version_info[1]: + out["sdk_venv_py"] = built_by + return out + + def _sdk_on_syspath() -> bool: # find_spec is ~10ms; actually importing the SDK pulls in # transitive deps and costs ~800ms — too heavy for a @@ -481,6 +592,15 @@ def main() -> tuple[int, str, str]: err_phase / err_kind are non-empty only on BUILD_FAILED — they let telemetry split bootstrap failures by root cause. """ + global _venv_was_emptied + _venv_was_emptied = False + # The SDK only serves the LLM reviews. With those switched off, or the + # whole plugin (SECURITY_GUIDANCE_DISABLE=1), a venv of a few hundred MB + # would be built for code that never runs, and the notice below would + # advertise a reviewer the user turned off. See #5331. + if llm_reviews_switched_off(): + return SKIP_REVIEWS_OFF, "", "" + # Honesty check (fixes the misleading NOOP_VENV in #2071): the SDK # requires Python >=3.10 and uses 3.10+ syntax (match statements, # PEP 604 unions). On a 3.9 hook interpreter we CANNOT import it no @@ -533,11 +653,12 @@ def main() -> tuple[int, str, str]: if venv_py.exists(): try: r = subprocess.run( - [str(venv_py), "-c", "import claude_agent_sdk"], - capture_output=True, timeout=10, + [str(venv_py), "-c", _SDK_IMPORT_PROBE], + capture_output=True, timeout=10, cwd=_spawn_cwd(), ) if r.returncode == 0: return NOOP_VENV, "", "" + _venv_was_emptied = r.returncode == _PROBE_EMPTIED except Exception: pass # broken venv; rebuild below @@ -559,7 +680,7 @@ def main() -> tuple[int, str, str]: err_kind = "" we_own_sentinel = False try: - state_dir.mkdir(parents=True, exist_ok=True) + make_state_dir(str(state_dir)) # O_EXCL makes the sentinel an atomic lock — if two SessionStarts # race past the exists() check above, only one creates it. try: @@ -570,7 +691,7 @@ def main() -> tuple[int, str, str]: err_phase = "venv" subprocess.run( [sys.executable, "-m", "venv", "--clear", str(venv)], - capture_output=True, timeout=60, check=True, + capture_output=True, timeout=60, check=True, cwd=_spawn_cwd(), ) # Some machines route pip through a private registry; we # don't pass --index-url here so we inherit that default. Outside @@ -594,7 +715,7 @@ def main() -> tuple[int, str, str]: [str(venv_py), "-m", "pip", "install", "--quiet", "--disable-pip-version-check", "--prefer-binary", "--no-cache-dir", "claude-agent-sdk"], - capture_output=True, timeout=120, check=True, + capture_output=True, timeout=120, check=True, cwd=_spawn_cwd(), ) return BUILT, "", "" except subprocess.CalledProcessError as e: @@ -735,7 +856,7 @@ def _maybe_emit_user_notice(outcome: int, pv: int) -> str | None: marker = state_dir / f".agentic_unavailable_notice_v{pv or 0}" if marker.exists(): return None - state_dir.mkdir(parents=True, exist_ok=True) + make_state_dir(str(state_dir)) # Write timestamp + Python version so the marker is self-documenting # if a user goes looking. O_EXCL would be racier with no real win # (two concurrent SessionStarts both showing the notice once is fine). @@ -765,7 +886,11 @@ if __name__ == "__main__": # SessionStart runs before the user's first prompt; doing this in the # background means the first commit-review of the session usually finds # the venv ready. - print(json.dumps({"async": True, "asyncTimeout": 180000}), flush=True) + # + # Not printed in portable mode: a host that reads stdout as one JSON + # document rejects a second line, and runs this hook to completion anyway. + if not portable_output(): + print(json.dumps({"async": True, "asyncTimeout": 180000}), flush=True) t0 = time.perf_counter() try: outcome, err_phase, err_kind = main() @@ -842,9 +967,16 @@ if __name__ == "__main__": # HOOK_PY_INCOMPATIBLE (whose "py_3.9" err_kind otherwise collapses to # err=99, losing the version). Cheap — no subprocess, just sys.version_info. metrics["sdk_hook_py"] = sys.version_info[0] * 100 + sys.version_info[1] + metrics.update(_install_context_metrics(outcome, _resolve_state_dir())) pv = _plugin_version_int() if pv: metrics["pv"] = pv + # See sg-python.sh pass 4: started by real path, the PATH names could + # not read the plugin files. + if os.environ.get("SG_LAUNCHER_FALLBACK") == "1": + metrics["launcher_fallback"] = 1 + metrics.update(launcher_probe_metrics()) + metrics.update(launcher_direct_metric()) response: dict[str, object] = {"metrics": metrics} # One-time user-visible notice when the agentic reviewer is dead on # arrival. Uses hookSpecificOutput.additionalContext (SessionStart's @@ -859,4 +991,4 @@ if __name__ == "__main__": "additionalContext": notice, } response["systemMessage"] = notice - print(json.dumps(response), flush=True) + write_hook_output(response) diff --git a/plugins/security-guidance/hooks/extensibility.py b/plugins/security-guidance/hooks/extensibility.py index a9c7f8fe..d27e9173 100644 --- a/plugins/security-guidance/hooks/extensibility.py +++ b/plugins/security-guidance/hooks/extensibility.py @@ -12,11 +12,13 @@ Two extensibility points, both additive only: Discovery, in precedence order (matching CLAUDE.md / settings.json): - ``~/.claude/<name>`` (user) + - ``$CLAUDE_CONFIG_DIR/<name>`` (user, when set to another directory) - ``<cwd>/.claude/<name>`` (project, committed) - ``<cwd>/.claude/<name>.local.<ext>`` (project local, gitignored) Managed delivery via ``managed-settings.json`` is not yet supported. -Org admins can still push files to ``~/.claude/`` via MDM/GPO. +Org admins can still push files to ``~/.claude/`` (or the directory +``CLAUDE_CONFIG_DIR`` names) via MDM/GPO. Trust model: - The ``.md`` is repo-controlled and goes into the USER prompt (not system), @@ -28,6 +30,9 @@ Trust model: built-in ones. Reminder length is capped. - Custom regexes are validated at load for catastrophic-backtracking structure and skipped (with a debug log) if they look ReDoS-prone. + - A project file that is a link, or sits under a ``.claude`` that is one, is + skipped (with a debug log). User files may be links, as dotfile managers + make them. - Built-in patterns cannot be disabled. ``ENABLE_PATTERN_RULES=0`` disables all pattern checks; there is no per-rule kill switch in v1. """ @@ -89,11 +94,32 @@ def user_patterns() -> List[Dict[str, Any]]: # ── claude-security-guidance.md ─────────────────────────────────────────────────────── +def _user_config_dirs() -> List[str]: + """User-level config directories. ``~/.claude`` always, so a file placed + there keeps loading; then ``CLAUDE_CONFIG_DIR`` when it names another + directory. Empty is treated as not set, as in ``_base.state_dir``.""" + dirs = [os.path.expanduser(os.path.join("~", ".claude"))] + configured = (os.environ.get("CLAUDE_CONFIG_DIR") or "").strip() + if configured: + configured = os.path.expanduser(configured) + if _same_dir(configured, dirs[0]): + return dirs + dirs.append(configured) + return dirs + + +def _same_dir(a: str, b: str) -> bool: + try: + return os.path.samefile(a, b) + except OSError: + return os.path.normcase(os.path.abspath(a)) == os.path.normcase(os.path.abspath(b)) + + def _config_paths(cwd: Optional[str], basename: str) -> List[Tuple[str, str]]: """Existing config file paths, lowest precedence first (so concat reads in precedence order user → project → project-local). Truncation is done on the concatenated string, so lowest-precedence content is dropped last.""" - paths = [("User", os.path.expanduser(os.path.join("~", ".claude", basename)))] + paths = [("User", os.path.join(d, basename)) for d in _user_config_dirs()] if cwd: paths.append(("Project", os.path.join(cwd, ".claude", basename))) # claude-security-guidance.local.md / security-patterns.local.yaml @@ -102,13 +128,30 @@ def _config_paths(cwd: Optional[str], basename: str) -> List[Tuple[str, str]]: return paths +def _readable_config(path: str, cwd: Optional[str]) -> bool: + """Whether a config file may be read. A project file (`cwd` given) must be + a regular file at that very place: a link, or a `.claude` that is one, + would bring in text from wherever it points.""" + if cwd is None or not os.path.lexists(path): + return True + expected = os.path.join(os.path.realpath(cwd), os.path.relpath(path, cwd)) + real = os.path.realpath(path) + if os.path.normcase(real) == os.path.normcase(expected) and os.path.isfile(real): + return True + debug_log(f"extensibility: skipping {path}: a link, or not a regular file") + return False + + def _load_guidance(cwd: Optional[str]) -> str: parts = [] for label, path in _config_paths(cwd, GUIDANCE_BASENAME): + if not _readable_config(path, cwd if label.startswith("Project") else None): + continue try: with open(path, encoding="utf-8") as f: txt = f.read().strip() - except OSError: + except (OSError, UnicodeDecodeError) as e: + debug_log(f"extensibility: skipping {path}: {type(e).__name__}") continue if txt: parts.append(f"### {label} security guidance\n{txt}") @@ -125,9 +168,14 @@ def _load_guidance(cwd: Optional[str]) -> str: return combined +_GUIDANCE_END_TAG = re.compile(r"<(\s*/\s*project-security-guidance)", re.IGNORECASE) + + def _wrap_guidance(guidance: str) -> str: if not guidance: return "" + # The block ends at its own closing tag, not at one in the text it holds. + guidance = _GUIDANCE_END_TAG.sub(r"<\1", guidance) return ( "\n\n<project-security-guidance>\n" "The user has provided project-specific security guidance below. " @@ -152,13 +200,18 @@ def _load_user_patterns(cwd: Optional[str]) -> List[Dict[str, Any]]: # try each supported extension. for ext in (".yaml", ".yml", ".json"): candidate = path + ext - data = _read_config(candidate) - if data is None: + # One unusable file is skipped on its own, so the rules from the + # other locations still load. + try: + if not _readable_config(candidate, cwd if label.startswith("Project") else None): + continue + found = _rules_from_file(candidate, label) + except Exception as e: + debug_log(f"extensibility: skipping {candidate}: {type(e).__name__}: {e}") + continue + if found is None: continue - for entry in (data or {}).get("patterns", []): - rule = _validate_pattern(entry, source=label) - if rule: - rules.append(rule) + rules.extend(found) break # found one extension; don't double-load .yaml AND .json if len(rules) >= PATTERN_MAX_RULES: break @@ -168,8 +221,32 @@ def _load_user_patterns(cwd: Optional[str]) -> List[Dict[str, Any]]: return rules +def _rules_from_file(path: str, label: str) -> Optional[List[Dict[str, Any]]]: + """The valid rules in one config file, or None when the file is missing + or does not hold a mapping with a ``patterns`` list.""" + data = _read_config(path) + if data is None: + return None + entries = data.get("patterns", []) + if entries is None: + entries = [] + if not isinstance(entries, list): + debug_log( + f"extensibility: skipping {path}: 'patterns' must be a list, " + f"got {type(entries).__name__}" + ) + return None + rules = [] + for entry in entries: + rule = _validate_pattern(entry, source=label) + if rule: + rules.append(rule) + return rules + + def _read_config(path: str) -> Optional[Dict[str, Any]]: - """Read a YAML or JSON config file. Returns None on missing/malformed.""" + """Read a YAML or JSON config file. Returns None on missing/malformed, + and when the file parses to something other than a mapping.""" try: with open(path, encoding="utf-8") as f: raw = f.read() @@ -179,7 +256,7 @@ def _read_config(path: str) -> Optional[Dict[str, Any]]: return None if path.endswith(".json"): try: - return json.loads(raw) + return _mapping_or_none(path, json.loads(raw)) except ValueError as e: debug_log(f"extensibility: skipping {path}: invalid JSON: {e}") return None @@ -190,12 +267,26 @@ def _read_config(path: str) -> Optional[Dict[str, Any]]: debug_log(f"extensibility: skipping {path}: PyYAML not installed (use .json)") return None try: - return yaml.safe_load(raw) + return _mapping_or_none(path, yaml.safe_load(raw)) except yaml.YAMLError as e: # type: ignore debug_log(f"extensibility: skipping {path}: invalid YAML: {e}") return None +def _mapping_or_none(path: str, data: Any) -> Optional[Dict[str, Any]]: + """A config file must hold a mapping. A scalar or a list is valid YAML or + JSON too (e.g. the pattern list written without its ``patterns:`` key); + skip that one file so the other locations still load.""" + if isinstance(data, dict): + return data + if data is not None: + debug_log( + f"extensibility: skipping {path}: expected a mapping, " + f"got {type(data).__name__}" + ) + return None + + def _validate_pattern(entry: Any, source: str) -> Optional[Dict[str, Any]]: """Validate one user pattern entry. Returns a rule dict in the same shape as the built-in SECURITY_PATTERNS, or None if invalid (logged).""" @@ -227,7 +318,7 @@ def _validate_pattern(entry: Any, source: str) -> Optional[Dict[str, Any]]: try: rule["regex"] = regex re.compile(regex) - except re.error as e: + except (re.error, OverflowError, RecursionError) as e: debug_log(f"extensibility: skipping {name}: invalid regex: {e}") return None diff --git a/plugins/security-guidance/hooks/gitutil.py b/plugins/security-guidance/hooks/gitutil.py index b6e8cd54..c80d0600 100644 --- a/plugins/security-guidance/hooks/gitutil.py +++ b/plugins/security-guidance/hooks/gitutil.py @@ -15,15 +15,51 @@ Functions that DO compose patched leaves (``compute_v2_review_set``, ``security_reminder_hook.py`` for that reason. """ import contextlib +import errno import os import re import subprocess +import sys +import tempfile +import threading +import time from _base import debug_log +def _git_program(windows=os.name == "nt"): + """How to start git. On Windows a bare name is looked up in the process's + working directory before PATH, and the hooks run in the project, so a + `git.exe` there would start in place of the installed one. Name the first + git.exe on PATH instead, leaving out relative entries, the working + directory, and folders inside the project (unless the project holds the + home folder, and with it the user's own tools). With none, a path that + does not exist: git is reported missing, as it would be anywhere else.""" + if not windows: + return "git" + def norm(p): + return os.path.normcase(os.path.normpath(p)) + try: + cwd = norm(os.getcwd()) + except OSError: + cwd = "" + home = norm(os.path.expanduser("~")) + os.sep + roots = [norm(r) for r in (cwd, os.environ.get("CLAUDE_PROJECT_DIR")) if r] + roots = [r.rstrip(os.sep) + os.sep for r in roots if not home.startswith(r.rstrip(os.sep) + os.sep)] + for d in os.environ.get("PATH", "").split(os.pathsep): + d = d.strip().strip('"') + if not d or not os.path.isabs(d) or norm(d) == cwd: + continue + if any((norm(d) + os.sep).startswith(r) for r in roots): + continue + path = os.path.join(d, "git.exe") + if os.path.isfile(path): + return path + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "git.exe") + + GIT_CMD = [ - "git", + _git_program(), "-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", # core.quotePath=false: emit raw UTF-8 in path-emitting commands instead @@ -37,13 +73,52 @@ GIT_CMD = [ # reflog %gs subjects, ls-files, status, etc. — without per-site # flag duplication. See #2082, #2099. "-c", "core.quotePath=false", + # A work-tree `git diff` that meets stat-stale entries (mtime changed, + # content same) rewrites the index under index.lock, GIT_OPTIONAL_LOCKS or + # not. Off, every diff here only reads the index, so none needs a copy. + "-c", "diff.autoRefreshIndex=false", + # The diff parsers read `diff --git a/<old> b/<new>` headers. A user's + # diff.mnemonicPrefix (c/ w/ i/), diff.noprefix or diff.srcPrefix / + # diff.dstPrefix changes that shape, and every file would be dropped. + "-c", "diff.mnemonicPrefix=false", + "-c", "diff.noprefix=false", + "-c", "diff.srcPrefix=a/", + "-c", "diff.dstPrefix=b/", + # A directory with its own `.git` inside the work tree (a submodule, a + # vendored checkout, an unpacked archive) is another repository with its + # own config. To report it as modified, or to show its changes inline + # (diff.submodule=diff), git starts a second git inside it, which reads + # that config and runs the commands it names. Such a directory is left + # out of the review set, and so is a commit that moves one. See also + # _drop_gitlinks for the commands that take no such setting. + "-c", "diff.ignoreSubmodules=all", + "-c", "diff.submodule=short", + # `git log` and `git show` print commit metadata for the hooks to parse. + # With log.showSignature set they would also verify each signed commit, + # adding lines to the output and starting the program gpg.program names. + "-c", "log.showSignature=false", ] +# diff.ignoreSubmodules is a default that the repository's own settings for +# one submodule (.gitmodules, submodule.<name>.ignore) override. The option +# cannot be overridden. Pass it to every `git status` and work-tree `git diff`. +IGNORE_SUBMODULES = "--ignore-submodules=all" + SAFE_GIT_CONFIG = ( ("core.fsmonitor", "false"), ("core.hooksPath", "/dev/null"), + ("diff.ignoreSubmodules", "all"), + ("diff.submodule", "short"), + ("log.showSignature", "false"), ) +# The hook's git runs in the background beside the session's own git. Without +# this, `git status` holds .git/index.lock while it writes back a refreshed +# index: a concurrent `git add` fails, and a timeout kill leaves the lock. +SAFE_GIT_ENV = { + "GIT_OPTIONAL_LOCKS": "0", +} + def git_config_env(pairs, base=None): base = os.environ if base is None else base @@ -61,6 +136,51 @@ def git_config_env(pairs, base=None): def apply_safe_git_env(): os.environ.update(git_config_env(SAFE_GIT_CONFIG)) + os.environ.update(SAFE_GIT_ENV) + + +def index_entry_count(path): + """The entry count in an index file's header (signature, version, count), + or None if it cannot be read. A split or sparse index holds fewer entries + than the repository has tracked files.""" + try: + with open(path, "rb") as f: + header = f.read(12) + except OSError: + return None + if header[:4] == b"DIRC" and len(header) == 12: + return int.from_bytes(header[8:12], "big") + return None + + +def windows_drive_mount(path, mountinfo): + """Whether `path` is on a Windows drive mounted into WSL, given the text + of /proc/self/mountinfo. Every file status there is a round trip to the + Windows host.""" + best, best_type = "", "" + for line in mountinfo.splitlines(): + # id parent dev root mountpoint options [tags] - fstype source options + before, _, after = line.partition(" - ") + fields, tail = before.split(), after.split() + if len(fields) < 5 or not tail: + continue + point = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4]) + inside = path == point or path.startswith(point.rstrip("/") + "/") + if inside and len(point) >= len(best): + best, best_type = point, tail[0] + return best_type in ("9p", "drvfs", "virtiofs") + + +def on_windows_drive(path): + """windows_drive_mount for this machine. False anywhere but WSL.""" + try: + with open("/proc/sys/kernel/osrelease") as f: + if "microsoft" not in f.read().lower(): + return False + with open("/proc/self/mountinfo") as f: + return windows_drive_mount(os.path.realpath(path), f.read()) + except (OSError, ValueError): + return False def _git_rev_parse_head(cwd): @@ -110,6 +230,35 @@ def _find_git_index(cwd): return None +def _find_git_index_and_top(cwd): + """_find_git_index plus the work tree's top directory, from one git + call. (None, None) when either is missing: no index file, a bare + repository, or a `cwd` inside the git directory.""" + try: + result = subprocess.run( + [*GIT_CMD, "rev-parse", "--git-dir", "--show-toplevel"], + cwd=cwd, capture_output=True, timeout=5 + ) + if result.returncode != 0: + return None, None + # Paths as bytes from git: decode by hand (see _find_git_index). + out = result.stdout.decode("utf-8", errors="replace") + parts = out[:-1].split("\n") if out.endswith("\n") else [] + if len(parts) != 2 or not all(parts): + # A newline in one of the paths: ask for them one at a time. + index_path, top = _find_git_index(cwd), _git_toplevel(cwd) + return (index_path, top) if index_path and top else (None, None) + git_dir, top = parts + if not os.path.isabs(git_dir): + git_dir = os.path.join(cwd, git_dir) + index_path = os.path.join(git_dir, "index") + if not os.path.isfile(index_path): + return None, None + return index_path, top + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + return None, None + + def _diff_pathspec(cwd, paths): """Convert absolute touched-paths to repo-relative pathspec args for git diff. Paths outside cwd (e.g. ~/.claude/…) are dropped. Returns the @@ -131,32 +280,528 @@ def _diff_pathspec(cwd, paths): return ["--"] + rel if rel else [] +# Bytes of file paths per git call once a path list has proved too long for +# one command. Windows caps a whole command line at 32,767 characters; +# elsewhere the limit is about 1 MB (macOS) or 2 MB (Linux), environment +# included. +_MAX_PATH_ARGS_BYTES = ( + 20000 if sys.platform in ("win32", "cygwin", "msys") else 500000) +# More commands than this and the path list is not worth diffing: each one +# matches its paths against the whole index, and a set that large is thrown +# away as pathological after the diff anyway. +_MAX_PATH_BATCHES = 4 + + +def _arg_cost(path): + # The exec limit also counts a terminator and a pointer per argument. + return len(os.fsencode(path)) + 9 + + +def _path_args_budget(env=None): + """_MAX_PATH_ARGS_BYTES, or less where this system's own limit is lower + (a small stack limit, a BSD, a very large environment). Half of what the + system leaves after the environment, so the rest of the command fits.""" + try: + arg_max = os.sysconf("SC_ARG_MAX") + except (AttributeError, ValueError, OSError): + return _MAX_PATH_ARGS_BYTES + env_bytes = sum(len(k) + len(v) + 2 for k, v in (env or os.environ).items()) + return min(_MAX_PATH_ARGS_BYTES, max(4096, (arg_max - env_bytes) // 2)) + + +def _path_batches(paths, budget=None): + if budget is None: + budget = _MAX_PATH_ARGS_BYTES + batch, size = [], 0 + for p in paths: + n = _arg_cost(p) + if batch and size + n > budget: + yield batch + batch, size = [], 0 + batch.append(p) + size += n + if batch: + yield batch + + +# Why a path list too long for one command left the last get_git_diff call +# without a diff, for the Stop review's skip reason: "gave_up" (still too +# long once cut down) or "none_reviewable". +_path_list_too_long = set() + +# A failed `git add --intent-to-add` during this hook run, for the Stop hook's +# metrics: kind * 100000 + the paths the add was given. One number because a +# review row is near CC's key cap. kind: 1 git could not write the empty blob +# to the object store and the retry failed too, 2 any other git error, 3 +# timeout, 4 the retry with a private object directory worked. A git that +# speaks another language reports a failed retry as 2. With 1 to 3 the new +# files are not in the diff, so a turn that only added files looks like a turn +# with no changes. `left_out` says so for the last get_git_diff alone: the code +# stays set through a fallback diff that worked. +_add_failed = {"code": 0, "left_out": False} + +# Next to an index copy: where a retried `git add -N` stores the empty blob. +_OBJECTS_DIR_SUFFIX = ".objects" + + +def _too_many_args(e): + """The OS refused to start the command because its arguments are too + long. Windows reports ERROR_FILENAME_EXCED_RANGE, which Python turns into + a FileNotFoundError.""" + return e.errno == errno.E2BIG or getattr(e, "winerror", None) == 206 + + +def _run_captured(cmd, timeout, **kwargs): + return subprocess.run(cmd, capture_output=True, timeout=timeout, **kwargs) + + +def _run_git_on_review_paths(args, paths, timeout, run=_run_captured, **kwargs): + """Run `git <args> -- <paths>` for the Stop review; return the results. + `run(cmd, timeout, **kwargs)` starts each command (_run_diff_bounded for + the diff, so its output is never held whole). + + One command with every path whenever the OS takes it. When the OS refuses + the argument list as too long, only reviewable source paths are kept (the + review drops the rest from the diff anyway) and they go out in as few + commands as they need, all within `timeout` seconds together. If even + those need more than _MAX_PATH_BATCHES commands, the refusal is raised + again. + + Secret-shaped paths that are gone from the work tree go into every one + of those commands: `git diff` must still pair a rename away from such a + name, or the secret's content would show as a new source file. A new file + that only resembles the secret may be paired with it too and left out of + the review. Any other rename is paired only when both of its paths are + reviewable source and in the same command; otherwise it shows as a delete + or an add. + """ + cmd = [*GIT_CMD, *args, "--"] + paths = list(paths) + deadline = time.monotonic() + timeout + try: + return [run(cmd + paths, timeout, **kwargs)] + except OSError as e: + if not _too_many_args(e): + raise + too_long = e + budget = _path_args_budget(kwargs.get("env")) + batches = list(_path_batches( + [p for p in paths if _is_reviewable_source(p)], budget)) + root = kwargs.get("cwd") or os.getcwd() + secret = [p for p in paths if _is_secret_shaped(p) + and not os.path.lexists(os.path.join(root, p))] + debug_log(f"git {args[0]}: {len(paths)} paths do not fit one command; " + f"{len(batches)} command(s) for the reviewable ones") + # The secret-shaped paths ride along with every batch, so they get a + # small share of a command of their own. + if (len(batches) > _MAX_PATH_BATCHES + or sum(map(_arg_cost, secret)) > budget // 4): + raise too_long + cmd += secret + results = [] + for batch in batches: + left = deadline - time.monotonic() + if left <= 0: + raise subprocess.TimeoutExpired(cmd, timeout) + results.append(run(cmd + batch, left, **kwargs)) + return results + + +_TEMP_INDEX_PREFIX = "security_hook_idx_" +# A copy is named <prefix>pid<hook pid>_<8 random chars>, so a later run can +# tell that the hook which made it is gone. Names from earlier releases have +# only the 8 random characters and never match. +_OWNER_PID = re.compile(re.escape(_TEMP_INDEX_PREFIX) + r"pid(\d+)_[^.]{8}(?:\.|$)") +# A live copy is in use for under a minute (a couple of bounded git calls), so +# anything this old belongs to a run that was killed. +_TEMP_INDEX_STALE_S = 1800 +# Unlinking index-sized files is slow; a large backlog drains over several +# calls instead of stalling one hook past its timeout. +_SWEEP_BUDGET_S = 2 + + +def _owner_is_gone(name): + """True when `name` records the pid of the hook that made it and no such + process exists. A reused pid reads as alive: the file then waits for the + age limit. Never probes on Windows, where os.kill would end the process.""" + if not hasattr(os, "getuid"): + return False + m = _OWNER_PID.match(name) + if not m: + return False + try: + os.kill(int(m.group(1)), 0) + except ProcessLookupError: + return True + except (OSError, OverflowError, ValueError): + return False + return False + + +def _sweep_stale_indexes(dirpath, uid): + """Delete index copies in `dirpath` that a killed run left behind, and + their private object directories. Only regular files and directories owned + by `uid` (None: no ownership check) are touched. + + A copy goes as soon as the hook that made it is gone: a killed hook can + leave one per run, and index-sized files fill a disk long before the age + limit. The age limit covers names without a pid and reused pids. + + Age is the newer of mtime and ctime: a copy keeps .git/index's mtime, + which may be days old, so mtime alone would delete a file that another + running hook is still using.""" + import stat + import time + cutoff = time.time() - _TEMP_INDEX_STALE_S + deadline = time.monotonic() + _SWEEP_BUDGET_S + try: + names = os.listdir(dirpath) + except OSError: + return + for name in names: + if not name.startswith(_TEMP_INDEX_PREFIX): + continue + p = os.path.join(dirpath, name) + try: + st = os.lstat(p) + is_dir = stat.S_ISDIR(st.st_mode) and name.endswith(_OBJECTS_DIR_SUFFIX) + if not (is_dir or stat.S_ISREG(st.st_mode)): + continue + if uid is not None and st.st_uid != uid: + continue + if max(st.st_mtime, st.st_ctime) < cutoff or _owner_is_gone(name): + if is_dir: + _remove_objects_dir(p) + else: + os.unlink(p) + if time.monotonic() > deadline: + return + except OSError: + pass + + +def _sweep_stale_stash_indexes(real_index): + """Delete `<index>.stash.<pid>` files a killed `git stash create` left + next to the real index. Releases before 2.0.10 ran it there, so a clone + can hold hundreds, each the size of the index, and nothing else removes + them. Only our own regular files past the age limit go: git keeps one for + seconds. `index.lock` is never touched; a live git may hold it.""" + import stat + import time + uid = os.getuid() if hasattr(os, "getuid") else None + cutoff = time.time() - _TEMP_INDEX_STALE_S + deadline = time.monotonic() + _SWEEP_BUDGET_S + dirpath = os.path.dirname(real_index) + stale = re.compile(re.escape(os.path.basename(real_index)) + r"\.stash\.\d+$") + try: + names = os.listdir(dirpath) + except OSError: + return + for name in names: + if not stale.match(name): + continue + p = os.path.join(dirpath, name) + try: + st = os.lstat(p) + if not stat.S_ISREG(st.st_mode): + continue + if uid is not None and st.st_uid != uid: + continue + if max(st.st_mtime, st.st_ctime) < cutoff: + os.unlink(p) + if time.monotonic() > deadline: + return + except OSError: + pass + + +# Index copies this process has made and not yet removed. +_live_copies = set() + +# A hook working on a copy needs up to three times the index's size for a +# moment (the copy, and git's rewrite and stash index next to it). With many +# agents in one clone that alone can fill a disk, so only so many hooks per +# user work on a copy at once; the others wait, then go without. +_MAX_LIVE_COPIES = 8 +_SLOT_WAIT_S = 5 + + +def _copy_slots(tmpdir, index_bytes): + """How many hooks may work on a copy at once: what fits into a quarter of + the free space at three times the index each, from 1 to + _MAX_LIVE_COPIES.""" + import shutil + try: + free = shutil.disk_usage(tmpdir).free + except OSError: + return 1 + return max(1, min(_MAX_LIVE_COPIES, free // (12 * max(index_bytes, 1)))) + + +@contextlib.contextmanager +def _copy_slot(tmpdir, index_bytes=0): + """Hold one of the copy slots; yields False when none came free in + _SLOT_WAIT_S. A slot is an flock, so the kernel frees it when its holder + dies, however it dies. No cap without flock or a private dir.""" + import time + try: + import fcntl + except ImportError: + fcntl = None + if fcntl is None or tmpdir is None: + yield True + return + deadline = time.monotonic() + _SLOT_WAIT_S + held = None + while held is None: + for i in range(_copy_slots(tmpdir, index_bytes)): + try: + fd = os.open(os.path.join(tmpdir, f"slot{i}.lock"), + os.O_RDWR | os.O_CREAT, 0o600) + except OSError: + continue + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + held = fd + break + except OSError: + os.close(fd) + if held is None: + if time.monotonic() >= deadline: + break + time.sleep(0.1) + try: + yield held is not None + finally: + if held is not None: + os.close(held) + + +def _remove_objects_dir(path): + """Remove a private object directory. git writes objects read-only, which + stops Windows from deleting them.""" + import shutil + import stat + + def writable_then_retry(func, p, _exc): + # Not a link: chmod would follow it to whatever it points at. + if func in (os.unlink, os.rmdir) and not os.path.islink(p): + os.chmod(p, stat.S_IWRITE) + func(p) + + try: + shutil.rmtree(path, onerror=writable_then_retry) + except OSError: + pass + + +def _remove_copy(tmp_index): + """Remove an index copy and the files git wrote next to it.""" + import glob + for p in glob.glob(glob.escape(tmp_index) + "*"): + try: + os.unlink(p) + except OSError: + pass + _remove_objects_dir(tmp_index + _OBJECTS_DIR_SUFFIX) + + +def remove_copies_on_termination(): + """Remove this process's index copies when it is told to terminate. + Python's default SIGTERM action ends the process without running + `finally`, which left an index-sized file behind each time.""" + import signal + + def _on_term(signum, _frame): + for tmp_index in list(_live_copies): + _remove_copy(tmp_index) + os._exit(128 + signum) + + try: + signal.signal(signal.SIGTERM, _on_term) + except (ValueError, OSError, AttributeError): + pass # not the main thread, or no SIGTERM here + + +def _hook_tmpdir(): + """Private per-user directory for throwaway index copies, or None to use + the bare temp dir. + + An index copy can be hundreds of MB and a killed hook cannot clean up, so + copies live in one 0700 directory and anything stale is swept on entry, + here and in the bare temp dir. The name is predictable and the temp dir + may be shared, so anything that is not a real directory we own is + refused. Windows has no uid and its temp dir is already per-user.""" + import stat + import tempfile + base = tempfile.gettempdir() + uid = os.getuid() if hasattr(os, "getuid") else None + _sweep_stale_indexes(base, uid) + if uid is None: + return None + d = os.path.join(base, f"claude-security-guidance-{uid}") + try: + os.mkdir(d, 0o700) + except FileExistsError: + pass + except OSError: + return None + try: + st = os.lstat(d) + except OSError: + return None + if not stat.S_ISDIR(st.st_mode) or st.st_uid != uid: + return None + try: + os.chmod(d, 0o700) + except OSError: + pass + _sweep_stale_indexes(d, uid) + return d + + +def _note_add_failure(kind, paths): + _add_failed["code"] = kind * 100000 + min(len(paths), 99999) + _add_failed["left_out"] = kind != 4 + + +def _env_with_private_objects(env, cwd, objects_dir): + """`env` for a git that writes objects to `objects_dir` and still reads + the repository's own, wherever git says they are (a linked worktree shares + them). A path holding the list separator is left out, since git would + split it; the add then fails only if it has to read an object.""" + out = {**env, "GIT_OBJECT_DIRECTORY": objects_dir} + try: + r = subprocess.run([*GIT_CMD, "rev-parse", "--git-path", "objects"], + cwd=cwd, capture_output=True, timeout=5, env=env) + except (subprocess.TimeoutExpired, OSError): + return out + own = os.path.join(cwd, os.fsdecode(r.stdout).strip()) + if r.returncode == 0 and os.pathsep not in own: + out["GIT_ALTERNATE_OBJECT_DIRECTORIES"] = os.pathsep.join( + p for p in (own, env.get("GIT_ALTERNATE_OBJECT_DIRECTORIES")) if p) + return out + + +def _add_new_files(cwd, paths, env, tmp_index): + """`git add --intent-to-add` the untracked `paths` to the index copy `env` + names, so the diff shows them. The add has to store the empty blob, so a + user who cannot write the object store gets a second try that stores it in + a private directory next to the copy. Nothing needs that blob afterwards: + `git diff` reads these files from the work tree. `git stash create` would + fail on such an index, and only runs on a copy with nothing added. + + No stdout used here (only returncode matters), but text=True still spawns + reader threads that decode stderr — git error messages can reference + non-ASCII filenames and crash on cp1252. See #2099. Drop text=True so bytes + stay raw.""" + # The paths are file names, not patterns, for this git add only: an + # untracked file named `:(glob)**` must add only itself. git refuses that + # together with a glob or icase setting from the user's environment. + env = {k: v for k, v in env.items() + if k not in ("GIT_GLOB_PATHSPECS", "GIT_ICASE_PATHSPECS")} + env["GIT_LITERAL_PATHSPECS"] = "1" + + def add(add_env): + results = _run_git_on_review_paths( + ["add", "--intent-to-add"], paths, timeout=10, cwd=cwd, env=add_env) + return next((r for r in results if r.returncode != 0), None) + + try: + failed = add(env) + except subprocess.TimeoutExpired: + # No retry: git is slow here, not refused. The diff goes on for the + # tracked files, so a timeout on every turn cannot stop their reviews. + _note_add_failure(3, paths) + return + if not failed: + return + debug_log(f"git add -N failed: {failed.stderr[:200].decode('utf-8', errors='replace')}") + # Whatever happens to the retry, the diff goes on for the tracked files. + try: + objects_dir = tmp_index + _OBJECTS_DIR_SUFFIX + os.mkdir(objects_dir, 0o700) + retry_failed = add(_env_with_private_objects(env, cwd, objects_dir)) + if retry_failed: + debug_log(f"git add -N retry failed: {retry_failed.stderr[:200].decode('utf-8', errors='replace')}") + except (subprocess.TimeoutExpired, OSError) as e: + debug_log(f"git add -N retry failed: {e}") + retry_failed = True + if retry_failed: + _note_add_failure(1 if b"empty blob" in failed.stderr else 2, paths) + else: + _note_add_failure(4, paths) + + @contextlib.contextmanager def _temp_index(cwd, untracked_paths=None): """Yield an env dict pointing GIT_INDEX_FILE at a throwaway copy of the repo's index with `git add --intent-to-add` applied, so untracked files show up in subsequent `git diff` calls without touching the user's real - index. Yields None if no index can be found (bare repo / not a repo); the - caller should fall back to a plain diff. Always cleans up the temp file. + index. The copy has no gitlink entry for a nested repository (see + _drop_gitlinks). Yields None if no index can be found (bare repo / not a + repo), no copy slot came free, or those entries could not be removed. + Callers then go on without the copy: the diff reads the real index, and + the baseline falls back to HEAD. Always cleans up the temp file. Perf: when `untracked_paths` is given, only those paths are added (O(n) in untracked count). The default `add -N .` stats every file in the worktree — slow in large repos vs fast targeted scan. v2 callers already know the untracked set from `git status --porcelain`, so they - pass it; v1 keeps the whole-tree scan since it has no prior list.""" - import shutil - import tempfile + pass it; v1 keeps the whole-tree scan since it has no prior list. - real_index = _find_git_index(cwd) + Cleanup removes `<tmp_index>*`, not just the copy: git writes + `<index>.lock` and `<index>.stash.<pid>` next to the index it is given and + cannot remove them itself when a subprocess timeout kills it.""" + real_index, top = _find_git_index_and_top(cwd) if not real_index: yield None return - tmp_fd, tmp_index = tempfile.mkstemp(prefix="security_hook_idx_") + _sweep_stale_stash_indexes(real_index) + tmpdir = _hook_tmpdir() + try: + index_bytes = os.path.getsize(real_index) + except OSError: + index_bytes = 0 + with _copy_slot(tmpdir, index_bytes) as got_slot: + if not got_slot: + debug_log("no index copy slot came free; going without a copy") + yield None + return + with _copied_index(cwd, top, real_index, tmpdir, untracked_paths) as env: + yield env + + +@contextlib.contextmanager +def _copied_index(cwd, top, real_index, tmpdir, untracked_paths): + import shutil + import tempfile + + tmp_fd, tmp_index = tempfile.mkstemp( + prefix=f"{_TEMP_INDEX_PREFIX}pid{os.getpid()}_", dir=tmpdir) os.close(tmp_fd) + _live_copies.add(tmp_index) try: - shutil.copy2(real_index, tmp_index) + # A hard link where the filesystem allows it: no bytes are copied, and + # git never writes an index in place (it renames a new file over it), + # so the real index cannot change through the link. Either way the + # copy must keep .git/index's mtime (copy2, not copyfile): git + # re-reads any file whose mtime is not older than the index file's, + # and with a fresh mtime it would miss a same-size edit made in the + # same second as the last index write. + try: + os.unlink(tmp_index) + os.link(real_index, tmp_index) + except (OSError, NotImplementedError, AttributeError): + shutil.copy2(real_index, tmp_index) env = {**os.environ, "GIT_INDEX_FILE": tmp_index} + # Before `git add` runs, since add reads the gitlinks its paths cover. + if not _drop_gitlinks(top, env, tmp_index): + debug_log("could not remove gitlinks from the index copy; going on " + "without the copy") + yield None + return if untracked_paths is None: add_args = ["."] elif untracked_paths: @@ -165,27 +810,157 @@ def _temp_index(cwd, untracked_paths=None): # `git status` and here would silently drop ALL untracked files # from the diff. --ignore-missing only works with --dry-run, so # filter to surviving paths (lexists so dangling symlinks count). + # A nested repository (`dir/`) would be added as a submodule + # entry; see diff.ignoreSubmodules in GIT_CMD. surviving = [p for p in untracked_paths - if os.path.lexists(os.path.join(cwd, p))] - add_args = ["--"] + surviving if surviving else None + if not p.endswith("/") + and os.path.lexists(os.path.join(cwd, p))] + add_args = surviving or None else: add_args = None if add_args: - # No stdout used here (only returncode matters), but text=True - # still spawns reader threads that decode stderr — git error - # messages can reference non-ASCII filenames and crash on - # cp1252. See #2099. Drop text=True so bytes stay raw. - subprocess.run( - [*GIT_CMD, "add", "--intent-to-add"] + add_args, - cwd=cwd, capture_output=True, timeout=10, - env=env, - ) + _add_new_files(cwd, add_args, env, tmp_index) + # A whole-tree add puts a nested repository back. + if untracked_paths is None and not _drop_gitlinks(top, env, tmp_index): + yield None + return yield env finally: - try: - os.unlink(tmp_index) - except OSError: - pass + _remove_copy(tmp_index) + _live_copies.discard(tmp_index) + + +# An index entry stores its mode as four bytes; this is a gitlink's. +_GITLINK_MODE = b"\x00\x00\xe0\x00" +# Signature of the index extension that says most entries are in a second, +# shared file (core.splitIndex). +_SPLIT_INDEX_EXT = b"link" + + +def _may_have_gitlinks(index_path): + """False only when the index file at `index_path` cannot hold a gitlink + entry: neither a gitlink's mode bytes nor the split-index signature occur + anywhere in it. Either can occur by chance (in an object id, a file + size, a path), so True means "list the entries and see". Most + repositories have no gitlink, and for them this saves a git process on + every index copy.""" + keep = max(len(_GITLINK_MODE), len(_SPLIT_INDEX_EXT)) - 1 + try: + with open(index_path, "rb") as f: + tail = b"" + while True: + chunk = f.read(1 << 20) + if not chunk: + return False + data = tail + chunk + if _GITLINK_MODE in data or _SPLIT_INDEX_EXT in data: + return True + tail = data[-keep:] + except OSError: + return True + + +_GITLINK_LIST_TIMEOUT = 10 +# `git ls-files --sparse` (git 2.35) lists a sparse index as it is stored; +# without it git first expands the index to every path in the repository. +# Set to False once this git has refused the option. +_ls_files_sparse = True + + +def _gitlink_paths(top, env): + """Paths (bytes, relative to `top`) of the gitlink entries, at any depth, + in the index `env` names. None if they could not be listed. The listing + is read as it comes, so a large index costs time but not memory.""" + global _ls_files_sparse + cmd = [*GIT_CMD, "ls-files", "-s", "-z"] + (["--sparse"] if _ls_files_sparse else []) + p = subprocess.Popen(cmd, cwd=top, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, env=env) + killer = threading.Timer(_GITLINK_LIST_TIMEOUT, p.kill) + killer.daemon = True + found = set() + tail = b"" + try: + killer.start() + while True: + chunk = p.stdout.read1(65536) + if not chunk: + break + data = tail + chunk + end = data.rfind(b"\0") + 1 + # "<mode> <object> <stage>\t<path>", NUL-terminated. Most chunks + # have no gitlink and are not split up. + if b"160000 " in data: + for rec in data[:end].split(b"\0"): + if rec.startswith(b"160000 "): + found.add(rec.split(b"\t", 1)[1]) + tail = data[end:] + rc = p.wait() + finally: + killer.cancel() + if p.poll() is None: + p.kill() + p.wait() + p.stdout.close() + if rc == 129 and _ls_files_sparse: + # Usage error: a git without --sparse. + _ls_files_sparse = False + return _gitlink_paths(top, env) + if rc != 0: + debug_log(f"git ls-files on the index copy failed (rc={rc})") + return None + return found + + +# True when the last _drop_gitlinks call removed at least one gitlink, so the +# index copy and HEAD can no longer have the same tree even on a clean tree. +last_dropped_gitlinks = False + + +def _drop_gitlinks(top, env, tmp_index): + """Remove the gitlink (mode 160000) entry of every nested repository from + the index copy that `env` names, never from the real index, before + anything else runs on the copy. A gitlink marks a directory that is, or + can be, another repository (see diff.ignoreSubmodules in GIT_CMD): + `git stash create` takes no flag like IGNORE_SUBMODULES, and `git add` + reads the gitlinks its paths cover. An entry whose directory has no + `.git` stays: git starts nothing there, and without the entry an older + `git stash create` fails on the empty directory. `top` is the work + tree's top directory. Returns True on success, False on any failure; the + copy must not be used then.""" + global last_dropped_gitlinks + last_dropped_gitlinks = False + try: + if not _may_have_gitlinks(tmp_index): + return True + gitlinks = _gitlink_paths(top, env) + if gitlinks is None: + return False + top_b = os.fsencode(top) + nested = sorted(p for p in gitlinks + if os.path.lexists(os.path.join(top_b, p, b".git"))) + if not nested: + return True + mtime_ns = os.stat(tmp_index).st_mtime_ns + # git writes a new file and renames it over the copy, so a copy that + # is a hard link to the real index leaves the real one as it was. + rm = subprocess.run( + [*GIT_CMD, "update-index", "--force-remove", "-z", "--stdin"], + cwd=top, input=b"".join(p + b"\0" for p in nested), + capture_output=True, timeout=10, env=env) + if rm.returncode != 0: + debug_log(f"git update-index on the index copy failed (rc={rm.returncode}): " + f"{rm.stderr[:200].decode('utf-8', errors='replace')}") + return False + # Keep the mtime the copy came with; see _copied_index. + os.utime(tmp_index, ns=(mtime_ns, mtime_ns)) + last_dropped_gitlinks = True + debug_log(f"gitlink entries (nested repositories) removed from the " + f"index copy: {len(nested)}; their contents are not reviewed") + return True + except (subprocess.TimeoutExpired, OSError, RuntimeError, ValueError) as e: + # RuntimeError: the timer thread could not start. + debug_log(f"removing gitlinks from the index copy failed: {e}") + return False def _git_toplevel(cwd): @@ -206,6 +981,41 @@ def _git_toplevel(cwd): return None +def _git_toplevel_and_head(cwd): + """(repo root, HEAD SHA) for `cwd` from one git call, or (None, None) + when git cannot give both (not a work tree, no commits).""" + try: + r = subprocess.run( + [*GIT_CMD, "rev-parse", "--show-toplevel", "HEAD"], + cwd=cwd, capture_output=True, timeout=5, + ) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + return None, None + if r.returncode != 0: + return None, None + # See #2099: the first line is a PATH, so decode the bytes manually. + top, _, head = r.stdout.decode("utf-8", errors="replace").strip().rpartition("\n") + return (top.strip() or None), (head.strip() or None) + + +def _git_head_at(cwd, epoch): + """The SHA this checkout's HEAD pointed at just before `epoch` (seconds), + read from its own reflog. A checkout created after `epoch` answers with + the commit it was created at. None when git cannot say (no reflog).""" + try: + # Reflog times are whole seconds: ask for the second before, so a + # commit made in the same second as `epoch` counts as after it. + r = subprocess.run( + [*GIT_CMD, "rev-parse", "--verify", "-q", f"HEAD@{{@{int(epoch) - 1}}}"], + cwd=cwd, capture_output=True, timeout=5, + ) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError, + TypeError, ValueError, OverflowError): + return None + sha = r.stdout.decode("utf-8", errors="replace").strip() + return sha if r.returncode == 0 and sha else None + + def _git_dir(repo_root): """Absolute shared `.git` directory for repo_root. @@ -373,12 +1183,28 @@ def _git_reflog_recent_commits(repo_root, max_age_s=120, max_n=5): return fresh, stale +# New name -> secret-shaped old name, for the renames and copies git reported +# during this hook run. The Stop review limits its diff to a list of paths: +# with only the new name in it, git shows the secret's content as a new file, +# and the diff parsers cannot tell it came from a name they leave out. +_secret_rename_origins = {} + + +def _note_rename(old_path, new_path): + if _is_secret_shaped(old_path): + _secret_rename_origins[new_path] = old_path + + def _git_name_only(cwd, base, include_untracked=False): """Return the set of repo-root-relative paths that differ from `base`, or None if git failed (unresolvable ref, not a repo, timeout). Callers must distinguish None (error → don't trust as a filter) from set() (genuinely nothing changed). `-c core.quotePath=false -z` keeps non-ASCII - and space-containing paths intact.""" + and space-containing paths intact. + + Against the work tree the set can also hold a file whose stat changed and + whose content did not: the diff does not refresh the index. Callers + intersect with `git status`, which compares content.""" # Decode stdout/stderr as UTF-8 with errors="replace" instead of using # text=True. core.quotePath=false makes git emit raw UTF-8 for non-ASCII # paths, and text=True on Windows decodes via cp1252 strict — a non-ASCII @@ -388,7 +1214,7 @@ def _git_name_only(cwd, base, include_untracked=False): def _run(env): # core.quotePath=false comes from GIT_CMD globally (see definition). result = subprocess.run( - [*GIT_CMD, "diff", "--name-only", "-z", base], + [*GIT_CMD, "diff", IGNORE_SUBMODULES, "--name-status", "-z", base], cwd=cwd, capture_output=True, timeout=30, env=env, ) @@ -397,9 +1223,22 @@ def _git_name_only(cwd, base, include_untracked=False): debug_log(f"_git_name_only({base!r}) rc={result.returncode}: {stderr_str[:200]}") return None stdout = (result.stdout or b"").decode("utf-8", errors="replace") - return {p for p in stdout.split("\0") if p} + # Fields are `status\0path\0`, or `R<score>\0old\0new\0` for a rename + # or copy, which is listed by its new name only. + fields = stdout.split("\0") + names = set() + i = 0 + while i + 1 < len(fields): + if fields[i][:1] in ("R", "C") and i + 2 < len(fields): + _note_rename(fields[i + 1], fields[i + 2]) + i += 1 + names.add(fields[i + 1]) + i += 2 + return names try: + # The diff only reads the index (diff.autoRefreshIndex is off in + # GIT_CMD); a copy is for `add -N` alone. if not include_untracked: return _run(None) with _temp_index(cwd) as env: @@ -427,7 +1266,7 @@ def _git_status_porcelain(cwd): try: # core.quotePath=false comes from GIT_CMD globally (see definition). r = subprocess.run( - [*GIT_CMD, "status", "--porcelain=v1", "-uall", "-z"], + [*GIT_CMD, "status", IGNORE_SUBMODULES, "--porcelain=v1", "-uall", "-z"], cwd=cwd, capture_output=True, timeout=30, ) if r.returncode != 0: @@ -445,7 +1284,10 @@ def _git_status_porcelain(cwd): continue xy, path = e[:2], e[3:] if xy == "??": - untracked.add(path) + # With -uall only a nested repository is listed as `dir/`. + # It is not a file to review; see diff.ignoreSubmodules. + if not path.endswith("/"): + untracked.add(path) else: tracked.add(path) # Rename/copy entries are XY old\0new\0 — second NUL field is @@ -453,6 +1295,8 @@ def _git_status_porcelain(cwd): # 2-char-status entry. if "R" in xy or "C" in xy: i += 1 + if i < len(entries): + _note_rename(entries[i], path) i += 1 return tracked, untracked except (subprocess.TimeoutExpired, FileNotFoundError, OSError, ValueError) as e: @@ -479,6 +1323,264 @@ def _is_ancestor(cwd, maybe_ancestor, descendant): return False +_DEFAULT_FILE_DIFF_BYTES = 4 * 1024 * 1024 +_LATE_FILE_DIFF_BYTES = 256 * 1024 +_LAST_FILE_DIFF_BYTES = 96 * 1024 + + +def _file_diff_cap(): + """SG_MAX_FILE_DIFF_BYTES, or the default when it is unset, not a number + or not positive. Never below what a late file keeps, which is well above + what the review prompt takes of one file.""" + try: + n = int(os.environ.get("SG_MAX_FILE_DIFF_BYTES", "")) + except ValueError: + return _DEFAULT_FILE_DIFF_BYTES + if n <= 0: + return _DEFAULT_FILE_DIFF_BYTES + return max(n, _LATE_FILE_DIFF_BYTES) + + +# How much of one file's diff the Stop review keeps while it reads git's +# output: this much from the start of the file's section, and as much again +# of the added lines after that. The review prompt takes far less of a file +# (DIFF_PER_FILE_BYTES); the rest is for filter_preexisting_from_diff, which +# needs the removed and the added side of a rewritten file. +MAX_FILE_DIFF_BYTES = _file_diff_cap() +# Past this much kept in total, a further file keeps _LATE_FILE_DIFF_BYTES +# the same two ways; past four times as much, _LAST_FILE_DIFF_BYTES from its +# start, which is still more than the review prompt takes of one file. +MAX_KEPT_DIFF_BYTES = 4 * MAX_FILE_DIFF_BYTES +_SECTION_MARK = b"\ndiff --git " +_MAX_HEADER_BYTES = 64 * 1024 +_CUT_MARK = b"\\ [cut by security-guidance: the rest of this file's diff was not read]\n" + +# What the last get_git_diff read, for the Stop hook's metrics: bytes of git +# output, reviewable files whose diff was cut and the bytes cut from them, +# and diffs that hit the timeout during this hook run. +_diff_read = {"bytes": 0, "cut_files": 0, "cut_bytes": 0, "timeouts": 0} + + +class _DiffKeeper: + """Takes `git diff` output as it arrives and keeps only what the review + can use: the header line of every file, and a bounded part of each file + parse_diff_into_files would keep. A turn that wrote large data files no + longer puts their whole diff in memory.""" + + def __init__(self): + self.parts = [] + self.kept = 0 + self.seen = 0 + self.cut_files = 0 + self.cut_bytes = 0 + # Unhandled input. Starts as a newline that is not git's, so a header + # at the very start of the output is found like any other. + self._pending = b"\n" + self._at_start = True + self._reviewable = True + self._room = self._added_room = MAX_FILE_DIFF_BYTES + self._cut = False + self._ends_nl = True + # Past the cut: whether the next byte starts a line, and whether the + # line being read is an added one. + self._bol = True + self._in_added = False + + def feed(self, data): + buf = self._pending + data + pos = 0 + while True: + i = buf.find(_SECTION_MARK, pos) + if i < 0: + # Hold back a tail that could be the start of a header. + end = max(pos, len(buf) - len(_SECTION_MARK) + 1) + self._keep(buf, pos, end) + self._pending = buf[end:] + return + j = buf.find(b"\n", i + 1) + if j < 0 and len(buf) - i <= _MAX_HEADER_BYTES: + # The header line is not complete yet. + self._keep(buf, pos, i) + self._pending = buf[i:] + return + self._keep(buf, pos, i + 1) + if j >= 0: + self._begin(buf[i + 1:j]) + pos = i + 1 + + def _begin(self, header): + """A new file's section starts with this `diff --git ` line.""" + self._end_section() + text = header[len(_SECTION_MARK) - 1:].decode("utf-8", errors="replace") + paths = _parse_diff_header(text) + # A section the parser drops, or whose header it cannot read and only + # counts, keeps its header line. + self._reviewable = bool(paths) and _section_is_reviewable( + paths[1] or paths[0], paths[0]) + if not self._reviewable: + self._room, self._added_room = len(header) + 1, 0 + elif self.kept > 4 * MAX_KEPT_DIFF_BYTES: + self._room, self._added_room = max(len(header) + 1, _LAST_FILE_DIFF_BYTES), 0 + elif self.kept > MAX_KEPT_DIFF_BYTES: + self._room = self._added_room = max(len(header) + 1, _LATE_FILE_DIFF_BYTES) + else: + self._room = self._added_room = MAX_FILE_DIFF_BYTES + self._cut = False + + def _end_section(self): + if not self._ends_nl: + # Cut inside a line: end it, so what follows starts a line. + self._append(b"\n") + if self._cut: + self._append(_CUT_MARK) + + def _append(self, piece): + self.parts.append(piece) + self.kept += len(piece) + self._ends_nl = piece.endswith(b"\n") + + def _keep(self, buf, start, end): + if self._at_start and end > start: + self._at_start = False + start += 1 + n = end - start + if n <= 0: + return + self.seen += n + take = min(n, self._room) + if take: + self._append(buf[start:start + take]) + self._room -= take + if take == n or not self._reviewable: + return + if not self._cut: + self._cut = True + self.cut_files += 1 + self._bol = self._ends_nl + self._in_added = False + self.cut_bytes += n - take - self._keep_added(buf, start + take, end) + + def _keep_added(self, buf, pos, end): + """Past a file's cut, keep its added lines only, up to as much again. + A rewritten file lists every removed line before the first added one, + and the review is about the added ones. Returns the bytes kept.""" + kept = 0 + run = -1 # where a run of added lines not yet appended starts + while pos < end and self._added_room > 0: + if self._bol: + self._bol = False + self._in_added = buf.startswith(b"+", pos) + if self._in_added and run < 0 and not self._ends_nl: + self._append(b"\n") + if not self._in_added: + if run >= 0: + self._append(buf[run:pos]) + run = -1 + nxt = buf.find(b"\n+", pos, end) + if nxt < 0: + self._bol = buf.endswith(b"\n", pos, end) + break + pos = nxt + 1 + self._bol = True + continue + nl = buf.find(b"\n", pos, end) + stop = end if nl < 0 else nl + 1 + take = min(stop - pos, self._added_room) + if run < 0: + run = pos + self._added_room -= take + kept += take + self._bol = nl >= 0 + if pos + take < stop: + self._append(buf[run:pos + take]) + run = -1 + pos = stop + if run >= 0: + self._append(buf[run:pos]) + return kept + + def result(self): + self._keep(self._pending, 0, len(self._pending)) + self._pending = b"" + self._end_section() + self._cut = False + return b"".join(self.parts) + + +def _run_diff_bounded(cmd, timeout, **kwargs): + """subprocess.run(cmd, capture_output=True, timeout=timeout) for a + `git diff`, except that stdout goes through _DiffKeeper instead of being + held whole.""" + keeper = _DiffKeeper() + deadline = time.monotonic() + timeout + try: + err = tempfile.TemporaryFile() + except OSError: + err = None + try: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=err if err is not None else subprocess.DEVNULL, + **kwargs) + failed = [] + given_up = [] + + def _read(): + try: + while not given_up: + chunk = p.stdout.read1(65536) + if not chunk or given_up: + break + keeper.feed(chunk) + except BaseException as e: + failed.append(e) + + # Read on a thread, so the timeout holds even while something git + # started still has the pipe open after git itself is gone. + reader = threading.Thread(target=_read, daemon=True) + timed_out = False + try: + try: + reader.start() + except RuntimeError as e: + failed.append(e) + else: + reader.join(timeout) + timed_out = reader.is_alive() + if timed_out or failed: + p.kill() + try: + returncode = p.wait(max(deadline - time.monotonic(), 0.1)) + except subprocess.TimeoutExpired: + timed_out = True + finally: + if timed_out or failed or p.returncode is None: + # The reader may be left waiting on a pipe something else + # holds: it keeps nothing more, and the pipe stays its. + given_up.append(True) + p.kill() + p.wait() + if reader.ident is not None: + reader.join(1) + if not reader.is_alive(): + p.stdout.close() + _diff_read["bytes"] += keeper.seen + if timed_out: + _diff_read["timeouts"] += 1 + raise subprocess.TimeoutExpired(cmd, timeout) + if failed: + raise OSError(f"reading the diff failed: {failed[0]!r}") + stdout = keeper.result() + _diff_read["cut_files"] += keeper.cut_files + _diff_read["cut_bytes"] += keeper.cut_bytes + stderr = b"" + if err is not None: + err.seek(0) + stderr = err.read(4096) + return subprocess.CompletedProcess(cmd, returncode, stdout, stderr) + finally: + if err is not None: + err.close() + def get_git_diff(cwd, baseline_sha, full_context=False, paths=None, untracked_paths=None): """ @@ -492,9 +1594,16 @@ def get_git_diff(cwd, baseline_sha, full_context=False, paths=None, untracked_pa If `paths` is given, the diff is restricted to those paths (relative to cwd; absolute paths are converted, paths outside cwd are dropped). + A path list too long for one command is cut down to reviewable source + files: see _run_git_on_review_paths. `untracked_paths` (repo-root-relative) is forwarded to _temp_index so it can add only those files instead of scanning the whole worktree. + The output is read through _DiffKeeper: files the review would drop come + back as their header line only, and a file's diff past + MAX_FILE_DIFF_BYTES is cut down to its added lines. """ + _path_list_too_long.clear() + _add_failed["left_out"] = False pathspec = _diff_pathspec(cwd, paths) if paths and not pathspec: # Caller restricted to specific paths but none are inside this repo @@ -504,19 +1613,36 @@ def get_git_diff(cwd, baseline_sha, full_context=False, paths=None, untracked_pa return "" # core.quotePath=false comes from GIT_CMD globally (see definition). - cmd = [*GIT_CMD, "diff", "--no-color", "--no-ext-diff", "--no-textconv", baseline_sha] + (["--unified=99999"] if full_context else []) + pathspec + args = ["diff", IGNORE_SUBMODULES, "--no-color", "--no-ext-diff", "--no-textconv", baseline_sha] + (["--unified=99999"] if full_context else []) + _diff_read.update(bytes=0, cut_files=0, cut_bytes=0) + + def _run(env): + if pathspec: + results = _run_git_on_review_paths(args, pathspec[1:], timeout=30, run=_run_diff_bounded, cwd=cwd, env=env) + else: + results = [_run_diff_bounded([*GIT_CMD, *args], 30, cwd=cwd, env=env)] + if not results: + _path_list_too_long.add("none_reviewable") + for result in results: + if result.returncode != 0: + debug_log(f"git diff failed: {result.stderr[:200].decode('utf-8', errors='replace')}") + return None + # Decode with errors='replace' so binary diffs don't crash + return b"".join(r.stdout for r in results).decode("utf-8", errors="replace") + try: + if untracked_paths is not None and not untracked_paths: + # Nothing to `add -N`, and the diff only reads the index. + return _run(None) with _temp_index(cwd, untracked_paths) as env: - # env is None when no index could be found (bare repo / not a - # repo) — diff still runs, just without untracked-file support. - result = subprocess.run(cmd, cwd=cwd, capture_output=True, timeout=30, env=env) - if result.returncode != 0: - debug_log(f"git diff failed: {result.stderr[:200].decode('utf-8', errors='replace')}") - return None - # Decode with errors='replace' so binary diffs don't crash - return result.stdout.decode("utf-8", errors="replace") + # env is None when there is no index (bare repo / not a repo) + # or no copy slot — diff still runs, just without + # untracked-file support. + return _run(env) except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e: debug_log(f"git diff error: {e}") + if isinstance(e, OSError) and _too_many_args(e): + _path_list_too_long.add("gave_up") return None @@ -571,6 +1697,116 @@ SKIP_FILE_SUFFIXES = ( '.lock', '_pb2.py', '.pb.go', ) +# Files that hold secrets by naming convention are never sent for review. The +# review covers everything dirty in the work tree, including files the user +# changed and Claude never opened, so a key file or an encrypted dotfile source +# would otherwise leave the machine in the review request. The list is kept to +# names that are secret stores by convention; ordinary source such as +# `private_api.py` or `id_generator.py` stays reviewable. +_SSH_PRIVATE_KEY_BASENAMES = frozenset(( + 'id_rsa', 'id_dsa', 'id_ecdsa', 'id_ecdsa_sk', 'id_ed25519', + 'id_ed25519_sk', 'id_xmss', +)) +_PRIVATE_KEY_NAME_SUFFIXES = ('_rsa', '_dsa', '_ecdsa', '_ed25519') +# `.credentials.json` and `.claude.json` are Claude Code's own files: sign-in +# tokens, a saved API key, MCP server secrets. Claude Code rewrites them as it +# runs, so they show up as changed when its config directory is a git repo. +_CREDENTIAL_BASENAMES = frozenset(( + 'credentials', '.netrc', '.pgpass', '.npmrc', '.pypirc', + '.git-credentials', '.dockercfg', '.credentials.json', '.claude.json', +)) +# chezmoi encodes a target file's attributes as stacked name prefixes +# (`private_dot_netrc`, `encrypted_private_token`); `dot_` stands for a +# leading dot. +_CHEZMOI_ATTR_PREFIXES = ( + 'after_', 'before_', 'create_', 'empty_', 'encrypted_', 'exact_', + 'executable_', 'external_', 'literal_', 'modify_', 'once_', 'onchange_', + 'private_', 'readonly_', 'remove_', 'run_', 'symlink_', +) +# An `encrypted_` source keeps its target's name, so `encrypted_store.py` could +# be either ciphertext or code about encryption. Treat it as ciphertext only +# when it has no extension or a data-file one. +_ENCRYPTED_DATA_EXTENSIONS = frozenset(( + '', '.json', '.yaml', '.yml', '.toml', '.properties', '.tfvars', +)) + + +# Paths skipped as secret-shaped during this hook run. The Stop hook reports +# the count (never the names) so the skip can be watched in telemetry. +_secret_shaped_skipped = set() + + +def _skip_secret_shaped(file_path): + if not _is_secret_shaped(file_path): + return False + _secret_shaped_skipped.add(file_path) + return True + + +def _is_secret_shaped(file_path): + base = os.path.basename(file_path.replace("\\", "/")).lower() + target = base + encrypted = False + stripped = True + while stripped: + stripped = False + for prefix in _CHEZMOI_ATTR_PREFIXES: + if target.startswith(prefix) and len(target) > len(prefix): + target = target[len(prefix):] + encrypted = encrypted or prefix == 'encrypted_' + stripped = True + break + if target.startswith('dot_'): + target = '.' + target[4:] + ext = os.path.splitext(target)[1] + if encrypted and ext in _ENCRYPTED_DATA_EXTENSIONS: + return True + for name in {base, target}: + if name.startswith('.env'): + return True + if name in _SSH_PRIVATE_KEY_BASENAMES or name in _CREDENTIAL_BASENAMES: + return True + if '.' not in name and name.endswith(_PRIVATE_KEY_NAME_SUFFIXES): + return True + return False + + +# Data files whose name says they hold secrets (`secrets.yml`, +# `client_secret.json`). They are reviewed like any other data file; the Stop +# hook only counts how many are in its review set (never the names), to show +# how common they are. +_SECRET_NAMED_STEMS = frozenset(( + 'secret', 'secrets', 'credential', 'credentials', + 'client_secret', 'client_secrets', 'client-secret', 'client-secrets', + 'service-account', 'service_account', 'serviceaccount', + 'service-account-key', 'service_account_key', 'serviceaccountkey', +)) +_SECRET_NAMED_DATA_EXTENSIONS = frozenset(( + '.json', '.yaml', '.yml', '.toml', '.properties', '.tfvars', +)) +# `secrets.example.yml` is a placeholder and `secrets.enc.yaml` is ciphertext. +_NOT_SECRET_NAME_PARTS = frozenset(( + 'example', 'sample', 'template', 'tmpl', 'dist', 'schema', + 'enc', 'encrypted', 'sops', +)) + + +def _is_secret_named(file_path): + base = os.path.basename(file_path.replace("\\", "/")).lower() + parts = base.lstrip('.').split('.') + if len(parts) < 2 or '.' + parts[-1] not in _SECRET_NAMED_DATA_EXTENSIONS: + return False + if _NOT_SECRET_NAME_PARTS.intersection(parts[1:-1]): + return False + stem = parts[0] + # Google's OAuth download is `client_secret_<client id>.json`. + return stem in _SECRET_NAMED_STEMS or stem.startswith('client_secret_') + + +def count_secret_named(diff_files): + return sum(1 for file_path, _ in diff_files if _is_secret_named(file_path)) + + # Path tokens that bump a file's review priority when a commit exceeds # MAX_DIFF_FILES and we have to pick a subset. These are exactly the surfaces # single-shot and agentic reviews disagree on most (auth, routing, IPC, @@ -648,6 +1884,8 @@ def _is_reviewable_source(file_path): norm = "/" + file_path.replace("\\", "/") if any(("/" + p) in norm for p in SKIP_PATH_PATTERNS): return False + if _skip_secret_shaped(file_path): + return False if file_path.endswith(SKIP_FILE_SUFFIXES): return False ext = os.path.splitext(file_path)[1].lower() @@ -677,32 +1915,110 @@ def _is_reviewable_source(file_path): return False +# A file's section starts at a `diff --git ` line. Inside a section every line +# of file content carries a `+`, `-` or space prefix, so only a real header can +# start a line with it; source text that merely contains the words cannot. +_DIFF_SECTION_RE = re.compile(r'^diff --git ', re.MULTILINE) +_UNQUOTED_HEADER_RE = re.compile(r'^a/(.+?) b/(.+)$') +_C_ESCAPES = { + 'a': b'\a', 'b': b'\b', 'f': b'\f', 'n': b'\n', 'r': b'\r', 't': b'\t', + 'v': b'\v', '\\': b'\\', '"': b'"', +} + + +def _read_quoted_path(text, start): + """Decode the C-quoted path whose opening quote is at text[start]. + Returns (path, index after the closing quote), or None if unterminated.""" + out = bytearray() + i = start + 1 + while i < len(text): + c = text[i] + if c == '"': + return out.decode("utf-8", errors="replace"), i + 1 + if c == '\\' and i + 1 < len(text): + n = text[i + 1] + if n in _C_ESCAPES: + out += _C_ESCAPES[n] + i += 2 + continue + octal = re.match(r'[0-7]{1,3}', text[i + 1:i + 4]) + if octal: + out.append(int(octal.group(0), 8) & 0xFF) + i += 1 + len(octal.group(0)) + continue + out += c.encode("utf-8") + i += 1 + return None + + +def _parse_diff_header(header): + """(old_path, new_path) from the text after `diff --git `, or None. + + core.quotePath=false keeps non-ASCII names raw, but git still C-quotes a + name that contains a double quote, a backslash or a control character: + `"a/we\\"ird.py" "b/we\\"ird.py"`. Either side can be quoted on its own. + """ + old = new = None + if header.startswith('"'): + first = _read_quoted_path(header, 0) + if not first: + return None + old, end = first + rest = header[end:].lstrip(' ') + if rest.startswith('"'): + second = _read_quoted_path(rest, 0) + new = second[0] if second else None + else: + new = rest + elif header.endswith('"') and ' "b/' in header: + split_at = header.index(' "b/') + old = header[:split_at] + second = _read_quoted_path(header, split_at + 1) + new = second[0] if second else None + else: + m = _UNQUOTED_HEADER_RE.match(header) + return (m.group(1), m.group(2)) if m else None + if not old or not new or not old.startswith('a/') or not new.startswith('b/'): + return None + return old[2:], new[2:] + + +# Section headers that could not be parsed during this hook run; that file is +# left out of the review. The Stop hook reports the count so a header shape +# this parser misses shows up in telemetry instead of passing silently. +_unparsed_diff_headers = set() + + +def _section_is_reviewable(file_path, old_path): + """Whether a diff section is sent for review: a source file, and not one + renamed away from a secret-shaped name, which still carries the secret's + content.""" + return _is_reviewable_source(file_path) and not _skip_secret_shaped(old_path) + + +def _iter_diff_sections(diff_output): + """Yield (new_path, old_path, lines after the header line) per file section.""" + for section in _DIFF_SECTION_RE.split(diff_output)[1:]: + lines = section.split('\n') + paths = _parse_diff_header(lines[0]) + if paths: + yield paths[1] or paths[0], paths[0], lines[1:] + else: + _unparsed_diff_headers.add(lines[0]) + + def extract_file_paths_from_diff(diff_output): """ Extract file paths from unified diff output (without content). Only includes files with source code extensions. - Returns a list of file paths. """ if not diff_output or not diff_output.strip(): return [] - paths = [] - file_diffs = diff_output.split("diff --git ") - - for file_diff in file_diffs: - if not file_diff.strip(): - continue - lines = file_diff.split('\n') - header_match = re.match(r'^a/(.+?) b/(.+)$', lines[0]) - if not header_match: - continue - file_path = header_match.group(2) or header_match.group(1) or '' - if not _is_reviewable_source(file_path): - continue - paths.append(file_path) - - return paths - + return [ + file_path for file_path, old_path, _ in _iter_diff_sections(diff_output) + if _section_is_reviewable(file_path, old_path) + ] def parse_diff_into_files(diff_output): @@ -714,28 +2030,16 @@ def parse_diff_into_files(diff_output): return [] files = [] - file_diffs = diff_output.split("diff --git ") - - for file_diff in file_diffs: - if not file_diff.strip(): - continue - - # Extract filename from first line: "a/path/to/file b/path/to/file" - lines = file_diff.split('\n') - header_match = re.match(r'^a/(.+?) b/(.+)$', lines[0]) - if not header_match: - continue - - file_path = header_match.group(2) or header_match.group(1) or '' - - # Filter to source code files only - if not _is_reviewable_source(file_path): + for file_path, old_path, lines in _iter_diff_sections(diff_output): + # Filter to source code files only. A rename away from a secret-shaped + # name still carries the secret's content, so the old name counts too. + if not _section_is_reviewable(file_path, old_path): continue # Extract the diff content (from first @@ onwards) diff_lines = [] in_hunks = False - for line in lines[1:]: + for line in lines: if line.startswith('@@'): in_hunks = True if in_hunks: diff --git a/plugins/security-guidance/hooks/hooks.json b/plugins/security-guidance/hooks/hooks.json index 22734270..eef54730 100644 --- a/plugins/security-guidance/hooks/hooks.json +++ b/plugins/security-guidance/hooks/hooks.json @@ -37,58 +37,26 @@ { "type": "command", "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/sg-python.sh\" \"${CLAUDE_PLUGIN_ROOT}/hooks/security_reminder_hook.py\"", - "if": "Bash(git commit:*)", + "if": "Bash(git*commit*)", "asyncRewake": true, - "rewakeMessage": "Background security review of commit — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", - "rewakeSummary": "Commit security review found issues" + "rewakeMessage": "Background security review of committed or pushed changes — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", + "rewakeSummary": "Security review found issues" }, { "type": "command", "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/sg-python.sh\" \"${CLAUDE_PLUGIN_ROOT}/hooks/security_reminder_hook.py\"", - "if": "Bash(git -C * commit *)", + "if": "Bash(git*push*)", "asyncRewake": true, - "rewakeMessage": "Background security review of commit — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", - "rewakeSummary": "Commit security review found issues" + "rewakeMessage": "Background security review of committed or pushed changes — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", + "rewakeSummary": "Security review found issues" }, { "type": "command", "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/sg-python.sh\" \"${CLAUDE_PLUGIN_ROOT}/hooks/security_reminder_hook.py\"", - "if": "Bash(git push:*)", + "if": "Bash(gt *)", "asyncRewake": true, - "rewakeMessage": "Background security review of pushed commits not yet reviewed — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", - "rewakeSummary": "Push security review found issues" - }, - { - "type": "command", - "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/sg-python.sh\" \"${CLAUDE_PLUGIN_ROOT}/hooks/security_reminder_hook.py\"", - "if": "Bash(git -C * push*)", - "asyncRewake": true, - "rewakeMessage": "Background security review of pushed commits not yet reviewed — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", - "rewakeSummary": "Push security review found issues" - }, - { - "type": "command", - "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/sg-python.sh\" \"${CLAUDE_PLUGIN_ROOT}/hooks/security_reminder_hook.py\"", - "if": "Bash(gt create:*)", - "asyncRewake": true, - "rewakeMessage": "Background security review of commit — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", - "rewakeSummary": "Commit security review found issues" - }, - { - "type": "command", - "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/sg-python.sh\" \"${CLAUDE_PLUGIN_ROOT}/hooks/security_reminder_hook.py\"", - "if": "Bash(gt modify:*)", - "asyncRewake": true, - "rewakeMessage": "Background security review of commit — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", - "rewakeSummary": "Commit security review found issues" - }, - { - "type": "command", - "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/sg-python.sh\" \"${CLAUDE_PLUGIN_ROOT}/hooks/security_reminder_hook.py\"", - "if": "Bash(gt submit:*)", - "asyncRewake": true, - "rewakeMessage": "Background security review of pushed commits not yet reviewed — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", - "rewakeSummary": "Push security review found issues" + "rewakeMessage": "Background security review of committed or pushed changes — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply:", + "rewakeSummary": "Security review found issues" } ], "matcher": "Bash" @@ -106,19 +74,6 @@ } ] } - ], - "SubagentStop": [ - { - "hooks": [ - { - "type": "command", - "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/sg-python.sh\" \"${CLAUDE_PLUGIN_ROOT}/hooks/security_reminder_hook.py\"", - "asyncRewake": true, - "rewakeMessage": "Background security review feedback — address or acknowledge the findings below, then continue with the user's original request or continue waiting for their reply. This is supplementary, not a replacement for your previous response:", - "rewakeSummary": "Background security review found issues" - } - ] - } ] } } diff --git a/plugins/security-guidance/hooks/llm.py b/plugins/security-guidance/hooks/llm.py index 1430dd4e..c0a878f4 100644 --- a/plugins/security-guidance/hooks/llm.py +++ b/plugins/security-guidance/hooks/llm.py @@ -17,18 +17,31 @@ Two reassignable globals here are read by handlers in ``_last_review_truncated_bytes``. Handlers reference them as ``llm.X`` (not via ``from``-import) so they observe reassignment. """ -import glob +import contextlib +import hashlib import json +import logging import os import re +import subprocess import sys +import tempfile +import threading +import time +import traceback +import urllib.parse import urllib.request -from typing import Optional, Tuple, Dict, Any, List +from typing import Optional, Tuple, Dict, Any, List, Callable + +try: + import ssl +except ImportError: # a Python built without TLS + ssl = None import extensibility import review_api -from _base import debug_log, _record_usage, _record_http_error, _PV, PROVENANCE_TAG, state_dir as _resolve_state_dir # noqa: F401 -from session_state import with_locked_state +from _base import debug_log, _record_usage, _record_http_error, _record_net_error, _clear_net_error, _PV, PROVENANCE_TAG, state_dir as _resolve_state_dir, agent_sdk_site_dirs, make_state_dir # noqa: F401 +from session_state import load_state, with_locked_state from gitutil import git_config_env, SAFE_GIT_CONFIG @@ -52,17 +65,13 @@ def _inject_agent_sdk_venv_into_syspath(state_dir): fails with `ModuleNotFoundError: pywintypes` and the agentic reviewer falls back to single-shot silently. Replicate what site.py would do. """ - venv_root = os.path.join(state_dir, "agent-sdk-venv") - candidates = ( - glob.glob(os.path.join(venv_root, "lib", "python*", "site-packages")) - + glob.glob(os.path.join(venv_root, "Lib", "site-packages")) - # `pip install --target` fallback (ensure_agent_sdk BUILT_TARGET, used - # when venv can't bootstrap pip): a FLAT layout — packages sit directly - # in agent-sdk-libs/, not under a site-packages subdir. See #2154 - # follow-up. The pywin32 .pth bootstrap below applies here too (target - # installs don't process .pth at runtime, same as a manual venv insert). - + [os.path.join(state_dir, "agent-sdk-libs")] - ) + # `agent-sdk-libs` is the `pip install --target` fallback + # (ensure_agent_sdk BUILT_TARGET, used when venv can't bootstrap pip): a + # FLAT layout, packages sit directly in it, not under a site-packages + # subdir. See #2154 follow-up. The pywin32 .pth bootstrap below applies + # there too (target installs don't process .pth at runtime, same as a + # manual venv insert). + candidates = agent_sdk_site_dirs(state_dir) added = False for sp in candidates: if not os.path.isdir(sp) or sp in sys.path: @@ -125,11 +134,19 @@ HAS_API_CREDENTIALS = bool( ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN or _HAS_3P_PROVIDER_AT_LOAD ) -# Model for security review. Default chosen for its precision profile on +# Model for security review. Opus is the default for its precision profile on # interruptive review surfaces — false positives are the dominant uninstall # driver, so the default favors precision over recall and over latency. # Override via the SECURITY_REVIEW_MODEL env var (see README). -SECURITY_REVIEW_MODEL = os.environ.get("SECURITY_REVIEW_MODEL", "").strip() or "claude-opus-4-7" +# +# Without an override the review follows the newest Opus rather than a +# release named here: default_review_model() resolves it per call path. +# _PINNED_REVIEW_MODEL is only what a review uses when that lookup fails. +_PINNED_REVIEW_MODEL = "claude-opus-5-5" +# Claude Code's own alias. The CLI the Agent SDK spawns resolves it to the +# newest Opus it knows how to drive, in the provider's id form on 3P. +_CLI_OPUS_ALIAS = "opus" +SECURITY_REVIEW_MODEL = os.environ.get("SECURITY_REVIEW_MODEL", "").strip() or _PINNED_REVIEW_MODEL # OAuth subscriber tokens (ANTHROPIC_AUTH_TOKEN) require this exact system prompt # for api.anthropic.com/v1/messages — the API checks for one of the known Claude @@ -142,6 +159,24 @@ CLAUDE_CODE_SYSTEM_PROMPT = "You are a Claude agent, built on Anthropic's Claude # each call. None = no error; int = HTTP status code; -1 = network/timeout; _last_call_claude_http_error = None +# Outcome of this thread's last analyze_code_security call. Per thread because +# the commit review's race runs that call on a thread of its own. +_review_status = threading.local() + + +def last_review_failed() -> bool: + """True when this thread's last analyze_code_security call got no verdict + from the model (every call failed, or the reply could not be read), which + a caller must not treat as "nothing found".""" + return getattr(_review_status, "failed", False) + + +def last_review_status() -> Optional[int]: + """A copy of the module-wide _last_call_claude_http_error, taken when this + thread's last analyze_code_security call got no verdict (an overlapping call + on another thread can change the global); None if the reply was unreadable.""" + return getattr(_review_status, "status", None) + # ===================================================================== # Outbound connectivity probe @@ -161,23 +196,221 @@ def _anthropic_base_url() -> str: Honors ANTHROPIC_BASE_URL (the convention the Anthropic SDK and CC itself use) so customers behind an LLM gateway (LiteLLM, Bifrost, self-hosted Anthropic-compatible proxy) can route the plugin's reviews through their - gateway. Defaults to https://api.anthropic.com. Always returns a string - with no trailing slash so callers can safely append /v1/messages etc. + gateway. Defaults to https://api.anthropic.com, also when the variable is + set but blank, which is how the SDK and CC read it. Always returns a + string with no trailing slash so callers can safely append /v1/messages + etc. """ - return os.environ.get("ANTHROPIC_BASE_URL", "https://api.anthropic.com").rstrip("/") + base = os.environ.get("ANTHROPIC_BASE_URL", "").strip() + return (base or "https://api.anthropic.com").rstrip("/") + + +# ── Windows certificate store with an entry OpenSSL cannot read ────────── +# Older Python releases (3.11 and 3.12 among them) hand each Windows +# certificate store to OpenSSL in one call, so a single certificate it cannot +# parse makes ssl.create_default_context() raise, and every HTTPS call fails +# before a byte is sent. SSL_CERT_FILE does not help: the store is read +# first. Newer releases load the store one certificate at a time and skip +# the ones that fail; _urlopen does the same on the releases that do not. + +_STORE_TLS_UNSET = object() +_store_tls_context = _STORE_TLS_UNSET # None once Python's default context is known to build +_store_tls_lock = threading.Lock() +# Store entries left out of that context; emit_metrics reports it. +_store_certs_skipped = 0 + + +def _context_without_unreadable_store_certs(): + """(context, skipped): Python's default TLS context built from the + Windows store entries OpenSSL can parse, and how many it could not. + The context is None when the default one builds as it is, off Windows, + or when no entry is readable. Trust is never widened: the certificates + come from the same stores, under the same server-authentication filter, + and the chain and host name are verified as in any default context.""" + enum_certificates = getattr(ssl, "enum_certificates", None) # Windows only + if enum_certificates is None: + return None, 0 + try: + ssl.create_default_context() + return None, 0 + except ssl.SSLError: + pass + server_auth = ssl.Purpose.SERVER_AUTH.oid + context, skipped = None, 0 + for store in getattr(ssl.SSLContext, "_windows_cert_stores", ("CA", "ROOT")): + try: + for cert, encoding, trust in enum_certificates(store): + if encoding != "x509_asn" or not (trust is True or server_auth in trust): + continue + try: + if context is None: + # With cadata, create_default_context applies every + # default except reading the store itself. + context = ssl.create_default_context(cadata=cert) + else: + context.load_verify_locations(cadata=cert) + except ssl.SSLError: + skipped += 1 + except PermissionError: + pass + if context is None: + return None, skipped + context.set_default_verify_paths() + # What http.client sets on the context it would have made. + context.set_alpn_protocols(["http/1.1"]) + if context.post_handshake_auth is not None: + context.post_handshake_auth = True + return context, skipped + + +def _urlopen(request, timeout): + """urllib.request.urlopen for the plugin's API calls. Uses Python's + default TLS context unless that cannot be built because of the Windows + certificate store (see above); checked once per process.""" + global _store_tls_context, _store_certs_skipped + if _store_tls_context is _STORE_TLS_UNSET: + with _store_tls_lock: + if _store_tls_context is _STORE_TLS_UNSET: + try: + context, skipped = _context_without_unreadable_store_certs() + except Exception as e: + debug_log(f"TLS: certificate store check failed: {type(e).__name__}") + context, skipped = None, 0 + if context is not None: + debug_log(f"TLS: the Windows certificate store has {skipped} certificate(s) " + "OpenSSL cannot read; using the store without them") + _store_certs_skipped = skipped + _store_tls_context = context + if _store_tls_context is None: + return urllib.request.urlopen(request, timeout=timeout) + return urllib.request.urlopen(request, timeout=timeout, context=_store_tls_context) + + +def _is_remote_session() -> bool: + return os.environ.get("CLAUDE_CODE_REMOTE", "").lower() in ("1", "true", "yes", "on") def _probe_anthropic(timeout: float = 5.0) -> bool: req = urllib.request.Request(_anthropic_base_url() + "/", method="HEAD") try: - with urllib.request.urlopen(req, timeout=timeout): + with _urlopen(req, timeout): return True except urllib.error.HTTPError: return True # got a status code → connected - except (urllib.error.URLError, TimeoutError, OSError): + except (urllib.error.URLError, TimeoutError, OSError) as e: + _record_net_error(_net_error_code(e)) return False +# ===================================================================== +# Why a call got no HTTP response +# ===================================================================== +# Classes for the `net_err` metric. Measure only: nothing reads these to +# decide what the plugin does. +NET_ERR_CERT_VERIFY = 1 # the server certificate did not verify +NET_ERR_TLS = 2 # any other TLS failure, including a default context that cannot be built +NET_ERR_DNS = 3 # the host name did not resolve +NET_ERR_CONNECT = 4 # refused, reset, closed early or unreachable +NET_ERR_TIMEOUT = 5 +NET_ERR_PROXY = 6 # the proxy refused the tunnel +NET_ERR_OTHER = 7 # another failure from the HTTP client +NET_ERR_INNER_CLI = 9 # the review ran through a CLI, which failed +NET_ERR_INNER_CLI_TIMEOUT = 10 +NET_ERR_NO_SDK = 11 # the review needed the agent SDK, which could not be imported + +_NET_FLAG_NO_CA_ON_DISK = 1 # Python's default CA file and directory are missing or empty +_NET_FLAG_PROXY = 2 # the request would go through a proxy +_NET_FLAG_NODE_EXTRA_CA = 4 # NODE_EXTRA_CA_CERTS is set (Python does not read it) +_NET_FLAG_SSL_CERT_ENV = 8 # SSL_CERT_FILE or SSL_CERT_DIR is set + + +def _net_error_class(exc) -> Tuple[int, int]: + """(class, X.509 verify code) for an exception from an API call; class 0 + for one that is not a network failure. The verify code is OpenSSL's + (20 = issuer not found locally, 19 = self-signed certificate in the + chain, 10 = expired), 0 when the class is not NET_ERR_CERT_VERIFY.""" + import errno + import http.client + import socket + try: + import ssl + cert_error, tls_error = ssl.SSLCertVerificationError, ssl.SSLError + except ImportError: + cert_error = tls_error = () + reason = getattr(exc, "reason", None) + err = reason if isinstance(reason, BaseException) else exc + if isinstance(err, cert_error): + code = getattr(err, "verify_code", 0) + return NET_ERR_CERT_VERIFY, code if isinstance(code, int) and 0 < code < 100 else 0 + if isinstance(err, tls_error): + return NET_ERR_TLS, 0 + if isinstance(err, socket.gaierror): + return NET_ERR_DNS, 0 + if isinstance(err, OSError): + text = str(err).lower() + if "tunnel connection failed" in text: + return NET_ERR_PROXY, 0 + if isinstance(err, TimeoutError) or "timed out" in text: + return NET_ERR_TIMEOUT, 0 + if isinstance(err, ConnectionError) or err.errno in ( + errno.ENETUNREACH, errno.EHOSTUNREACH, errno.ENETDOWN, + errno.EHOSTDOWN, errno.EADDRNOTAVAIL): + return NET_ERR_CONNECT, 0 + return NET_ERR_OTHER, 0 + if isinstance(err, http.client.HTTPException) or isinstance(exc, urllib.error.URLError): + return NET_ERR_OTHER, 0 + return 0, 0 + + +def _net_error_flags(tls: bool) -> int: + """What the machine's setup says about a failed call. Each check is + best effort: one that cannot be made leaves its flag off.""" + flags = 0 + if tls: + try: + import ssl + paths = ssl.get_default_verify_paths() + has_file = bool(paths.cafile) and os.path.getsize(paths.cafile) > 0 + has_dir = False + if paths.capath: + with os.scandir(paths.capath) as it: + has_dir = any(True for _ in it) + # Windows also reads the system store, which this cannot see. + if not has_file and not has_dir and sys.platform != "win32": + flags |= _NET_FLAG_NO_CA_ON_DISK + except Exception: + pass + if os.environ.get("NODE_EXTRA_CA_CERTS"): + flags |= _NET_FLAG_NODE_EXTRA_CA + if os.environ.get("SSL_CERT_FILE") or os.environ.get("SSL_CERT_DIR"): + flags |= _NET_FLAG_SSL_CERT_ENV + try: + url = urllib.parse.urlsplit(_anthropic_base_url()) + # As urllib chooses: the proxy for the URL's scheme, bypass by host:port. + if (urllib.request.getproxies().get(url.scheme) + and not urllib.request.proxy_bypass(url.netloc.rpartition("@")[2])): + flags |= _NET_FLAG_PROXY + except Exception: + pass + return flags + + +def _net_error_code(exc) -> int: + """`net_err` for an exception: class + 100 * verify code + 10000 * flags, + or 0 for an exception that is not a network failure.""" + try: + cls, verify = _net_error_class(exc) + except Exception: + return 0 + if not cls: + return 0 + try: + flags = _net_error_flags(cls in (NET_ERR_CERT_VERIFY, NET_ERR_TLS)) + except Exception: + flags = 0 + return cls + 100 * verify + 10000 * flags + + def _strip_anthropic_from_no_proxy() -> None: for var in ("NO_PROXY", "no_proxy"): val = os.environ.get(var) @@ -195,7 +428,7 @@ def ensure_anthropic_reachable() -> bool: global _anthropic_reachable if _anthropic_reachable is not None: return _anthropic_reachable - if os.environ.get("CLAUDE_CODE_REMOTE", "").lower() not in ("1", "true", "yes", "on"): + if not _is_remote_session(): _anthropic_reachable = True return True if _probe_anthropic(): @@ -256,8 +489,62 @@ def _cap_files_for_prompt(files): _auth_prefer_token = False -def _build_auth_headers(use_token): - betas = ["structured-outputs-2025-11-13"] +# A valid HTTP header name (an RFC 9110 token). +_HEADER_NAME = re.compile(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+") +_custom_header_lines_skipped = set() + + +def _custom_headers() -> List[Tuple[str, str]]: + """ANTHROPIC_CUSTOM_HEADERS as (name, value) pairs, parsed the way Claude + Code parses it: one `Name: Value` per line, split at the first colon, + trimmed, lines with no name dropped. Lines http.client cannot send (an + invalid name, a carriage return or non-Latin-1 text in the value) are + skipped. Values are never logged: they are often credentials.""" + pairs = [] + raw = os.environ.get("ANTHROPIC_CUSTOM_HEADERS", "") + for number, line in enumerate(re.split(r"\r?\n", raw), 1): + name, sep, value = line.partition(":") + name, value = name.strip(), value.strip() + if not sep or not name: + continue + try: + value.encode("latin-1") + sendable = bool(_HEADER_NAME.fullmatch(name)) and "\r" not in value + except UnicodeEncodeError: + sendable = False + if not sendable: + if number not in _custom_header_lines_skipped: + _custom_header_lines_skipped.add(number) + debug_log(f"ANTHROPIC_CUSTOM_HEADERS: skipped line {number}, not a valid HTTP header") + continue + pairs.append((name, value)) + return pairs + + +def gateway_cfg() -> int: + """How the plugin's reviews are routed, for telemetry: bit 1 when + ANTHROPIC_BASE_URL names a host other than api.anthropic.com, bit 2 when + at least one ANTHROPIC_CUSTOM_HEADERS line is sent, bit 4 when reviews + run through the user's claude on a Claude apps gateway sign-in (see + gateway_review_cli). 0 when none. A number only: no host, header name + or value.""" + cfg = 0 + base = _anthropic_base_url() + try: + host = (urllib.parse.urlsplit(base).hostname or "").lower() + except ValueError: + host = "" + if base and host != "api.anthropic.com": + cfg |= 1 + if any(name.lower() != "anthropic-beta" for name, _ in _custom_headers()): + cfg |= 2 + if _gateway_cli: + cfg |= 4 + return cfg + + +def _build_auth_headers(use_token, betas=("structured-outputs-2025-11-13",)): + betas = list(betas) headers = { "Content-Type": "application/json", "anthropic-version": "2023-06-01", @@ -267,7 +554,20 @@ def _build_auth_headers(use_token): betas.append("oauth-2025-04-20") else: headers["x-api-key"] = ANTHROPIC_API_KEY - headers["anthropic-beta"] = ",".join(betas) + for name, value in _custom_headers(): + lowered = name.lower() + if lowered == "anthropic-beta": + # Not sent: the review needs none of the user's betas, and Claude + # Code drops anthropic-beta lines when the organization's + # compliance policy disables experimental betas. + continue + # As in Claude Code, a custom line wins over a header of the same + # name that the plugin set, including the credential header. + for existing in [k for k in headers if k.lower() == lowered]: + del headers[existing] + headers[name] = value + if betas: + headers["anthropic-beta"] = ",".join(betas) return headers @@ -278,7 +578,10 @@ def _build_auth_headers(use_token): _ADAPTIVE_THINKING_MODELS = ( "claude-opus-4-6", "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", "claude-sonnet-4-6", + "claude-sonnet-5", ) _LEGACY_THINKING_MODELS = ( "claude-3-", @@ -339,16 +642,372 @@ def _is_3p_provider() -> bool: return False -def _call_claude_via_sdk(prompt, output_schema, *, max_tokens=16000, model=None): +# ── Default review model ───────────────────────────────────────────────── +# The API has no alias for "newest Opus", so the raw-HTTP path asks the +# endpoint it is about to call (GET /v1/models) and keeps the answer in the +# state dir. Asking that endpoint, not api.anthropic.com, means a gateway +# that serves an older Opus gets a model it can actually route. + +_REVIEW_MODEL_CACHE_FILE = "review_model.json" +_REVIEW_MODEL_TTL_SECONDS = 24 * 3600 +# A failed lookup is remembered too, for less long: otherwise an endpoint +# without /v1/models costs every hook fire the lookup timeout. +_REVIEW_MODEL_RETRY_SECONDS = 3600 +_REVIEW_MODEL_LOOKUP_TIMEOUT = 5 +_REVIEW_MODEL_LISTING_MAX_BYTES = 4 * 1024 * 1024 + +# claude-opus-<major>[-<minor>][-<yyyymmdd>]. Anything else a listing or the +# cache file holds (other families, gateway-prefixed names, free text) is +# never sent as a model id. +_OPUS_ID = re.compile(r"claude-opus-(\d{1,2})(?:-(\d{1,2}))?(?:-(\d{8}))?") + +_review_model_memo: Optional[str] = None + + +def _newest_opus(model_ids) -> Optional[str]: + """The highest-versioned Opus id in `model_ids`, or None.""" + best = None + for model_id in model_ids: + m = _OPUS_ID.fullmatch(model_id) if isinstance(model_id, str) else None + if not m: + continue + version = (int(m.group(1)), int(m.group(2) or 0), m.group(3) or "") + if best is None or version > best[0]: + best = (version, model_id) + return best[1] if best else None + + +def _fetch_newest_opus() -> Optional[str]: + """Newest Opus the configured endpoint lists; None if it can't say.""" + # Newest first, so one modest page is enough; an org with many models + # otherwise returns a megabyte-sized listing. + url = _anthropic_base_url() + "/v1/models?limit=100" + use_token = _auth_prefer_token or not ANTHROPIC_API_KEY + for _ in range(2): + request = urllib.request.Request( + url, headers=_build_auth_headers(use_token, betas=()), method="GET", + ) + try: + with _urlopen(request, _REVIEW_MODEL_LOOKUP_TIMEOUT) as response: + listing = json.loads( + response.read(_REVIEW_MODEL_LISTING_MAX_BYTES).decode("utf-8") + ) + entries = listing.get("data") if isinstance(listing, dict) else None + if not isinstance(entries, list): + debug_log("review model lookup: no model list in response") + return None + ids = [e.get("id") for e in entries if isinstance(e, dict)] + if listing.get("has_more"): + # A truncated page may hold no current Opus at all (an org's + # own models can fill it); never let it pick an older one. + ids.append(_PINNED_REVIEW_MODEL) + return _newest_opus(ids) + except urllib.error.HTTPError as e: + if e.code == 401 and not use_token and ANTHROPIC_AUTH_TOKEN: + use_token = True + continue + debug_log(f"review model lookup: HTTP {e.code}") + return None + except Exception as e: + debug_log(f"review model lookup failed: {type(e).__name__}") + return None + return None + + +def _read_review_model_cache(path, endpoint_key, now): + """(hit, model) from the cache file. `model` is None for a remembered + failed lookup. The file is user-writable state: every field is checked + and a model id is accepted only if it has the shape of an Opus id.""" + try: + with open(path, "r", encoding="utf-8") as f: + entry = json.loads(f.read(4096)) + except (OSError, ValueError): + return False, None + if not isinstance(entry, dict) or entry.get("endpoint") != endpoint_key: + return False, None + at = entry.get("at") + if isinstance(at, bool) or not isinstance(at, (int, float)): + return False, None + model = entry.get("model") + if model is None: + ttl = _REVIEW_MODEL_RETRY_SECONDS + elif isinstance(model, str) and _OPUS_ID.fullmatch(model): + ttl = _REVIEW_MODEL_TTL_SECONDS + else: + return False, None + if not 0 <= now - at < ttl: + return False, None + return True, model + + +def _write_review_model_cache(path, endpoint_key, model, now): + tmp = None + try: + make_state_dir(os.path.dirname(path)) + fd, tmp = tempfile.mkstemp(dir=os.path.dirname(path), prefix=".review_model.") + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump({"endpoint": endpoint_key, "model": model, "at": now}, f) + os.replace(tmp, path) + except OSError as e: + debug_log(f"review model cache not written: {type(e).__name__}") + if tmp: + try: + os.unlink(tmp) + except OSError: + pass + + +def default_review_model() -> str: + """Model for a review the user hasn't pinned with SECURITY_REVIEW_MODEL. + + 3P providers and Claude apps gateway sign-ins go through the Agent SDK, + so the CLI alias does the work. 1P and LLM gateways set with + ANTHROPIC_BASE_URL get the newest Opus the endpoint lists, looked up at + most once per _REVIEW_MODEL_TTL_SECONDS; _PINNED_REVIEW_MODEL when the + lookup can't answer. Never raises. + """ + global _review_model_memo + explicit = os.environ.get("SECURITY_REVIEW_MODEL", "").strip() + if explicit: + return explicit + if _is_3p_provider() or _gateway_cli: + return _CLI_OPUS_ALIAS + if not HAS_API_CREDENTIALS: + return _PINNED_REVIEW_MODEL + if _review_model_memo: + return _review_model_memo + try: + path = os.path.join(_resolve_state_dir(), _REVIEW_MODEL_CACHE_FILE) + # Hashed: a gateway URL can carry credentials. + endpoint_key = hashlib.sha256( + _anthropic_base_url().encode("utf-8") + ).hexdigest()[:16] + now = time.time() + hit, model = _read_review_model_cache(path, endpoint_key, now) + if not hit: + model = _fetch_newest_opus() + _write_review_model_cache(path, endpoint_key, model, now) + except Exception as e: + debug_log(f"review model resolution failed: {type(e).__name__}") + model = None + _review_model_memo = model or _PINNED_REVIEW_MODEL + return _review_model_memo + + +# A session signed in to a Claude apps gateway with /login gives hooks no +# credential, but a child `claude` restores that sign-in by itself, so reviews +# can run through the user's CLI without a credential entering this process. + +# `claude auth status` may wait for a settings helper script an administrator +# configured, which Claude Code gives 10 s by default. +_GATEWAY_CHECK_TIMEOUT_S = 15 +_GATEWAY_STATE_KEY = "gateway_sign_in" +# The CLI that reviews for a signed-in gateway session, "" when this process +# decided not to use the gateway route, None before it has decided. +_gateway_cli: Optional[str] = None +# True when this process asked the CLI about the sign-in and got no usable +# answer: the reviews are skipped, but the session may well be signed in. +_gateway_check_unanswered = False +# True once a review through the gateway route ran out of time. What makes +# it do so (a rejected sign-in, a gateway that cannot be reached) does not +# depend on the model, so the Sonnet attempt is not made after it. +_gateway_review_timed_out = False + +_WINDOWS = os.name == "nt" + + +def _user_cli_path() -> Optional[str]: + """The user's own `claude`: SG_AGENTIC_CLI_PATH if set ("" names none), else + the first of the absolute paths below that exists, else None.""" + cli_path = os.environ.get("SG_AGENTIC_CLI_PATH") + if cli_path is not None: + return cli_path + # CLAUDE_CODE_EXECPATH is the absolute path to the running CC binary + # itself (e.g. ~/.local/share/claude/versions/2.1.x — that's the binary, + # not a directory). Claude Code sets it for the Bash tool's shells, not for + # hooks, so hooks usually find the CLI at a fixed path below. We do NOT + # fall back to shutil.which("claude") because the hook's cwd is the + # user's (potentially attacker-supplied) repo, and Windows shutil.which + # searches cwd first — a checked-in ./claude.exe would get spawned. + # Absolute-path probes only. + probes = [os.environ.get("CLAUDE_CODE_EXECPATH")] + if not _WINDOWS: + # Where Claude Code is installed on macOS and Linux. + probes += [ + os.path.expanduser("~/.local/bin/claude"), + "/root/.local/bin/claude", + # Claude Code Remote container install path. + "/opt/claude-code/bin/claude", + ] + for p in probes: + if p and os.path.isfile(p): + return p + return None + + +def _windows_user_cli() -> Optional[str]: + """Windows: the full path of the user's claude.exe, from the native + installer's folder, else from PATH; None when neither has one. Skips any + folder that is not given as a full path, and the working directory and + project root, and any folder inside them (unless that root holds the home + folder, and with it the user's own tools).""" + skipped = [d for d in (os.getcwd(), os.environ.get("CLAUDE_PROJECT_DIR")) if d and os.path.isdir(d)] + home = os.path.expanduser("~") + real_home = os.path.normcase(os.path.realpath(home)).rstrip(os.sep) + os.sep + roots = [r.rstrip(os.sep) + os.sep for r in (os.path.normcase(os.path.realpath(d)) for d in skipped)] + roots = [r for r in roots if not real_home.startswith(r)] + home_bin = os.path.join(home, ".local", "bin") + for folder in [home_bin] + os.environ.get("PATH", "").split(os.pathsep): + folder = folder.strip().strip('"') + cli = os.path.join(folder, "claude.exe") + try: + real = os.path.normcase(os.path.realpath(folder)) + # A full path is one that abspath leaves as it is. + if (os.path.normcase(os.path.abspath(folder)) == os.path.normcase(os.path.normpath(folder)) + and os.path.isfile(cli) + and not any(os.path.samefile(folder, d) for d in skipped) + and not any(real.startswith(r) for r in roots)): + return cli + except OSError: + pass + return None + + +def _cli_for_sdk(cli_path: Optional[str]) -> Optional[str]: + """What to pass the Agent SDK as `cli_path`: a given one as it is. With none + given, the SDK chooses, except on Windows. There this names the CLI bundled + with the SDK, else the user's own, by its full path, and when neither exists + the bundled path anyway, so that the start fails.""" + if cli_path or not _WINDOWS: + return cli_path or None + # When the SDK module has no file, any full path at which nothing exists will do. + sdk_file = getattr(sys.modules.get("claude_agent_sdk"), "__file__", None) or __file__ + bundled = os.path.join(os.path.dirname(os.path.abspath(sdk_file)), "_bundled", "claude.exe") + if os.path.isfile(bundled): + return bundled + cli_path = _windows_user_cli() + if not cli_path: + debug_log("no Claude Code for the Agent SDK: none is bundled with it, and there is no claude.exe " + "in ~\\.local\\bin or on PATH outside this project (SG_AGENTIC_CLI_PATH can name one)") + return bundled + debug_log(f"the Agent SDK has no bundled Claude Code; using {cli_path}") + _tolerate_unknown_sdk_messages() + return cli_path + + +def _tolerate_unknown_sdk_messages() -> None: + """Make the SDK's message parser pass unknown message types through + instead of raising: a CLI newer than the SDK emits types the SDK does not + know (e.g. rate_limit_event).""" + try: + from claude_agent_sdk._internal import message_parser as _mp + import claude_agent_sdk._internal.client as _sdk_client + from claude_agent_sdk import SystemMessage as _SysMsg + + _orig_parse = _mp.parse_message + if getattr(_orig_parse, "_sg_tolerant", False): + return + + def _tolerant(data): + try: + return _orig_parse(data) + except Exception: + return _SysMsg(subtype=data.get("type", "unknown"), data=data) + + _tolerant._sg_tolerant = True + _mp.parse_message = _tolerant + _sdk_client.parse_message = _tolerant + except Exception: + pass + + +def _gateway_signed_in(cli: str) -> Optional[bool]: + """Whether `cli auth status --json` reports a session signed in to a + Claude apps gateway. None when the CLI gave no usable answer (an error, a + timeout, or output that is not a JSON object).""" + try: + # A process the CLI starts can keep stdout open after the CLI exits; + # reading a pipe would then wait out the timeout and lose the answer. + with tempfile.TemporaryFile() as out: + subprocess.run( + [cli, "auth", "status", "--json"], stdin=subprocess.DEVNULL, + stdout=out, stderr=subprocess.DEVNULL, + env={**os.environ, **_agentic_spawn_env()}, + cwd=os.path.expanduser("~"), + timeout=_GATEWAY_CHECK_TIMEOUT_S, check=False, + ) + out.seek(0) + status = json.loads(out.read(64 * 1024)) + except Exception as e: + debug_log(f"gateway check: no usable answer from {cli} ({type(e).__name__})") + return None + if not isinstance(status, dict): + debug_log("gateway check: answer was not a JSON object") + return None + provider, logged_in = status.get("apiProvider"), status.get("loggedIn") + debug_log(f"gateway check: apiProvider={str(provider)[:40]!r} loggedIn={repr(logged_in)[:10]}") + # A managed machine that names a gateway but was never signed in also + # reports "gateway". + return provider == "gateway" and logged_in is True + + +def gateway_review_cli(session_id) -> Optional[str]: + """The CLI to review through when the plugin has no credential but the + session is signed in to a Claude apps gateway; None otherwise. A yes or + no from the CLI is kept for the rest of the session, so signing in or + out mid-session takes effect in the next one. When the CLI gives no + usable answer, nothing is saved, so the next hook asks again.""" + global _gateway_cli, _gateway_check_unanswered + if _gateway_cli is not None: + return _gateway_cli or None + cli = _user_cli_path() or "" + ok = None + # No skip for CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC, as on Bedrock and Vertex: + # the gateway docs suggest setting it, and the review goes through the + # gateway like the session's own requests. + if not os.path.isabs(cli): + debug_log("gateway check: skipped, SG_AGENTIC_CLI_PATH is not a full path" + if "SG_AGENTIC_CLI_PATH" in os.environ + else "gateway check: skipped, no claude at ~/.local/bin/claude or the other " + "usual full paths (SG_AGENTIC_CLI_PATH can name one)") + else: + cached = load_state(session_id).get(_GATEWAY_STATE_KEY) + if (isinstance(cached, dict) and cached.get("cli") == cli + and isinstance(cached.get("ok"), bool)): + ok = cached["ok"] + debug_log(f"gateway check: using this session's saved answer (signed in: {ok})") + else: + ok = _gateway_signed_in(cli) + _gateway_check_unanswered = ok is None + if ok is not None: + def _save(state): + state[_GATEWAY_STATE_KEY] = {"cli": cli, "ok": ok} + with_locked_state(session_id, _save) + _gateway_cli = cli if ok else "" + return _gateway_cli or None + + +def can_review(session_id) -> bool: + """True when the plugin has a credential, or has none but the session is + signed in to a Claude apps gateway. Call it before reviewing: the review + takes the gateway route only after it has said yes in this process.""" + return HAS_API_CREDENTIALS or gateway_review_cli(session_id) is not None + + +def _call_claude_via_sdk(prompt, output_schema, *, max_tokens=16000, model=None, + cli_path=None): """Single-turn SDK call as a substitute for the HTTP _call_claude path on - 3P providers. Uses the same `output_format` JSON-schema contract so the - return value shape is identical (parsed dict or None). + 3P providers, and, with `cli_path` set, for a session signed in to a + Claude apps gateway. Uses the same `output_format` JSON-schema contract so + the return value shape is identical (parsed dict or None). - No tools (`allowed_tools=[]`) — the security review only needs structured - output, not Read/Grep/Glob. Single turn keeps cost predictable. + No tools (see _only_tools_args) — the security review only needs + structured output, not Read/Grep/Glob. Single turn keeps cost predictable. """ - global _last_call_claude_http_error + global _last_call_claude_http_error, _gateway_review_timed_out _last_call_claude_http_error = None + _clear_net_error() + route = "gateway" if cli_path else "3P" try: import asyncio as _asyncio @@ -373,33 +1032,42 @@ def _call_claude_via_sdk(prompt, output_schema, *, max_tokens=16000, model=None) query, ) except Exception as e: - debug_log(f"3P sdk-single-turn: SDK unavailable ({e})") + debug_log(f"{route} sdk-single-turn: SDK unavailable ({e})") _last_call_claude_http_error = -1 _record_http_error(-1) + _record_net_error(NET_ERR_NO_SDK) return None - cli_path = os.environ.get("SG_AGENTIC_CLI_PATH") or None - chosen_model = model or SECURITY_REVIEW_MODEL + _send_sdk_logs_to_debug_log() + if cli_path: + _tolerate_unknown_sdk_messages() + else: + cli_path = _cli_for_sdk(os.environ.get("SG_AGENTIC_CLI_PATH") or None) + default_model = default_review_model() + chosen_model = model or default_model # Capture child claude stderr so a failing 3P call surfaces the real # error (auth missing, model id wrong, etc.) in the debug log instead # of just "exit code 1". _captured_stderr: List[str] = [] + # HTTP statuses (or "no response") the CLI said it retried an API request after. + _retried_after: set = set() async def _arun(): opts = ClaudeAgentOptions( system_prompt=CLAUDE_CODE_SYSTEM_PROMPT, cli_path=cli_path, allowed_tools=[], + extra_args={**_only_tools_args(), **_reviewer_session_args()}, setting_sources=[], max_turns=2, model=chosen_model, output_format={"type": "json_schema", "schema": output_schema}, # Identical --model/--fallback-model is rejected by the CLI at - # startup; chosen_model defaults to SECURITY_REVIEW_MODEL, so - # only pass a fallback when it actually differs. + # startup; chosen_model defaults to default_model, so only pass + # a fallback when it actually differs. fallback_model=( - SECURITY_REVIEW_MODEL if chosen_model != SECURITY_REVIEW_MODEL else None + default_model if chosen_model != default_model else None ), env=_agentic_spawn_env(), stderr=lambda l: _captured_stderr.append(l), @@ -416,41 +1084,59 @@ def _call_claude_via_sdk(prompt, output_schema, *, max_tokens=16000, model=None) structured = msg.structured_output _record_usage(getattr(msg, "usage", None) or {}, chosen_model, cost_usd=getattr(msg, "total_cost_usd", None)) + elif getattr(msg, "subtype", None) == "api_retry": + status = _api_retry_status(msg) + _retried_after.add(str(status) if status is not None else "no response") return structured # 60s ceiling: a single review request on a healthy 3P endpoint completes # in 5-15s; >60s means the child claude is hung (e.g. user has the 3P env # var set but no provider creds → child waits for an auth that never # comes). Bound the wait so a misconfigured 3P session doesn't stall the - # whole hook. - try: - result = _asyncio.run(_asyncio.wait_for(_arun(), timeout=60)) + # whole hook. A gateway review gets the 120 s a direct HTTP review gets. + timeout_s = 120 if route == "gateway" else 60 + + def _log_child_stderr(): if _captured_stderr: - debug_log(f"3P sdk-single-turn child stderr ({len(_captured_stderr)} lines):") + debug_log(f"{route} sdk-single-turn child stderr ({len(_captured_stderr)} lines):") for _l in _captured_stderr[:20]: debug_log(f" | {_l.rstrip()}") + + try: + result = _run_inner_cli( + lambda: _asyncio.run(_asyncio.wait_for(_arun(), timeout=timeout_s)), + _captured_stderr) + _log_child_stderr() return result except _asyncio.TimeoutError: - debug_log("3P sdk-single-turn: timeout after 60s") + # The CLI retries a refused or unreachable API by itself and reports + # each retry in its output, not on stderr. + debug_log(f"{route} sdk-single-turn: timeout after {timeout_s}s" + + (f"; the CLI's API requests were retried after status " + f"{', '.join(sorted(_retried_after))}" if _retried_after else "")) + _log_child_stderr() + if route == "gateway": + _gateway_review_timed_out = True _last_call_claude_http_error = -1 _record_http_error(-1) + _record_net_error(NET_ERR_INNER_CLI_TIMEOUT) return None except Exception as e: - debug_log(f"3P sdk-single-turn: query failed ({e})") - if _captured_stderr: - debug_log(f"3P sdk-single-turn child stderr ({len(_captured_stderr)} lines):") - for _l in _captured_stderr[:20]: - debug_log(f" | {_l.rstrip()}") + debug_log(f"{route} sdk-single-turn: query failed ({e})") + _log_child_stderr() _last_call_claude_http_error = -1 _record_http_error(-1) + _record_net_error(NET_ERR_INNER_CLI) return None def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000, model=None, - retry_5xx=True): + retry_5xx=True, cacheable_prefix=None): """ Call the configured LLM model with extended thinking and structured outputs. - Model defaults to Sonnet 4.6 but can be overridden via SECURITY_REVIEW_MODEL env var. + cacheable_prefix: static text sent ahead of `prompt` as its own content block + with cache_control, so repeated calls share a cached request prefix. + Model defaults to default_review_model(); SECURITY_REVIEW_MODEL overrides it. Returns parsed JSON response or None on failure. On failure, sets module-level _last_call_claude_http_error to the HTTP status (or -1 for network/timeout) so callers can distinguish API failure from an @@ -463,6 +1149,7 @@ def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000, """ global _last_call_claude_http_error _last_call_claude_http_error = None + _clear_net_error() if _is_3p_provider(): # On Bedrock/Vertex/Foundry/Mantle the api.anthropic.com path below @@ -471,10 +1158,19 @@ def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000, # child claude CLI. Note: thinking_budget/retry_5xx don't pass # through — the SDK manages retries (529) and thinking config # internally per the chosen model. + if cacheable_prefix: + prompt = cacheable_prefix + "\n\n" + prompt return _call_claude_via_sdk(prompt, output_schema, max_tokens=max_tokens, model=model) if not HAS_API_CREDENTIALS: + if _gateway_cli: + # Signed in to a Claude apps gateway: the user's CLI reviews with + # its own stored sign-in (see gateway_review_cli). + if cacheable_prefix: + prompt = cacheable_prefix + "\n\n" + prompt + return _call_claude_via_sdk(prompt, output_schema, max_tokens=max_tokens, + model=model, cli_path=_gateway_cli) return None global _auth_prefer_token @@ -484,11 +1180,20 @@ def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000, use_token = _auth_prefer_token or not ANTHROPIC_API_KEY headers = _build_auth_headers(use_token) + if cacheable_prefix: + content = [ + {"type": "text", "text": cacheable_prefix, + "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": prompt}, + ] + else: + content = prompt + payload = { - "model": model or SECURITY_REVIEW_MODEL, + "model": model or default_review_model(), "max_tokens": max_tokens, "system": CLAUDE_CODE_SYSTEM_PROMPT, - "messages": [{"role": "user", "content": prompt}], + "messages": [{"role": "user", "content": content}], # API moved the structured-output schema from top-level `output_format` # to `output_config.format` per # https://platform.claude.com/docs/en/build-with-claude/structured-outputs. @@ -530,7 +1235,7 @@ def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000, headers=headers, method="POST", ) - with urllib.request.urlopen(request, timeout=120) as response: + with _urlopen(request, 120) as response: response_body = response.read().decode("utf-8") response_data = json.loads(response_body) _record_usage(response_data.get("usage") or {}, @@ -563,6 +1268,7 @@ def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000, debug_log(f"Request failed after retries: {e}") _last_call_claude_http_error = -1 _record_http_error(-1) + _record_net_error(_net_error_code(e)) return None if not response_data: @@ -599,7 +1305,7 @@ def _dual_or_enabled() -> bool: def _call_claude_dual_or(prompt, output_schema, *, bool_key: str, list_key: str, - thinking_budget=10000, max_tokens=16000): + thinking_budget=10000, max_tokens=16000, cacheable_prefix=None): """Run prompt through the model 2× in parallel and OR-merge the results. The second look samples the model again on the same prompt — independent @@ -622,28 +1328,33 @@ def _call_claude_dual_or(prompt, output_schema, *, bool_key: str, list_key: str, from concurrent.futures import ThreadPoolExecutor explicit = os.environ.get("SECURITY_REVIEW_MODEL", "").strip() - primary = explicit or SECURITY_REVIEW_MODEL + primary = default_review_model() + # Through the user's CLI, the alias gives the Sonnet id the user's own + # sessions use on the gateway, including any ANTHROPIC_DEFAULT_SONNET_MODEL. + fallback = "sonnet" if _gateway_cli else "claude-sonnet-4-6" if not _dual_or_enabled(): # Single-call path. Reuse the same sonnet-fallback retry as a dual_or # leg so a 529/400 on the primary doesn't drop recall to zero. r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget, - max_tokens=max_tokens, model=primary, retry_5xx=False) - if r is None and not explicit: + max_tokens=max_tokens, model=primary, retry_5xx=False, + cacheable_prefix=cacheable_prefix) + if r is None and not explicit and not _gateway_review_timed_out: debug_log(f"single: {primary} failed, falling back to sonnet") r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget, - max_tokens=max_tokens, model="claude-sonnet-4-6", - retry_5xx=True) + max_tokens=max_tokens, model=fallback, + retry_5xx=True, cacheable_prefix=cacheable_prefix) return r def _leg(): r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget, - max_tokens=max_tokens, model=primary, retry_5xx=False) - if r is None and not explicit: + max_tokens=max_tokens, model=primary, retry_5xx=False, + cacheable_prefix=cacheable_prefix) + if r is None and not explicit and not _gateway_review_timed_out: debug_log(f"dual_or: {primary} leg failed, falling back to sonnet") r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget, - max_tokens=max_tokens, model="claude-sonnet-4-6", - retry_5xx=True) + max_tokens=max_tokens, model=fallback, + retry_5xx=True, cacheable_prefix=cacheable_prefix) return r with ThreadPoolExecutor(max_workers=2) as ex: @@ -787,94 +1498,9 @@ def _dedup_against_state(session_id: str, vulns: List[Dict[str, Any]], return kept, len(vulns) - len(kept) -def analyze_code_security(files: List[Tuple[str, str]], is_diff: bool = False, previous_findings: Optional[List[str]] = None) -> Tuple[Optional[str], List[Dict[str, Any]]]: - """ - Use Haiku to perform a security review of code. - files: list of (file_path, content_or_diff) tuples - is_diff: if True, the content is a unified diff rather than full file contents - previous_findings: list of category strings from earlier stop hook firings this turn, - used to prompt the reviewer to verify those issues were actually fixed. - Returns (formatted guidance string or None, list of vuln dicts with severity/category). - """ - if not HAS_API_CREDENTIALS or not files: - return None, [] - - # Build language context from file extensions - lang_hints = { - ".go": "Go", ".java": "Java/Spring Boot", ".py": "Python", - ".rb": "Ruby", ".php": "PHP", ".rs": "Rust", - ".ts": "TypeScript", ".js": "JavaScript", ".jsx": "JavaScript/React", - ".tsx": "TypeScript/React", ".ejs": "EJS templates", - ".html": "HTML/templates", ".properties": "Java properties", - ".yaml": "YAML config", ".yml": "YAML config", - } - languages = set() - for fp, _ in files: - ext = os.path.splitext(fp)[1].lower() - if ext in lang_hints: - languages.add(lang_hints[ext]) - language = ", ".join(sorted(languages)) if languages else "server-side" - - files = _cap_files_for_prompt(files) - - # Build the files section - files_section = [] - for fp, content in files: - ext = os.path.splitext(fp)[1].lower() - label = "DIFF" if is_diff else "FILE" - files_section.append(f"=== {label}: {fp} ===\n```{ext.lstrip('.')}\n{content}\n```") - files_text = "\n\n".join(files_section) - - content_desc = "diff" if is_diff else "code" - - if is_diff: - diff_instruction = """Note: You are reviewing a unified diff. Unmarked lines (starting with a space) are UNCHANGED context — they were already in the file before this session. Lines starting with + are ADDITIONS made in this session. Lines starting with - are REMOVALS. - -CRITICAL: ONLY flag vulnerabilities that are NEWLY INTRODUCED in + lines. Do NOT flag: -- Issues in unmarked context lines (space-prefixed = pre-existing code). Even if a context line contains SECRET_KEY = 'hardcoded', DEBUG=True, hardcoded passwords, SQL injection, or any other vulnerability — it is PRE-EXISTING and must be ignored. -- Issues where the SAME pattern existed in the removed (-) lines and was re-added in + lines (this means the code was rewritten/reformatted but the pattern is pre-existing) -- Pre-existing patterns that Claude simply preserved when rewriting a file -- Any vulnerability whose vulnerable code snippet appears in context (space-prefixed) lines -- Vulnerabilities in the ORIGINAL/STARTER code that the developer was given to work with. If a file was fully rewritten (all lines show as - then +), compare the + content against the - content. Only flag NEWLY INTRODUCED patterns that did NOT exist in the - lines. -- Issues OUTSIDE THE SCOPE of what the developer was asked to do. If the task was "add logging middleware" and the starter code has a hardcoded SECRET_KEY, that is pre-existing and out of scope — do NOT flag it. - -A vulnerability is ONLY new if the + lines introduce a pattern that did NOT exist anywhere in the - lines or context lines of the same file. - -EXCEPTION — data flow to pre-existing sinks: If + lines route user-controlled data to a PRE-EXISTING dangerous sink (like `new Function()`, `eval()`, `exec()`, or shell string interpolation in context lines), this IS a new vulnerability. The sink was already there, but the new code created a new attack path to it. Flag this as a new vulnerability in the + lines.""" - else: - diff_instruction = "" - - structured_prev = [f for f in (previous_findings or []) if isinstance(f, dict)] - if structured_prev: - prev_lines = "\n".join( - f" - {f.get('filePath', '?')} [{f.get('category', '?')}]: {f.get('vulnerableCode', '?')}" - for f in structured_prev - ) - prev_section = ( - "PREVIOUS FINDINGS (already surfaced to the developer earlier this turn — DO NOT re-flag):\n" - "The exact findings below were already shown to the developer, who has either fixed them or " - "acknowledged them as not applicable. DO NOT report any finding whose (filePath, category) pair " - "matches an entry below — it was already handled. The vulnerableCode may differ slightly from " - "what you see now (diff context lines shift between fires) — match on file + category, not exact " - "code bytes. ONLY re-flag a (filePath, category) from this list if the code at that location was " - "CHANGED since the prior review and the change is an incomplete fix or introduces a new issue.\n" - f"{prev_lines}\n" - ) - else: - prev_section = "" - - prompt = """You are a security expert reviewing {language} {content_desc}. Analyze the {content_desc} below for CONCRETE security vulnerabilities that an attacker could exploit. - -{diff_instruction} - -{prev_section} - -For each vulnerability found, provide: -1. The file path where it occurs (use the exact path from the === {file_type}: header) -2. The vulnerability category -3. The specific vulnerable code (quote the exact line(s)) -4. How an attacker would exploit it -5. A specific code fix +# Static review rubric, sent as its own content block with cache_control so the +# request prefix is byte-identical across calls. Keep it free of per-call text. +_CODE_REVIEW_RUBRIC = """You are a security expert reviewing code for CONCRETE security vulnerabilities that an attacker could exploit. IMPORTANT vulnerability categories to check: @@ -882,50 +1508,50 @@ IMPORTANT vulnerability categories to check: **Path Traversal**: Is user input used to construct file paths? Key insight: filepath.Join() in Go does NOT prevent path traversal — filepath.Join("/var/log", "../../etc/passwd") returns "/etc/passwd". Same for Python's os.path.join() and Java's Paths.get().resolve(). CRITICAL: `path.resolve()`/`filepath.Clean()`/`normalize()` are LEXICAL — they collapse `..` but do NOT dereference symlinks, so `startsWith(baseDir)` after them is symlink-bypassable. Call `fs.realpathSync()`/`os.path.realpath()`/`filepath.EvalSymlinks()` FIRST, then check the result starts with the realpath of baseDir. -**SQL Injection**: Is user input concatenated into SQL queries instead of using parameterized queries? This includes f-string interpolation (e.g., `f"WHERE name = '{{user_input}}'"`) and string concatenation (e.g., `"WHERE name = '" + user_input + "'"`). Even if input appears to be validated upstream, use parameterized queries. In Python: `cursor.execute('WHERE name = %s', (user_input,))`. In Go: `db.Query('WHERE name = $1', userInput)`. +**SQL Injection**: Is user input concatenated into SQL queries instead of using parameterized queries? This includes f-string interpolation (e.g., `f"WHERE name = '{user_input}'"`) and string concatenation (e.g., `"WHERE name = '" + user_input + "'"`). Even if input appears to be validated upstream, use parameterized queries. In Python: `cursor.execute('WHERE name = %s', (user_input,))`. In Go: `db.Query('WHERE name = $1', userInput)`. A NEW security-gate parameter (group/role/tool/permission/scope) is safe only if (a) the gate is enforced unconditionally, OR (b) when its enabling condition is False the function raises/denies. If execution can continue past the new gate unchecked, flag fail-open — a later check may be vacuous when the new gate was the caller's only constraint. -**Authorization (IDOR / scoping / visibility)**: A handler that returns or modifies a tenant-, owner-, role-, or visibility-scoped resource MUST verify the requester is in that scope. Missing-authz patterns: `findById(id)` / `Model.objects.get(id=id)` without an ownership check; `Model.objects.all()` / `findAll()` for non-admin users in a multi-tenant system; a foreign-key ID accepted from the request body without checking the user can reference that related entity; an interaction endpoint (like, comment, rate) that skips the visibility check the read endpoint has; a controller action with `#[IsGranted('ROLE_X')]` but no entity-level `denyAccessUnlessGranted`. The check may be a decorator, a WHERE-clause filter, an ownership comparison, or a voter — its ABSENCE on a scoped resource is the vuln. Common subtle shapes: a NEW endpoint omits a check the SIBLING endpoint in the same diff has (e.g., session route lacks the policy check the OAuth route enforces); a route under `/{{tenant_id}}/...` whose handler never references that path param (queries only by `auth.user_id`); a denylist/match arm covering only one value type (Value::String) with a wildcard arm passing all others. +**Authorization (IDOR / scoping / visibility)**: A handler that returns or modifies a tenant-, owner-, role-, or visibility-scoped resource MUST verify the requester is in that scope. Missing-authz patterns: `findById(id)` / `Model.objects.get(id=id)` without an ownership check; `Model.objects.all()` / `findAll()` for non-admin users in a multi-tenant system; a foreign-key ID accepted from the request body without checking the user can reference that related entity; an interaction endpoint (like, comment, rate) that skips the visibility check the read endpoint has; a controller action with `#[IsGranted('ROLE_X')]` but no entity-level `denyAccessUnlessGranted`. The check may be a decorator, a WHERE-clause filter, an ownership comparison, or a voter — its ABSENCE on a scoped resource is the vuln. Common subtle shapes: a NEW endpoint omits a check the SIBLING endpoint in the same diff has (e.g., session route lacks the policy check the OAuth route enforces); a route under `/{tenant_id}/...` whose handler never references that path param (queries only by `auth.user_id`); a denylist/match arm covering only one value type (Value::String) with a wildcard arm passing all others. **Secrets/PII in Logs, URLs, or Errors**: Any sink that persists or transmits values an observer of logs/URLs/errors shouldn't see. Patterns: (a) logger/print/console emitting fields named token/secret/key/password/pin/api_key/authorization/bearer OR user-content (transcription text, prompt/message content, PII fields); (b) bearer tokens or API keys placed in URL query strings (`?key=`, `?token=`, `?access_token=`) — leaks to access logs/referer/history; (c) `str(exc)`/`repr(exc)`/`fmt.Errorf("...%s", respBody)`/`traceback.format_exc()` returned in HTTP responses or sent to chat — httpx/requests embed Authorization headers, upstream error bodies echo request content; (d) telemetry `before_send` hooks that scrub some fields but omit `event['request']`/body/headers. **Unsafe Deserialization**: Untrusted bytes/paths reaching pickle deserialization including via wrappers — `pickle.load`/`pickle.loads`, `torch.load` or `.torch_load()` without `weights_only=True`, `yaml.load` without `SafeLoader`, `joblib.load`, `cloudpickle.load`/`.cloudpickle_load()`, `marshal.loads`, PHP `unserialize`, Java `ObjectInputStream`. Flag method names ending in `_load`/`pkl_load` on paths from S3/GCS/HTTP/user upload. -**TLS Verification Disabled / Plaintext Transport**: An explicit literal that disables transport encryption or peer-cert validation for a non-loopback connection. Client-side: Python `requests.*(verify=False)` / `httpx.Client(verify=False)` / `ssl._create_unverified_context()`; Go `tls.Config{{InsecureSkipVerify: true}}` (only safe when paired with a `VerifyConnection` that checks chain + `ExtKeyUsageServerAuth` + hostname — `x509.ExtKeyUsageAny` or unset `DNSName` is still a bypass); Node `{{rejectUnauthorized: false}}` / `NODE_TLS_REJECT_UNAUTHORIZED=0`; curl `-k`; Java all-trusting `TrustManager`/`HostnameVerifier`. Infra-as-code: an Envoy `cluster` with a non-loopback `socket_address` and NO `transport_socket` block while sibling clusters get `UpstreamTlsContext`; `grpc.insecure_channel()` / `grpc.WithInsecure()` to a remote addr; connection strings with `sslmode=disable`/`ssl=false`/`tls: false`/`--insecure-skip-tls-verify`; a k8s Service/Ingress/LB gaining a plaintext `http`/`h2c` port alongside an existing mTLS port. Do NOT flag `localhost`/`127.0.0.1`/unix-socket targets or test fixtures. +**TLS Verification Disabled / Plaintext Transport**: An explicit literal that disables transport encryption or peer-cert validation for a non-loopback connection. Client-side: Python `requests.*(verify=False)` / `httpx.Client(verify=False)` / `ssl._create_unverified_context()`; Go `tls.Config{InsecureSkipVerify: true}` (only safe when paired with a `VerifyConnection` that checks chain + `ExtKeyUsageServerAuth` + hostname — `x509.ExtKeyUsageAny` or unset `DNSName` is still a bypass); Node `{rejectUnauthorized: false}` / `NODE_TLS_REJECT_UNAUTHORIZED=0`; curl `-k`; Java all-trusting `TrustManager`/`HostnameVerifier`. Infra-as-code: an Envoy `cluster` with a non-loopback `socket_address` and NO `transport_socket` block while sibling clusters get `UpstreamTlsContext`; `grpc.insecure_channel()` / `grpc.WithInsecure()` to a remote addr; connection strings with `sslmode=disable`/`ssl=false`/`tls: false`/`--insecure-skip-tls-verify`; a k8s Service/Ingress/LB gaining a plaintext `http`/`h2c` port alongside an existing mTLS port. Do NOT flag `localhost`/`127.0.0.1`/unix-socket targets or test fixtures. **SSRF (Server-Side Request Forgery)**: A user-influenceable URL/host/path reaching an outbound fetch — `requests.get`/`httpx`/`aiohttp`/`urllib`/`fetch`/`axios`/`http.Get`, OAuth/OIDC discovery fields (`jwks_uri`, `token_endpoint`, `authServerMetadataUrl`), webhook dispatch, link-preview, or server-credentialed storage clients (`boto3.get_object`, `gcs.Blob.from_string`) on a bucket/key from an attacker-authored manifest. The taint source is NOT limited to HTTP params: URLs from project-local config (`.mcp.json`, `.vscode/settings.json`, `package.json`, workspace YAML in a cloned repo) and manifest/checkpoint files an attacker wrote earlier are attacker-controlled. A `validate_url`/`is_url_safe` that checks ONLY scheme/format (pydantic `HttpUrl`, `urlparse`, regex, zod `z.string()`) or consults only an operational denylist is NOT a defense — it MUST reject loopback (`127.0.0.0/8`, `::1`, `0.0.0.0`), RFC1918 private, and link-local `169.254.0.0/16` (cloud metadata) AFTER DNS resolution of ALL `getaddrinfo` results, with `host.rstrip('.').lower()` before any `.endswith()` compare (FQDN trailing-dot and `evilgoogle.com` bypasses). Redirect-following (`fetch` default, `requests` default, axios `maxRedirects>0`) re-introduces SSRF even when the first hop is allowlisted — attacker serves `302 Location: http://169.254.169.254/`; fix is `redirect: 'manual'` + re-validate each hop. **Argument Injection (argv flag smuggling)**: User input as a positional argv element — `spawn(bin,[...])`, `execFile`, `subprocess.run([...])`, `exec.Command(bin, args...)` — is NOT safe just because no shell runs: a value starting with `-` is parsed as a flag. Exec-capable flags: ripgrep `--pre=CMD`, git `--upload-pack=CMD`/`-c core.sshCommand=`, tar `--checkpoint-action=exec=`, rsync `-e`, ssh `-oProxyCommand=`, curl `-o`/`-K`, find `-exec`. Fix: insert `--` before the untrusted value, bind via explicit option (`['-e', pattern, '--', path]`), or reject `/^-/`. -**OAuth/OIDC Flow Weaknesses**: (a) **Forgeable state** — an OAuth callback's `state` is CSRF-protective ONLY if unguessable AND bound to the session (compared against a cookie/server-session, or HMAC-verified). A `state` decoded as plain base64 JSON (`JSON.parse(Buffer.from(state,'base64url'))`, `json.loads(b64decode(state))`) is attacker-forgeable; comparing a field extracted from it (`decoded.email === identity.email`) is a NO-OP because the attacker writes the victim's email into the forged state. Flag callbacks decoding `state` without `crypto.createHmac` verify, `cookies.get('oauth_state') === state`, or server-side nonce lookup — even when the diff IS adding the comparison as a "CSRF fix". (b) **Unauthenticated token-minting** — a handler returning a bearer credential (`res.json({{sessionId / access_token / apiKey}})`, `JSONResponse({{'access_token': ...}})`) that reads only `req.query`/`req.body` and never references `req.user`/`req.auth`/`Authorization`/auth middleware. +**OAuth/OIDC Flow Weaknesses**: (a) **Forgeable state** — an OAuth callback's `state` is CSRF-protective ONLY if unguessable AND bound to the session (compared against a cookie/server-session, or HMAC-verified). A `state` decoded as plain base64 JSON (`JSON.parse(Buffer.from(state,'base64url'))`, `json.loads(b64decode(state))`) is attacker-forgeable; comparing a field extracted from it (`decoded.email === identity.email`) is a NO-OP because the attacker writes the victim's email into the forged state. Flag callbacks decoding `state` without `crypto.createHmac` verify, `cookies.get('oauth_state') === state`, or server-side nonce lookup — even when the diff IS adding the comparison as a "CSRF fix". (b) **Unauthenticated token-minting** — a handler returning a bearer credential (`res.json({sessionId / access_token / apiKey})`, `JSONResponse({'access_token': ...})`) that reads only `req.query`/`req.body` and never references `req.user`/`req.auth`/`Authorization`/auth middleware. -**XSS — Autoescape Off / Incomplete or Wrong Escaper**: (a) `jinja2.Environment()`/`jinja2.Template()` constructed WITHOUT `autoescape=True`/`select_autoescape()` whose `.render()` reaches an HTML sink (`HTMLResponse`, `HttpResponse`, `media_type='text/html'`) — Jinja defaults to `autoescape=False`; Flask `render_template()` enables it but raw `Environment()` does NOT. Same: Go `text/template` (vs `html/template`) to `http.ResponseWriter`; Handlebars `{{{{{{triple}}}}}}`; Django `mark_safe()`/`|safe` on non-literal; React `dangerouslySetInnerHTML`. (b) The `div.textContent=s; return div.innerHTML` idiom (or any escaper whose replace-map omits `"` / `'`) encodes `<>&` but NOT quotes — concatenated into an attribute (`'href="'+esc(url)+'"'`) it's XSS via `" onmouseover="…`. A protocol allowlist `/^https?:/` does NOT stop attribute breakout. (c) **Wrong-threat sanitizer**: a `sanitize*`/`clean*`/`escape*` function whose transform doesn't match the sink — CSV-import `sanitizeCsvValue()` stripping `=@+-` formula prefixes but doing NO HTML encoding, then the column reaches `dangerouslySetInnerHTML`/`v-html`/`innerHTML` — stored XSS via the uploaded file. The misleading function name is the false-safety signal. +**XSS — Autoescape Off / Incomplete or Wrong Escaper**: (a) `jinja2.Environment()`/`jinja2.Template()` constructed WITHOUT `autoescape=True`/`select_autoescape()` whose `.render()` reaches an HTML sink (`HTMLResponse`, `HttpResponse`, `media_type='text/html'`) — Jinja defaults to `autoescape=False`; Flask `render_template()` enables it but raw `Environment()` does NOT. Same: Go `text/template` (vs `html/template`) to `http.ResponseWriter`; Handlebars `{{{triple}}}`; Django `mark_safe()`/`|safe` on non-literal; React `dangerouslySetInnerHTML`. (b) The `div.textContent=s; return div.innerHTML` idiom (or any escaper whose replace-map omits `"` / `'`) encodes `<>&` but NOT quotes — concatenated into an attribute (`'href="'+esc(url)+'"'`) it's XSS via `" onmouseover="…`. A protocol allowlist `/^https?:/` does NOT stop attribute breakout. (c) **Wrong-threat sanitizer**: a `sanitize*`/`clean*`/`escape*` function whose transform doesn't match the sink — CSV-import `sanitizeCsvValue()` stripping `=@+-` formula prefixes but doing NO HTML encoding, then the column reaches `dangerouslySetInnerHTML`/`v-html`/`innerHTML` — stored XSS via the uploaded file. The misleading function name is the false-safety signal. **Sibling Validator/Sanitizer Asymmetry**: A diff where ONE field/argument receives a security refinement (regex/`.refine()`/sanitizer like `escapeHtml`/`stripBidiChars`/`DOMPurify.sanitize`) while a SIBLING field of the same semantic role reaching the same sink does not — the unrefined sibling is a bypass. The `+` line adding the refinement to one place is the cue: check every sibling. -**Orchestrator Template Injection (Airflow/Argo/Tekton)**: Airflow `{{{{ run_id }}}}`/`{{{{ dag_run.conf[...] }}}}`/`{{{{ params.* }}}}`, Argo `{{{{workflow.parameters.*}}}}`, or Tekton `$(params.*)` rendered into a shell string (`bash_command=`, `cmds=["bash","-c", ...]`, `script:`) — these are user-settable via the trigger API. Fix: pass as a separate argv element or env var. Do NOT flag scheduler-only macros like `{{{{ ds }}}}`. +**Orchestrator Template Injection (Airflow/Argo/Tekton)**: Airflow `{{ run_id }}`/`{{ dag_run.conf[...] }}`/`{{ params.* }}`, Argo `{{workflow.parameters.*}}`, or Tekton `$(params.*)` rendered into a shell string (`bash_command=`, `cmds=["bash","-c", ...]`, `script:`) — these are user-settable via the trigger API. Fix: pass as a separate argv element or env var. Do NOT flag scheduler-only macros like `{{ ds }}`. -**SSRF URL-Allowlist Bypass**: Host allowlists are bypassable via: (a) USERINFO — `url.startswith(allowed_prefix)` or comparing `urlparse().netloc`/`url.host` (which include `user:pass@`) lets `https://trusted.com@evil.com/x` through; compare ONLY `urlparse(u).hostname` / `new URL(u).hostname` / `u.Hostname()`. (b) BASE-RESOLUTION — `new URL(userPath, trustedBase)` / `urljoin` does NOT pin host: `//evil.com/x` is protocol-relative, absolute `http://evil.com` ignores base; check `result.hostname === expectedHost` AFTER resolution. (c) STRING-SUFFIX — `host.endswith('.trusted.com')` on a value later interpolated into `f"https://{{host}}"` passes `evil.com/.trusted.com` and `evil.com#.trusted.com`. (d) NORMALIZATION — missing `.lower().rstrip('.')` lets `Trusted.COM.` slip; falsy-netloc short-circuit `if parsed.netloc and parsed.netloc != allowed:` lets `http:evil.com` through. (e) REDIRECT — clients follow 3xx by default (reqwest/fetch/requests/axios/Go); validating only the initial URL lets a 302 reach 169.254.169.254. Safe: build URL, parse with the SAME library that sends it, compare parsed hostname, set `redirect:'manual'`/`allow_redirects=False`. +**SSRF URL-Allowlist Bypass**: Host allowlists are bypassable via: (a) USERINFO — `url.startswith(allowed_prefix)` or comparing `urlparse().netloc`/`url.host` (which include `user:pass@`) lets `https://trusted.com@evil.com/x` through; compare ONLY `urlparse(u).hostname` / `new URL(u).hostname` / `u.Hostname()`. (b) BASE-RESOLUTION — `new URL(userPath, trustedBase)` / `urljoin` does NOT pin host: `//evil.com/x` is protocol-relative, absolute `http://evil.com` ignores base; check `result.hostname === expectedHost` AFTER resolution. (c) STRING-SUFFIX — `host.endswith('.trusted.com')` on a value later interpolated into `f"https://{host}"` passes `evil.com/.trusted.com` and `evil.com#.trusted.com`. (d) NORMALIZATION — missing `.lower().rstrip('.')` lets `Trusted.COM.` slip; falsy-netloc short-circuit `if parsed.netloc and parsed.netloc != allowed:` lets `http:evil.com` through. (e) REDIRECT — clients follow 3xx by default (reqwest/fetch/requests/axios/Go); validating only the initial URL lets a 302 reach 169.254.169.254. Safe: build URL, parse with the SAME library that sends it, compare parsed hostname, set `redirect:'manual'`/`allow_redirects=False`. **XXE / XML Entity Expansion**: Untrusted XML (uploaded .docx/.xlsx/.pptx/.svg, SOAP/SAML bodies, feed/webhook payloads, OOXML extracted from a zip) parsed with Python stdlib `xml.etree.ElementTree`, `xml.dom.minidom.parse`/`parseString`, `xml.sax.make_parser`, or `xml.dom.pulldom` — these do NOT disable DTDs or external entities, so `<!ENTITY x SYSTEM "file:///etc/passwd">` reads local files and a billion-laughs entity bomb DoS's the process. Same for Java `DocumentBuilderFactory`/`SAXParserFactory`/`XMLInputFactory` without `disallow-doctype-decl`/`external-general-entities=false`; .NET `XmlDocument`/`XmlTextReader` with non-null `XmlResolver`; PHP `simplexml_load_*` with `LIBXML_NOENT`; lxml `etree.parse` with `resolve_entities=True`. Fix: Python → swap import to `defusedxml.*`; Java → `factory.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true)`; lxml → `XMLParser(resolve_entities=False, no_network=True)`. Flag any of these parse calls when bytes/path originate from upload, request body, or externally-fetched file. **Substring/Unanchored Allowlist Bypass**: A security gate — allowlist, host/origin check, redirect-target validation, or SIEM/detection-rule exclusion — that matches by substring (`allowed in value`, `value.includes(allowed)`, `strings.Contains`, unanchored `re.search`) or unanchored prefix/suffix (`value.startswith("https://trusted.com")` with no trailing `/`; `value.endswith("trusted.com")` with no leading `.`) is bypassable: `trusted.com.evil.com`, `eviltrusted.com`, `evil.com/?x=trusted.com`. URL string-match on RAW `requestURI`: `/proxy/exec?_=/proxy/metrics` ends with `/proxy/metrics`; `/public/../admin` contains `/public/`. ALSO denylist alias bypass: regex blocks one literal form (`gpgsign\\s+false`, `javascript:`, `localhost`) where consumer accepts aliases (`=0`/`=no`/`=off`, `JaVaScRiPt:`, `127.0.0.1`/`[::1]`). ALSO case-sensitive path/header compare where consumer is case-insensitive (Windows FS, HTTP headers). Fix: parse the structured field (`urlparse().path`/`.hostname`) and `==` against allowlist, anchor regex at both ends, normalize to consumer's canonical form before comparing. -**XSS via Manual HTML/Markdown Building**: Code assembling HTML by string formatting — `format!("<a href='{{x}}'>")`, `f"<div>{{val}}</div>"`, `fmt.Fprintf(w, "<span>%s</span>", v)`, `"<li>" + s + "</li>"` — is XSS at EVERY interpolated `{{var}}` lacking escape. INCONSISTENT: function calls `html.escape()` on SOME fields but interpolates others raw — audit each `{{...}}` individually; one `html.escape` nearby is NOT proof of safety. ATTRIBUTE-CONTEXT: data concatenated into quoted attribute (`'<a href="' + x + '">'`) is XSS unless escaper encodes `"` AND `'`; the `div.textContent=s; div.innerHTML` trick and `.replace(/[<>&]/g,...)` escape only `< > &` — NOT quotes; `[x](https://a" onmouseover=alert(1))` breaks out of `href`. MARKDOWN: `<MDEditor.Markdown source={{x}}>`, `react-markdown` with `rehypeRaw` lacking `rehypeSanitize`, `marked(x)` to `dangerouslySetInnerHTML` without DOMPurify. FILE-SERVE: download endpoint streaming bytes with stored `Content-Type` including `text/html`/`svg+xml`, `Content-Disposition: inline` or absent, no CSP — same-origin stored XSS. UNTRUSTED-ON-FIRST-PARTY: `httputil.ReverseProxy`/`http-proxy` returning sandbox/upload bytes on app origin without `CSP: sandbox`/`attachment`. +**XSS via Manual HTML/Markdown Building**: Code assembling HTML by string formatting — `format!("<a href='{x}'>")`, `f"<div>{val}</div>"`, `fmt.Fprintf(w, "<span>%s</span>", v)`, `"<li>" + s + "</li>"` — is XSS at EVERY interpolated `{var}` lacking escape. INCONSISTENT: function calls `html.escape()` on SOME fields but interpolates others raw — audit each `{...}` individually; one `html.escape` nearby is NOT proof of safety. ATTRIBUTE-CONTEXT: data concatenated into quoted attribute (`'<a href="' + x + '">'`) is XSS unless escaper encodes `"` AND `'`; the `div.textContent=s; div.innerHTML` trick and `.replace(/[<>&]/g,...)` escape only `< > &` — NOT quotes; `[x](https://a" onmouseover=alert(1))` breaks out of `href`. MARKDOWN: `<MDEditor.Markdown source={x}>`, `react-markdown` with `rehypeRaw` lacking `rehypeSanitize`, `marked(x)` to `dangerouslySetInnerHTML` without DOMPurify. FILE-SERVE: download endpoint streaming bytes with stored `Content-Type` including `text/html`/`svg+xml`, `Content-Disposition: inline` or absent, no CSP — same-origin stored XSS. UNTRUSTED-ON-FIRST-PARTY: `httputil.ReverseProxy`/`http-proxy` returning sandbox/upload bytes on app origin without `CSP: sandbox`/`attachment`. **Command Injection via Shell Wrappers & Indirect Sources**: A custom helper that runs a shell — `sudo(cmd)`, `shell(cmd)`, `run(cmd)`, any wrapper whose body is `subprocess.run(cmd, shell=True)` / `Popen(["sh","-c",cmd])` — is the SAME sink as `os.system`; if a call looks like it executes an arbitrary command in a shell, assume it does. Any f-string/`+` building its argument from a non-literal is injectable. Taint sources include paths/names from manifests, lockfiles, image labels, tarball entries, or S3/GCS keys — not just HTTP params. `Path(x).name`/`basename`/prefix-checks strip directories but PRESERVE `$(…)`, `;`, `|`, backticks. Fix: `shlex.quote()` every segment, or pass an argv list without a shell. -**Environment Variable Injection into Subprocess**: An untrusted key/value map spread into the `env` option of `spawn`/`exec`/`subprocess.Popen`/`exec.Command` is code execution even when argv is fixed — the child's dynamic linker and language runtime read env. Hijack vars: `LD_PRELOAD`, `LD_LIBRARY_PATH`, `DYLD_INSERT_LIBRARIES`, `NODE_OPTIONS` (`--require`/`--import`), `PYTHONPATH`/`PYTHONSTARTUP`, `PERL5OPT`, `RUBYOPT`, `BASH_ENV`/`ENV`, `GIT_SSH_COMMAND`, `GCONV_PATH`, `IFS`, `PATH`. Shape: `spawn(cmd, args, {{env: {{...process.env, ...untrusted}}}})`, `Popen(..., env={{**os.environ, **untrusted}})`. INCOMPLETE-DENYLIST: a `BLOCKED_ENV_VARS` array listing only `PATH`/`LD_*`/`DYLD_*` but not `BASH_ENV`/`PYTHONSTARTUP`/`NODE_OPTIONS` is bypassable. INHERITED-LEAK: `process.env.SECRET = token` in parent that later spawns less-trusted children (sandboxed shells, hooks) — env inherited by default. Fix: `env_clear()` + explicit allowlist, or deny by prefix family. +**Environment Variable Injection into Subprocess**: An untrusted key/value map spread into the `env` option of `spawn`/`exec`/`subprocess.Popen`/`exec.Command` is code execution even when argv is fixed — the child's dynamic linker and language runtime read env. Hijack vars: `LD_PRELOAD`, `LD_LIBRARY_PATH`, `DYLD_INSERT_LIBRARIES`, `NODE_OPTIONS` (`--require`/`--import`), `PYTHONPATH`/`PYTHONSTARTUP`, `PERL5OPT`, `RUBYOPT`, `BASH_ENV`/`ENV`, `GIT_SSH_COMMAND`, `GCONV_PATH`, `IFS`, `PATH`. Shape: `spawn(cmd, args, {env: {...process.env, ...untrusted}})`, `Popen(..., env={**os.environ, **untrusted})`. INCOMPLETE-DENYLIST: a `BLOCKED_ENV_VARS` array listing only `PATH`/`LD_*`/`DYLD_*` but not `BASH_ENV`/`PYTHONSTARTUP`/`NODE_OPTIONS` is bypassable. INHERITED-LEAK: `process.env.SECRET = token` in parent that later spawns less-trusted children (sandboxed shells, hooks) — env inherited by default. Fix: `env_clear()` + explicit allowlist, or deny by prefix family. **Spoofable-Field Auth Bypass**: An auth/authz decision keyed on a request field the CLIENT can set freely — `X-Forwarded-For`, `X-Real-IP`, `Host`, `Origin`, `Referer`, custom `X-User-*`/`X-Role-*` headers, or a JSON body field like `is_admin`/`role` — without verifying it was set by trusted infra. ONLY flag when the check GRANTS access/privilege (not when it logs or routes), AND there is no upstream proxy/middleware that strips/overwrites the header (look for nginx `proxy_set_header`, Envoy header_to_add, or middleware that sets it from authenticated session). -**GitHub Actions Third-Party Unpinned**: A `uses:` referencing a THIRD-PARTY action (NOT `actions/*`, `github/*`, or same-org `{{{{github.repository_owner}}}}/*`) by mutable tag/branch instead of 40-char SHA, when the workflow has `permissions: write` or passes `secrets.*`. Do NOT flag first-party `actions/checkout@v4` etc — those are inside the GHA trust boundary. +**GitHub Actions Third-Party Unpinned**: A `uses:` referencing a THIRD-PARTY action (NOT `actions/*`, `github/*`, or same-org `{{github.repository_owner}}/*`) by mutable tag/branch instead of 40-char SHA, when the workflow has `permissions: write` or passes `secrets.*`. Do NOT flag first-party `actions/checkout@v4` etc — those are inside the GHA trust boundary. **Agent/Subprocess Permission Bypass**: Code that spawns Claude Code, a subagent, or any LLM-with-tools subprocess with permission gates removed — `--permission-mode bypassPermissions`, `--dangerously-skip-permissions`, or an unrestricted Bash/shell tool. Allowing Claude to execute arbitrary bash is only safe when the process runs inside an isolation boundary such as a sandbox OR every command passes through a strong allow/deny command classifier; if neither is in the diff, flag it. -**Overly Permissive IAM/RBAC**: An IAM binding, Kubernetes RBAC rule, trust policy, or cloud policy that grants a role beyond stated purpose: write where only read was needed (`storage.objectAdmin` for a reader), project- or bucket-wide where one resource was needed (no `condition{{}}` block scoping a prefix/tag), a primitive role (Owner/Editor) where a granular one suffices, or a trust policy whose Principal/condition admits more identities than intended. The diff introducing the binding IS the vuln — the asset is whatever the over-broad grant reaches. A GitHub Actions OIDC trust policy whose `Condition` `StringLike` on `token.actions.githubusercontent.com:sub` ends in `:*` (e.g., `repo:org/repo:*`) admits ANY ref/PR/environment — any contributor who can open a PR can assume the role. +**Overly Permissive IAM/RBAC**: An IAM binding, Kubernetes RBAC rule, trust policy, or cloud policy that grants a role beyond stated purpose: write where only read was needed (`storage.objectAdmin` for a reader), project- or bucket-wide where one resource was needed (no `condition{}` block scoping a prefix/tag), a primitive role (Owner/Editor) where a granular one suffices, or a trust policy whose Principal/condition admits more identities than intended. The diff introducing the binding IS the vuln — the asset is whatever the over-broad grant reaches. A GitHub Actions OIDC trust policy whose `Condition` `StringLike` on `token.actions.githubusercontent.com:sub` ends in `:*` (e.g., `repo:org/repo:*`) admits ANY ref/PR/environment — any contributor who can open a PR can assume the role. **Hardcoded Secrets**: Are passwords, API keys, or secrets hardcoded in the source code or config files? @@ -935,11 +1561,11 @@ A NEW security-gate parameter (group/role/tool/permission/scope) is safe only if **Boolean Type Coercion (Python)**: In Python, multipart form data sends all values as strings. `bool("false")` returns `True` because any non-empty string is truthy. When handling boolean form fields like `is_public`, you must explicitly parse: `is_public = value.lower() in ('true', '1', 'yes')`. Simply doing `is_public = request.form.get('is_public', True)` or `is_public = bool(request.form.get('is_public'))` is INSECURE because the string "false" evaluates to True. -**Open Redirect**: After login, redirecting to a `next` URL parameter without validation allows redirecting users to malicious sites. In Python/Flask: `redirect(request.args.get('next'))` is ALWAYS vulnerable. In Django: `redirect(request.GET.get('next'))` is ALWAYS vulnerable. Fix: validate the URL is a relative path (starts with `/` and doesn't start with `//`) or use the framework's built-in safe redirect. Django: use `url_has_allowed_host_and_scheme(url, allowed_hosts={{request.get_host()}})`. Flask: check `url.startswith('/') and not url.startswith('//')`. +**Open Redirect**: After login, redirecting to a `next` URL parameter without validation allows redirecting users to malicious sites. In Python/Flask: `redirect(request.args.get('next'))` is ALWAYS vulnerable. In Django: `redirect(request.GET.get('next'))` is ALWAYS vulnerable. Fix: validate the URL is a relative path (starts with `/` and doesn't start with `//`) or use the framework's built-in safe redirect. Django: use `url_has_allowed_host_and_scheme(url, allowed_hosts={request.get_host()})`. Flask: check `url.startswith('/') and not url.startswith('//')`. **Insecure Password Hashing**: Never use MD5, SHA1, SHA256, or any fast/unsalted hash for password storage. Use bcrypt, scrypt, argon2, or PBKDF2. In Python: use `werkzeug.security.generate_password_hash()` or `bcrypt.hashpw()`. In Django: use `User.objects.create_user()` which handles hashing automatically. -**Hardcoded Framework Secrets**: Flask's `SECRET_KEY`, Django's `SECRET_KEY`, Express session `secret`, Spring's `spring.datasource.password`, and `DEBUG = True` must not be hardcoded with static strings. Read from environment variables: `os.environ.get('SECRET_KEY', os.urandom(32))`, `process.env.SESSION_SECRET`, `${{DB_PASSWORD}}`. A static/hardcoded string is INSECURE regardless of its complexity. +**Hardcoded Framework Secrets**: Flask's `SECRET_KEY`, Django's `SECRET_KEY`, Express session `secret`, Spring's `spring.datasource.password`, and `DEBUG = True` must not be hardcoded with static strings. Read from environment variables: `os.environ.get('SECRET_KEY', os.urandom(32))`, `process.env.SESSION_SECRET`, `${DB_PASSWORD}`. A static/hardcoded string is INSECURE regardless of its complexity. **Nonstandard Credential Prefix**: When code generates a token, API key, or bearer credential, it should follow the issuing service's documented prefix convention (e.g. `sk-` for OpenAI/Anthropic-style API keys, `ghp_` for GitHub, `AKIA` for AWS). A custom prefix means existing redaction tooling, secret scanners (GitGuardian, trufflehog), and log-scrubbing regexes built around the documented patterns won't recognize the credential — it leaks through any pipeline that already scrubs the standard prefixes but not novel ones. Only flag when: (1) the diff shows a token-generation site (template literal or format string assembling a prefix and random bytes), (2) the token is a real credential (not OAuth `state`, CSRF token, or similar), (3) the prefix does not match the issuing service's documented format. @@ -991,7 +1617,7 @@ DO NOT flag: - Development fallback secrets like `os.environ.get('SECRET_KEY', 'dev-fallback')` or `process.env.SECRET || 'dev-default'` — these are legitimate development patterns - Flask/Django SECRET_KEY or session secrets in development/example code, seed scripts, or test files — only flag in production config files - Path traversal in code where the path is NOT user-controlled (e.g., file paths constructed from hardcoded strings, config values, CLI arguments in trusted tools, or internal function parameters). Environment variables and CLI arguments are trusted input sources. -- XSS in code that does not handle HTTP requests or render HTML to browsers (e.g., CLI tools, backend services, data processing scripts, seed files). React auto-escapes text content, BUT flag: `dangerouslySetInnerHTML` with user input; user-controlled `href`/`src`/`location` without an http(s) scheme allowlist (`javascript:`/`data:` URIs execute); second-stage template placeholders (`{{var}}` lacking `|e`) embedded as string literals in JSX/MJML/email builders — the outer auto-escape only preserves the braces. +- XSS in code that does not handle HTTP requests or render HTML to browsers (e.g., CLI tools, backend services, data processing scripts, seed files). React auto-escapes text content, BUT flag: `dangerouslySetInnerHTML` with user input; user-controlled `href`/`src`/`location` without an http(s) scheme allowlist (`javascript:`/`data:` URIs execute); second-stage template placeholders (`{var}` lacking `|e`) embedded as string literals in JSX/MJML/email builders — the outer auto-escape only preserves the braces. - Open redirect in code that does not handle HTTP requests - SSRF in code where URLs are not user-controlled (e.g., hardcoded API endpoints, config-driven URLs). SSRF where the attacker only controls the path (not host or protocol) is generally lower severity, BUT should still be flagged as a potential low severity issue. - SQL injection in code using parameterized queries, ORMs, or query builders (these are safe by design) @@ -1001,7 +1627,101 @@ DO NOT flag: - Crashes from undefined variables, missing keys, or type errors — these are bugs, not security vulnerabilities - Telemetry/analytics API keys (Honeycomb, Datadog, Sentry, etc.) — these are designed to be client-side - Open redirect in URL shorteners, link redirectors, or proxy endpoints where redirecting to user-provided URLs IS the intended feature -- Vulnerabilities in pre-existing starter/template code that was not written by the developer in this session +- Vulnerabilities in pre-existing starter/template code that was not written by the developer in this session""" + + +def analyze_code_security(files: List[Tuple[str, str]], is_diff: bool = False, previous_findings: Optional[List[str]] = None) -> Tuple[Optional[str], List[Dict[str, Any]]]: + """ + Use Haiku to perform a security review of code. + files: list of (file_path, content_or_diff) tuples + is_diff: if True, the content is a unified diff rather than full file contents + previous_findings: list of category strings from earlier stop hook firings this turn, + used to prompt the reviewer to verify those issues were actually fixed. + Returns (formatted guidance string or None, list of vuln dicts with severity/category). + Also returns (None, []) when the model gave no verdict; last_review_failed() + tells the two apart. + """ + _review_status.failed = False + _review_status.status = None + if not (HAS_API_CREDENTIALS or _gateway_cli) or not files: + return None, [] + + # Build language context from file extensions + lang_hints = { + ".go": "Go", ".java": "Java/Spring Boot", ".py": "Python", + ".rb": "Ruby", ".php": "PHP", ".rs": "Rust", + ".ts": "TypeScript", ".js": "JavaScript", ".jsx": "JavaScript/React", + ".tsx": "TypeScript/React", ".ejs": "EJS templates", + ".html": "HTML/templates", ".properties": "Java properties", + ".yaml": "YAML config", ".yml": "YAML config", + } + languages = set() + for fp, _ in files: + ext = os.path.splitext(fp)[1].lower() + if ext in lang_hints: + languages.add(lang_hints[ext]) + language = ", ".join(sorted(languages)) if languages else "server-side" + + files = _cap_files_for_prompt(files) + + # Build the files section + files_section = [] + for fp, content in files: + ext = os.path.splitext(fp)[1].lower() + label = "DIFF" if is_diff else "FILE" + files_section.append(f"=== {label}: {fp} ===\n```{ext.lstrip('.')}\n{content}\n```") + files_text = "\n\n".join(files_section) + + content_desc = "diff" if is_diff else "code" + + if is_diff: + diff_instruction = """Note: You are reviewing a unified diff. Unmarked lines (starting with a space) are UNCHANGED context — they were already in the file before this session. Lines starting with + are ADDITIONS made in this session. Lines starting with - are REMOVALS. + +CRITICAL: ONLY flag vulnerabilities that are NEWLY INTRODUCED in + lines. Do NOT flag: +- Issues in unmarked context lines (space-prefixed = pre-existing code). Even if a context line contains SECRET_KEY = 'hardcoded', DEBUG=True, hardcoded passwords, SQL injection, or any other vulnerability — it is PRE-EXISTING and must be ignored. +- Issues where the SAME pattern existed in the removed (-) lines and was re-added in + lines (this means the code was rewritten/reformatted but the pattern is pre-existing) +- Pre-existing patterns that Claude simply preserved when rewriting a file +- Any vulnerability whose vulnerable code snippet appears in context (space-prefixed) lines +- Vulnerabilities in the ORIGINAL/STARTER code that the developer was given to work with. If a file was fully rewritten (all lines show as - then +), compare the + content against the - content. Only flag NEWLY INTRODUCED patterns that did NOT exist in the - lines. +- Issues OUTSIDE THE SCOPE of what the developer was asked to do. If the task was "add logging middleware" and the starter code has a hardcoded SECRET_KEY, that is pre-existing and out of scope — do NOT flag it. + +A vulnerability is ONLY new if the + lines introduce a pattern that did NOT exist anywhere in the - lines or context lines of the same file. + +EXCEPTION — data flow to pre-existing sinks: If + lines route user-controlled data to a PRE-EXISTING dangerous sink (like `new Function()`, `eval()`, `exec()`, or shell string interpolation in context lines), this IS a new vulnerability. The sink was already there, but the new code created a new attack path to it. Flag this as a new vulnerability in the + lines.""" + else: + diff_instruction = "" + + structured_prev = [f for f in (previous_findings or []) if isinstance(f, dict)] + if structured_prev: + prev_lines = "\n".join( + f" - {f.get('filePath', '?')} [{f.get('category', '?')}]: {f.get('vulnerableCode', '?')}" + for f in structured_prev + ) + prev_section = ( + "PREVIOUS FINDINGS (already surfaced to the developer earlier this turn — DO NOT re-flag):\n" + "The exact findings below were already shown to the developer, who has either fixed them or " + "acknowledged them as not applicable. DO NOT report any finding whose (filePath, category) pair " + "matches an entry below — it was already handled. The vulnerableCode may differ slightly from " + "what you see now (diff context lines shift between fires) — match on file + category, not exact " + "code bytes. ONLY re-flag a (filePath, category) from this list if the code at that location was " + "CHANGED since the prior review and the change is an incomplete fix or introduces a new issue.\n" + f"{prev_lines}\n" + ) + else: + prev_section = "" + + prompt = """You are a security expert reviewing {language} {content_desc}. Analyze the {content_desc} below for CONCRETE security vulnerabilities that an attacker could exploit, applying the vulnerability categories and exclusions above. + +{diff_instruction} + +{prev_section} + +For each vulnerability found, provide: +1. The file path where it occurs (use the exact path from the === {file_type}: header) +2. The vulnerability category +3. The specific vulnerable code (quote the exact line(s)) +4. How an attacker would exploit it +5. A specific code fix {files_text} @@ -1040,10 +1760,22 @@ Respond with a JSON object. If vulnerabilities are found, set hasVulnerabilities } prompt += extensibility.guidance_block() - analysis = _call_claude_dual_or(prompt, output_schema, - bool_key="hasVulnerabilities", - list_key="vulnerabilities") - if not analysis or not analysis.get("hasVulnerabilities") or not analysis.get("vulnerabilities"): + try: + analysis = _call_claude_dual_or(prompt, output_schema, + bool_key="hasVulnerabilities", + list_key="vulnerabilities", + cacheable_prefix=_CODE_REVIEW_RUBRIC) + except Exception as e: + debug_log(f"LLM code review: call raised {type(e).__name__}: {e}") + _record_net_error(_net_error_code(e)) + analysis = None + # A reply without the schema's required flag is not a verdict either. + if not isinstance(analysis, dict) or "hasVulnerabilities" not in analysis: + _review_status.failed = True + _review_status.status = _last_call_claude_http_error + debug_log("LLM code review: no verdict from the model") + return None, [] + if not analysis.get("hasVulnerabilities") or not analysis.get("vulnerabilities"): debug_log("LLM code review: no vulnerabilities found") return None, [] @@ -1089,6 +1821,145 @@ _FINDINGS_SCHEMA = review_api.FINDINGS_SCHEMA _SURVIVED_SCHEMA = review_api.SURVIVED_SCHEMA +class _DebugLogHandler(logging.Handler): + def emit(self, record): + try: + text = record.getMessage() + if record.exc_info: + text += " | " + "".join( + traceback.format_exception_only(*record.exc_info[:2])).strip() + except Exception: + return # never handleError(): it prints to stderr + debug_log(f"agent sdk: {text}") + + +_sdk_log_lock = threading.Lock() + + +def _send_sdk_logs_to_debug_log() -> None: + """Route the Agent SDK's own log records to the debug log, and stop them + propagating. With no handler, Python prints its errors to this hook's + stderr, which Claude Code shows Claude as the review notice.""" + sdk_logger = logging.getLogger("claude_agent_sdk") + with _sdk_log_lock: + if not any(isinstance(h, _DebugLogHandler) for h in sdk_logger.handlers): + sdk_logger.addHandler(_DebugLogHandler()) + sdk_logger.propagate = False + + +# Claude Code older than 2.0.63 exits at startup with "unknown option" when +# given --no-session-persistence. The flag is sent until an inner CLI +# refuses it. +_SESSION_FLAG = "no-session-persistence" +_send_session_flag = True + + +def _reviewer_session_args() -> Dict[str, Optional[str]]: + """Extra CLI flags for the SDK-spawned inner `claude`. + + Review runs are one-shot and never resumed, so by default the inner CLI + is told not to save them as session transcripts (they hold the full + review prompt, including diff hunks). SECURITY_GUIDANCE_PERSIST_SESSIONS=1 + keeps the transcripts, e.g. to debug a review. + """ + keep = os.environ.get("SECURITY_GUIDANCE_PERSIST_SESSIONS", "").strip().lower() + if not _send_session_flag or keep in ("1", "on", "true", "yes"): + return {} + return {_SESSION_FLAG: None} + + +def _names_session_flag(stderr_line: Any) -> bool: + text = str(stderr_line) + return _SESSION_FLAG in text and "unknown option" in text.lower() + + +# The agentic reviewer reads a diff it must not trust, so it gets +# file-reading tools and nothing else. +_REVIEWER_TOOLS = ("Read", "Grep", "Glob") + + +def _only_tools_args(tools: Tuple[str, ...] = ()) -> Dict[str, Optional[str]]: + """extra_args that make `tools` the inner CLI's whole tool set. + + allowed_tools only pre-approves tools. Without these flags the inner CLI + still offers every Claude Code tool (shell, file edits, web, subagents) + and any configured MCP server, and whether a call to one runs is left to + the permission mode that CLI starts in. Passed as CLI flags because not + every Agent SDK build that can run a review has a `tools` option. + """ + args: Dict[str, Optional[str]] = {"tools": ",".join(tools)} + if _send_strict_mcp: + args["strict-mcp-config"] = None + else: + # The servers an administrator delivers still start; their tools are + # kept from the reviewer. + args["disallowedTools"] = "mcp__*" + return args + + +# Claude Code exits at startup when given --strict-mcp-config on a machine +# whose administrator delivers the MCP servers (managed-mcp.json). The flag +# is sent until an inner CLI refuses it. +_send_strict_mcp = True + + +def _names_strict_mcp_flag(stderr_line: Any) -> bool: + return "--strict-mcp-config" in str(stderr_line) + + +def _refused_flags(stderr_lines: List[Any]) -> List[str]: + """Which of the flags this plugin can leave out the lines refuse.""" + refused = [] + if any(_names_strict_mcp_flag(l) for l in stderr_lines): + refused.append("--strict-mcp-config") + if any(_names_session_flag(l) for l in stderr_lines): + refused.append("--" + _SESSION_FLAG) + return refused + + +def _run_inner_cli(start: Callable[[], Any], stderr_lines: List[str]) -> Any: + """start() runs one inner CLI to its result; stderr_lines receives what + that CLI writes to stderr. A start the CLI refused over + --strict-mcp-config or --no-session-persistence is made again without + that flag, and the flag is left out for the rest of this process. A CLI + names one flag per start, so one that refuses both is started three + times.""" + global _send_strict_mcp, _send_session_flag + left_out: List[str] = [] + while True: + del stderr_lines[:] + try: + return start() + except Exception: + refused = [f for f in _refused_flags(stderr_lines) if f not in left_out] + if not refused: + raise + if "--strict-mcp-config" in refused: + _send_strict_mcp = False + if "--" + _SESSION_FLAG in refused: + _send_session_flag = False + left_out += refused + debug_log(f"inner CLI refused {' and '.join(refused)}; " + "starting it again without") + + +class _ReviewerAuthFailed(Exception): + """The agentic reviewer's credentials were refused; retrying cannot help.""" + + +# Retry notices in a row, each after a 401, at which the review stops. One is +# normal: the CLI renews a stored login after a 401 and tries again. Not used +# in a remote session, where the CLI waits out a short sign-in outage on purpose. +_AGENTIC_MAX_401_RETRIES = 3 + + +def _api_retry_status(msg: Any) -> Optional[int]: + """The HTTP status of the CLI's api_retry notice, if it gives one.""" + data = getattr(msg, "data", None) + status = data.get("error_status") if isinstance(data, dict) else None + return status if isinstance(status, int) else None + + def _agentic_spawn_env() -> Dict[str, str]: """opts.env for the SDK-spawned inner `claude` CLI. @@ -1107,15 +1978,18 @@ def _agentic_spawn_env() -> Dict[str, str]: is the only credential. """ env = { + # Policy settings can enable this plugin inside the reviewing CLI too; + # its hooks there must not start reviews of their own. + "SECURITY_GUIDANCE_DISABLE": "1", "FALLBACK_FOR_ALL_PRIMARY_MODELS": "1", "CLAUDE_CODE_WEBSOCKET_AUTH_FILE_DESCRIPTOR": "", "CLAUDE_CODE_OAUTH_TOKEN_FILE_DESCRIPTOR": "", "CLAUDE_CODE_INCLUDE_PARTIAL_MESSAGES": "", - # Neutralize git config/env hijack vectors so an allowlisted - # `git diff/log/show` cannot be turned into RCE via diff.external, - # core.pager, core.sshCommand, or an inherited GIT_* var. The agentic - # session only needs read-only history inspection; it never needs an - # external diff driver, a pager, or a remote. + # Neutralize git config/env hijack vectors so a `git diff/log/show` + # cannot be turned into RCE via diff.external, core.pager, + # core.sshCommand, or an inherited GIT_* var. The inner CLI has no + # shell tool (see _only_tools_args); this stays as a second line of + # defence should it ever run git. "GIT_CONFIG_NOSYSTEM": "1", "GIT_CONFIG_GLOBAL": "/dev/null", "GIT_EXTERNAL_DIFF": "", @@ -1137,13 +2011,50 @@ def _agentic_spawn_env() -> Dict[str, str]: return env +# The tools the agentic reviewer is given to look around the repo. +_REVIEWER_FILE_TOOLS = ("Read", "Grep", "Glob") + + +def _agentic_context_dir(repo_dir: str) -> str: + """The directory the agentic reviewer runs in.""" + return os.environ.get("SG_AGENTIC_CONTEXT_DIR") or repo_dir + + def agentic_review( repo_dir: str, diff_files: List[Tuple[str, str]], touched_paths: List[str], ) -> Tuple[Optional[str], List[Dict[str, Any]], Dict[str, Any]]: + """Run the two-stage agentic review (_agentic_review) and add reviewer + context diagnostics to its metrics. Counts only, no paths: + + rv_cwd_gone 1 = the directory the reviewer runs in was missing + before the review, 2 = it went missing during it + rv_tool_err Read/Grep/Glob calls that came back as errors + rv_tool_calls Read/Grep/Glob calls made + rv_turns turns the reviewer took, as the CLI counts them + rv_api_ms time the reviewer spent waiting on the model, in ms + + The counts cover all stages of the review. Keys are left out when 0. + rv_turns and rv_api_ms come from each stage's result message, so a stage + that ends without one (the CLI exits early) adds nothing to them.""" + context_dir = _agentic_context_dir(repo_dir) + gone = 0 if os.path.isdir(context_dir) else 1 + ctx = {"rv_tool_err": 0, "rv_tool_calls": 0, "rv_turns": 0, "rv_api_ms": 0} + guidance, vulns, metrics = _agentic_review(repo_dir, diff_files, touched_paths, ctx) + if not gone and not os.path.isdir(context_dir): + gone = 2 + ctx_m = {"rv_cwd_gone": gone, **ctx} + return guidance, vulns, {**metrics, **{k: v for k, v in ctx_m.items() if v}} + + +def _agentic_review( + repo_dir: str, diff_files: List[Tuple[str, str]], touched_paths: List[str], + ctx: Dict[str, int], +) -> Tuple[Optional[str], List[Dict[str, Any]], Dict[str, Any]]: """Two-stage Agent-SDK review: investigate (Read/Grep/Glob over the repo) then a self-refute filter pass. Returns (guidance_or_None, vulns, metrics). On SDK unavailability returns (None, [], {"agentic_fallback": - reason}) so the caller can fall back to the single-shot path.""" + reason}) so the caller can fall back to the single-shot path. Adds to + the `ctx` counters (see agentic_review) as the reviewer makes tool calls.""" import time as _t # Note: do NOT pop ANTHROPIC_AUTH_TOKEN from os.environ here. The race @@ -1187,16 +2098,15 @@ def agentic_review( debug_log(f"agentic_review: SDK unavailable ({e}); falling back") return None, [], {"agentic_fallback": f"import:{type(e).__name__}"} - # Default to the documented public model. Overridable via SG_AGENTIC_MODEL. - # The bundled SDK CLI only knows public model names. - _DEFAULT_PUBLIC_MODEL = "claude-opus-4-7" - model = os.environ.get("SG_AGENTIC_MODEL") or _DEFAULT_PUBLIC_MODEL + _send_sdk_logs_to_debug_log() + # Overridable via SG_AGENTIC_MODEL. + model = os.environ.get("SG_AGENTIC_MODEL") or _CLI_OPUS_ALIAS max_turns = int(os.environ.get("SG_AGENTIC_MAX_TURNS", "18")) # In production repo_dir is the user's working tree (full repo). Under the # eval harness it's a temp dir with ONLY touched_paths — the agent can't # trace cross-file data flow. The harness sets SG_AGENTIC_CONTEXT_DIR to a # full repo (worktree at the commit, or the live clone at HEAD). - context_dir = os.environ.get("SG_AGENTIC_CONTEXT_DIR") or repo_dir + context_dir = _agentic_context_dir(repo_dir) context_note = "" if context_dir != repo_dir: context_note = ( @@ -1221,57 +2131,41 @@ def agentic_review( "the findings list." ) - # Always prefer the user's installed `claude` over the SDK's bundled CLI. + # Prefer the `claude` that _user_cli_path finds over the SDK's bundled CLI. # The bundled CLI is whatever shipped with the pip-installed SDK version # and can lag the user's CLI by months — protocol skew between them is a # top cause of agentic_fallback=2 in production (the SDK reads # `[Request interrupted by user]` and gives up). The CLI that launched # this hook is by definition >= the plugin's tested floor, so it's # always at least as capable. - # - # CLAUDE_CODE_EXECPATH is the absolute path to the running CC binary - # itself (e.g. ~/.local/share/claude/versions/2.1.x — that's the binary, - # not a directory). It is the exact CLI that loaded this hook. We do NOT - # fall back to shutil.which("claude") because the hook's cwd is the - # user's (potentially attacker-supplied) repo, and Windows shutil.which - # searches cwd first — a checked-in ./claude.exe would get spawned. - # Absolute-path probes only. - # - # Also monkeypatch the SDK's message parser to tolerate unknown message - # types (newer CLI emits rate_limit_event which older SDK raises on). - cli_path = os.environ.get("SG_AGENTIC_CLI_PATH") - if cli_path is None: - for p in ( - os.environ.get("CLAUDE_CODE_EXECPATH"), - os.path.expanduser("~/.local/bin/claude"), - "/root/.local/bin/claude", - # Claude Code Remote container install path. CLAUDE_CODE_EXECPATH - # is not exported to hook subprocesses there, so without this - # candidate cli_path resolves to None and the SDK uses its - # bundled CLI — which lags the running CC by builds. - "/opt/claude-code/bin/claude", - ): - if p and os.path.isfile(p): - cli_path = p - break + cli_path = _gateway_cli or _user_cli_path() if cli_path: - try: - from claude_agent_sdk._internal import message_parser as _mp - import claude_agent_sdk._internal.client as _sdk_client - from claude_agent_sdk import SystemMessage as _SysMsg - - _orig_parse = _mp.parse_message - - def _tolerant(data): - try: - return _orig_parse(data) - except Exception: - return _SysMsg(subtype=data.get("type", "unknown"), data=data) - - _mp.parse_message = _tolerant - _sdk_client.parse_message = _tolerant - except Exception: - pass + _tolerate_unknown_sdk_messages() + else: + cli_path = _cli_for_sdk(cli_path) + + # Without a callback the inner CLI writes to this hook's stderr, which + # Claude Code shows Claude as the review notice. Log it instead. + child_stderr_lines = [0] + # Lines of the current start that refuse a flag (_run_inner_cli). + flag_refusals: List[str] = [] + + def _child_stderr_metric() -> Dict[str, int]: + return {"child_stderr": child_stderr_lines[0]} if child_stderr_lines[0] else {} + + def _on_child_stderr(line: str) -> None: + if _refused_flags([line]): + # Not counted: where a flag is refused, every review gets this + # line once, from the start that is then made again. + flag_refusals.append(str(line)) + debug_log(f"agentic reviewer's CLI stderr | {str(line).rstrip()}") + return + child_stderr_lines[0] += 1 + metrics["child_stderr"] = child_stderr_lines[0] + if child_stderr_lines[0] <= 20: + debug_log(f"agentic reviewer's CLI stderr | {str(line).rstrip()}") + elif child_stderr_lines[0] == 21: + debug_log("agentic reviewer's CLI stderr | (further lines not logged)") async def _arun(system: str, prompt: str, *, schema: Dict[str, Any], turns: Optional[int] = None @@ -1284,12 +2178,11 @@ def agentic_review( system_prompt=system, cwd=context_dir, cli_path=cli_path, - allowed_tools=["Read", "Grep", "Glob"], - # Read/Grep/Glob within cwd are auto-approved in default - # permission mode, so bypassPermissions is unnecessary (and - # would trip our own agent-permission-bypass guidance). Leaving - # permission_mode unset means an accidental future addition of - # a write/exec tool to allowed_tools is caught by the gate. + # Pre-approved, and the only tools the reviewer has: no + # permission mode can then let it write, run or fetch anything. + allowed_tools=list(_REVIEWER_TOOLS), + extra_args={**_only_tools_args(_REVIEWER_TOOLS), + **_reviewer_session_args()}, setting_sources=[], max_turns=turns if turns is not None else max_turns, model=model, @@ -1302,11 +2195,11 @@ def agentic_review( # Identical --model/--fallback-model is rejected by the inner CLI # at startup ("Fallback model cannot be the same as the main # model", exit 1 → ProcessError). The default model here IS - # _DEFAULT_PUBLIC_MODEL, so an unconditional fallback_model would + # _CLI_OPUS_ALIAS, so an unconditional fallback_model would # kill every spawn before the first API call. Omit the fallback # when it would equal the primary. fallback_model=( - _DEFAULT_PUBLIC_MODEL if model != _DEFAULT_PUBLIC_MODEL else None + _CLI_OPUS_ALIAS if model != _CLI_OPUS_ALIAS else None ), # Plugin-hook subprocesses get ANTHROPIC_AUTH_TOKEN (the user's # OAuth token) injected by Claude Code. The SDK builds the child @@ -1329,6 +2222,7 @@ def agentic_review( # the review run end-to-end; the others are belt-and-suspenders # for the same fd-passing pattern. env=_agentic_spawn_env(), + stderr=_on_child_stderr, ) n = 0 structured: Optional[Dict[str, Any]] = None @@ -1345,23 +2239,63 @@ def agentic_review( yield {"type": "user", "message": {"role": "user", "content": prompt}} - async for msg in query(prompt=_once(), options=opts): - if isinstance(msg, AssistantMessage): - n += 1 - elif isinstance(msg, ResultMessage): - subtype = msg.subtype - if msg.structured_output is not None: - structured = msg.structured_output - # SDK ResultMessage carries aggregate usage + cache-aware - # cost across the whole multi-turn run; prefer its cost over - # the price-table estimate. getattr guards older SDK builds. - _record_usage(getattr(msg, "usage", None) or {}, model, - cost_usd=getattr(msg, "total_cost_usd", None)) + # Blocks are matched by shape so older SDK builds need no extra import. + file_tool_ids = set() + auth_retries = 0 + # Closed on the way out, also on a raise, so the CLI stops with the review. + async with contextlib.aclosing(query(prompt=_once(), options=opts)) as messages: + async for msg in messages: + if isinstance(msg, AssistantMessage): + # Old CLIs answer each refused request by asking the model + # again, hundreds of times; the message is the same on all. + if getattr(msg, "error", None) == "authentication_failed": + raise _ReviewerAuthFailed("authentication_failed") + auth_retries = 0 + n += 1 + for b in getattr(msg, "content", None) or []: + if (getattr(b, "name", None) in _REVIEWER_FILE_TOOLS + and getattr(b, "id", None)): + file_tool_ids.add(b.id) + ctx["rv_tool_calls"] += 1 + elif isinstance(msg, ResultMessage): + subtype = msg.subtype + if msg.structured_output is not None: + structured = msg.structured_output + # SDK ResultMessage carries aggregate usage + cache-aware + # cost across the whole multi-turn run; prefer its cost over + # the price-table estimate. getattr guards older SDK builds. + _record_usage(getattr(msg, "usage", None) or {}, model, + cost_usd=getattr(msg, "total_cost_usd", None)) + for key, attr in (("rv_turns", "num_turns"), ("rv_api_ms", "duration_api_ms")): + value = getattr(msg, attr, None) + # Counting must never fail the review: anything that + # is not a positive finite number adds nothing. + if (isinstance(value, (int, float)) and not isinstance(value, bool) + and 0 < value < float("inf")): + ctx[key] += int(value) + elif getattr(msg, "subtype", None) == "api_retry": + # A gateway's 401 without the API's x-should-retry: false + # keeps the CLI retrying for about three minutes. + auth_retries = auth_retries + 1 if _api_retry_status(msg) == 401 else 0 + if (auth_retries >= _AGENTIC_MAX_401_RETRIES + and not _is_remote_session()): + raise _ReviewerAuthFailed(f"{auth_retries} retries after HTTP 401") + else: + # Tool results come back as user messages. + content = getattr(msg, "content", None) + if isinstance(content, list): + ctx["rv_tool_err"] += sum( + 1 for b in content + if getattr(b, "is_error", None) is True + and getattr(b, "tool_use_id", None) in file_tool_ids + ) return structured, n, subtype def _run(system: str, prompt: str, *, schema: Dict[str, Any] ) -> Tuple[Optional[Dict[str, Any]], int, Optional[str]]: - return _asyncio.run(_arun(system, prompt, schema=schema)) + return _run_inner_cli( + lambda: _asyncio.run(_arun(system, prompt, schema=schema)), + flag_refusals) # Stage 1: investigate — SDK enforces _FINDINGS_SCHEMA and retries the # agent on mismatch, so `inv` is either a validated dict or None. @@ -1380,12 +2314,14 @@ def agentic_review( f"{json.dumps(inv, indent=2) if inv else '<none>'}") except Exception as e: debug_log(f"agentic_review: investigate failed ({e}); falling back") - return None, [], {"agentic_fallback": f"investigate:{type(e).__name__}"} + return None, [], {"agentic_fallback": f"investigate:{type(e).__name__}", + **_child_stderr_metric()} metrics["investigate_ms"] = int((_t.time() - t0) * 1000) metrics["investigate_turns"] = inv_turns if inv is None: reason = inv_subtype or "no_structured_output" - return None, [], {"agentic_fallback": f"investigate:{reason}"} + return None, [], {"agentic_fallback": f"investigate:{reason}", + **_child_stderr_metric()} # Keep medium-severity candidates through self-refute — that pass is the # real precision gate, and the model's investigate-stage severity rating # is conservative (it defaults to "medium"). Filtering to high/critical diff --git a/plugins/security-guidance/hooks/security_reminder_hook.py b/plugins/security-guidance/hooks/security_reminder_hook.py index 48a7eb49..7c84357f 100755 --- a/plugins/security-guidance/hooks/security_reminder_hook.py +++ b/plugins/security-guidance/hooks/security_reminder_hook.py @@ -23,7 +23,7 @@ The plugin has two layers: ## How the git baseline works -On each UserPromptSubmit, the plugin runs `git stash create` to get a SHA representing +On each UserPromptSubmit, the plugin takes a snapshot commit to get a SHA representing the current working tree state (HEAD + any uncommitted changes). This SHA is saved to the session state file. When the Stop hook fires, it runs `git diff <baseline_sha>` to get only the changes made since that snapshot. After analysis, the baseline is updated @@ -46,10 +46,12 @@ Per-feature toggles (all default enabled; set to "0" to disable): - ENABLE_COMMIT_REVIEW: PostToolUse[Bash] commit security review Other: -- SECURITY_REVIEW_MODEL: Model for LLM review (default: claude-opus-4-7) +- SECURITY_REVIEW_MODEL: Model for LLM review (default: the newest Opus) - ANTHROPIC_API_KEY: Required for LLM-based reviews - ANTHROPIC_AUTH_TOKEN: Alternative to API key — OAuth access token sent as Bearer auth. - Claude Code passes this automatically for OAuth-authenticated users. + Claude Code passes this to the plugin's hooks for signed-in users, unless + telemetry or nonessential traffic is turned off. With neither, the LLM + reviews are skipped and the user is told once (see README, Credentials). """ try: @@ -66,6 +68,7 @@ import re import subprocess import sys import threading +import time import urllib.request from datetime import datetime from enum import IntEnum @@ -82,8 +85,11 @@ from _base import ( # noqa: E402,F401 DEBUG_LOG_FILE, DEBUG_LOG_MAX_BYTES, debug_log, PROVENANCE_TAG, PROVENANCE_BANNER, _read_plugin_version_int, _PV, _USAGE, _USAGE_LOCK, - _PRICE_PER_MTOK, _PRICE_DEFAULT, _record_usage, _usage_metrics, - state_dir as _resolve_state_dir, + _PRICE_PER_MTOK, _PRICE_DEFAULT, _record_usage, _usage_metrics, _net_error_metric, + _USAGE_CONTEXT, _log_usage, + launcher_direct_metric, launcher_probe_metrics, state_dir as _resolve_state_dir, + write_hook_output as _write_hook_output, portable_output, + llm_reviews_switched_off, agent_sdk_installed, make_state_dir, ) import extensibility # noqa: E402 from patterns import ( # noqa: E402,F401 @@ -97,9 +103,11 @@ from session_state import ( # noqa: E402,F401 load_state, save_state, with_locked_state, ) from gitutil import ( # noqa: E402,F401 - GIT_CMD, apply_safe_git_env, + GIT_CMD, apply_safe_git_env, remove_copies_on_termination, + on_windows_drive, _git_rev_parse_head, _find_git_index, _diff_pathspec, _temp_index, _git_toplevel, _git_dir, _git_rev_list_range, _git_diff_range, + _git_toplevel_and_head, _git_head_at, _detect_main_branch, _git_reflog_recent_commits, _git_name_only, _git_status_porcelain, _is_ancestor, get_git_diff, SOURCE_CODE_EXTENSIONS, SOURCE_CODE_BASENAMES, @@ -117,7 +125,12 @@ from diffstate import ( # noqa: E402,F401 get_baseline_file_content, capture_git_baseline, _REVIEWED_SHAS_BASENAME, _REVIEWED_SHAS_CAP, _reviewed_shas_path, _load_reviewed_shas, _append_reviewed_shas, + _full_shas, _mark_inflight, _load_inflight_shas, _reviewed_after_inflight, + _diff_review_started, _diff_review_done, UNTRACKED_BASELINE_CAP, _list_untracked, compute_v2_review_set, + touched_scope, review_scope_code, + baseline_origin, baseline_origin_metric, FOREIGN_ORIGINS, + KEPT_NO_CAPTURE, KEPT_UNREVIEWED, ) from reporesolve import ( # noqa: E402,F401 RES_NONE, RES_CWD, RES_COMMAND, RES_SHA_SCAN, RES_TOUCHED_PATHS, RES_HINT, @@ -125,6 +138,8 @@ from reporesolve import ( # noqa: E402,F401 toplevel_from_command, repo_containing_commit, scan_roots, resolve_repo_root, save_repo_hint, load_repo_hint, ) +import diffstate # noqa: E402 module ref for last_baseline_fail +import gitutil # noqa: E402 module ref for per-run counters import llm # noqa: E402 module ref for reassignable globals (_last_call_claude_http_error etc.) from llm import ( # noqa: E402,F401 ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN, HAS_API_CREDENTIALS, @@ -189,6 +204,13 @@ MAX_STOP_HOOK_FIRINGS = int(os.environ.get("MAX_STOP_HOOK_FIRINGS", "3")) # diff; unbounded diffs burn tokens and risk 400 on context length. MAX_DIFF_FILES = int(os.environ.get("MAX_DIFF_FILES", "30")) +# Changed source paths past which the Stop review is skipped before any diff. +# A set this large is a repo at $HOME or an unignored cache, not edits; no +# platform takes that many paths in one git command, and only resolving them +# takes seconds. Paths the review never reads (an unignored node_modules) do +# not count: they must not hide the one edited source file beside them. +MAX_REVIEW_SET_PATHS = int(os.environ.get("MAX_REVIEW_SET_PATHS", "100000")) + # Appended to all exit(2) guidance so the asyncRewake auto-turn doesn't # cause the model to abandon the user's original request. CONTINUATION_SUFFIX = ( @@ -198,6 +220,48 @@ CONTINUATION_SUFFIX = ( "response." ) +# New files named in the notice for a failed `git add -N`. +_LEFT_OUT_NAMES_SHOWN = 5 + +# Extra keys emit_metrics appends to every row this invocation emits. +_TAIL_METRICS: Dict[str, Any] = {} +# Of those, the ones that go after the others whenever they were set: a row +# over CC's key cap loses them before any older review key. Only the launcher +# keys below come later; those are also on every smaller row of the session. +_LAST_METRIC_KEYS = ("diff_seen",) + +# The launcher's slow interpreter probes, if any. After the keys above. +_LAUNCHER_METRICS: Dict[str, Any] = launcher_probe_metrics() +# Whether the launcher skipped a packaged alias. Only on rows sent anyway. +_LAUNCHER_DIRECT: Dict[str, Any] = launcher_direct_metric() + +# Reviewer context diagnostics from llm.agentic_review, most useful first. +_REVIEWER_CONTEXT_KEYS = ("rv_cwd_gone", "rv_tool_err", "rv_tool_calls") +# How much work the agentic reviewer did, from llm.agentic_review: turns, and +# time waiting on the model. They tell more turns from slower turns when +# review_ms moves. +_REVIEWER_EFFORT_KEYS = ("rv_turns", "rv_api_ms") +# A clean commit review row has one free key under CC's key cap once +# rv_tool_calls is on it, so the two are sent as one number: +# rv_effort = rv_turns * _EFFORT_TURNS_UNIT + rv_api_ms (capped below the unit) +_EFFORT_TURNS_UNIT = 10_000_000 +# The packed value queued for this row. Emitted after every other key, the +# launcher's included, so it never displaces one at the cap. +_REVIEWER_EFFORT_METRICS: Dict[str, Any] = {} + + +def _note_reviewer_context(agentic_metrics): + """Queue the agentic reviewer's context and effort diagnostics for this row.""" + for key in _REVIEWER_CONTEXT_KEYS: + if agentic_metrics.get(key): + _TAIL_METRICS[key] = agentic_metrics[key] + turns = agentic_metrics.get("rv_turns") or 0 + api_ms = agentic_metrics.get("rv_api_ms") or 0 + if turns or api_ms: + _REVIEWER_EFFORT_METRICS["rv_effort"] = ( + turns * _EFFORT_TURNS_UNIT + min(api_ms, _EFFORT_TURNS_UNIT - 1)) + + def emit_metrics( metrics, rewake_summary=None, @@ -228,31 +292,22 @@ def emit_metrics( task-notification one-liner. Must be in the same JSON line as the metrics because CC stops scanning stdout after the first {-prefixed line. - `additional_context` (asyncRewake findings): model-visible guidance text. - Delivery channel depends on `hook_event_name` because CC's hook-output - contract is NOT symmetric across events: - - - PostToolUse (commit-review, push-sweep): surfaced via the modern - hookSpecificOutput.additionalContext protocol. `PostToolUse` is a - member of CC's hookSpecificOutput discriminated union - (coreSchemas.ts), so the JSON validates and metrics/rewakeSummary - are consumed. See #1375 / #1783 for why this replaced the legacy - stderr + exit(2) shape for PostToolUse. - - - Stop / SubagentStop: there is NO `Stop` member in that union, so - emitting hookSpecificOutput{hookEventName:"Stop"} makes the whole - line fail isSyncHookJSONOutput validation — which on the asyncRewake - path silently drops metrics AND rewakeSummary, and (because the - legacy stderr write was removed) leaks the raw JSON to the model as - the rewake body. CC's asyncRewake delivery actually reads - `stderr || stdout` for the model-visible body and only scans stdout - JSON for metrics+rewakeSummary — it never reads additionalContext - on this path. So for Stop we use the documented clean pattern: - guidance on stderr, valid JSON (metrics + rewakeSummary + - top-level decision/reason) on stdout. The top-level decision:"block" - + reason also covers the sync-fallback path (single-shot `claude -p`, - where asyncRewake degrades to a sync Stop hook that reads - decision/reason). See #2159. + `additional_context` (asyncRewake findings, exit-2 callers only): + model-visible guidance text, written to stderr for every event. CC's + asyncRewake delivery reads `stderr || stdout` for the model-visible body + and only scans stdout JSON for metrics+rewakeSummary — it never reads + additionalContext on this path. stdout keeps one valid JSON line so + metrics + rewakeSummary survive: + + - Stop / SubagentStop: also top-level decision:"block" + reason, for + the sync-fallback path (single-shot `claude -p`, where asyncRewake + degrades to a sync Stop hook that reads decision/reason). There is + no `Stop` member in CC's hookSpecificOutput union, so that field + must not be used here. + + - PostToolUse (commit-review, push-sweep): stderr only. In the + sync-fallback path CC shows the model an exit-2 PostToolUse hook's + stderr, so additionalContext would repeat the text. Empty/None additional_context emits neither channel (back-compat for metrics-only callers). @@ -263,40 +318,147 @@ def emit_metrics( surface; systemMessage adds a per-fire override when the static rewakeMessage isn't specific enough for the finding being shown. - `hook_event_name` (used only when additional_context is set): selects the - delivery channel above. Defaults to "PostToolUse" (commit-review and + `hook_event_name` (used only when additional_context is set): whether to + add decision/reason as above. Defaults to "PostToolUse" (commit-review and push-sweep are the most common callers); handle_stop_hook passes "Stop". """ + _log_usage(_usage_metrics()) head = {} if _PV and "pv" not in metrics: head["pv"] = _PV - head.update(_usage_metrics()) + # sg-python.sh could only start this interpreter by its real path: the + # names on PATH were launcher aliases that cannot read the plugin files. + if os.environ.get("SG_LAUNCHER_FALLBACK") == "1": + head["launcher_fallback"] = 1 + if _precheck_miss: + # Up front: this is the key that says a text check would have lost a + # review, so it must not be the one cut by the key cap. + head["precheck_miss"] = 1 + usage = _usage_metrics() + head.update(usage) + # Why the last call got no HTTP response (api_error -1 says only that it + # did not). Ahead of the row's own keys: on a failed review this is the + # key that says what failed, so it is not the one to lose to the key cap. + net_err = _net_error_metric() + if net_err: + head["net_err"] = net_err if head: metrics = {**head, **metrics} + # Diagnostic keys go last, so they are the first dropped when a row is + # over CC's key cap and never displace an existing key. + tail = {} + if llm._store_certs_skipped: + # The Windows certificate store held entries OpenSSL cannot read and + # the plugin's API calls ran on the store without them. + tail["tls_store_skipped"] = llm._store_certs_skipped + if usage or "review_ms" in metrics or metrics.get("review_failed"): + # Review rows say whether the plugin's calls went through a gateway, + # so the error rate behind gateways can be read per plugin version. + cfg = llm.gateway_cfg() + if cfg: + tail["gateway_cfg"] = cfg + if not llm._send_strict_mcp: + # An inner CLI refused --strict-mcp-config (a machine with a + # managed-mcp.json) and the review was started again without it. + tail["strict_mcp_off"] = 1 + tail.update(_TAIL_METRICS) + for k in _LAST_METRIC_KEYS: + if k in tail: + tail[k] = tail.pop(k) + tail.update(_LAUNCHER_METRICS) + tail.update(_REVIEWER_EFFORT_METRICS) + # Last of all: the first key a full row loses. + tail.update(_LAUNCHER_DIRECT) + metrics = {**metrics, **{k: v for k, v in tail.items() if k not in metrics}} out = {"metrics": metrics} if rewake_summary: out["rewakeSummary"] = rewake_summary if additional_context: + # Guidance on stderr, the asyncRewake body channel (`stderr || stdout`). + sys.stderr.write(additional_context) + sys.stderr.flush() if hook_event_name in ("Stop", "SubagentStop"): - # Stop is NOT in CC's hookSpecificOutput union — emitting it there - # fails schema validation and drops metrics+rewakeSummary (#2159). - # Clean pattern: guidance on stderr (the asyncRewake body channel, - # delivered via `stderr || stdout`), top-level decision/reason for - # the sync-fallback path. stdout JSON stays valid so metrics + - # rewakeSummary survive. - sys.stderr.write(additional_context) - sys.stderr.flush() + # Sync-fallback path for Stop reads decision/reason. out["decision"] = "block" out["reason"] = additional_context - else: - # PostToolUse et al. — valid union member; modern protocol. - out["hookSpecificOutput"] = { - "hookEventName": hook_event_name, - "additionalContext": additional_context, - } if system_message: out["systemMessage"] = system_message - print(json.dumps(out), flush=True) + _write_hook_output(out) + + +def _llm_review_off(session_id): + """Why no LLM review can run: (log text, extra metrics), or None.""" + if not ENABLE_CODE_SECURITY_REVIEW: + return "LLM review disabled", {} + if not llm.can_review(session_id): + return ("no API credentials and no Claude apps gateway sign-in", + {"no_creds": True}) + return None + + +# Claude Code hands a signed-in user's credentials to this plugin's hooks, +# but not in a session started with one of these set. +_NO_CREDENTIALS_NOTICE_INTERVAL_SEC = 7 * 24 * 3600 + + +def _credential_withholding_env_vars(): + names = [v for v in ("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", + "DISABLE_TELEMETRY") if os.environ.get(v)] + if os.environ.get("DO_NOT_TRACK", "").strip().lower() in ( + "1", "true", "yes", "on"): + names.append("DO_NOT_TRACK") + return names + + +def _no_credentials_notice(file_path, session_id): + """Text telling the user the LLM reviews are not running, or None. + + With no credential and no Claude apps gateway sign-in, every Stop, + commit and push review is skipped, and nothing else tells the user. + Returned for an edit inside a git repo (the reviews only look at those), + at most once a week per install: the marker file in the state dir holds + the time of the last notice, or of the last check that found the reviews + running through a Claude apps gateway sign-in. A marker that can't be + written means no notice rather than one on every edit. The caller is the + Edit/Write hook because the review hooks run in the background, where + Claude Code does not show a systemMessage. + """ + import time as _time + try: + marker = os.path.join(_resolve_state_dir(), ".no_credentials_notice") + try: + age = _time.time() - os.path.getmtime(marker) + if 0 <= age < _NO_CREDENTIALS_NOTICE_INTERVAL_SEC: + return None + except OSError: + pass + if not _git_toplevel(os.path.dirname(os.path.abspath(file_path))): + return None + make_state_dir(os.path.dirname(marker)) + with open(marker, "w"): + pass + # In a session signed in to a Claude apps gateway the reviews do run, + # through the user's claude (llm.gateway_review_cli). `claude auth + # status` runs only after the marker is written, so an edit waits for + # it at most once a week. + if llm.can_review(session_id): + return None + except OSError: + return None + withheld_by = _credential_withholding_env_vars() + if withheld_by: + why = ("Claude Code does not pass your sign-in to plugins while " + + " or ".join(withheld_by) + " is set") + else: + why = "No Anthropic credentials were passed to the plugin's hooks" + rest = (", so only the pattern warnings on edits are active" + if ENABLE_PATTERN_RULES else "") + return ( + "security-guidance: the AI security review of your changes is not " + f"running. {why}{rest}. See \"Credentials\" in the plugin README. " + "Set ENABLE_CODE_SECURITY_REVIEW=0 to turn the review off and hide " + "this notice, which is shown at most once a week." + ) # ===================================================================== # State management @@ -405,7 +567,10 @@ def record_pending_warnings(session_id, file_path, rule_names): def sweep_pending_warnings(session_id): """ Stop-hook final sweep. Re-read every file in pending_warnings, re-check - patterns, and return (fixed, unresolved, unresolved_mask). Clears state. + patterns, and return (fixed, unresolved, unresolved_mask, fired_mask). + fired_mask has a bit for every built-in rule that matched an edit since + the last sweep (whether or not its reminder was shown again), so the two + masks give, per rule, fired and still matching. Clears state. A file that's been deleted counts as fixed — the dangerous code is gone. Never raises — this is telemetry and must not break the Stop hook. """ @@ -413,18 +578,24 @@ def sweep_pending_warnings(session_id): try: pending = state.get("pending_warnings") if not isinstance(pending, dict) or not pending: - return 0, 0, 0 + return 0, 0, 0, 0 by_file = {} for key in pending: if not isinstance(key, str) or ":" not in key: continue fp, _, rule = key.rpartition(":") + if fp.endswith(":user"): + # User rules are named "user:<name>"; keep them apart + # from a built-in rule of the same name. + fp, rule = fp[:-len(":user")], "user:" + rule by_file.setdefault(fp, set()).add(rule) unresolved = [] + fired = set() fixed = 0 for fp, rules in by_file.items(): + fired |= rules try: with open(fp, "r", errors="replace") as f: still_matching = {r for r, _ in check_patterns(fp, f.read())} @@ -440,13 +611,14 @@ def sweep_pending_warnings(session_id): # Filter to known rules so a renamed/removed rule in old state # doesn't KeyError rule_names_to_mask. known = [r for r in unresolved if r in _RULE_NAME_TO_ID] - return fixed, len(unresolved), rule_names_to_mask(known) + return (fixed, len(unresolved), rule_names_to_mask(known), + rule_names_to_mask(fired)) except Exception as e: debug_log(f"sweep_pending_warnings failed: {e}") - return 0, 0, 0 + return 0, 0, 0, 0 result = with_locked_state(session_id, _sweep) - return result if result is not None else (0, 0, 0) + return result if result is not None else (0, 0, 0, 0) # ===================================================================== # Git baseline management @@ -456,11 +628,30 @@ def sweep_pending_warnings(session_id): # Pattern matching # ===================================================================== +def _notebook_cell_sources(content): + """Cell sources of a saved notebook, or None when content is not one (a + single cell's source, say).""" + if not isinstance(content, str) or not content.lstrip("\ufeff \t\r\n").startswith("{"): + return None + try: + sources = [cell.get("source", "") for cell in json.loads(content.lstrip("\ufeff"))["cells"]] + return "\n".join(src if isinstance(src, str) else "".join(src) for src in sources) + except Exception: + return None + def check_patterns(file_path, content): """Check if file path or content matches any security patterns. Returns ALL matches.""" normalized_path = file_path.lstrip("/") matches = [] + # A saved notebook is JSON: its string escapes hide code from the rules + # ("\nos.system(" has no word boundary) and its outputs can run to + # megabytes. Check the cells as written instead. + if normalized_path.endswith(".ipynb"): + cell_sources = _notebook_cell_sources(content) + if cell_sources is not None: + content = cell_sources + for pattern in list(SECURITY_PATTERNS) + extensibility.user_patterns(): # path_filter is a gate: when present, the rule only applies to # matching paths. Distinct from path_check, which is itself a @@ -499,7 +690,7 @@ def check_patterns(file_path, content): return matches -def extract_content_from_input(tool_name, tool_input): +def extract_content_from_input(tool_name, tool_input, tool_response=None): """Extract content to check from tool input based on tool type.""" if tool_name == "Write": return tool_input.get("content", "") @@ -510,12 +701,54 @@ def extract_content_from_input(tool_name, tool_input): if edits: return " ".join(edit.get("new_string", "") for edit in edits) return "" + elif tool_name == "NotebookEdit": + # A deleted cell adds no code. A failed edit comes back as an error + # in the response, with the notebook unchanged. + if tool_input.get("edit_mode") == "delete": + return "" + if isinstance(tool_response, dict) and tool_response.get("error"): + return "" + new_source = tool_input.get("new_source", "") + return new_source if isinstance(new_source, str) else "" return "" # ===================================================================== # Hook handlers # ===================================================================== +def _store_baseline_fail(state, fail): + """Keep diffstate.last_baseline_fail next to the baseline_sha it describes.""" + if fail: + state["baseline_fail"] = fail + else: + state.pop("baseline_fail", None) + + +# A prompt waits for this hook, so one this slow is worth a metrics row. Faster +# prompts send none: the row would be one more event per prompt for every user. +SLOW_PROMPT_MS = 1000 + + +def _slow_prompt_metrics(cwd, total, baseline, untracked, captured, baseline_fail): + """The metrics row for a slow prompt. Times are seconds, spent in + handle_user_prompt_submit (the launcher reports its own). Counts and flags + only, never a path. Telemetry only.""" + metrics = { + "ups_ms": int(total * 1000), + "ups_baseline_ms": int(baseline * 1000), + "ups_untracked_ms": int(untracked * 1000), + } + if diffstate.last_index_entries is not None: + metrics["ups_index_entries"] = diffstate.last_index_entries + if on_windows_drive(cwd): + metrics["ups_wsl_drive"] = 1 + if not captured: + metrics["ups_no_baseline"] = 1 + if baseline_fail: + metrics["ups_baseline_fail"] = baseline_fail + return metrics + + def handle_user_prompt_submit(input_data): """ Handle UserPromptSubmit — capture git baseline SHA. @@ -528,36 +761,65 @@ def handle_user_prompt_submit(input_data): to be lost when the wipe landed before Stop's state read. """ + import time as _time + started = _time.monotonic() cwd = input_data.get("cwd", "") if not cwd: debug_log("UPS: no cwd, skipping baseline capture") sys.exit(0) session_id = input_data.get("session_id", "default") - # stash-create and ls-files both walk the worktree (~2-5s each in a very - # large repo). Run them concurrently so UPS latency stays ≈ max(both). - import concurrent.futures as _cf - with _cf.ThreadPoolExecutor(max_workers=2) as _ex: - _f_sha = _ex.submit(capture_git_baseline, cwd) - _f_ut = _ex.submit(_list_untracked, cwd) - sha = _f_sha.result() - # Always capture the untracked snapshot. `git stash create` returns - # empty when there are no TRACKED changes, but pre-existing untracked - # files still need to be excluded from the next Stop's review_set — - # otherwise an untracked-only working tree gets every untracked file - # reviewed on every turn until something tracked is dirtied. - untracked_now = _f_ut.result() or {} + # The untracked snapshot has one reader: the Stop-hook diff review, which + # skips before reading it when either of these switches is off (see + # handle_stop_hook). Listing the worktree on every prompt then buys + # nothing, so only the turn baseline (still read by the Write pattern + # filter) is captured. + snapshot_untracked = ENABLE_STOP_REVIEW and ENABLE_CODE_SECURITY_REVIEW + spent = {"baseline": 0.0, "untracked": 0.0} + + def _timed(key, fn, repo_cwd): + began = _time.monotonic() + try: + return fn(repo_cwd) + finally: + spent[key] += _time.monotonic() - began + + def _capture(repo_cwd): + if not snapshot_untracked: + return _timed("baseline", capture_git_baseline, repo_cwd), None + # the baseline snapshot and ls-files both walk the worktree (~2-5s each in a + # very large repo). Run them concurrently so UPS latency stays + # ≈ max(both). + import concurrent.futures as _cf + with _cf.ThreadPoolExecutor(max_workers=2) as _ex: + _f_sha = _ex.submit(_timed, "baseline", capture_git_baseline, repo_cwd) + _f_ut = _ex.submit(_timed, "untracked", _list_untracked, repo_cwd) + # Taken whether or not the snapshot produced a SHA. The baseline + # falls back to HEAD when there are no TRACKED changes, but + # pre-existing untracked files still need to be excluded from the + # next Stop's review_set — otherwise an untracked-only working + # tree gets every untracked file reviewed on every turn until + # something tracked is dirtied. + return _f_sha.result(), _f_ut.result() or {} + + captured_at = _time.time() + sha, untracked_now = _capture(cwd) + # No repo here and none remembered: the listing above failed, it did not + # come back empty. + no_repo = False if not sha and not _git_toplevel(cwd): hint = load_repo_hint(session_id) if hint: debug_log(f"UPS: cwd is not a git repo; using repo hint {hint!r}") cwd = hint - with _cf.ThreadPoolExecutor(max_workers=2) as _ex: - _f_sha = _ex.submit(capture_git_baseline, cwd) - _f_ut = _ex.submit(_list_untracked, cwd) - sha = _f_sha.result() - untracked_now = _f_ut.result() or {} - head = _git_rev_parse_head(cwd) + sha, untracked_now = _capture(cwd) + else: + no_repo = True + baseline_fail = diffstate.last_baseline_fail + # The repo root rides on the same git call as HEAD. + root, head = (None, None) if no_repo else _git_toplevel_and_head(cwd) + if sha and not head: + head = _git_rev_parse_head(cwd) # If the previous turn's Stop hook never ran (user interrupt, follow-up # during work, tool-reject, model crash, maxTurns, PostToolUse block…), @@ -575,14 +837,44 @@ def handle_user_prompt_submit(input_data): # we still need to capture a baseline. if state.get("touched_paths") and state.get("baseline_sha"): preserved["value"] = True + # A baseline captured while the Stop review was off has no + # snapshot beside it. Without one the next Stop would take every + # untracked file in the tree as written this turn. + if untracked_now is not None and "untracked_at_baseline" not in state: + state["untracked_at_baseline"] = untracked_now + # Once a prompt failed to capture, the baseline is old whatever + # later prompts did. + state["baseline_kept"] = ( + KEPT_NO_CAPTURE + if not sha or state.get("baseline_kept") == KEPT_NO_CAPTURE + else KEPT_UNREVIEWED) return if sha: state["baseline_sha"] = sha state["head_at_capture"] = head - # untracked_at_baseline is independent of whether the stash produced - # a SHA — write it unconditionally so compute_v2_review_set's - # preexisting-untracked exclusion works in untracked-only trees. - state["untracked_at_baseline"] = untracked_now + state.pop("turn_touched", None) + _store_baseline_fail(state, baseline_fail) + # A Stop review that ends up in another checkout does not use + # this baseline. + state["baseline_root"] = root + state.pop("baseline_kept", None) + elif state.get("baseline_sha"): + state["baseline_kept"] = KEPT_NO_CAPTURE + # When the turn began: what a Stop review in another checkout has + # to go on. + state["prompt_at"] = captured_at + # untracked_at_baseline is independent of whether the snapshot produced + # a SHA, so compute_v2_review_set's preexisting-untracked exclusion + # works in untracked-only trees. None means no snapshot was taken + # (Stop review off): drop the previous one, which belongs to an older + # baseline. + if untracked_now is None: + state.pop("untracked_at_baseline", None) + elif not no_repo: + # no_repo: the earlier baseline stays (above), and so does the + # snapshot that belongs to it. An empty one in its place would + # make every untracked file in that repo look new. + state["untracked_at_baseline"] = untracked_now with_locked_state(session_id, _save) if preserved["value"]: @@ -599,6 +891,13 @@ def handle_user_prompt_submit(input_data): debug_log(f"Failed to capture git baseline (cwd={cwd!r}) — not a git repo, " f"or git invocation failed (check log entries above)") + total = _time.monotonic() - started + debug_log(f"UPS: took {total:.2f}s (baseline {spent['baseline']:.2f}s, " + f"untracked {spent['untracked']:.2f}s)") + if (total * 1000 >= SLOW_PROMPT_MS or _LAUNCHER_METRICS) and not portable_output(): + emit_metrics(_slow_prompt_metrics( + cwd, total, spent["baseline"], spent["untracked"], bool(sha), baseline_fail)) + sys.exit(0) def _resolve_amend_pre_sha(repo_root, expected_post_sha=None): @@ -677,26 +976,33 @@ _COMMIT_DIFFSTAT_PATTERNS = [ # or trailing hook output isn't picked up and fed to `git show`. _COMMIT_SHA_RE = re.compile(r'^\[[^\]]*?\b([0-9a-f]{7,40})\]', re.MULTILINE) -# Regex matching `git commit` commands. Mirrors Claude Code's own commit -# detection — it does NOT tolerate `git -c k=v commit` global options, which -# keeps this hook aligned with CC's commit attribution on what counts as a -# commit. +# Regex matching `git commit` commands, global options included (see below). # # Also matches `gt create` and `gt modify` — Graphite's stacked-PR wrapper # around git. `gt create` produces a new commit (mapped to git commit # semantics); `gt modify` amends the current commit (mapped to git commit -# --amend, also flagged by _GIT_AMEND_RE below). The hooks.json matcher -# widening for `gt create:*` / `gt modify:*` / `gt submit:*` ships in the -# same change set — without that widening this regex change is dead code -# because the hook subprocess never spawns for gt invocations. See #2048. +# --amend, also flagged by _GIT_AMEND_RE below). hooks.json has a +# `Bash(gt *)` entry so the hook subprocess spawns for gt invocations. +# +# Global options allowed between `git` and the subcommand, shared by +# _GIT_COMMIT_RE and _GIT_PUSH_RE: `-C <path>`, `-c key=val`, `--opt=val`, +# the long options that take a separate value, and flags that take none +# (`--no-pager`, `-p`). A value may be quoted in part (`-c user.name="A B"`). +_GIT_OPT_VALUE = r'''(?:[^\s"']|"[^"]*"|'[^']*')+''' +_GIT_VALUE_OPTS = r'git-dir|work-tree|namespace|config-env|attr-source' +_GIT_GLOBAL_OPTS = ( + r'(?:\s+-[Cc]\s+' + _GIT_OPT_VALUE + + r'|\s+--[^\s=]+=' + _GIT_OPT_VALUE + + r'|\s+--(?:' + _GIT_VALUE_OPTS + r')\s+' + _GIT_OPT_VALUE + # `--help` / `--version` run no subcommand. + + r'|\s+(?:--(?!(?:' + _GIT_VALUE_OPTS + r'|help|version)\s)[^\s=]+|-[pP])(?=\s))*' +) _GIT_COMMIT_RE = re.compile( - # `git -C <path>` and `git -c key=val` global options are allowed between - # `git` and `commit` (mirrors the long-standing tolerance in - # _GIT_PUSH_RE). Without this, `git -C /repo commit` is silently dropped - # by the handler — see #2089's secondary finding. The gt branch has no - # global-option layer to worry about. - r'\bgit(?:\s+-[Cc]\s+(?:"[^"]*"|\'[^\']*\'|[^\s"\']\S*)|\s+--[^\s=]+=\S+' - r'|\s+--(?:git-dir|work-tree)\s+\S+)*\s+commit\b' + # Without the global-option tolerance, `git -C /repo commit` is silently + # dropped by the handler — see #2089's secondary finding. The gt branch + # has no global-option layer to worry about. + # `(?!-)`: not the plumbing commands `git commit-graph` / `git commit-tree`. + r'\bgit' + _GIT_GLOBAL_OPTS + r'\s+commit\b(?!-)' r'|\bgt\s+(?:create|modify)\b' ) # Match either the `--amend` flag (with the leading whitespace boundary @@ -721,11 +1027,9 @@ COMMIT_REVIEW_RATE_WINDOW_S = int( # ─── push-sweep ───────────────────────────────────────────────────────────── # # Mirrors Claude Code's own push-command matching — tolerates `git -C <p>` / -# `git -c k=v` global options. The hooks.json `Bash(git push:*)` matcher -# (subcommand prefix) doesn't, but those forms are rare in practice -# and the python only ever runs after CC's matcher fired, so this regex is a -# defensive re-gate, not a widening — `git -C path push` won't reach python -# unless chained with a plain `git push` in the same compound command. +# `git -c k=v` global options, as the hooks.json `Bash(git*push*)` matcher +# does. That matcher also lets through e.g. `git stash push`; this regex is +# what decides whether the command is a push. # # `gh pr create` is intentionally NOT a separate hooks.json matcher: gh runs # `git push` as a child process, which CC's matcher doesn't observe (it sees @@ -733,14 +1037,13 @@ COMMIT_REVIEW_RATE_WINDOW_S = int( # entry would buy minimal extra coverage (sessions that push only via gh) at # the cost of an extra python spawn on every `... && gh pr create` compound # (the common case). Those sessions are caught on their next standalone `git push`. -# Matches `git push` (with optional `-c k=v` / `-C path` global options -# CC's hooks.json matcher doesn't tolerate) OR `gt submit` — Graphite's +# Matches `git push` (with optional `-c k=v` / `-C path` global options) +# OR `gt submit` — Graphite's # stacked-PR push command. gt submit forwards to `git push` internally, # but the bash hook fires on Claude's top-level command so we need to # recognize gt submit at the matcher level. See #2048. _GIT_PUSH_RE = re.compile( - r'(?:\bgit(?:\s+-[cC]\s+(?:"[^"]*"|\'[^\']*\'|[^\s"\']\S*)|\s+--[^\s=]+=\S+' - r'|\s+--(?:git-dir|work-tree)\s+\S+)*\s+push\b|\bgt\s+submit\b)' + r'(?:\bgit' + _GIT_GLOBAL_OPTS + r'\s+push\b|\bgt\s+submit\b)' ) # `git push` stdout: "abc1234..def5678 branch -> branch" (or `+abc..def` on @@ -757,12 +1060,61 @@ _PUSH_RANGE_RE = re.compile( MAX_PUSH_SWEEP_FILES = int(os.environ.get("SG_PUSH_SWEEP_MAX_FILES", "30")) MAX_PUSH_SWEEP_RANGE = int(os.environ.get("SG_PUSH_SWEEP_MAX_RANGE", "50")) PUSH_SWEEP_REPORT_CAP = int(os.environ.get("SG_PUSH_SWEEP_REPORT_CAP", "3")) +# How long a push review waits for a commit review that is still running on +# the pushed commits, before reviewing them itself. Longer than the agentic +# race delay, so a commit review that fell back to single-shot can finish. +try: + INFLIGHT_WAIT_S = int(os.environ.get("SG_INFLIGHT_WAIT_S", "") or 240) +except ValueError: + INFLIGHT_WAIT_S = 240 + +# Word pairs a shell `case` pattern (`*git*commit*|*git*push*|...`) would look +# for in the raw hook input: wider than _GIT_COMMIT_RE / _GIT_PUSH_RE, so a +# command those match always contains one of these pairs in this order. +_SHELL_PRECHECK_PAIRS = ( + ("git", "commit"), ("git", "push"), + ("gt", "create"), ("gt", "modify"), ("gt", "submit"), +) +# One in this many unmatched Bash calls reports, to keep the row count down. +_UNMATCHED_BASH_SAMPLE = 10 +# Set when a command that is reviewed would not have passed the text check. +_precheck_miss = False + + +def _shell_precheck_passes(raw): + """True when `raw` holds one of _SHELL_PRECHECK_PAIRS, first word first. + + Measurement only. Tells how many hook starts that match no commit or push + a text check in the launcher could avoid (`precheck_skip`), and whether + that check would ever drop a command that is reviewed (`precheck_miss`). + """ + for first, second in _SHELL_PRECHECK_PAIRS: + at = raw.find(first) + if at != -1 and raw.find(second, at + len(first)) != -1: + return True + return False + + +def _unmatched_bash_sampled(input_data): + """Pick the same one call in _UNMATCHED_BASH_SAMPLE in every start of it. + + One Bash call can start this hook once per hooks.json entry, so the choice + is made from the call's id: all its starts report or none do. A call with + no id is not reported. + """ + tool_use_id = input_data.get("tool_use_id") + if not isinstance(tool_use_id, str) or not tool_use_id: + return False + digest = hashlib.sha256(tool_use_id.encode("utf-8", "replace")).digest() + return int.from_bytes(digest[:4], "big") % _UNMATCHED_BASH_SAMPLE == 0 + def _claim_bash_hook_once(input_data): """De-dupe across hooks.json `if` matchers firing for the same Bash call. - `git commit -m x && git push` matches both `Bash(git commit:*)` and - `Bash(git push:*)` `if` configs → CC spawns this script twice with the + `git commit -m x && git push` matches both `Bash(git*commit*)` and + `Bash(git*push*)` `if` configs, and a command CC cannot parse runs every + `if` config → CC spawns this script more than once with the SAME `tool_use_id`. The first spawn atomically creates a sentinel under `.git/`; subsequent spawns see it and exit early. Avoids redundant LLM calls (and the redundant asyncRewake) on compound commands. @@ -893,6 +1245,17 @@ def _push_section(bash_output): section = section[:end] return section +def _push_in_output(bash_output): + """True when Bash output shows a git push that updated a remote branch.""" + section = _push_section(bash_output or "") + if not section.lstrip("\n").startswith("To "): + return False + return bool(_PUSH_RANGE_RE.search(section)) or "[new branch]" in section + +# Only words the failed-review notice; `gh pr create` alone is not routed to +# the push review. +_MAY_PUSH_VIA_GH_RE = re.compile(r"\bgh\s+pr\s+create\b") + def _detect_prev_upstream(repo_root, bash_output): """Where the remote was BEFORE this push. @@ -956,26 +1319,90 @@ def is_commit_review_enabled(): COMMIT_REVIEW_ENABLED = is_commit_review_enabled() +_HOOK_STARTED = time.monotonic() + +# Seconds into the hook after which neither side of a commit or push review's +# race waits for the other any longer; well inside Claude Code's 10-minute hook +# timeout. +_RACE_MAX_WAIT_S = 480 +# Seconds after which the review ends even if a side is still waiting on a +# request that does not answer, with time left to tell the session before +# Claude Code kills the hook. The inner CLI exits with the hook. The same limit +# holds for a commit or push review's single-shot review outside a race. +_RACE_END_S = 570 +# api_error for a race that hit that deadline with no verdict. +_RACE_TIMED_OUT = -2 + +def _hook_spent_s() -> float: + return time.monotonic() - _HOOK_STARTED + +def _single_shot_review( + diff_files: List[Tuple[str, str]], + previous_findings: List[Dict[str, Any]], +) -> Tuple[Optional[str], List[Dict[str, Any]], Dict[str, Any]]: + """A single-shot review as (guidance, vulns, metrics); failures are in the + metrics, not raised.""" + try: + g, v = analyze_code_security( + diff_files, is_diff=True, previous_findings=previous_findings + ) + # Read on this thread: the outcome is per thread. + failed = llm.last_review_failed() + return g, v, {"agentic": False, "review_failed": failed, + "api_error": llm.last_review_status() if failed else None} + except Exception as e: + debug_log(f"single-shot review crashed: {type(e).__name__}: {e}") + return None, [], {"agentic": False, "review_failed": True, "api_error": None} + +def _single_shot_review_in_time( + diff_files: List[Tuple[str, str]], + previous_findings: List[Dict[str, Any]], +) -> Tuple[Optional[str], List[Dict[str, Any]], bool, Optional[int]]: + """A single-shot review outside a race, as (guidance, vulns, failed, status). + + It ends at _RACE_END_S into the hook, whatever is pending. + """ + done: List[Any] = [] + worker = threading.Thread( + target=lambda: done.append(_single_shot_review(diff_files, previous_findings)), + daemon=True, + ) + worker.start() + worker.join(timeout=max(0.0, _RACE_END_S - _hook_spent_s())) + if not done: + debug_log("single-shot review: nothing returned in time") + return None, [], True, _RACE_TIMED_OUT + g, v, m = done[0] + return g, v, m["review_failed"], m["api_error"] + def _agentic_review_with_race( repo_root: str, diff_files: List[Tuple[str, str]], rel_touched: List[str], previous_findings: List[Dict[str, Any]], + spent_s: float = 0.0, ) -> Tuple[Optional[str], List[Dict[str, Any]], Dict[str, Any]]: """Race the agentic reviewer against a delayed single-shot fallback. Agentic starts at t=0. After SG_AGENTIC_RACE_DELAY_S (default 180s), the - single-shot diff reviewer also starts. Whichever finishes first wins. If - agentic finishes before the delay elapses, the fallback never runs. + single-shot diff reviewer also starts. Whichever finishes first wins, + except that a single-shot review with no verdict waits for agentic up to + _RACE_MAX_WAIT_S from the start, and that if agentic fails after the + fallback started, the fallback's result (or, past the deadline, no + verdict) is returned so the caller never starts another review. If agentic + finishes before the delay, the fallback never runs. + + `spent_s`, the time the hook has already run, comes off the delay and both + limits: the hook's time limit does not restart with the race. Metrics added: race_winner : 1 = agentic won, 2 = fallback won (CC accepts only bool/finite-number metric values — strings would discard the dict) - race_delay_s : the configured delay + race_delay_s : the delay used race_started : 1 if the fallback was actually launched, else 0 - Only the commit-review handler calls this — external harnesses invoke - agentic_review() directly and are unaffected. SG_AGENTIC_NO_RACE=1 + Only the commit-review and push-sweep handlers call this — external + harnesses invoke agentic_review() directly and are unaffected. SG_AGENTIC_NO_RACE=1 disables the race for any other caller that wants pure agentic. """ import queue as _queue @@ -985,9 +1412,16 @@ def _agentic_review_with_race( if os.environ.get("SG_AGENTIC_NO_RACE") == "1": return agentic_review(repo_root, diff_files, rel_touched) - delay_s = int(os.environ.get("SG_AGENTIC_RACE_DELAY_S", "180")) + delay_s = max(0, int(os.environ.get("SG_AGENTIC_RACE_DELAY_S", "180")) - int(spent_s)) + if spent_s >= 1: + debug_log(f"Review race: {spent_s:.0f}s already spent, fallback starts after {delay_s}s") q: "_queue.Queue[Tuple[str, Any]]" = _queue.Queue(maxsize=1) fallback_started = _th.Event() + fallback_done = _th.Event() + agentic_done = _th.Event() + fallback_result: Dict[str, Any] = {} + deadline = _t.monotonic() + max(0.0, _RACE_MAX_WAIT_S - spent_s) + timed_out = (None, [], {"agentic": False, "review_failed": True, "api_error": _RACE_TIMED_OUT}) def _agentic() -> None: try: @@ -998,33 +1432,173 @@ def _agentic_review_with_race( q.put_nowait(("agentic", r)) except _queue.Full: pass + agentic_done.set() def _fallback() -> None: _t.sleep(delay_s) - if not q.empty(): + if agentic_done.is_set(): return # agentic finished within the delay — never start fallback fallback_started.set() + result = _single_shot_review(diff_files, previous_findings) + fallback_result["r"] = result + fallback_done.set() + if result[2]["review_failed"]: + # A failed fallback does not win: the agentic review's inner + # CLI may still get a verdict where direct calls cannot. + agentic_done.wait(timeout=max(0.0, deadline - _t.monotonic())) try: - g, v = analyze_code_security( - diff_files, is_diff=True, previous_findings=previous_findings - ) - except Exception as e: # pragma: no cover - g, v = None, [] - try: - q.put_nowait(("fallback", (g, v, {"agentic": False}))) + q.put_nowait(("fallback", result)) except _queue.Full: pass _th.Thread(target=_agentic, daemon=True).start() _th.Thread(target=_fallback, daemon=True).start() - winner, (g, v, m) = q.get() - m = dict(m) # don't mutate the callee's metrics dict + try: + winner, (g, v, m) = q.get(timeout=max(0.0, _RACE_END_S - spent_s)) + except _queue.Empty: + debug_log("review race: nothing returned in time") + winner, (g, v, m) = "fallback", timed_out + # What the agentic run saw of its directory, and how much work it did, + # is kept when the fallback's result replaces it below. + reviewer_context = ( + {k: m[k] for k in _REVIEWER_CONTEXT_KEYS + _REVIEWER_EFFORT_KEYS if k in m} + if winner == "agentic" else {} + ) + if winner == "agentic" and m.get("agentic_fallback") and fallback_started.is_set(): + # The caller would rerun the single-shot review the fallback has + # already started; use its result instead. + if fallback_done.wait(timeout=max(0.0, deadline - _t.monotonic())): + winner, (g, v, m) = "fallback", fallback_result["r"] + else: + winner, (g, v, m) = "fallback", timed_out + m = {**m, **reviewer_context} # a copy: don't mutate the callee's metrics dict m["race_winner"] = 1 if winner == "agentic" else 2 m["race_delay_s"] = delay_s m["race_started"] = 1 if fallback_started.is_set() else 0 return g, v, m +def _review_failure_reason(status): + """(failure type, reason, likely cause or "") for a review with no verdict. + + `status` is llm.last_review_status(): an HTTP status, -1 for a network + error or timeout, None when the reply could not be read; or + _RACE_TIMED_OUT. + """ + gateway = (", or an LLM gateway may need its key in ANTHROPIC_CUSTOM_HEADERS" + if os.environ.get("ANTHROPIC_BASE_URL", "").strip() else "") + if status == _RACE_TIMED_OUT: + return ("timeout", "the review did not finish in time", "") + if llm._is_3p_provider(): + return ("provider", "the review through the cloud provider got no answer", + "The plugin's log has the provider's error.") + if llm._gateway_cli: + pinned = os.environ.get("SECURITY_REVIEW_MODEL", "").strip() + return ("gateway", "the review through Claude Code on the Claude apps gateway returned no result", + "The plugin's log has the reason. If it says the sign-in has expired, run /login again." + + (f" SECURITY_REVIEW_MODEL ({pinned}) may also name a model the gateway does " + "not serve." if pinned else "")) + if status == 401: + return ("auth", "the API rejected the plugin's credentials (HTTP 401)", + f"The sign-in may have expired or been revoked{gateway}.") + if status == 403: + return ("auth", "the API refused access (HTTP 403)", + f"The account may lack access to the review model{gateway}.") + if status == 429 or (isinstance(status, int) and status >= 500): + return ("busy", f"the API was busy or unavailable (HTTP {status})", + "This is usually temporary.") + if status == -1: + return ("network", "the model could not be reached (network error or timeout)", "") + if isinstance(status, int): + pinned = os.environ.get("SECURITY_REVIEW_MODEL", "").strip() + return ("request", f"the API refused the request (HTTP {status})", + f"SECURITY_REVIEW_MODEL ({pinned}) may name a model this account " + "cannot use." if pinned else "") + return ("reply", "the model's reply could not be read", "") + + +def _left_out_files_notice(session_id, repo_root, untracked, diff_output): + """(one line for the terminal, text for Claude) naming the source files a + failed `git add -N` kept out of this Stop review, or None. + + Each set of files gets one notice a session: the wake ends in another Stop + that fails the same way, and a notice there would wake Claude for ever. So + no notice unless the mark is saved. Paths are JSON quoted in ASCII: a file + name can hold a line break. + """ + in_diff = set(extract_file_paths_from_diff(diff_output or "")) + + def has_content(p): + try: + return os.path.getsize(os.path.join(repo_root, p)) > 0 + except OSError: + return False + + names = sorted(p for p in untracked or () + if p not in in_diff and _is_reviewable_source(p) and has_content(p)) + if not names: + return None + key = "new_files_left_out:" + hashlib.sha256( + "\0".join(names).encode("utf-8", "replace")).hexdigest()[:16] + if not atomic_check_and_mark_warning(session_id, key): + return None + if key not in load_state(session_id).get("shown_warnings", ()): + return None + _TAIL_METRICS["add_fail_told"] = len(names) + shown = ", ".join(json.dumps(p) for p in names[:_LEFT_OUT_NAMES_SHOWN]) + if len(names) > _LEFT_OUT_NAMES_SHOWN: + shown += f" and {len(names) - _LEFT_OUT_NAMES_SHOWN} more" + return (f"Security review left out new files: {shown}. Log: {DEBUG_LOG_FILE}", + f"The security review did not see these new files, so it did not " + f"review them: {shown}. git could not add them to the review's " + f"diff. Tell the user this in one sentence, and that details are " + f"in {DEBUG_LOG_FILE}.") + + +def _exit_review_failed(kind, session_id, metrics, status, pushed="no"): + """End a commit ("Commit") or push ("Push") review that got no verdict. + + Never reports the code as clean. Tells Claude once per session for each + kind of review and failure; repeats are only logged. `pushed` ("yes", + "maybe" or "no"): whether the commit's own command also pushed it. Exits. + """ + failure, reason, cause = _review_failure_reason(status) + debug_log(f"{kind} review: got no verdict ({reason}); not marked reviewed") + metrics = {"review_failed": True, + **({"api_error": status} if status is not None else {}), + **metrics, "vulns_found": 0} + # A commit its own command pushed gets its own notice: no later push + # re-checks it. + told_key = f"review_failed:{kind}:{failure}" + (":pushed" if pushed == "yes" else "") + if not atomic_check_and_mark_warning(session_id, told_key): + emit_metrics(metrics) + sys.exit(0) + subject = "this push" if kind == "Push" else "this commit" + if kind == "Push": + after = ("It was for the pushed commits not already reviewed when they " + "were committed, and a later push will not re-check them.") + elif pushed == "yes": + after = "The same command pushed it, so a later push will not re-check it." + elif not PUSH_SWEEP_ENABLED: + after = "" + elif pushed == "maybe": + after = ("If the same command pushed it, a later push will not re-check " + "it; otherwise a later push from Claude Code will.") + else: + after = "If it is pushed later from Claude Code, the push review will check it." + notice = " ".join(part for part in ( + f"The security review of {subject} did not complete: {reason}.", cause, after, + f"Tell the user this in one sentence, and that details are in {DEBUG_LOG_FILE}.", + ) if part) + # The notice goes on stderr: Claude Code shows Claude an asyncRewake + # hook's stderr, else its stdout, which carries the metrics line. + sys.stderr.write(PROVENANCE_BANNER + "\n\n" + notice + "\n") + sys.stderr.flush() + emit_metrics(metrics, + rewake_summary=(f"{kind} security review did not complete: {reason}. " + f"Log: {DEBUG_LOG_FILE}")) + sys.exit(2) + def handle_commit_review_posttooluse(input_data): """PostToolUse handler for Bash — reviews git commits for security issues. @@ -1036,15 +1610,17 @@ def handle_commit_review_posttooluse(input_data): the Stop hook won't re-flag the same (filePath, vulnerableCode) pair. """ session_id = input_data.get("session_id", "default") + _USAGE_CONTEXT["review"] = "commit" tool_input = input_data.get("tool_input", {}) tool_response = input_data.get("tool_response", {}) cwd = input_data.get("cwd", "") command = tool_input.get("command", "") if not isinstance(command, str) or not _GIT_COMMIT_RE.search(command): - # Defensive only — hooks.json's `"if": "Bash(git commit:*)"` is the - # real gate so CC never spawns python3 for ls/grep/etc. This catches - # cases where CC's command matching fails open and spawns the hook anyway. + # hooks.json's `"if": "Bash(git*commit*)"` only keeps CC from + # spawning python3 for ls/grep/etc. It lets through other git commands + # with "commit" in them (`git merge --no-commit`), and CC runs the hook + # anyway when it cannot parse the command; this regex decides. sys.exit(0) debug_log(f"Commit review: detected git commit in command") @@ -1113,8 +1689,13 @@ def handle_commit_review_posttooluse(input_data): _root = repo_root _fresh, _stale = _git_reflog_recent_commits(_root) if _fresh: + # Commits an earlier call's review is still working on are that + # review's: it records them when it has a verdict. (Read before + # the reviewed set, so one that ends in between counts as reviewed.) + _inflight = _load_inflight_shas(_root, _fresh) _already = _load_reviewed_shas(_root) - _reflog_shas = [s for s in _fresh if s not in _already] + _reflog_shas = [s for s in _fresh + if s not in _already and s not in _inflight] if _reflog_shas: commit_succeeded = True debug_log( @@ -1126,8 +1707,9 @@ def handle_commit_review_posttooluse(input_data): # Fresh commit(s) in reflog but all already in # sg-reviewed-shas — likely a Bash retry or the commit was # reviewed via a prior fire. Correct to skip; sub=2 lets telemetry - # split this from genuine fails. - _skip_21_sub = 2 + # split this from genuine fails. sub=5: at least one is still + # being reviewed by an earlier call. + _skip_21_sub = 5 if any(s not in _already for s in _fresh) else 2 elif _stale: _skip_21_sub = 3 # commit entries exist but all >120s old else: @@ -1146,9 +1728,10 @@ def handle_commit_review_posttooluse(input_data): emit_metrics({"skipped": True, "skip_reason": 32, **_base}) sys.exit(0) - if not ENABLE_CODE_SECURITY_REVIEW or not HAS_API_CREDENTIALS: - debug_log("Commit review: LLM review disabled or no API credentials") - emit_metrics({"skipped": True, "skip_reason": 22, **_base}) + off = _llm_review_off(session_id) + if off: + debug_log(f"Commit review: {off[0]}") + emit_metrics({"skipped": True, "skip_reason": 22, **off[1], **_base}) sys.exit(0) if not ensure_anthropic_reachable(): @@ -1387,52 +1970,48 @@ def handle_commit_review_posttooluse(input_data): previous_findings = with_locked_state(session_id, _read_previous) or [] + # Tell a push review (or the next commit's reflog scan) that these commits + # are being reviewed, so it does not start a second review of them. + _mark_inflight(repo_root, _full_shas(repo_root, shas)) + # Count only: keyed on what the reviewer is sent, after the byte caps. + _diff_token, _seen_m = _diff_review_started( + repo_root, _cap_files_for_prompt(diff_files), tip=shas[0]) + _TAIL_METRICS.update(_seen_m) + review_start = _time.time() agentic_metrics: Dict[str, Any] = {} if _agentic_commit_review_enabled(): rel_touched = [fp for fp, _ in diff_files] concrete_guidance, vulns, _am = _agentic_review_with_race( - repo_root, diff_files, rel_touched, previous_findings + repo_root, diff_files, rel_touched, previous_findings, spent_s=_hook_spent_s() ) agentic_metrics.update(_am) + if _am.get("child_stderr"): + # Lines the inner CLI wrote to stderr during the review: how often + # the condition that used to replace the findings body occurs. Not + # counted when the single-shot fallback wins the race. + _TAIL_METRICS["child_stderr"] = _am["child_stderr"] + _note_reviewer_context(agentic_metrics) + review_failed = bool(agentic_metrics.get("review_failed")) + api_status = agentic_metrics.get("api_error") # Fall back to single-shot only on agentic FAILURE (SDK/investigate # crash). If agentic completed and returned 0 findings, trust that. if agentic_metrics.get("agentic_fallback"): - concrete_guidance, vulns = analyze_code_security( - diff_files, is_diff=True, previous_findings=previous_findings + concrete_guidance, vulns, review_failed, api_status = _single_shot_review_in_time( + diff_files, previous_findings ) else: - concrete_guidance, vulns = analyze_code_security( - diff_files, is_diff=True, previous_findings=previous_findings + concrete_guidance, vulns, review_failed, api_status = _single_shot_review_in_time( + diff_files, previous_findings ) - # push-sweep state: record this commit as reviewed (full 40-hex sha) so a - # later `git push` can advance its diff base past it. Recorded here — after - # the review ran but before any exit path — so it's marked regardless of - # whether findings were emitted. `shas` holds abbreviated refs from - # `[branch sha]`; resolve to full so set-membership in the push-sweep is - # exact. Best-effort; failures here never block the review result. - try: - full_shas = [] - for s in shas: - # See #2099: drop text=True; decode manually for cp1252 safety. - r = subprocess.run( - [*GIT_CMD, "rev-parse", "--verify", "-q", s], - cwd=repo_root, capture_output=True, timeout=5, - ) - if r.returncode == 0: - full_shas.append(r.stdout.decode("utf-8", errors="replace").strip()) - _append_reviewed_shas(repo_root, full_shas, vulns_found=len(vulns or [])) - except Exception: - pass - review_ms = int((_time.time() - review_start) * 1000) # `survived` is the raw self-refute count BEFORE the high/critical-only # severity filter; `survived_after_sev` is the count the user actually # sees. Include `survived_after_sev` ONLY when the filter actually # dropped candidates — otherwise it's redundant with `survived` and eats - # into CC's 10-key emit cap, pushing files_reviewed/review_ms out of the + # into CC's metric-key cap, pushing files_reviewed/review_ms out of the # emitted metrics. # # CC accepts only booleans and finite numbers as metric values. @@ -1468,6 +2047,37 @@ def handle_commit_review_posttooluse(input_data): else {} ) + if review_failed: + # Not marked reviewed, so a later push review from Claude Code still + # covers it. + _exit_review_failed("Commit", session_id, { + **_base, **_agentic_m, + "files_reviewed": len(diff_files), "review_ms": review_ms, + }, api_status, pushed=( + "yes" if _push_in_output(bash_output) + else "maybe" if _GIT_PUSH_RE.search(command) or _MAY_PUSH_VIA_GH_RE.search(command) + else "no")) + + # push-sweep state: record this commit as reviewed (full 40-hex sha) so a + # later `git push` can advance its diff base past it. Recorded only after a + # review that got a verdict, whether or not it found anything. `shas` holds abbreviated refs from + # `[branch sha]`; resolve to full so set-membership in the push-sweep is + # exact. Best-effort; failures here never block the review result. + try: + full_shas = [] + for s in shas: + # See #2099: drop text=True; decode manually for cp1252 safety. + r = subprocess.run( + [*GIT_CMD, "rev-parse", "--verify", "-q", s], + cwd=repo_root, capture_output=True, timeout=5, + ) + if r.returncode == 0: + full_shas.append(r.stdout.decode("utf-8", errors="replace").strip()) + _append_reviewed_shas(repo_root, full_shas, vulns_found=len(vulns or [])) + except Exception: + pass + _diff_review_done(_diff_token) + if not concrete_guidance: debug_log("Commit review: no security issues found") emit_metrics({ @@ -1525,9 +2135,8 @@ def handle_commit_review_posttooluse(input_data): sev[s] += 1 # Rebuild guidance from new_vulns only — concrete_guidance from the LLM - # still lists deduped entries. Pass via additional_context so CC surfaces - # the reason via hookSpecificOutput.additionalContext instead of empty - # stdout (#1783) / stderr-only "json output validation failed" (#1375). + # still lists deduped entries. emit_metrics writes it to stderr, the body + # of the asyncRewake notice. _commit_guidance = (PROVENANCE_BANNER + "\n\n" + _format_vulns_guidance(new_vulns) + CONTINUATION_SUFFIX + "\n") @@ -1542,8 +2151,6 @@ def handle_commit_review_posttooluse(input_data): # exit(2) is preserved per the asyncRewake protocol — it's what CC # uses as the "force fix" signal that triggers the rewakeMessage flow. - # The stderr.write was removed; additional_context above now carries - # the same text via the modern JSON channel. See #1358/#1375/#1783. sys.exit(2) def handle_push_sweep_posttooluse(input_data): @@ -1570,6 +2177,7 @@ def handle_push_sweep_posttooluse(input_data): command = tool_input.get("command", "") or "" cwd = input_data.get("cwd") session_id = input_data.get("session_id", "") + _USAGE_CONTEXT["review"] = "push" bash_output = ( (tool_response.get("stdout", "") or "") + "\n" @@ -1577,9 +2185,8 @@ def handle_push_sweep_posttooluse(input_data): ) interrupted = tool_response.get("interrupted", False) - # Re-gate: hooks.json `if` matched, but confirm with the broader regex - # (defensive — `git -C`/`-c` forms won't reach here via the hooks.json - # prefix matcher alone, but a compound with a plain `git push` would). + # Re-gate: hooks.json `if` matched (or CC could not parse the command + # and ran every entry), so confirm with the regex. if not _GIT_PUSH_RE.search(command): sys.exit(0) @@ -1591,8 +2198,9 @@ def handle_push_sweep_posttooluse(input_data): if interrupted: emit_metrics({"skipped": True, "skip_reason": 21, **_base}) sys.exit(0) - if not ENABLE_CODE_SECURITY_REVIEW or not HAS_API_CREDENTIALS: - emit_metrics({"skipped": True, "skip_reason": 22, **_base}) + off = _llm_review_off(session_id) + if off: + emit_metrics({"skipped": True, "skip_reason": 22, **off[1], **_base}) sys.exit(0) if not cwd: emit_metrics({"skipped": True, "skip_reason": 25, **_base}) @@ -1722,13 +2330,27 @@ def handle_push_sweep_posttooluse(input_data): "pushed": len(push_range)}) sys.exit(0) - reviewed = _load_reviewed_shas(repo_root) + # A commit review that is still running on these commits records them + # once it has a verdict. Wait for it instead of reviewing the same diff a + # second time; if it ends without one, they are reviewed here. + reviewed, _waited = _reviewed_after_inflight( + repo_root, push_range, INFLIGHT_WAIT_S) + _wait_m = {} + if _waited: + _wait_m = {"inflight_wait_s": max(1, int(round(_waited)))} + debug_log(f"Push sweep: waited {_waited:.0f}s for a running commit review") base, tail = _compute_push_sweep_base(prev_upstream, push_range, reviewed) prefix_advanced = len(push_range) - len(tail) if base is None: debug_log("Push sweep: every pushed commit already reviewed") emit_metrics({**_base, "pushed": len(push_range), "unreviewed": 0, - "prefix_advanced": prefix_advanced}) + "prefix_advanced": prefix_advanced, **_wait_m}) + sys.exit(0) + if _waited and head and _git_rev_parse_head(repo_root) != head: + # Same guard as above: the diff and the reviewer's file reads are + # both taken at HEAD, and HEAD left the pushed tip during the wait. + debug_log("Push sweep: HEAD moved while waiting, skipping") + emit_metrics({"skipped": True, "skip_reason": 44, **_base, **_wait_m}) sys.exit(0) debug_log(f"Push sweep: range={len(push_range)} prefix_advanced=" @@ -1781,6 +2403,11 @@ def handle_push_sweep_posttooluse(input_data): emit_metrics({"skipped": True, "skip_reason": 23, **_base}) sys.exit(0) + _mark_inflight(repo_root, tail) + _diff_token, _seen_m = _diff_review_started( + repo_root, _cap_files_for_prompt(diff_files)) + _TAIL_METRICS.update(_seen_m) + import time as _time now = _time.time() previous_findings = with_locked_state( @@ -1794,40 +2421,54 @@ def handle_push_sweep_posttooluse(input_data): rel_touched = [fp for fp, _ in diff_files] if _agentic_commit_review_enabled(): concrete_guidance, vulns, agentic_metrics = _agentic_review_with_race( - repo_root, diff_files, rel_touched, previous_findings + repo_root, diff_files, rel_touched, previous_findings, spent_s=_hook_spent_s(), ) + if agentic_metrics.get("child_stderr"): + _TAIL_METRICS["child_stderr"] = agentic_metrics["child_stderr"] + _note_reviewer_context(agentic_metrics) + review_failed = bool(agentic_metrics.get("review_failed")) + api_status = agentic_metrics.get("api_error") if agentic_metrics.get("agentic_fallback"): - concrete_guidance, vulns = analyze_code_security( - diff_files, is_diff=True, previous_findings=previous_findings + concrete_guidance, vulns, review_failed, api_status = _single_shot_review_in_time( + diff_files, previous_findings ) else: - concrete_guidance, vulns = analyze_code_security( - diff_files, is_diff=True, previous_findings=previous_findings + concrete_guidance, vulns, review_failed, api_status = _single_shot_review_in_time( + diff_files, previous_findings ) agentic_metrics = {} review_ms = int((_time.time() - review_start) * 1000) + if review_failed: + # Not marked reviewed. A later push does not re-check these commits: + # its range starts from the new upstream. + _exit_review_failed("Push", session_id, { + **_base, **_wait_m, "pushed": len(push_range), "unreviewed": len(tail), + "files_reviewed": len(diff_files), "review_ms": review_ms, + }, api_status) # The tail is now covered by this net-diff review. _append_reviewed_shas(repo_root, tail, vulns_found=len(vulns or [])) + _diff_review_done(_diff_token) new_vulns, n_deduped = _dedup_against_state( session_id, vulns or [], prompted=_finding_keys(previous_findings) ) - # Metrics — keep within the 10-key cap; agentic sub-metrics are dropped + # Metrics — keep within CC's metric-key cap; agentic sub-metrics are dropped # here in favour of the push-sweep funnel keys (telemetry can join on session_id # to the per-commit fires for agentic detail). rewake_summary must ride # this line (CC reads only the first {-prefixed stdout line); the emit # is deferred to the two exit points below so the with-vulns path can - # also pass additional_context in the same JSON line (#1375/#1783) — - # the by-design "CC keeps only the first JSON line" constraint means - # we can't emit twice. Builds the shared metrics dict here; vulns path + # also pass additional_context in the same emit_metrics call — the + # by-design "CC keeps only the first JSON line" constraint means we + # can't emit twice. Builds the shared metrics dict here; vulns path # adds additional_context, no-vulns path emits as-is. _push_metrics = { **_base, "pushed": len(push_range), "unreviewed": len(tail), "prefix_advanced": prefix_advanced, "vulns_found": len(new_vulns), "files_reviewed": len(diff_files), "review_ms": review_ms, **({"deduped": n_deduped} if n_deduped else {}), + **_wait_m, } _push_rewake_summary = _format_vulns_summary(new_vulns, prefix="Push security review found") @@ -1887,15 +2528,23 @@ def handle_push_sweep_posttooluse(input_data): else: guidance = concrete_guidance or _format_vulns_guidance(reported) or "" # Emit metrics + additional_context together — single JSON line is the - # contract CC's hook parser expects. exit(2) preserved as the asyncRewake - # "force fix" trigger (see comment near handle_commit_review_posttooluse). - # See #1358 / #1375 / #1783. + # contract CC's hook parser expects; the guidance itself goes to stderr. + # exit(2) preserved as the asyncRewake "force fix" trigger (see comment + # near handle_commit_review_posttooluse). emit_metrics(_push_metrics, rewake_summary=_push_rewake_summary, additional_context=(PROVENANCE_BANNER + "\n\n" + guidance + CONTINUATION_SUFFIX + "\n"), hook_event_name="PostToolUse") sys.exit(2) +def _stop_diff_truncated(): + """Bytes of reviewable diff the Stop review did not send: cut while + reading git's output, plus what the prompt caps dropped after that. + As the metric to emit, or nothing when there were none.""" + n = llm._last_review_truncated_bytes + gitutil._diff_read["cut_bytes"] + return {"diff_truncated": n} if n else {} + + def handle_stop_hook(input_data): """ Handle the Stop hook — final security check using git diff. @@ -1935,23 +2584,61 @@ def handle_stop_hook(input_data): head_at_capture = snap["head_at_capture"] untracked_at_baseline = snap.get("untracked_at_baseline") or {} previous_findings = snap["previous_findings"] + # Only when it happened: state lost to an unreadable file, and saves that + # overwrote another hook's (possible only without locking). SubagentStop + # does not clear them, so it does not report them either. + if not is_subagent: + for _k in ("state_resets", "state_overwrites"): + if isinstance(snap.get(_k), int) and snap[_k] > 0: + _TAIL_METRICS[_k] = snap[_k] # Sweep pattern-warning outcomes (pure local work; stop_hook_active is # already guaranteed False here so no double-count guard needed). sweep = {} - warn_fixed, warn_unresolved, warn_unresolved_mask = sweep_pending_warnings(session_id) + (warn_fixed, warn_unresolved, warn_unresolved_mask, + warn_fired_mask) = sweep_pending_warnings(session_id) if warn_fixed or warn_unresolved: sweep = { "warn_fixed": warn_fixed, "warn_unresolved": warn_unresolved, "warn_unresolved_mask": warn_unresolved_mask, } + # The two masks also ride the tail, so the reviewed rows below, which + # leave `sweep` or its mask out, carry them when there is room under the + # key cap. Per rule they give how often it fired and how often the + # flagged code was still there at Stop. warn_fired_mask goes after + # warn_unresolved_mask: a row cut at the cap loses it first, and a row + # without it is left out of the per-rule read instead of counting its + # rules as fixed. + if warn_unresolved_mask: + _TAIL_METRICS["warn_unresolved_mask"] = warn_unresolved_mask + if warn_fired_mask: + _TAIL_METRICS["warn_fired_mask"] = warn_fired_mask v2_metrics = {} + left_out = False + restore_paths = touched_paths + + def _left_out_notice(): + # Asked for as the row goes out: the mark must not outlive a hook + # killed during the review. + if not left_out: + return None + return _left_out_files_notice(session_id, repo_root, untracked, diff_output) + + def _left_out_wake(): + """emit_metrics arguments that wake Claude to say new files were left out.""" + notice = _left_out_notice() + if not notice: + return {} + return {"rewake_summary": notice[0], "hook_event_name": hook_event_name, + "additional_context": (PROVENANCE_BANNER + "\n\n" + notice[1] + + CONTINUATION_SUFFIX + "\n")} def _skip(reason, restore=False, **extra): if restore: - restore_unreviewed_stop_state(session_id, touched_paths, snap_baseline) + restore_unreviewed_stop_state(session_id, restore_paths, snap_baseline) + wake = _left_out_wake() # CC truncates metrics to 10 keys by # insertion order. v2_metrics (3) must precede sweep (3) so the v2 # diagnostics survive when extra adds touched_paths_count + ip_* keys. @@ -1959,8 +2646,12 @@ def handle_stop_hook(input_data): "skipped": True, "skip_reason": reason, "fire_index": fire_count + 1, "diff_strategy_v2": True, **v2_metrics, **extra, **sweep, - }) - sys.exit(0) + **({"diff_hdr_unparsed": len(gitutil._unparsed_diff_headers)} + if gitutil._unparsed_diff_headers else {}), + **({"secret_shaped_skipped": len(gitutil._secret_shaped_skipped)} + if gitutil._secret_shaped_skipped else {}), + }, **wake) + sys.exit(2 if wake else 0) # Limit stop hook firings per asyncRewake loop to prevent infinite loops. # fire_count auto-expires after STOP_LOOP_STATE_TTL_SEC so a stale count @@ -1969,9 +2660,12 @@ def handle_stop_hook(input_data): debug_log(f"Stop hook: already fired {fire_count} times (max {MAX_STOP_HOOK_FIRINGS}), skipping") _skip(2) - if not ENABLE_CODE_SECURITY_REVIEW or not HAS_API_CREDENTIALS: - debug_log("Stop hook: LLM review disabled or no API credentials") - _skip(3) + off = _llm_review_off(session_id) + if off: + debug_log(f"Stop hook: {off[0]}") + # When the sign-in check gave no answer, this turn's changes are kept + # for the next Stop review. + _skip(3, restore=llm._gateway_check_unanswered, **off[1]) # Stop-hook-only kill switch — placed after consume_stop_state so # touched_paths is still cleared each turn (a disabled Stop hook that @@ -2008,37 +2702,116 @@ def handle_stop_hook(input_data): save_repo_hint(session_id, repo_cwd) cwd = repo_cwd + # The baseline belongs to the checkout the prompt hook captured it in. + # When the turn ends in another one (the session entered a worktree, or + # the edits were in a different repo), it says nothing about what changed + # here: this checkout's older commits would look like commits made this + # turn, and its files would be diffed against the other checkout's. Use + # what this checkout can say for itself instead: its own HEAD when the + # turn began (from its reflog) for commits, HEAD for file contents, and + # file times for which uncommitted files were already there. + origin = None + if repo_cwd: + try: + origin = baseline_origin(repo_cwd, snap_baseline, snap["baseline_root"]) + except Exception as e: + debug_log(f"Stop hook: baseline origin check failed: {e}") + foreign_baseline = origin in FOREIGN_ORIGINS + since_ns = None + if foreign_baseline: + at = snap["prompt_at"] + at = at if isinstance(at, (int, float)) and 0 < at < 1e11 else None + baseline_sha = None + head_at_capture = _git_head_at(repo_cwd, at) if at else None + untracked_at_baseline = {} + since_ns = int(at * 1e9) if at else None + debug_log(f"Stop hook: baseline {str(snap_baseline)[:12]} is from another " + f"checkout (origin {origin}); this one was at " + f"{str(head_at_capture)[:12]} when the turn began") + review_paths, diff_base, repo_root, untracked, v2_metrics = compute_v2_review_set( - cwd, baseline_sha, head_at_capture, untracked_at_baseline + cwd, baseline_sha, head_at_capture, untracked_at_baseline, since_ns=since_ns ) + # Sent even when 0, once git status has worked. + if origin is not None and v2_metrics.get("dirty_now_count", -1) >= 0: + _TAIL_METRICS["bl_origin"] = baseline_origin_metric(origin, snap["baseline_kept"]) + v2_metrics = {**res_metrics, **v2_metrics} + # Count-only view of how the review set relates to what the session's + # edit tools wrote. Only when a repo was read; never fails the review. + def _note_scope(files): + try: + if "scope" not in _scope: + _scope["scope"] = touched_scope( + repo_root, input_data.get("cwd", ""), snap["turn_touched"]) + # -1: the stored baseline could not be used in this repo. + fail = (3 if v2_metrics.get("changed_since_count") == -1 + else snap.get("baseline_fail")) + _TAIL_METRICS["review_scope"] = review_scope_code( + repo_root, *_scope["scope"], files, fail) + except Exception as e: + debug_log(f"review scope telemetry skipped: {e}") + _scope = {} + if v2_metrics.get("dirty_now_count", -1) >= 0: + _note_scope(review_paths) if not review_paths: debug_log("Stop hook: empty review set") _skip(9, touched_paths_count=len(touched_paths)) debug_log(f"Stop hook: review_set={len(review_paths)} base={diff_base[:12]} dirty_now={v2_metrics['dirty_now_count']} changed_since={v2_metrics['changed_since_count']}") + if (len(review_paths) > MAX_REVIEW_SET_PATHS + and sum(_is_reviewable_source(os.path.relpath(p, repo_root)) + for p in review_paths) > MAX_REVIEW_SET_PATHS): + debug_log(f"Stop hook: pathological review set (more than " + f"{MAX_REVIEW_SET_PATHS} paths), skipping") + _skip(8) # Run from repo_root so the toplevel-relative review_paths resolve. - # Diff CONTENT against the turn-start stash (baseline_sha) so the LLM + # Diff CONTENT against the turn-start snapshot (baseline_sha) so the LLM # sees only this-turn edits — diffing against HEAD includes the user's # pre-turn uncommitted WIP, which inflates review_ms and can re-flag # the same pre-existing pattern every turn. The file LIST still comes # from git state (compute_v2_review_set), so Bash/subagent edits are # caught either way. Fall back to diff_base (HEAD/head_at_capture) - # when the stash is missing or pruned. + # when the baseline commit is missing or pruned. content_base = baseline_sha or diff_base diff_output = get_git_diff(repo_root, content_base, full_context=False, paths=review_paths, untracked_paths=untracked) - if diff_output is None and content_base != diff_base: + if (diff_output is None and content_base != diff_base + and "gave_up" not in gitutil._path_list_too_long): debug_log(f"Stop hook: diff against {content_base[:12]} failed — falling back to {diff_base}") diff_output = get_git_diff(repo_root, diff_base, full_context=False, paths=review_paths, untracked_paths=untracked) + # Only when it happened: a failed `git add -N` (first, so a row at CC's key + # cap keeps it), MiB of git diff output read, reviewable files whose diff + # was cut while reading it, diffs that hit the timeout. + for _k, _v in (("add_fail", gitutil._add_failed["code"]), + ("diff_mb", gitutil._diff_read["bytes"] >> 20), + ("diff_cut_files", gitutil._diff_read["cut_files"]), + ("diff_timeout", gitutil._diff_read["timeouts"])): + if _v: + _TAIL_METRICS[_k] = _v # filter_preexisting_from_diff needs a resolvable pre-turn ref; fall # back to HEAD when UPS never captured a baseline (print mode). if not baseline_sha: baseline_sha = "HEAD" + # New files a failed `git add -N` hid: when nothing gets reviewed they are + # put back, listed as touched so the next prompt keeps this baseline even + # if an edit tool never wrote them. A review of the tracked files goes on + # without them, rather than repeating that review every turn. + left_out = gitutil._add_failed["left_out"] + if left_out: + restore_paths = list(touched_paths) + [ + os.path.join(repo_root, p) for p in untracked or ()] + if not diff_output or not diff_output.strip(): + if "gave_up" in gitutil._path_list_too_long: + debug_log("Stop hook: changed paths do not fit the git commands, skipping") + _skip(8) + if "none_reviewable" in gitutil._path_list_too_long: + debug_log("Stop hook: no source code files among the changed paths") + _skip(7) debug_log("Stop hook: no changes since baseline") - _skip(6) + _skip(6, restore=left_out) diff_hash = hashlib.sha256(diff_output.encode("utf-8", "replace")).hexdigest() if snap.get("reviewed_diff_hash") == diff_hash: @@ -2049,7 +2822,7 @@ def handle_stop_hook(input_data): diff_files = parse_diff_into_files(diff_output) if not diff_files: debug_log("Stop hook: no source code files in diff") - _skip(7) + _skip(7, restore=left_out) # Mirror commit-review: hard-bail only on pathological diffs (>300 files, # usually a bad baseline), otherwise prioritize by security-risk path @@ -2070,8 +2843,13 @@ def handle_stop_hook(input_data): # Filter out pre-existing content from file rewrites diff_files = filter_preexisting_from_diff(diff_files, cwd, baseline_sha) + if "scope" in _scope: + _note_scope([fp for fp, _ in diff_files]) debug_log(f"Stop hook: reviewing {len(diff_files)} changed files (standard diff)") + secret_named = gitutil.count_secret_named(diff_files) + if secret_named: + _TAIL_METRICS["secret_named_reviewed"] = secret_named import time as _time stop_review_start = _time.time() @@ -2116,7 +2894,12 @@ def handle_stop_hook(input_data): ] # Update baseline so next stop hook iteration only sees new changes new_sha = None if is_subagent else capture_git_baseline(cwd) + new_base_fail = diffstate.last_baseline_fail new_untracked_baseline = _list_untracked(cwd) if new_sha else None + # After a baseline from another checkout, the rest of it is replaced + # too, below. + new_head = (_git_rev_parse_head(cwd) + if new_sha and foreign_baseline and not head_at_capture else None) def _record_fire(state): if is_subagent: @@ -2141,6 +2924,14 @@ def handle_stop_hook(input_data): if new_sha: state["baseline_sha"] = new_sha state["untracked_at_baseline"] = new_untracked_baseline + _store_baseline_fail(state, new_base_fail) + state["baseline_root"] = repo_root + state.pop("baseline_kept", None) + if foreign_baseline: + # The stored HEAD is the other checkout's. Keep the one + # this review used, so a later Stop this turn still sees + # commits made (or amended) here since the prompt. + state["head_at_capture"] = head_at_capture or new_head with_locked_state(session_id, _record_fire) if new_sha: @@ -2155,11 +2946,10 @@ def handle_stop_hook(input_data): # untracked_baseline_n is the signal for whether the UPS-time # untracked-snapshot capture actually ran. sweep_trimmed = {k: v for k, v in sweep.items() if k != "warn_unresolved_mask"} - # Pass guidance via additional_context so CC surfaces the findings via - # hookSpecificOutput.additionalContext instead of stderr-only (which - # was the cause of "json output validation failed" / empty-reason UI in - # #1375 / #1783). exit(2) preserved as the asyncRewake "force fix" - # signal — that's the documented mechanism. See #1358 / #1375 / #1783. + left_out_notice = _left_out_notice() + # Guidance goes to stderr plus top-level decision/reason (see + # emit_metrics). exit(2) preserved as the asyncRewake "force fix" + # signal — that's the documented mechanism. emit_metrics({ "vulns_found": len(vulns), "untracked_baseline_n": len(untracked_at_baseline), @@ -2170,30 +2960,38 @@ def handle_stop_hook(input_data): "touched_paths_count": len(touched_paths), "review_ms": review_ms, "fire_index": fire_index, - **({"diff_truncated": llm._last_review_truncated_bytes} - if llm._last_review_truncated_bytes else {}), + **_stop_diff_truncated(), **({"repo_resolution": repo_res} if res_metrics else {}), **sweep_trimmed, + **({"diff_hdr_unparsed": len(gitutil._unparsed_diff_headers)} + if gitutil._unparsed_diff_headers else {}), + **({"secret_shaped_skipped": len(gitutil._secret_shaped_skipped)} + if gitutil._secret_shaped_skipped else {}), }, rewake_summary=_format_vulns_summary(vulns), - additional_context=(PROVENANCE_BANNER + "\n\n" - + concrete_guidance + CONTINUATION_SUFFIX + "\n"), + additional_context=(PROVENANCE_BANNER + "\n\n" + concrete_guidance + + ("\n\n" + left_out_notice[1] if left_out_notice else "") + + CONTINUATION_SUFFIX + "\n"), hook_event_name=hook_event_name) sys.exit(2) - if llm._last_call_claude_http_error is not None: - debug_log(f"Stop hook: API call failed with status {llm._last_call_claude_http_error}") - restore_unreviewed_stop_state(session_id, touched_paths, snap_baseline) + stop_review_failed = llm.last_review_failed() + if stop_review_failed: + debug_log(f"Stop hook: review got no verdict " + f"({_review_failure_reason(llm.last_review_status())[1]})") + restore_unreviewed_stop_state(session_id, restore_paths, snap_baseline) else: debug_log("Stop hook: no security issues found") if is_subagent: record_reviewed_diff(session_id, diff_hash) - # CC truncates metrics to 10 keys by - # insertion order. The previous **sweep,**v2_metrics tail meant the 3 - # v2_metrics keys were always sliced off this most-common path, so the - # diff-strategy diagnostics never reached telemetry. Drop sweep here (it's - # PostToolUse-warning state, orthogonal to diff-strategy comparison). - # 6 base + optional api_error + 3 v2_metrics = ≤10. + wake = _left_out_wake() + # CC keeps the first 20 metric keys (10 in older versions), in insertion + # order, after the pv and usage keys emit_metrics puts first. The previous + # **sweep,**v2_metrics tail meant the 3 v2_metrics keys were always sliced + # off this most-common path, so the diff-strategy diagnostics never reached + # telemetry. Drop sweep here (it's PostToolUse-warning state, orthogonal to + # diff-strategy comparison). emit_metrics({ + **({"review_failed": True} if stop_review_failed else {}), "vulns_found": 0, "diff_strategy_v2": True, "files_reviewed": len(diff_files), @@ -2201,11 +2999,14 @@ def handle_stop_hook(input_data): "review_ms": review_ms, "fire_index": fire_index, **({"api_error": llm._last_call_claude_http_error} if llm._last_call_claude_http_error is not None else {}), - **({"diff_truncated": llm._last_review_truncated_bytes} - if llm._last_review_truncated_bytes else {}), + **_stop_diff_truncated(), **v2_metrics, - }) - sys.exit(0) + **({"diff_hdr_unparsed": len(gitutil._unparsed_diff_headers)} + if gitutil._unparsed_diff_headers else {}), + **({"secret_shaped_skipped": len(gitutil._secret_shaped_skipped)} + if gitutil._secret_shaped_skipped else {}), + }, **wake) + sys.exit(2 if wake else 0) _SDK_BOOTSTRAP_THROTTLE = os.path.join(_resolve_state_dir(), ".sdk_bootstrap_spawned") @@ -2223,19 +3024,22 @@ def _maybe_bootstrap_agent_sdk_async(): is idempotent and O_EXCL-locked, so concurrent/repeat spawns are safe; the throttle file only avoids spawning dozens of subprocesses during the build window. No-ops in ~10ms on local installs (SDK already - importable). + importable, or already installed under the state dir, which is not on + sys.path here) and when the environment turns the LLM reviews off. """ try: import importlib.util if importlib.util.find_spec("claude_agent_sdk") is not None: return + if llm_reviews_switched_off() or agent_sdk_installed(_resolve_state_dir()): + return import time as _t try: if _t.time() - os.path.getmtime(_SDK_BOOTSTRAP_THROTTLE) < 300: return except OSError: pass - os.makedirs(os.path.dirname(_SDK_BOOTSTRAP_THROTTLE), exist_ok=True) + make_state_dir(os.path.dirname(_SDK_BOOTSTRAP_THROTTLE)) # Touch the throttle BEFORE spawning so a burst of PostToolUse # fires in the same second don't each spawn a subprocess. open(_SDK_BOOTSTRAP_THROTTLE, "w").close() @@ -2253,6 +3057,7 @@ def main(): """Main hook function.""" debug_log(f"Hook called with args: {sys.argv}") apply_safe_git_env() + remove_copies_on_termination() # Master kill switch — honors ENABLE_SECURITY_REMINDER=0 (legacy) and # SECURITY_GUIDANCE_DISABLE=1 (clearer name, no double negative). Emit @@ -2276,6 +3081,7 @@ def main(): sys.exit(0) session_id = input_data.get("session_id", "default") + _USAGE_CONTEXT["session_id"] = re.sub(r"[^A-Za-z0-9_.-]", "_", str(session_id))[:80] tool_name = input_data.get("tool_name", "") tool_input = input_data.get("tool_input", {}) hook_event_name = input_data.get("hook_event_name", "") @@ -2307,11 +3113,15 @@ def main(): # Handle PostToolUse[Bash] — commit review or push sweep (asyncRewake). # - # hooks.json has two `if` configs under the Bash matcher (`git commit:*` - # and `git push:*`). CC evaluates each `if` independently and spawns this + # hooks.json has three `if` configs under the Bash matcher (`git*commit*`, + # `git*push*` and `gt *`), kept to one per kind because CC runs EVERY + # entry when it cannot parse the command (loops, some substitutions): one + # entry per spelling meant one python per entry on those calls. CC + # evaluates each `if` independently and spawns this # script ONCE PER MATCH — so `git commit -m x && git push` spawns python # twice with the same command string and the same tool_use_id. The python - # cannot tell which `if` fired it. + # cannot tell which `if` fired it, which is also why the three entries + # share one rewakeMessage. # # Routing therefore MUST check commit FIRST so that compound commit+push # commands continue to hit commit-review (the pre-existing behaviour) on @@ -2329,12 +3139,27 @@ def main(): if tool_name == "Bash" and hook_event_name == "PostToolUse": cmd = (input_data.get("tool_input") or {}).get("command", "") or "" if not (_GIT_COMMIT_RE.search(cmd) or _GIT_PUSH_RE.search(cmd)): + # Measurement only: this start matched no commit or push. Count + # it, and say whether a plain text check could have told so before + # Python. Both values carry the sample weight, so they sum to + # estimates of all such starts. + if _unmatched_bash_sampled(input_data) and not portable_output(): + skip = 0 if _shell_precheck_passes(raw_input) else _UNMATCHED_BASH_SAMPLE + emit_metrics({"bash_no_match": _UNMATCHED_BASH_SAMPLE, "precheck_skip": skip}) return + if not _shell_precheck_passes(raw_input): + # The text check would have dropped a command that is reviewed. + # Rides on whichever row this start emits; nothing is skipped. + global _precheck_miss + _precheck_miss = True if not _claim_bash_hook_once(input_data): # Another spawn for this same tool_use_id already claimed the # work (compound matched multiple `if` configs). Emit a single # metric so telemetry can count how often the de-dupe kicks in. - print(json.dumps({"metrics": {"bash_hook_dedup": True}}), flush=True) + dedup = {"bash_hook_dedup": True} + if _precheck_miss and not portable_output(): + dedup = {"pv": _PV, "precheck_miss": 1, **dedup} + _write_hook_output({"metrics": dedup}) sys.exit(0) if _GIT_COMMIT_RE.search(cmd): handle_commit_review_posttooluse(input_data) @@ -2355,7 +3180,8 @@ def main(): record_touched_path(session_id, file_path) - content = extract_content_from_input(tool_name, tool_input) + content = extract_content_from_input( + tool_name, tool_input, input_data.get("tool_response")) all_guidance = [] raw_pattern_matches = [] @@ -2395,30 +3221,32 @@ def main(): # rule_id: RuleId of the first raw match (values stay small/enumerable in telemetry) # rule_mask: bitmask of ALL raw matches — POPCOUNT gives raw hit count, # (mask >> N) & 1 tests for a specific rule + output = {} if raw_pattern_matches: raw_names = [r for r, _ in raw_pattern_matches] - output = {"metrics": { + output["metrics"] = { "pattern_hits": len(all_guidance), # User-defined patterns (rule_name="user:*") have no static # RuleId; emit -1 so the metrics pipeline can distinguish. "rule_id": int(_RULE_NAME_TO_ID.get(raw_names[0], -1)), "rule_mask": rule_names_to_mask(raw_names), **({"pv": _PV} if _PV else {}), - }} - if all_guidance: - output["hookSpecificOutput"] = { - "hookEventName": "PostToolUse", - "additionalContext": PROVENANCE_TAG + "\n\n" + "\n\n".join(all_guidance), - } - print(json.dumps(output)) - elif all_guidance: - # Defensive: pattern rules disabled but guidance somehow set (shouldn't happen) - print(json.dumps({ - "hookSpecificOutput": { - "hookEventName": "PostToolUse", - "additionalContext": PROVENANCE_TAG + "\n\n" + "\n\n".join(all_guidance), - } - })) + **_LAUNCHER_METRICS, + } + if all_guidance: + output["hookSpecificOutput"] = { + "hookEventName": "PostToolUse", + "additionalContext": PROVENANCE_TAG + "\n\n" + "\n\n".join(all_guidance), + } + # The LLM reviews are on but can't run: say so. + if (ENABLE_CODE_SECURITY_REVIEW and not HAS_API_CREDENTIALS + and (ENABLE_STOP_REVIEW or COMMIT_REVIEW_ENABLED + or PUSH_SWEEP_ENABLED)): + notice = _no_credentials_notice(file_path, session_id) + if notice: + output["systemMessage"] = notice + if output: + _write_hook_output(output) sys.exit(0) diff --git a/plugins/security-guidance/hooks/session_state.py b/plugins/security-guidance/hooks/session_state.py index 8dbadfd2..330abd28 100644 --- a/plugins/security-guidance/hooks/session_state.py +++ b/plugins/security-guidance/hooks/session_state.py @@ -17,9 +17,10 @@ except ImportError: import json import os import re +import tempfile from datetime import datetime -from _base import debug_log, state_dir as _state_dir +from _base import debug_log, make_state_dir, state_dir as _state_dir def _state_key(session_id): @@ -47,7 +48,7 @@ def get_lock_file(session_id): def cleanup_old_state_files(): - """Remove state files and lock files older than 30 days.""" + """Remove state files, lock files and leftover temporary files older than 30 days.""" try: state_dir = _state_dir() if not os.path.exists(state_dir): @@ -58,7 +59,7 @@ def cleanup_old_state_files(): for filename in os.listdir(state_dir): if filename.startswith("security_warnings_state_") and ( - filename.endswith(".json") or filename.endswith(".lock") + filename.endswith((".json", ".lock", ".tmp")) ): file_path = os.path.join(state_dir, filename) try: @@ -96,25 +97,90 @@ def load_state(session_id): if isinstance(data, dict): data.setdefault("shown_warnings", []) return data - except (json.JSONDecodeError, IOError, KeyError, TypeError): + except FileNotFoundError: + return {"shown_warnings": []} + except (ValueError, IOError, KeyError, TypeError): + # ValueError: not JSON, or bytes that do not decode. pass - return {"shown_warnings": []} + # The file is there but cannot be used, so everything the session had + # recorded is gone. Leave a mark for the Stop hook to report: a flag, not + # a count, since an earlier mark is lost with the rest. + debug_log(f"State file {state_file} is unreadable, starting from an empty state") + return {"shown_warnings": [], "state_resets": 1} + + +# Windows still writes the state file in place: a rename there is refused +# while another process has the file open and can make a concurrent open fail, +# and hooks run without a state lock on Windows (see with_locked_state). +_RENAME_INTO_PLACE = os.name != "nt" + + +def _write_then_rename(state_file, state): + """Write `state` to a temporary file and rename it over `state_file`. + Returns False, with nothing changed, when this directory does not allow + it; any other failure raises OSError and leaves the state file as it was.""" + tmp = None + try: + fd, tmp = tempfile.mkstemp( + dir=os.path.dirname(state_file), + prefix=os.path.basename(state_file) + ".", suffix=".tmp") + with os.fdopen(fd, "w") as f: + json.dump(state, f) + os.replace(tmp, state_file) + return True + except BaseException as e: + if tmp: + try: + os.unlink(tmp) + except OSError: + pass + if isinstance(e, PermissionError): + return False + raise + + +def _write_in_place(state_file, state): + """The fallback (Windows, or a folder that refuses the rename) writes the + state file itself. As with the rename, a link there is replaced, not + followed, and a new file is the user's only, as the temporary file is.""" + if os.path.islink(state_file): + os.unlink(state_file) + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(state_file, flags, 0o600) + with os.fdopen(fd, "w") as f: + json.dump(state, f) def save_state(session_id, state): - """Save the full state dict to file.""" + """Save the full state dict to file. + + The new contents go to a temporary file that is then renamed over the + state file, so a hook killed part-way through a save (a timeout, an + interrupt) leaves the previous state. Written in place it left a truncated + file, which load_state reads as an empty state: the turn baseline, touched + paths and shown warnings were lost. + """ state_file = get_state_file(session_id) try: state_dir = os.path.dirname(state_file) if state_dir: - os.makedirs(state_dir, exist_ok=True) + make_state_dir(state_dir) - with open(state_file, "w") as f: - json.dump(state, f) + if _RENAME_INTO_PLACE and _write_then_rename(state_file, state): + return + _write_in_place(state_file, state) except (IOError, OSError) as e: debug_log(f"Failed to save state file {state_file}: {e}") +def _raw_state(state_file): + try: + with open(state_file, "rb") as f: + return f.read() + except OSError: + return None + + def with_locked_state(session_id, callback): """ Execute callback with exclusive access to the state file. @@ -126,14 +192,21 @@ def with_locked_state(session_id, callback): state_dir = os.path.dirname(lock_file) try: - os.makedirs(state_dir, exist_ok=True) + make_state_dir(state_dir) except OSError: pass if fcntl is None: # No file locking available (Windows) — run without locking + state_file = get_state_file(session_id) + before = _raw_state(state_file) state = load_state(session_id) result = callback(state) + if _raw_state(state_file) != before: + # Another hook changed the state after it was read here, and this + # save is about to drop that change. Count how often it happens; + # the Stop hook reports the count. + state["state_overwrites"] = state.get("state_overwrites", 0) + 1 save_state(session_id, state) return result diff --git a/plugins/security-guidance/hooks/sg-python.sh b/plugins/security-guidance/hooks/sg-python.sh index 9cf8cc07..b1a5d5b9 100755 --- a/plugins/security-guidance/hooks/sg-python.sh +++ b/plugins/security-guidance/hooks/sg-python.sh @@ -33,6 +33,10 @@ set -e # changing it from inside the interpreter has no effect. export PYTHONUTF8=1 +# Set to 1 below only when the interpreter had to be found by its real path +# (pass 4); the hook reports it as the `launcher_fallback` metric. +unset SG_LAUNCHER_FALLBACK + # Git Bash / MSYS on Windows hands script paths to this shim in POSIX form # (`/c/Users/...`). When we exec a Windows `python.exe` (which we do on # Windows since `python3` is the Microsoft Store stub), python interprets the @@ -41,14 +45,30 @@ export PYTHONUTF8=1 # fails with ENOENT, and every Edit/Write/MultiEdit tool use blocks until the # session restarts. See anthropics/claude-plugins-official#2043. # -# Fix: convert absolute path args to native Windows form via `cygpath -w` -# before exec. `cygpath` is a Git Bash builtin; it's absent on macOS/Linux, -# where the `command -v` guard makes this a no-op. `cygpath -w` is idempotent -# for already-Windows paths so the rare mixed-form case is safe. +# Fix: convert absolute path args to native Windows form before exec. +# `cygpath` ships with Git Bash; it's absent on macOS/Linux, where the +# `command -v` guard makes this a no-op. `cygpath -w` is idempotent for +# already-Windows paths so the rare mixed-form case is safe. +# +# Under Git Bash every $(...) is a new Windows process, and this shim runs on +# every hook, so drive paths (`/c/Users/...`, where the plugin lives) are +# converted in-shell. Git Bash and MSYS2 (OSTYPE=msys) mount drives at +# /<letter>; anything else (`/tmp`, `/usr`, Cygwin's /cygdrive) needs cygpath +# to resolve. See #6470. if command -v cygpath >/dev/null 2>&1; then + sg_drive_mounts="" + case "${OSTYPE:-}" in msys*) sg_drive_mounts=1 ;; esac converted=() for a in "$@"; do case "$a" in + /[A-Za-z]/*) + if [ -n "$sg_drive_mounts" ]; then + drive="${a:1:1}" + rest="${a:3}" + converted+=("${drive^^}:\\${rest//\//\\}") + else + converted+=("$(cygpath -w "$a")") + fi ;; /*) converted+=("$(cygpath -w "$a")") ;; *) converted+=("$a") ;; esac @@ -56,10 +76,98 @@ if command -v cygpath >/dev/null 2>&1; then set -- "${converted[@]}" fi +# The hook script the chosen interpreter has to run (already in native form). +SG_TARGET="" +case "${1:-}" in + ""|-*) ;; + *) SG_TARGET="$1" ;; +esac + +# Names whose interpreter runs but cannot see the hook script ("|"-separated). +sg_unreadable="" + probe() { # $1..N: the interpreter command (may be multi-word like `py -3`) - # Writes "<major>.<minor>" to stdout and exits 0 iff at least Python 3. - "$@" -c 'import sys; print(f"{sys.version_info[0]}.{sys.version_info[1]}")' 2>/dev/null + # Writes "<major>.<minor>" to stdout and exits 0 iff it is at least + # Python 3 AND it can read the hook script; exits 3 if it cannot. + # + # The second half matters on Windows: an interpreter started through a + # packaged launcher alias (Microsoft Store Python, Python Install Manager; + # the python/python3/py entries under ...\Microsoft\WindowsApps) runs with + # a different view of %APPDATA% than the process that started it. It + # answers `-c` fine, then fails on the script with "can't open file ... + # [Errno 2]" and exit code 2 -- and exit 2 from an asyncRewake hook wakes + # the model, whose reply runs the Stop hook again: an endless loop that + # never reviewed anything. See #6251, #5781. + # + # On Windows a second line carries sys.executable (see sg_own_path). + # + # Only for `$(probe ...)`: the `exec` replaces that subshell instead of + # forking a second process from it. + [ "${BASH_SUBSHELL:-0}" -gt 0 ] || return 1 + exec "$@" -c 'import os, sys; t = (sys.argv[1:] or [""])[0]; sys.exit(3) if t and not (os.path.isfile(t) and os.access(t, os.R_OK)) else print(f"{sys.version_info[0]}.{sys.version_info[1]}"); sys.platform == "win32" and sys.version_info >= (3, 10) and print(sys.executable)' "$SG_TARGET" 2>/dev/null +} + +# Use as `ask ... || { note_unreadable "<name>"; continue; }` (see ask below). +note_unreadable() { + [ $? -eq 3 ] && sg_unreadable="$sg_unreadable|$1|" + return 0 +} + +# True iff the command's executable is on PATH ($1 of a possibly multi-word +# command like `py -3`). A builtin lookup: `$(probe ...)` costs a process even +# when the interpreter does not exist, so missing names are skipped first. +on_path() { + command -v "$1" >/dev/null 2>&1 +} + +# How long the probes took, for the launcher_probe_max_ms / launcher_probe_ms +# metrics the hook reports. Measurement only: `ask` is `v=$(probe ...)` with a +# clock around it, and changes neither which interpreter runs nor how long a +# probe may take. EPOCHREALTIME needs bash 5; older bash (macOS /bin/bash) only +# has whole seconds, where a difference of 1 can be a few milliseconds: there +# a single probe counts from 2 s, and the total is first probe to last. +unset SG_LAUNCHER_PROBE_MS SG_LAUNCHER_PROBE_MAX_MS +sg_probe_ms=0 +sg_probe_max_ms=0 +sg_probe_first="" +sg_coarse="" +sg_clock() { + # Sets $sg_now (ms) without starting a process. + case "${EPOCHREALTIME:-}" in + [0-9]*[.,][0-9][0-9][0-9][0-9][0-9][0-9]) # the decimal point follows the locale + sg_now=${EPOCHREALTIME//[!0-9]/} + sg_now=$((10#$sg_now / 1000)) ;; + *) + sg_coarse=1 + sg_now=$((SECONDS * 1000)) ;; + esac +} +ask() { + # Sets $v and $sg_exe; same exit status as probe. + local rc=0 t0 ms + sg_clock; t0=$sg_now + [ -n "$sg_probe_first" ] || sg_probe_first=$t0 + v=$(probe "$@") || rc=$? + sg_exe="" + case "$v" in + *$'\n'*) + sg_exe=${v#*$'\n'}; sg_exe=${sg_exe%$'\r'} + v=${v%%$'\n'*} ;; + esac + sg_clock; ms=$((sg_now - t0)) + if [ -n "$sg_coarse" ]; then + [ "$ms" -ge 2000 ] || ms=0 + sg_probe_ms=$((sg_now - sg_probe_first)) + [ "$sg_probe_ms" -ge 2000 ] || sg_probe_ms=0 + elif [ "$ms" -gt 0 ]; then + sg_probe_ms=$((sg_probe_ms + ms)) + fi + [ "$ms" -le "$sg_probe_max_ms" ] || sg_probe_max_ms=$ms + if [ "$sg_probe_ms" -gt 0 ]; then + export SG_LAUNCHER_PROBE_MS=$sg_probe_ms SG_LAUNCHER_PROBE_MAX_MS=$sg_probe_max_ms + fi + return $rc } # True iff arg is a "M.m" version string >= 3.10. claude_agent_sdk requires @@ -75,13 +183,156 @@ is_sdk_compatible() { esac } +# Packaged launcher aliases on Windows (python, python3 and py under +# ...\Microsoft\WindowsApps): starting one starts more than Python. The +# Python Install Manager's aliases make Windows check the package for an +# update on each start, and the launcher started the chosen name twice per +# hook, on every Edit, prompt and Stop. See anthropics/claude-code#98929. +# +# So when the name that answers is such an alias and its interpreter is a +# regular file elsewhere, the hook runs with that file instead, and its path +# is kept in the plugin's state folder (`launcher-python`) for a day. Later +# hooks start it directly: one process, and none through the alias. The +# remembered path is used only while PATH and the hook folder are what they +# were when it was checked and the file is still there; anything else is the +# full search below, which writes the entry again. A hook that fails on the +# remembered path empties the entry, so the next one searches again; deleting +# the file does the same by hand. One entry: sessions with different PATHs +# take turns checking, which is still one alias start per hook instead of two. +# +# Not while a virtual environment is active: the alias may be answering for +# that environment's interpreter, which other sessions should not inherit. +# +# Builtins only: nothing here starts a process on the remembered route. +unset SG_LAUNCHER_DIRECT +sg_args=("$@") +sg_exe="" # sys.executable from the last `ask` (Windows, Python 3.10+) +sg_own="" # set by sg_recall and sg_own_path +sg_memo="" # the entry's file; empty where none of this applies +sg_target_dir="" +case "${OSTYPE:-}" in + msys*|cygwin*) + if [ "${BASH_VERSINFO[0]:-0}" -ge 5 ] && [ -n "$SG_TARGET" ] && [ -z "${VIRTUAL_ENV:-}" ]; then + # The folder _base.state_dir() resolves to. + sg_home="${USERPROFILE:-${HOME:-}}" + sg_memo="${SECURITY_WARNINGS_STATE_DIR:-}" + if [ -z "$sg_memo" ] && [ -n "${CLAUDE_CONFIG_DIR:-}" ]; then + sg_memo="$CLAUDE_CONFIG_DIR/security" + fi + [ -n "$sg_memo" ] || sg_memo="~/.claude/security" + case "$sg_memo" in + "~"|"~"[\\/]*) + if [ -n "$sg_home" ]; then sg_memo="$sg_home${sg_memo:1}"; else sg_memo=""; fi ;; + esac + [ -z "$sg_memo" ] || sg_memo="$sg_memo/launcher-python" + sg_target_dir="${SG_TARGET%[\\/]*}" + fi ;; +esac + +sg_is_alias_path() { + case "${1,,}" in + */microsoft/windowsapps/*|*\\microsoft\\windowsapps\\*) return 0 ;; + esac + return 1 +} + +# An interpreter the remembered route may start: python*.exe by absolute +# path, a regular file, and not an alias itself. +sg_startable() { + local name="${1##*/}" + case "$1" in + /*) ;; + *) return 1 ;; + esac + case "${name,,}" in + python*.exe) ;; + *) return 1 ;; + esac + ! sg_is_alias_path "$1" || return 1 + [ -f "$1" ] && [ -x "$1" ] +} + +sg_recall() { + # Sets $sg_own to the remembered interpreter iff it may run this hook. + local tag saved path exe dir end age + [ -n "$sg_memo" ] && [ -r "$sg_memo" ] || return 1 + { + IFS= read -r tag && IFS= read -r saved && IFS= read -r path && + IFS= read -r exe && IFS= read -r dir && IFS= read -r end + } 2>/dev/null < "$sg_memo" || return 1 + [ "$tag" = "sg-launcher-python 1" ] && [ "$end" = "end" ] || return 1 + case "$saved" in + ""|*[!0-9]*|????????????*) return 1 ;; + esac + age=$((EPOCHSECONDS - 10#$saved)) + [ "$age" -ge 0 ] && [ "$age" -lt 86400 ] || return 1 + [ "$path" = "$PATH" ] && [ "$dir" = "$sg_target_dir" ] || return 1 + sg_startable "$exe" && [ -r "$SG_TARGET" ] || return 1 + sg_own="$exe" +} + +sg_own_path() { + # $1: the name that just answered `ask` with version $v. Succeeds, with + # $sg_own set, iff that name is a packaged alias whose interpreter is a + # regular file elsewhere that reads the hook script when started by + # itself and is the same version. Remembers it for the next hooks. + local alias="" own="$sg_exe" want="$v" drive + [ -n "$sg_memo" ] && [ -n "$own" ] || return 1 + hash "$1" 2>/dev/null && alias="${BASH_CMDS[$1]:-}" + sg_is_alias_path "$alias" || return 1 + case "$own" in + [A-Za-z]:[\\/]*) + if [ -n "${sg_drive_mounts:-}" ]; then + drive="${own:0:1}"; own="/${drive,,}/${own:3}"; own="${own//\\//}" + else + own=$(cygpath -u "$own" 2>/dev/null) || return 1 + fi ;; + esac + sg_startable "$own" || return 1 + ask "$own" || { v=$want; return 1; } + v=${v%$'\r'} + [ "$v" = "$want" ] || { v=$want; return 1; } + sg_own="$own" + if [ -d "${sg_memo%/*}" ]; then + { printf '%s\n' "sg-launcher-python 1" "$EPOCHSECONDS" "$PATH" "$own" "$sg_target_dir" "end"; } \ + 2>/dev/null > "$sg_memo" || true + fi + return 0 +} + +# Use right before `exec <name> "$@"`: runs the hook with the interpreter +# behind <name> instead when sg_own_path finds one, else returns. +sg_direct() { + if sg_own_path "$@"; then + export SG_LAUNCHER_DIRECT=2 + exec "$sg_own" "${sg_args[@]}" + fi + return 0 +} + +if sg_recall; then + export SG_LAUNCHER_DIRECT=1 + # Not exec'd, so a failure can empty the entry: no probe checked this + # interpreter today, and a broken one must not fail every hook for a day. + sg_rc=0 + "$sg_own" "$@" || sg_rc=$? + if [ "$sg_rc" -ne 0 ]; then + { : > "$sg_memo"; } 2>/dev/null || true + fi + exit "$sg_rc" +fi + # Pass 1 — try minor-versioned binaries in descending order. These are only # present if the user explicitly installed them (Homebrew / python.org / pyenv), # so picking one here always upgrades over the system `python3`. Highest -# available wins; the user doesn't have to PATH-prefer it. +# of these wins; the user doesn't have to PATH-prefer it. Releases newer +# than this list are pass 2b. for cmd in "python3.13" "python3.12" "python3.11" "python3.10"; do - v=$(probe "$cmd") || continue + on_path "$cmd" || continue + ask "$cmd" || { note_unreadable "$cmd"; continue; } + v=${v%$'\r'} if is_sdk_compatible "$v"; then + sg_direct "$cmd" exec "$cmd" "$@" fi done @@ -89,13 +340,40 @@ done # Pass 2 — bare interpreters, but only if SDK-compatible. Covers Linux distros # that ship 3.10+ as the default `python3`, and Windows where `python` / # `py -3` resolves to the user's python.org install. +sg_older="" # first of these that is a readable Python 3 below 3.10 for cmd in "python3" "python" "py -3"; do # shellcheck disable=SC2086 - v=$(probe $cmd) || continue + on_path $cmd || continue + # shellcheck disable=SC2086 + ask $cmd || { note_unreadable "$cmd"; continue; } + v=${v%$'\r'} if is_sdk_compatible "$v"; then # shellcheck disable=SC2086 + sg_direct $cmd + # shellcheck disable=SC2086 exec $cmd "$@" fi + # Anything that successfully reported a "M.m" string. + case "$v" in + [0-9]*.[0-9]*) [ -n "$sg_older" ] || sg_older="$cmd" ;; + esac +done + +# Pass 2b — minor versions newer than the pass 1 list, so a machine whose +# only SDK-compatible Python is a recent release (system `python3` 3.9 next to +# Homebrew's python3.14) is not left on pass 3. Tried only when nothing +# above is compatible: an interpreter that already works is never traded for +# a newer one, which may not have the SDK or wheels for its dependencies yet. +# Oldest first for the same reason. Missing names cost no process (on_path). +# See #4907, #5024. +for cmd in "python3.14" "python3.15" "python3.16" "python3.17" "python3.18" "python3.19"; do + on_path "$cmd" || continue + v=$(probe "$cmd") || { note_unreadable "$cmd"; continue; } + v=${v%%$'\n'*} + v=${v%$'\r'} + if is_sdk_compatible "$v"; then + exec "$cmd" "$@" + fi done # Pass 3 — fallback to any Python 3, even <3.10. Pattern-based checks @@ -103,20 +381,74 @@ done # own; the SDK-dependent paths will detect the version mismatch and degrade # inside the Python code. Without this fallback, the entire plugin would # stop working on default macOS, which is a regression vs today. +if [ -n "$sg_older" ]; then + # shellcheck disable=SC2086 + exec $sg_older "$@" +fi + +# Pass 4 — nothing above could run the hook script. If that is because the +# names on PATH resolve to packaged launcher aliases that cannot see the +# plugin directory (see probe), the same interpreter usually works when it is +# started by its real path instead of through the alias. Candidates: +# - every further python3/python on PATH, by full path (a regular install +# often sits behind the alias in PATH order; the first match is the one +# the bare name resolved to above), +# - the real executable each alias reports as sys.executable. +# The list is built before the loop that execs, and from a here-document: a +# redirect around that loop would become the hook's stdin. +sg_candidates=() +for name in "python3" "python"; do + first=1 + while IFS= read -r c; do + if [ -n "$first" ]; then first=""; continue; fi + [ -n "$c" ] && sg_candidates+=("$c") + done <<EOF +$(type -ap "$name" 2>/dev/null) +EOF +done for cmd in "python3" "python" "py -3"; do + case "$sg_unreadable" in *"|$cmd|"*) ;; *) continue ;; esac # shellcheck disable=SC2086 - v=$(probe $cmd) || continue - # Accept anything that successfully reported a "M.m" string. + c=$($cmd -c 'import sys; print(sys.executable)' 2>/dev/null) || continue + c=${c%$'\r'} # native Windows Python prints CRLF + [ -n "$c" ] || continue + if command -v cygpath >/dev/null 2>&1; then + c=$(cygpath -u "$c" 2>/dev/null) || continue + fi + sg_candidates+=("$c") +done +export SG_LAUNCHER_FALLBACK=1 +sg_fallback="" +sg_tried="|" +for cmd in "${sg_candidates[@]}"; do + case "$sg_tried" in *"|$cmd|"*) continue ;; esac + sg_tried="$sg_tried$cmd|" + ask "$cmd" || { note_unreadable "$cmd"; continue; } + v=${v%$'\r'} + if is_sdk_compatible "$v"; then + exec "$cmd" "$@" + fi case "$v" in - [0-9]*.[0-9]*) - # shellcheck disable=SC2086 - exec $cmd "$@" - ;; + [0-9]*.[0-9]*) [ -n "$sg_fallback" ] || sg_fallback="$cmd" ;; esac done +if [ -n "$sg_fallback" ]; then + exec "$sg_fallback" "$@" +fi +# Exit 1, never 2: on an asyncRewake hook exit 2 means "the review has +# findings, wake the model", and a launcher failure is not a finding. +if [ -n "$sg_unreadable" ]; then + echo "security-guidance: found Python, but it cannot read the hook script (missing, or not visible to Python), so no security checks ran." >&2 + echo " hook script: $SG_TARGET" >&2 + echo " on Windows this happens when python, python3 and py are Microsoft Store or Python Install Manager aliases;" >&2 + echo " install Python from https://python.org and put it ahead of ...\\Microsoft\\WindowsApps in PATH" >&2 + # No Python to report it, so the launcher emits the metric itself. + echo '{"metrics": {"launcher_unreadable": 1}}' + exit 1 +fi echo "security-guidance: no working Python 3 interpreter found." >&2 -echo " tried: python3.13, python3.12, python3.11, python3.10, python3, python, py -3" >&2 +echo " tried: python3.13 to python3.10, python3, python, py -3, python3.14 to python3.19" >&2 echo " on Windows, install Python from https://python.org (NOT the Microsoft Store)" >&2 echo " on macOS, install Python 3.10+ via Homebrew (\`brew install python\`)" >&2 exit 1 |
