diff options
| author | Claude <noreply@anthropic.com> | 2026-11-06 00:54:48 +0000 |
|---|---|---|
| committer | Claude <noreply@anthropic.com> | 2026-11-06 00:54:48 +0000 |
| commit | 7aba3deae3ff17d032c37e07d2929d1426b34278 (patch) | |
| tree | 9c482f9acda1599bd2f9bed95a6a46b75e634e80 /plugins/security-guidance/hooks/gitutil.py | |
| parent | 5472f5f56e0070feeeb464318b316714fe3e9105 (diff) | |
| parent | b8331dd69ab3e132bbf8cd6e89b5d8c1747e21b1 (diff) | |
Merge remote-tracking branch 'origin/main' into add-plugin/xcodeadd-plugin/xcode
Diffstat (limited to 'plugins/security-guidance/hooks/gitutil.py')
| -rw-r--r-- | plugins/security-guidance/hooks/gitutil.py | 1450 |
1 files changed, 1377 insertions, 73 deletions
diff --git a/plugins/security-guidance/hooks/gitutil.py b/plugins/security-guidance/hooks/gitutil.py index b6e8cd54..c80d0600 100644 --- a/plugins/security-guidance/hooks/gitutil.py +++ b/plugins/security-guidance/hooks/gitutil.py @@ -15,15 +15,51 @@ Functions that DO compose patched leaves (``compute_v2_review_set``, ``security_reminder_hook.py`` for that reason. """ import contextlib +import errno import os import re import subprocess +import sys +import tempfile +import threading +import time from _base import debug_log +def _git_program(windows=os.name == "nt"): + """How to start git. On Windows a bare name is looked up in the process's + working directory before PATH, and the hooks run in the project, so a + `git.exe` there would start in place of the installed one. Name the first + git.exe on PATH instead, leaving out relative entries, the working + directory, and folders inside the project (unless the project holds the + home folder, and with it the user's own tools). With none, a path that + does not exist: git is reported missing, as it would be anywhere else.""" + if not windows: + return "git" + def norm(p): + return os.path.normcase(os.path.normpath(p)) + try: + cwd = norm(os.getcwd()) + except OSError: + cwd = "" + home = norm(os.path.expanduser("~")) + os.sep + roots = [norm(r) for r in (cwd, os.environ.get("CLAUDE_PROJECT_DIR")) if r] + roots = [r.rstrip(os.sep) + os.sep for r in roots if not home.startswith(r.rstrip(os.sep) + os.sep)] + for d in os.environ.get("PATH", "").split(os.pathsep): + d = d.strip().strip('"') + if not d or not os.path.isabs(d) or norm(d) == cwd: + continue + if any((norm(d) + os.sep).startswith(r) for r in roots): + continue + path = os.path.join(d, "git.exe") + if os.path.isfile(path): + return path + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "git.exe") + + GIT_CMD = [ - "git", + _git_program(), "-c", "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", # core.quotePath=false: emit raw UTF-8 in path-emitting commands instead @@ -37,13 +73,52 @@ GIT_CMD = [ # reflog %gs subjects, ls-files, status, etc. — without per-site # flag duplication. See #2082, #2099. "-c", "core.quotePath=false", + # A work-tree `git diff` that meets stat-stale entries (mtime changed, + # content same) rewrites the index under index.lock, GIT_OPTIONAL_LOCKS or + # not. Off, every diff here only reads the index, so none needs a copy. + "-c", "diff.autoRefreshIndex=false", + # The diff parsers read `diff --git a/<old> b/<new>` headers. A user's + # diff.mnemonicPrefix (c/ w/ i/), diff.noprefix or diff.srcPrefix / + # diff.dstPrefix changes that shape, and every file would be dropped. + "-c", "diff.mnemonicPrefix=false", + "-c", "diff.noprefix=false", + "-c", "diff.srcPrefix=a/", + "-c", "diff.dstPrefix=b/", + # A directory with its own `.git` inside the work tree (a submodule, a + # vendored checkout, an unpacked archive) is another repository with its + # own config. To report it as modified, or to show its changes inline + # (diff.submodule=diff), git starts a second git inside it, which reads + # that config and runs the commands it names. Such a directory is left + # out of the review set, and so is a commit that moves one. See also + # _drop_gitlinks for the commands that take no such setting. + "-c", "diff.ignoreSubmodules=all", + "-c", "diff.submodule=short", + # `git log` and `git show` print commit metadata for the hooks to parse. + # With log.showSignature set they would also verify each signed commit, + # adding lines to the output and starting the program gpg.program names. + "-c", "log.showSignature=false", ] +# diff.ignoreSubmodules is a default that the repository's own settings for +# one submodule (.gitmodules, submodule.<name>.ignore) override. The option +# cannot be overridden. Pass it to every `git status` and work-tree `git diff`. +IGNORE_SUBMODULES = "--ignore-submodules=all" + SAFE_GIT_CONFIG = ( ("core.fsmonitor", "false"), ("core.hooksPath", "/dev/null"), + ("diff.ignoreSubmodules", "all"), + ("diff.submodule", "short"), + ("log.showSignature", "false"), ) +# The hook's git runs in the background beside the session's own git. Without +# this, `git status` holds .git/index.lock while it writes back a refreshed +# index: a concurrent `git add` fails, and a timeout kill leaves the lock. +SAFE_GIT_ENV = { + "GIT_OPTIONAL_LOCKS": "0", +} + def git_config_env(pairs, base=None): base = os.environ if base is None else base @@ -61,6 +136,51 @@ def git_config_env(pairs, base=None): def apply_safe_git_env(): os.environ.update(git_config_env(SAFE_GIT_CONFIG)) + os.environ.update(SAFE_GIT_ENV) + + +def index_entry_count(path): + """The entry count in an index file's header (signature, version, count), + or None if it cannot be read. A split or sparse index holds fewer entries + than the repository has tracked files.""" + try: + with open(path, "rb") as f: + header = f.read(12) + except OSError: + return None + if header[:4] == b"DIRC" and len(header) == 12: + return int.from_bytes(header[8:12], "big") + return None + + +def windows_drive_mount(path, mountinfo): + """Whether `path` is on a Windows drive mounted into WSL, given the text + of /proc/self/mountinfo. Every file status there is a round trip to the + Windows host.""" + best, best_type = "", "" + for line in mountinfo.splitlines(): + # id parent dev root mountpoint options [tags] - fstype source options + before, _, after = line.partition(" - ") + fields, tail = before.split(), after.split() + if len(fields) < 5 or not tail: + continue + point = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4]) + inside = path == point or path.startswith(point.rstrip("/") + "/") + if inside and len(point) >= len(best): + best, best_type = point, tail[0] + return best_type in ("9p", "drvfs", "virtiofs") + + +def on_windows_drive(path): + """windows_drive_mount for this machine. False anywhere but WSL.""" + try: + with open("/proc/sys/kernel/osrelease") as f: + if "microsoft" not in f.read().lower(): + return False + with open("/proc/self/mountinfo") as f: + return windows_drive_mount(os.path.realpath(path), f.read()) + except (OSError, ValueError): + return False def _git_rev_parse_head(cwd): @@ -110,6 +230,35 @@ def _find_git_index(cwd): return None +def _find_git_index_and_top(cwd): + """_find_git_index plus the work tree's top directory, from one git + call. (None, None) when either is missing: no index file, a bare + repository, or a `cwd` inside the git directory.""" + try: + result = subprocess.run( + [*GIT_CMD, "rev-parse", "--git-dir", "--show-toplevel"], + cwd=cwd, capture_output=True, timeout=5 + ) + if result.returncode != 0: + return None, None + # Paths as bytes from git: decode by hand (see _find_git_index). + out = result.stdout.decode("utf-8", errors="replace") + parts = out[:-1].split("\n") if out.endswith("\n") else [] + if len(parts) != 2 or not all(parts): + # A newline in one of the paths: ask for them one at a time. + index_path, top = _find_git_index(cwd), _git_toplevel(cwd) + return (index_path, top) if index_path and top else (None, None) + git_dir, top = parts + if not os.path.isabs(git_dir): + git_dir = os.path.join(cwd, git_dir) + index_path = os.path.join(git_dir, "index") + if not os.path.isfile(index_path): + return None, None + return index_path, top + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + return None, None + + def _diff_pathspec(cwd, paths): """Convert absolute touched-paths to repo-relative pathspec args for git diff. Paths outside cwd (e.g. ~/.claude/…) are dropped. Returns the @@ -131,32 +280,528 @@ def _diff_pathspec(cwd, paths): return ["--"] + rel if rel else [] +# Bytes of file paths per git call once a path list has proved too long for +# one command. Windows caps a whole command line at 32,767 characters; +# elsewhere the limit is about 1 MB (macOS) or 2 MB (Linux), environment +# included. +_MAX_PATH_ARGS_BYTES = ( + 20000 if sys.platform in ("win32", "cygwin", "msys") else 500000) +# More commands than this and the path list is not worth diffing: each one +# matches its paths against the whole index, and a set that large is thrown +# away as pathological after the diff anyway. +_MAX_PATH_BATCHES = 4 + + +def _arg_cost(path): + # The exec limit also counts a terminator and a pointer per argument. + return len(os.fsencode(path)) + 9 + + +def _path_args_budget(env=None): + """_MAX_PATH_ARGS_BYTES, or less where this system's own limit is lower + (a small stack limit, a BSD, a very large environment). Half of what the + system leaves after the environment, so the rest of the command fits.""" + try: + arg_max = os.sysconf("SC_ARG_MAX") + except (AttributeError, ValueError, OSError): + return _MAX_PATH_ARGS_BYTES + env_bytes = sum(len(k) + len(v) + 2 for k, v in (env or os.environ).items()) + return min(_MAX_PATH_ARGS_BYTES, max(4096, (arg_max - env_bytes) // 2)) + + +def _path_batches(paths, budget=None): + if budget is None: + budget = _MAX_PATH_ARGS_BYTES + batch, size = [], 0 + for p in paths: + n = _arg_cost(p) + if batch and size + n > budget: + yield batch + batch, size = [], 0 + batch.append(p) + size += n + if batch: + yield batch + + +# Why a path list too long for one command left the last get_git_diff call +# without a diff, for the Stop review's skip reason: "gave_up" (still too +# long once cut down) or "none_reviewable". +_path_list_too_long = set() + +# A failed `git add --intent-to-add` during this hook run, for the Stop hook's +# metrics: kind * 100000 + the paths the add was given. One number because a +# review row is near CC's key cap. kind: 1 git could not write the empty blob +# to the object store and the retry failed too, 2 any other git error, 3 +# timeout, 4 the retry with a private object directory worked. A git that +# speaks another language reports a failed retry as 2. With 1 to 3 the new +# files are not in the diff, so a turn that only added files looks like a turn +# with no changes. `left_out` says so for the last get_git_diff alone: the code +# stays set through a fallback diff that worked. +_add_failed = {"code": 0, "left_out": False} + +# Next to an index copy: where a retried `git add -N` stores the empty blob. +_OBJECTS_DIR_SUFFIX = ".objects" + + +def _too_many_args(e): + """The OS refused to start the command because its arguments are too + long. Windows reports ERROR_FILENAME_EXCED_RANGE, which Python turns into + a FileNotFoundError.""" + return e.errno == errno.E2BIG or getattr(e, "winerror", None) == 206 + + +def _run_captured(cmd, timeout, **kwargs): + return subprocess.run(cmd, capture_output=True, timeout=timeout, **kwargs) + + +def _run_git_on_review_paths(args, paths, timeout, run=_run_captured, **kwargs): + """Run `git <args> -- <paths>` for the Stop review; return the results. + `run(cmd, timeout, **kwargs)` starts each command (_run_diff_bounded for + the diff, so its output is never held whole). + + One command with every path whenever the OS takes it. When the OS refuses + the argument list as too long, only reviewable source paths are kept (the + review drops the rest from the diff anyway) and they go out in as few + commands as they need, all within `timeout` seconds together. If even + those need more than _MAX_PATH_BATCHES commands, the refusal is raised + again. + + Secret-shaped paths that are gone from the work tree go into every one + of those commands: `git diff` must still pair a rename away from such a + name, or the secret's content would show as a new source file. A new file + that only resembles the secret may be paired with it too and left out of + the review. Any other rename is paired only when both of its paths are + reviewable source and in the same command; otherwise it shows as a delete + or an add. + """ + cmd = [*GIT_CMD, *args, "--"] + paths = list(paths) + deadline = time.monotonic() + timeout + try: + return [run(cmd + paths, timeout, **kwargs)] + except OSError as e: + if not _too_many_args(e): + raise + too_long = e + budget = _path_args_budget(kwargs.get("env")) + batches = list(_path_batches( + [p for p in paths if _is_reviewable_source(p)], budget)) + root = kwargs.get("cwd") or os.getcwd() + secret = [p for p in paths if _is_secret_shaped(p) + and not os.path.lexists(os.path.join(root, p))] + debug_log(f"git {args[0]}: {len(paths)} paths do not fit one command; " + f"{len(batches)} command(s) for the reviewable ones") + # The secret-shaped paths ride along with every batch, so they get a + # small share of a command of their own. + if (len(batches) > _MAX_PATH_BATCHES + or sum(map(_arg_cost, secret)) > budget // 4): + raise too_long + cmd += secret + results = [] + for batch in batches: + left = deadline - time.monotonic() + if left <= 0: + raise subprocess.TimeoutExpired(cmd, timeout) + results.append(run(cmd + batch, left, **kwargs)) + return results + + +_TEMP_INDEX_PREFIX = "security_hook_idx_" +# A copy is named <prefix>pid<hook pid>_<8 random chars>, so a later run can +# tell that the hook which made it is gone. Names from earlier releases have +# only the 8 random characters and never match. +_OWNER_PID = re.compile(re.escape(_TEMP_INDEX_PREFIX) + r"pid(\d+)_[^.]{8}(?:\.|$)") +# A live copy is in use for under a minute (a couple of bounded git calls), so +# anything this old belongs to a run that was killed. +_TEMP_INDEX_STALE_S = 1800 +# Unlinking index-sized files is slow; a large backlog drains over several +# calls instead of stalling one hook past its timeout. +_SWEEP_BUDGET_S = 2 + + +def _owner_is_gone(name): + """True when `name` records the pid of the hook that made it and no such + process exists. A reused pid reads as alive: the file then waits for the + age limit. Never probes on Windows, where os.kill would end the process.""" + if not hasattr(os, "getuid"): + return False + m = _OWNER_PID.match(name) + if not m: + return False + try: + os.kill(int(m.group(1)), 0) + except ProcessLookupError: + return True + except (OSError, OverflowError, ValueError): + return False + return False + + +def _sweep_stale_indexes(dirpath, uid): + """Delete index copies in `dirpath` that a killed run left behind, and + their private object directories. Only regular files and directories owned + by `uid` (None: no ownership check) are touched. + + A copy goes as soon as the hook that made it is gone: a killed hook can + leave one per run, and index-sized files fill a disk long before the age + limit. The age limit covers names without a pid and reused pids. + + Age is the newer of mtime and ctime: a copy keeps .git/index's mtime, + which may be days old, so mtime alone would delete a file that another + running hook is still using.""" + import stat + import time + cutoff = time.time() - _TEMP_INDEX_STALE_S + deadline = time.monotonic() + _SWEEP_BUDGET_S + try: + names = os.listdir(dirpath) + except OSError: + return + for name in names: + if not name.startswith(_TEMP_INDEX_PREFIX): + continue + p = os.path.join(dirpath, name) + try: + st = os.lstat(p) + is_dir = stat.S_ISDIR(st.st_mode) and name.endswith(_OBJECTS_DIR_SUFFIX) + if not (is_dir or stat.S_ISREG(st.st_mode)): + continue + if uid is not None and st.st_uid != uid: + continue + if max(st.st_mtime, st.st_ctime) < cutoff or _owner_is_gone(name): + if is_dir: + _remove_objects_dir(p) + else: + os.unlink(p) + if time.monotonic() > deadline: + return + except OSError: + pass + + +def _sweep_stale_stash_indexes(real_index): + """Delete `<index>.stash.<pid>` files a killed `git stash create` left + next to the real index. Releases before 2.0.10 ran it there, so a clone + can hold hundreds, each the size of the index, and nothing else removes + them. Only our own regular files past the age limit go: git keeps one for + seconds. `index.lock` is never touched; a live git may hold it.""" + import stat + import time + uid = os.getuid() if hasattr(os, "getuid") else None + cutoff = time.time() - _TEMP_INDEX_STALE_S + deadline = time.monotonic() + _SWEEP_BUDGET_S + dirpath = os.path.dirname(real_index) + stale = re.compile(re.escape(os.path.basename(real_index)) + r"\.stash\.\d+$") + try: + names = os.listdir(dirpath) + except OSError: + return + for name in names: + if not stale.match(name): + continue + p = os.path.join(dirpath, name) + try: + st = os.lstat(p) + if not stat.S_ISREG(st.st_mode): + continue + if uid is not None and st.st_uid != uid: + continue + if max(st.st_mtime, st.st_ctime) < cutoff: + os.unlink(p) + if time.monotonic() > deadline: + return + except OSError: + pass + + +# Index copies this process has made and not yet removed. +_live_copies = set() + +# A hook working on a copy needs up to three times the index's size for a +# moment (the copy, and git's rewrite and stash index next to it). With many +# agents in one clone that alone can fill a disk, so only so many hooks per +# user work on a copy at once; the others wait, then go without. +_MAX_LIVE_COPIES = 8 +_SLOT_WAIT_S = 5 + + +def _copy_slots(tmpdir, index_bytes): + """How many hooks may work on a copy at once: what fits into a quarter of + the free space at three times the index each, from 1 to + _MAX_LIVE_COPIES.""" + import shutil + try: + free = shutil.disk_usage(tmpdir).free + except OSError: + return 1 + return max(1, min(_MAX_LIVE_COPIES, free // (12 * max(index_bytes, 1)))) + + +@contextlib.contextmanager +def _copy_slot(tmpdir, index_bytes=0): + """Hold one of the copy slots; yields False when none came free in + _SLOT_WAIT_S. A slot is an flock, so the kernel frees it when its holder + dies, however it dies. No cap without flock or a private dir.""" + import time + try: + import fcntl + except ImportError: + fcntl = None + if fcntl is None or tmpdir is None: + yield True + return + deadline = time.monotonic() + _SLOT_WAIT_S + held = None + while held is None: + for i in range(_copy_slots(tmpdir, index_bytes)): + try: + fd = os.open(os.path.join(tmpdir, f"slot{i}.lock"), + os.O_RDWR | os.O_CREAT, 0o600) + except OSError: + continue + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + held = fd + break + except OSError: + os.close(fd) + if held is None: + if time.monotonic() >= deadline: + break + time.sleep(0.1) + try: + yield held is not None + finally: + if held is not None: + os.close(held) + + +def _remove_objects_dir(path): + """Remove a private object directory. git writes objects read-only, which + stops Windows from deleting them.""" + import shutil + import stat + + def writable_then_retry(func, p, _exc): + # Not a link: chmod would follow it to whatever it points at. + if func in (os.unlink, os.rmdir) and not os.path.islink(p): + os.chmod(p, stat.S_IWRITE) + func(p) + + try: + shutil.rmtree(path, onerror=writable_then_retry) + except OSError: + pass + + +def _remove_copy(tmp_index): + """Remove an index copy and the files git wrote next to it.""" + import glob + for p in glob.glob(glob.escape(tmp_index) + "*"): + try: + os.unlink(p) + except OSError: + pass + _remove_objects_dir(tmp_index + _OBJECTS_DIR_SUFFIX) + + +def remove_copies_on_termination(): + """Remove this process's index copies when it is told to terminate. + Python's default SIGTERM action ends the process without running + `finally`, which left an index-sized file behind each time.""" + import signal + + def _on_term(signum, _frame): + for tmp_index in list(_live_copies): + _remove_copy(tmp_index) + os._exit(128 + signum) + + try: + signal.signal(signal.SIGTERM, _on_term) + except (ValueError, OSError, AttributeError): + pass # not the main thread, or no SIGTERM here + + +def _hook_tmpdir(): + """Private per-user directory for throwaway index copies, or None to use + the bare temp dir. + + An index copy can be hundreds of MB and a killed hook cannot clean up, so + copies live in one 0700 directory and anything stale is swept on entry, + here and in the bare temp dir. The name is predictable and the temp dir + may be shared, so anything that is not a real directory we own is + refused. Windows has no uid and its temp dir is already per-user.""" + import stat + import tempfile + base = tempfile.gettempdir() + uid = os.getuid() if hasattr(os, "getuid") else None + _sweep_stale_indexes(base, uid) + if uid is None: + return None + d = os.path.join(base, f"claude-security-guidance-{uid}") + try: + os.mkdir(d, 0o700) + except FileExistsError: + pass + except OSError: + return None + try: + st = os.lstat(d) + except OSError: + return None + if not stat.S_ISDIR(st.st_mode) or st.st_uid != uid: + return None + try: + os.chmod(d, 0o700) + except OSError: + pass + _sweep_stale_indexes(d, uid) + return d + + +def _note_add_failure(kind, paths): + _add_failed["code"] = kind * 100000 + min(len(paths), 99999) + _add_failed["left_out"] = kind != 4 + + +def _env_with_private_objects(env, cwd, objects_dir): + """`env` for a git that writes objects to `objects_dir` and still reads + the repository's own, wherever git says they are (a linked worktree shares + them). A path holding the list separator is left out, since git would + split it; the add then fails only if it has to read an object.""" + out = {**env, "GIT_OBJECT_DIRECTORY": objects_dir} + try: + r = subprocess.run([*GIT_CMD, "rev-parse", "--git-path", "objects"], + cwd=cwd, capture_output=True, timeout=5, env=env) + except (subprocess.TimeoutExpired, OSError): + return out + own = os.path.join(cwd, os.fsdecode(r.stdout).strip()) + if r.returncode == 0 and os.pathsep not in own: + out["GIT_ALTERNATE_OBJECT_DIRECTORIES"] = os.pathsep.join( + p for p in (own, env.get("GIT_ALTERNATE_OBJECT_DIRECTORIES")) if p) + return out + + +def _add_new_files(cwd, paths, env, tmp_index): + """`git add --intent-to-add` the untracked `paths` to the index copy `env` + names, so the diff shows them. The add has to store the empty blob, so a + user who cannot write the object store gets a second try that stores it in + a private directory next to the copy. Nothing needs that blob afterwards: + `git diff` reads these files from the work tree. `git stash create` would + fail on such an index, and only runs on a copy with nothing added. + + No stdout used here (only returncode matters), but text=True still spawns + reader threads that decode stderr — git error messages can reference + non-ASCII filenames and crash on cp1252. See #2099. Drop text=True so bytes + stay raw.""" + # The paths are file names, not patterns, for this git add only: an + # untracked file named `:(glob)**` must add only itself. git refuses that + # together with a glob or icase setting from the user's environment. + env = {k: v for k, v in env.items() + if k not in ("GIT_GLOB_PATHSPECS", "GIT_ICASE_PATHSPECS")} + env["GIT_LITERAL_PATHSPECS"] = "1" + + def add(add_env): + results = _run_git_on_review_paths( + ["add", "--intent-to-add"], paths, timeout=10, cwd=cwd, env=add_env) + return next((r for r in results if r.returncode != 0), None) + + try: + failed = add(env) + except subprocess.TimeoutExpired: + # No retry: git is slow here, not refused. The diff goes on for the + # tracked files, so a timeout on every turn cannot stop their reviews. + _note_add_failure(3, paths) + return + if not failed: + return + debug_log(f"git add -N failed: {failed.stderr[:200].decode('utf-8', errors='replace')}") + # Whatever happens to the retry, the diff goes on for the tracked files. + try: + objects_dir = tmp_index + _OBJECTS_DIR_SUFFIX + os.mkdir(objects_dir, 0o700) + retry_failed = add(_env_with_private_objects(env, cwd, objects_dir)) + if retry_failed: + debug_log(f"git add -N retry failed: {retry_failed.stderr[:200].decode('utf-8', errors='replace')}") + except (subprocess.TimeoutExpired, OSError) as e: + debug_log(f"git add -N retry failed: {e}") + retry_failed = True + if retry_failed: + _note_add_failure(1 if b"empty blob" in failed.stderr else 2, paths) + else: + _note_add_failure(4, paths) + + @contextlib.contextmanager def _temp_index(cwd, untracked_paths=None): """Yield an env dict pointing GIT_INDEX_FILE at a throwaway copy of the repo's index with `git add --intent-to-add` applied, so untracked files show up in subsequent `git diff` calls without touching the user's real - index. Yields None if no index can be found (bare repo / not a repo); the - caller should fall back to a plain diff. Always cleans up the temp file. + index. The copy has no gitlink entry for a nested repository (see + _drop_gitlinks). Yields None if no index can be found (bare repo / not a + repo), no copy slot came free, or those entries could not be removed. + Callers then go on without the copy: the diff reads the real index, and + the baseline falls back to HEAD. Always cleans up the temp file. Perf: when `untracked_paths` is given, only those paths are added (O(n) in untracked count). The default `add -N .` stats every file in the worktree — slow in large repos vs fast targeted scan. v2 callers already know the untracked set from `git status --porcelain`, so they - pass it; v1 keeps the whole-tree scan since it has no prior list.""" - import shutil - import tempfile + pass it; v1 keeps the whole-tree scan since it has no prior list. - real_index = _find_git_index(cwd) + Cleanup removes `<tmp_index>*`, not just the copy: git writes + `<index>.lock` and `<index>.stash.<pid>` next to the index it is given and + cannot remove them itself when a subprocess timeout kills it.""" + real_index, top = _find_git_index_and_top(cwd) if not real_index: yield None return - tmp_fd, tmp_index = tempfile.mkstemp(prefix="security_hook_idx_") + _sweep_stale_stash_indexes(real_index) + tmpdir = _hook_tmpdir() + try: + index_bytes = os.path.getsize(real_index) + except OSError: + index_bytes = 0 + with _copy_slot(tmpdir, index_bytes) as got_slot: + if not got_slot: + debug_log("no index copy slot came free; going without a copy") + yield None + return + with _copied_index(cwd, top, real_index, tmpdir, untracked_paths) as env: + yield env + + +@contextlib.contextmanager +def _copied_index(cwd, top, real_index, tmpdir, untracked_paths): + import shutil + import tempfile + + tmp_fd, tmp_index = tempfile.mkstemp( + prefix=f"{_TEMP_INDEX_PREFIX}pid{os.getpid()}_", dir=tmpdir) os.close(tmp_fd) + _live_copies.add(tmp_index) try: - shutil.copy2(real_index, tmp_index) + # A hard link where the filesystem allows it: no bytes are copied, and + # git never writes an index in place (it renames a new file over it), + # so the real index cannot change through the link. Either way the + # copy must keep .git/index's mtime (copy2, not copyfile): git + # re-reads any file whose mtime is not older than the index file's, + # and with a fresh mtime it would miss a same-size edit made in the + # same second as the last index write. + try: + os.unlink(tmp_index) + os.link(real_index, tmp_index) + except (OSError, NotImplementedError, AttributeError): + shutil.copy2(real_index, tmp_index) env = {**os.environ, "GIT_INDEX_FILE": tmp_index} + # Before `git add` runs, since add reads the gitlinks its paths cover. + if not _drop_gitlinks(top, env, tmp_index): + debug_log("could not remove gitlinks from the index copy; going on " + "without the copy") + yield None + return if untracked_paths is None: add_args = ["."] elif untracked_paths: @@ -165,27 +810,157 @@ def _temp_index(cwd, untracked_paths=None): # `git status` and here would silently drop ALL untracked files # from the diff. --ignore-missing only works with --dry-run, so # filter to surviving paths (lexists so dangling symlinks count). + # A nested repository (`dir/`) would be added as a submodule + # entry; see diff.ignoreSubmodules in GIT_CMD. surviving = [p for p in untracked_paths - if os.path.lexists(os.path.join(cwd, p))] - add_args = ["--"] + surviving if surviving else None + if not p.endswith("/") + and os.path.lexists(os.path.join(cwd, p))] + add_args = surviving or None else: add_args = None if add_args: - # No stdout used here (only returncode matters), but text=True - # still spawns reader threads that decode stderr — git error - # messages can reference non-ASCII filenames and crash on - # cp1252. See #2099. Drop text=True so bytes stay raw. - subprocess.run( - [*GIT_CMD, "add", "--intent-to-add"] + add_args, - cwd=cwd, capture_output=True, timeout=10, - env=env, - ) + _add_new_files(cwd, add_args, env, tmp_index) + # A whole-tree add puts a nested repository back. + if untracked_paths is None and not _drop_gitlinks(top, env, tmp_index): + yield None + return yield env finally: - try: - os.unlink(tmp_index) - except OSError: - pass + _remove_copy(tmp_index) + _live_copies.discard(tmp_index) + + +# An index entry stores its mode as four bytes; this is a gitlink's. +_GITLINK_MODE = b"\x00\x00\xe0\x00" +# Signature of the index extension that says most entries are in a second, +# shared file (core.splitIndex). +_SPLIT_INDEX_EXT = b"link" + + +def _may_have_gitlinks(index_path): + """False only when the index file at `index_path` cannot hold a gitlink + entry: neither a gitlink's mode bytes nor the split-index signature occur + anywhere in it. Either can occur by chance (in an object id, a file + size, a path), so True means "list the entries and see". Most + repositories have no gitlink, and for them this saves a git process on + every index copy.""" + keep = max(len(_GITLINK_MODE), len(_SPLIT_INDEX_EXT)) - 1 + try: + with open(index_path, "rb") as f: + tail = b"" + while True: + chunk = f.read(1 << 20) + if not chunk: + return False + data = tail + chunk + if _GITLINK_MODE in data or _SPLIT_INDEX_EXT in data: + return True + tail = data[-keep:] + except OSError: + return True + + +_GITLINK_LIST_TIMEOUT = 10 +# `git ls-files --sparse` (git 2.35) lists a sparse index as it is stored; +# without it git first expands the index to every path in the repository. +# Set to False once this git has refused the option. +_ls_files_sparse = True + + +def _gitlink_paths(top, env): + """Paths (bytes, relative to `top`) of the gitlink entries, at any depth, + in the index `env` names. None if they could not be listed. The listing + is read as it comes, so a large index costs time but not memory.""" + global _ls_files_sparse + cmd = [*GIT_CMD, "ls-files", "-s", "-z"] + (["--sparse"] if _ls_files_sparse else []) + p = subprocess.Popen(cmd, cwd=top, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, env=env) + killer = threading.Timer(_GITLINK_LIST_TIMEOUT, p.kill) + killer.daemon = True + found = set() + tail = b"" + try: + killer.start() + while True: + chunk = p.stdout.read1(65536) + if not chunk: + break + data = tail + chunk + end = data.rfind(b"\0") + 1 + # "<mode> <object> <stage>\t<path>", NUL-terminated. Most chunks + # have no gitlink and are not split up. + if b"160000 " in data: + for rec in data[:end].split(b"\0"): + if rec.startswith(b"160000 "): + found.add(rec.split(b"\t", 1)[1]) + tail = data[end:] + rc = p.wait() + finally: + killer.cancel() + if p.poll() is None: + p.kill() + p.wait() + p.stdout.close() + if rc == 129 and _ls_files_sparse: + # Usage error: a git without --sparse. + _ls_files_sparse = False + return _gitlink_paths(top, env) + if rc != 0: + debug_log(f"git ls-files on the index copy failed (rc={rc})") + return None + return found + + +# True when the last _drop_gitlinks call removed at least one gitlink, so the +# index copy and HEAD can no longer have the same tree even on a clean tree. +last_dropped_gitlinks = False + + +def _drop_gitlinks(top, env, tmp_index): + """Remove the gitlink (mode 160000) entry of every nested repository from + the index copy that `env` names, never from the real index, before + anything else runs on the copy. A gitlink marks a directory that is, or + can be, another repository (see diff.ignoreSubmodules in GIT_CMD): + `git stash create` takes no flag like IGNORE_SUBMODULES, and `git add` + reads the gitlinks its paths cover. An entry whose directory has no + `.git` stays: git starts nothing there, and without the entry an older + `git stash create` fails on the empty directory. `top` is the work + tree's top directory. Returns True on success, False on any failure; the + copy must not be used then.""" + global last_dropped_gitlinks + last_dropped_gitlinks = False + try: + if not _may_have_gitlinks(tmp_index): + return True + gitlinks = _gitlink_paths(top, env) + if gitlinks is None: + return False + top_b = os.fsencode(top) + nested = sorted(p for p in gitlinks + if os.path.lexists(os.path.join(top_b, p, b".git"))) + if not nested: + return True + mtime_ns = os.stat(tmp_index).st_mtime_ns + # git writes a new file and renames it over the copy, so a copy that + # is a hard link to the real index leaves the real one as it was. + rm = subprocess.run( + [*GIT_CMD, "update-index", "--force-remove", "-z", "--stdin"], + cwd=top, input=b"".join(p + b"\0" for p in nested), + capture_output=True, timeout=10, env=env) + if rm.returncode != 0: + debug_log(f"git update-index on the index copy failed (rc={rm.returncode}): " + f"{rm.stderr[:200].decode('utf-8', errors='replace')}") + return False + # Keep the mtime the copy came with; see _copied_index. + os.utime(tmp_index, ns=(mtime_ns, mtime_ns)) + last_dropped_gitlinks = True + debug_log(f"gitlink entries (nested repositories) removed from the " + f"index copy: {len(nested)}; their contents are not reviewed") + return True + except (subprocess.TimeoutExpired, OSError, RuntimeError, ValueError) as e: + # RuntimeError: the timer thread could not start. + debug_log(f"removing gitlinks from the index copy failed: {e}") + return False def _git_toplevel(cwd): @@ -206,6 +981,41 @@ def _git_toplevel(cwd): return None +def _git_toplevel_and_head(cwd): + """(repo root, HEAD SHA) for `cwd` from one git call, or (None, None) + when git cannot give both (not a work tree, no commits).""" + try: + r = subprocess.run( + [*GIT_CMD, "rev-parse", "--show-toplevel", "HEAD"], + cwd=cwd, capture_output=True, timeout=5, + ) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + return None, None + if r.returncode != 0: + return None, None + # See #2099: the first line is a PATH, so decode the bytes manually. + top, _, head = r.stdout.decode("utf-8", errors="replace").strip().rpartition("\n") + return (top.strip() or None), (head.strip() or None) + + +def _git_head_at(cwd, epoch): + """The SHA this checkout's HEAD pointed at just before `epoch` (seconds), + read from its own reflog. A checkout created after `epoch` answers with + the commit it was created at. None when git cannot say (no reflog).""" + try: + # Reflog times are whole seconds: ask for the second before, so a + # commit made in the same second as `epoch` counts as after it. + r = subprocess.run( + [*GIT_CMD, "rev-parse", "--verify", "-q", f"HEAD@{{@{int(epoch) - 1}}}"], + cwd=cwd, capture_output=True, timeout=5, + ) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError, + TypeError, ValueError, OverflowError): + return None + sha = r.stdout.decode("utf-8", errors="replace").strip() + return sha if r.returncode == 0 and sha else None + + def _git_dir(repo_root): """Absolute shared `.git` directory for repo_root. @@ -373,12 +1183,28 @@ def _git_reflog_recent_commits(repo_root, max_age_s=120, max_n=5): return fresh, stale +# New name -> secret-shaped old name, for the renames and copies git reported +# during this hook run. The Stop review limits its diff to a list of paths: +# with only the new name in it, git shows the secret's content as a new file, +# and the diff parsers cannot tell it came from a name they leave out. +_secret_rename_origins = {} + + +def _note_rename(old_path, new_path): + if _is_secret_shaped(old_path): + _secret_rename_origins[new_path] = old_path + + def _git_name_only(cwd, base, include_untracked=False): """Return the set of repo-root-relative paths that differ from `base`, or None if git failed (unresolvable ref, not a repo, timeout). Callers must distinguish None (error → don't trust as a filter) from set() (genuinely nothing changed). `-c core.quotePath=false -z` keeps non-ASCII - and space-containing paths intact.""" + and space-containing paths intact. + + Against the work tree the set can also hold a file whose stat changed and + whose content did not: the diff does not refresh the index. Callers + intersect with `git status`, which compares content.""" # Decode stdout/stderr as UTF-8 with errors="replace" instead of using # text=True. core.quotePath=false makes git emit raw UTF-8 for non-ASCII # paths, and text=True on Windows decodes via cp1252 strict — a non-ASCII @@ -388,7 +1214,7 @@ def _git_name_only(cwd, base, include_untracked=False): def _run(env): # core.quotePath=false comes from GIT_CMD globally (see definition). result = subprocess.run( - [*GIT_CMD, "diff", "--name-only", "-z", base], + [*GIT_CMD, "diff", IGNORE_SUBMODULES, "--name-status", "-z", base], cwd=cwd, capture_output=True, timeout=30, env=env, ) @@ -397,9 +1223,22 @@ def _git_name_only(cwd, base, include_untracked=False): debug_log(f"_git_name_only({base!r}) rc={result.returncode}: {stderr_str[:200]}") return None stdout = (result.stdout or b"").decode("utf-8", errors="replace") - return {p for p in stdout.split("\0") if p} + # Fields are `status\0path\0`, or `R<score>\0old\0new\0` for a rename + # or copy, which is listed by its new name only. + fields = stdout.split("\0") + names = set() + i = 0 + while i + 1 < len(fields): + if fields[i][:1] in ("R", "C") and i + 2 < len(fields): + _note_rename(fields[i + 1], fields[i + 2]) + i += 1 + names.add(fields[i + 1]) + i += 2 + return names try: + # The diff only reads the index (diff.autoRefreshIndex is off in + # GIT_CMD); a copy is for `add -N` alone. if not include_untracked: return _run(None) with _temp_index(cwd) as env: @@ -427,7 +1266,7 @@ def _git_status_porcelain(cwd): try: # core.quotePath=false comes from GIT_CMD globally (see definition). r = subprocess.run( - [*GIT_CMD, "status", "--porcelain=v1", "-uall", "-z"], + [*GIT_CMD, "status", IGNORE_SUBMODULES, "--porcelain=v1", "-uall", "-z"], cwd=cwd, capture_output=True, timeout=30, ) if r.returncode != 0: @@ -445,7 +1284,10 @@ def _git_status_porcelain(cwd): continue xy, path = e[:2], e[3:] if xy == "??": - untracked.add(path) + # With -uall only a nested repository is listed as `dir/`. + # It is not a file to review; see diff.ignoreSubmodules. + if not path.endswith("/"): + untracked.add(path) else: tracked.add(path) # Rename/copy entries are XY old\0new\0 — second NUL field is @@ -453,6 +1295,8 @@ def _git_status_porcelain(cwd): # 2-char-status entry. if "R" in xy or "C" in xy: i += 1 + if i < len(entries): + _note_rename(entries[i], path) i += 1 return tracked, untracked except (subprocess.TimeoutExpired, FileNotFoundError, OSError, ValueError) as e: @@ -479,6 +1323,264 @@ def _is_ancestor(cwd, maybe_ancestor, descendant): return False +_DEFAULT_FILE_DIFF_BYTES = 4 * 1024 * 1024 +_LATE_FILE_DIFF_BYTES = 256 * 1024 +_LAST_FILE_DIFF_BYTES = 96 * 1024 + + +def _file_diff_cap(): + """SG_MAX_FILE_DIFF_BYTES, or the default when it is unset, not a number + or not positive. Never below what a late file keeps, which is well above + what the review prompt takes of one file.""" + try: + n = int(os.environ.get("SG_MAX_FILE_DIFF_BYTES", "")) + except ValueError: + return _DEFAULT_FILE_DIFF_BYTES + if n <= 0: + return _DEFAULT_FILE_DIFF_BYTES + return max(n, _LATE_FILE_DIFF_BYTES) + + +# How much of one file's diff the Stop review keeps while it reads git's +# output: this much from the start of the file's section, and as much again +# of the added lines after that. The review prompt takes far less of a file +# (DIFF_PER_FILE_BYTES); the rest is for filter_preexisting_from_diff, which +# needs the removed and the added side of a rewritten file. +MAX_FILE_DIFF_BYTES = _file_diff_cap() +# Past this much kept in total, a further file keeps _LATE_FILE_DIFF_BYTES +# the same two ways; past four times as much, _LAST_FILE_DIFF_BYTES from its +# start, which is still more than the review prompt takes of one file. +MAX_KEPT_DIFF_BYTES = 4 * MAX_FILE_DIFF_BYTES +_SECTION_MARK = b"\ndiff --git " +_MAX_HEADER_BYTES = 64 * 1024 +_CUT_MARK = b"\\ [cut by security-guidance: the rest of this file's diff was not read]\n" + +# What the last get_git_diff read, for the Stop hook's metrics: bytes of git +# output, reviewable files whose diff was cut and the bytes cut from them, +# and diffs that hit the timeout during this hook run. +_diff_read = {"bytes": 0, "cut_files": 0, "cut_bytes": 0, "timeouts": 0} + + +class _DiffKeeper: + """Takes `git diff` output as it arrives and keeps only what the review + can use: the header line of every file, and a bounded part of each file + parse_diff_into_files would keep. A turn that wrote large data files no + longer puts their whole diff in memory.""" + + def __init__(self): + self.parts = [] + self.kept = 0 + self.seen = 0 + self.cut_files = 0 + self.cut_bytes = 0 + # Unhandled input. Starts as a newline that is not git's, so a header + # at the very start of the output is found like any other. + self._pending = b"\n" + self._at_start = True + self._reviewable = True + self._room = self._added_room = MAX_FILE_DIFF_BYTES + self._cut = False + self._ends_nl = True + # Past the cut: whether the next byte starts a line, and whether the + # line being read is an added one. + self._bol = True + self._in_added = False + + def feed(self, data): + buf = self._pending + data + pos = 0 + while True: + i = buf.find(_SECTION_MARK, pos) + if i < 0: + # Hold back a tail that could be the start of a header. + end = max(pos, len(buf) - len(_SECTION_MARK) + 1) + self._keep(buf, pos, end) + self._pending = buf[end:] + return + j = buf.find(b"\n", i + 1) + if j < 0 and len(buf) - i <= _MAX_HEADER_BYTES: + # The header line is not complete yet. + self._keep(buf, pos, i) + self._pending = buf[i:] + return + self._keep(buf, pos, i + 1) + if j >= 0: + self._begin(buf[i + 1:j]) + pos = i + 1 + + def _begin(self, header): + """A new file's section starts with this `diff --git ` line.""" + self._end_section() + text = header[len(_SECTION_MARK) - 1:].decode("utf-8", errors="replace") + paths = _parse_diff_header(text) + # A section the parser drops, or whose header it cannot read and only + # counts, keeps its header line. + self._reviewable = bool(paths) and _section_is_reviewable( + paths[1] or paths[0], paths[0]) + if not self._reviewable: + self._room, self._added_room = len(header) + 1, 0 + elif self.kept > 4 * MAX_KEPT_DIFF_BYTES: + self._room, self._added_room = max(len(header) + 1, _LAST_FILE_DIFF_BYTES), 0 + elif self.kept > MAX_KEPT_DIFF_BYTES: + self._room = self._added_room = max(len(header) + 1, _LATE_FILE_DIFF_BYTES) + else: + self._room = self._added_room = MAX_FILE_DIFF_BYTES + self._cut = False + + def _end_section(self): + if not self._ends_nl: + # Cut inside a line: end it, so what follows starts a line. + self._append(b"\n") + if self._cut: + self._append(_CUT_MARK) + + def _append(self, piece): + self.parts.append(piece) + self.kept += len(piece) + self._ends_nl = piece.endswith(b"\n") + + def _keep(self, buf, start, end): + if self._at_start and end > start: + self._at_start = False + start += 1 + n = end - start + if n <= 0: + return + self.seen += n + take = min(n, self._room) + if take: + self._append(buf[start:start + take]) + self._room -= take + if take == n or not self._reviewable: + return + if not self._cut: + self._cut = True + self.cut_files += 1 + self._bol = self._ends_nl + self._in_added = False + self.cut_bytes += n - take - self._keep_added(buf, start + take, end) + + def _keep_added(self, buf, pos, end): + """Past a file's cut, keep its added lines only, up to as much again. + A rewritten file lists every removed line before the first added one, + and the review is about the added ones. Returns the bytes kept.""" + kept = 0 + run = -1 # where a run of added lines not yet appended starts + while pos < end and self._added_room > 0: + if self._bol: + self._bol = False + self._in_added = buf.startswith(b"+", pos) + if self._in_added and run < 0 and not self._ends_nl: + self._append(b"\n") + if not self._in_added: + if run >= 0: + self._append(buf[run:pos]) + run = -1 + nxt = buf.find(b"\n+", pos, end) + if nxt < 0: + self._bol = buf.endswith(b"\n", pos, end) + break + pos = nxt + 1 + self._bol = True + continue + nl = buf.find(b"\n", pos, end) + stop = end if nl < 0 else nl + 1 + take = min(stop - pos, self._added_room) + if run < 0: + run = pos + self._added_room -= take + kept += take + self._bol = nl >= 0 + if pos + take < stop: + self._append(buf[run:pos + take]) + run = -1 + pos = stop + if run >= 0: + self._append(buf[run:pos]) + return kept + + def result(self): + self._keep(self._pending, 0, len(self._pending)) + self._pending = b"" + self._end_section() + self._cut = False + return b"".join(self.parts) + + +def _run_diff_bounded(cmd, timeout, **kwargs): + """subprocess.run(cmd, capture_output=True, timeout=timeout) for a + `git diff`, except that stdout goes through _DiffKeeper instead of being + held whole.""" + keeper = _DiffKeeper() + deadline = time.monotonic() + timeout + try: + err = tempfile.TemporaryFile() + except OSError: + err = None + try: + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=err if err is not None else subprocess.DEVNULL, + **kwargs) + failed = [] + given_up = [] + + def _read(): + try: + while not given_up: + chunk = p.stdout.read1(65536) + if not chunk or given_up: + break + keeper.feed(chunk) + except BaseException as e: + failed.append(e) + + # Read on a thread, so the timeout holds even while something git + # started still has the pipe open after git itself is gone. + reader = threading.Thread(target=_read, daemon=True) + timed_out = False + try: + try: + reader.start() + except RuntimeError as e: + failed.append(e) + else: + reader.join(timeout) + timed_out = reader.is_alive() + if timed_out or failed: + p.kill() + try: + returncode = p.wait(max(deadline - time.monotonic(), 0.1)) + except subprocess.TimeoutExpired: + timed_out = True + finally: + if timed_out or failed or p.returncode is None: + # The reader may be left waiting on a pipe something else + # holds: it keeps nothing more, and the pipe stays its. + given_up.append(True) + p.kill() + p.wait() + if reader.ident is not None: + reader.join(1) + if not reader.is_alive(): + p.stdout.close() + _diff_read["bytes"] += keeper.seen + if timed_out: + _diff_read["timeouts"] += 1 + raise subprocess.TimeoutExpired(cmd, timeout) + if failed: + raise OSError(f"reading the diff failed: {failed[0]!r}") + stdout = keeper.result() + _diff_read["cut_files"] += keeper.cut_files + _diff_read["cut_bytes"] += keeper.cut_bytes + stderr = b"" + if err is not None: + err.seek(0) + stderr = err.read(4096) + return subprocess.CompletedProcess(cmd, returncode, stdout, stderr) + finally: + if err is not None: + err.close() + def get_git_diff(cwd, baseline_sha, full_context=False, paths=None, untracked_paths=None): """ @@ -492,9 +1594,16 @@ def get_git_diff(cwd, baseline_sha, full_context=False, paths=None, untracked_pa If `paths` is given, the diff is restricted to those paths (relative to cwd; absolute paths are converted, paths outside cwd are dropped). + A path list too long for one command is cut down to reviewable source + files: see _run_git_on_review_paths. `untracked_paths` (repo-root-relative) is forwarded to _temp_index so it can add only those files instead of scanning the whole worktree. + The output is read through _DiffKeeper: files the review would drop come + back as their header line only, and a file's diff past + MAX_FILE_DIFF_BYTES is cut down to its added lines. """ + _path_list_too_long.clear() + _add_failed["left_out"] = False pathspec = _diff_pathspec(cwd, paths) if paths and not pathspec: # Caller restricted to specific paths but none are inside this repo @@ -504,19 +1613,36 @@ def get_git_diff(cwd, baseline_sha, full_context=False, paths=None, untracked_pa return "" # core.quotePath=false comes from GIT_CMD globally (see definition). - cmd = [*GIT_CMD, "diff", "--no-color", "--no-ext-diff", "--no-textconv", baseline_sha] + (["--unified=99999"] if full_context else []) + pathspec + args = ["diff", IGNORE_SUBMODULES, "--no-color", "--no-ext-diff", "--no-textconv", baseline_sha] + (["--unified=99999"] if full_context else []) + _diff_read.update(bytes=0, cut_files=0, cut_bytes=0) + + def _run(env): + if pathspec: + results = _run_git_on_review_paths(args, pathspec[1:], timeout=30, run=_run_diff_bounded, cwd=cwd, env=env) + else: + results = [_run_diff_bounded([*GIT_CMD, *args], 30, cwd=cwd, env=env)] + if not results: + _path_list_too_long.add("none_reviewable") + for result in results: + if result.returncode != 0: + debug_log(f"git diff failed: {result.stderr[:200].decode('utf-8', errors='replace')}") + return None + # Decode with errors='replace' so binary diffs don't crash + return b"".join(r.stdout for r in results).decode("utf-8", errors="replace") + try: + if untracked_paths is not None and not untracked_paths: + # Nothing to `add -N`, and the diff only reads the index. + return _run(None) with _temp_index(cwd, untracked_paths) as env: - # env is None when no index could be found (bare repo / not a - # repo) — diff still runs, just without untracked-file support. - result = subprocess.run(cmd, cwd=cwd, capture_output=True, timeout=30, env=env) - if result.returncode != 0: - debug_log(f"git diff failed: {result.stderr[:200].decode('utf-8', errors='replace')}") - return None - # Decode with errors='replace' so binary diffs don't crash - return result.stdout.decode("utf-8", errors="replace") + # env is None when there is no index (bare repo / not a repo) + # or no copy slot — diff still runs, just without + # untracked-file support. + return _run(env) except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e: debug_log(f"git diff error: {e}") + if isinstance(e, OSError) and _too_many_args(e): + _path_list_too_long.add("gave_up") return None @@ -571,6 +1697,116 @@ SKIP_FILE_SUFFIXES = ( '.lock', '_pb2.py', '.pb.go', ) +# Files that hold secrets by naming convention are never sent for review. The +# review covers everything dirty in the work tree, including files the user +# changed and Claude never opened, so a key file or an encrypted dotfile source +# would otherwise leave the machine in the review request. The list is kept to +# names that are secret stores by convention; ordinary source such as +# `private_api.py` or `id_generator.py` stays reviewable. +_SSH_PRIVATE_KEY_BASENAMES = frozenset(( + 'id_rsa', 'id_dsa', 'id_ecdsa', 'id_ecdsa_sk', 'id_ed25519', + 'id_ed25519_sk', 'id_xmss', +)) +_PRIVATE_KEY_NAME_SUFFIXES = ('_rsa', '_dsa', '_ecdsa', '_ed25519') +# `.credentials.json` and `.claude.json` are Claude Code's own files: sign-in +# tokens, a saved API key, MCP server secrets. Claude Code rewrites them as it +# runs, so they show up as changed when its config directory is a git repo. +_CREDENTIAL_BASENAMES = frozenset(( + 'credentials', '.netrc', '.pgpass', '.npmrc', '.pypirc', + '.git-credentials', '.dockercfg', '.credentials.json', '.claude.json', +)) +# chezmoi encodes a target file's attributes as stacked name prefixes +# (`private_dot_netrc`, `encrypted_private_token`); `dot_` stands for a +# leading dot. +_CHEZMOI_ATTR_PREFIXES = ( + 'after_', 'before_', 'create_', 'empty_', 'encrypted_', 'exact_', + 'executable_', 'external_', 'literal_', 'modify_', 'once_', 'onchange_', + 'private_', 'readonly_', 'remove_', 'run_', 'symlink_', +) +# An `encrypted_` source keeps its target's name, so `encrypted_store.py` could +# be either ciphertext or code about encryption. Treat it as ciphertext only +# when it has no extension or a data-file one. +_ENCRYPTED_DATA_EXTENSIONS = frozenset(( + '', '.json', '.yaml', '.yml', '.toml', '.properties', '.tfvars', +)) + + +# Paths skipped as secret-shaped during this hook run. The Stop hook reports +# the count (never the names) so the skip can be watched in telemetry. +_secret_shaped_skipped = set() + + +def _skip_secret_shaped(file_path): + if not _is_secret_shaped(file_path): + return False + _secret_shaped_skipped.add(file_path) + return True + + +def _is_secret_shaped(file_path): + base = os.path.basename(file_path.replace("\\", "/")).lower() + target = base + encrypted = False + stripped = True + while stripped: + stripped = False + for prefix in _CHEZMOI_ATTR_PREFIXES: + if target.startswith(prefix) and len(target) > len(prefix): + target = target[len(prefix):] + encrypted = encrypted or prefix == 'encrypted_' + stripped = True + break + if target.startswith('dot_'): + target = '.' + target[4:] + ext = os.path.splitext(target)[1] + if encrypted and ext in _ENCRYPTED_DATA_EXTENSIONS: + return True + for name in {base, target}: + if name.startswith('.env'): + return True + if name in _SSH_PRIVATE_KEY_BASENAMES or name in _CREDENTIAL_BASENAMES: + return True + if '.' not in name and name.endswith(_PRIVATE_KEY_NAME_SUFFIXES): + return True + return False + + +# Data files whose name says they hold secrets (`secrets.yml`, +# `client_secret.json`). They are reviewed like any other data file; the Stop +# hook only counts how many are in its review set (never the names), to show +# how common they are. +_SECRET_NAMED_STEMS = frozenset(( + 'secret', 'secrets', 'credential', 'credentials', + 'client_secret', 'client_secrets', 'client-secret', 'client-secrets', + 'service-account', 'service_account', 'serviceaccount', + 'service-account-key', 'service_account_key', 'serviceaccountkey', +)) +_SECRET_NAMED_DATA_EXTENSIONS = frozenset(( + '.json', '.yaml', '.yml', '.toml', '.properties', '.tfvars', +)) +# `secrets.example.yml` is a placeholder and `secrets.enc.yaml` is ciphertext. +_NOT_SECRET_NAME_PARTS = frozenset(( + 'example', 'sample', 'template', 'tmpl', 'dist', 'schema', + 'enc', 'encrypted', 'sops', +)) + + +def _is_secret_named(file_path): + base = os.path.basename(file_path.replace("\\", "/")).lower() + parts = base.lstrip('.').split('.') + if len(parts) < 2 or '.' + parts[-1] not in _SECRET_NAMED_DATA_EXTENSIONS: + return False + if _NOT_SECRET_NAME_PARTS.intersection(parts[1:-1]): + return False + stem = parts[0] + # Google's OAuth download is `client_secret_<client id>.json`. + return stem in _SECRET_NAMED_STEMS or stem.startswith('client_secret_') + + +def count_secret_named(diff_files): + return sum(1 for file_path, _ in diff_files if _is_secret_named(file_path)) + + # Path tokens that bump a file's review priority when a commit exceeds # MAX_DIFF_FILES and we have to pick a subset. These are exactly the surfaces # single-shot and agentic reviews disagree on most (auth, routing, IPC, @@ -648,6 +1884,8 @@ def _is_reviewable_source(file_path): norm = "/" + file_path.replace("\\", "/") if any(("/" + p) in norm for p in SKIP_PATH_PATTERNS): return False + if _skip_secret_shaped(file_path): + return False if file_path.endswith(SKIP_FILE_SUFFIXES): return False ext = os.path.splitext(file_path)[1].lower() @@ -677,32 +1915,110 @@ def _is_reviewable_source(file_path): return False +# A file's section starts at a `diff --git ` line. Inside a section every line +# of file content carries a `+`, `-` or space prefix, so only a real header can +# start a line with it; source text that merely contains the words cannot. +_DIFF_SECTION_RE = re.compile(r'^diff --git ', re.MULTILINE) +_UNQUOTED_HEADER_RE = re.compile(r'^a/(.+?) b/(.+)$') +_C_ESCAPES = { + 'a': b'\a', 'b': b'\b', 'f': b'\f', 'n': b'\n', 'r': b'\r', 't': b'\t', + 'v': b'\v', '\\': b'\\', '"': b'"', +} + + +def _read_quoted_path(text, start): + """Decode the C-quoted path whose opening quote is at text[start]. + Returns (path, index after the closing quote), or None if unterminated.""" + out = bytearray() + i = start + 1 + while i < len(text): + c = text[i] + if c == '"': + return out.decode("utf-8", errors="replace"), i + 1 + if c == '\\' and i + 1 < len(text): + n = text[i + 1] + if n in _C_ESCAPES: + out += _C_ESCAPES[n] + i += 2 + continue + octal = re.match(r'[0-7]{1,3}', text[i + 1:i + 4]) + if octal: + out.append(int(octal.group(0), 8) & 0xFF) + i += 1 + len(octal.group(0)) + continue + out += c.encode("utf-8") + i += 1 + return None + + +def _parse_diff_header(header): + """(old_path, new_path) from the text after `diff --git `, or None. + + core.quotePath=false keeps non-ASCII names raw, but git still C-quotes a + name that contains a double quote, a backslash or a control character: + `"a/we\\"ird.py" "b/we\\"ird.py"`. Either side can be quoted on its own. + """ + old = new = None + if header.startswith('"'): + first = _read_quoted_path(header, 0) + if not first: + return None + old, end = first + rest = header[end:].lstrip(' ') + if rest.startswith('"'): + second = _read_quoted_path(rest, 0) + new = second[0] if second else None + else: + new = rest + elif header.endswith('"') and ' "b/' in header: + split_at = header.index(' "b/') + old = header[:split_at] + second = _read_quoted_path(header, split_at + 1) + new = second[0] if second else None + else: + m = _UNQUOTED_HEADER_RE.match(header) + return (m.group(1), m.group(2)) if m else None + if not old or not new or not old.startswith('a/') or not new.startswith('b/'): + return None + return old[2:], new[2:] + + +# Section headers that could not be parsed during this hook run; that file is +# left out of the review. The Stop hook reports the count so a header shape +# this parser misses shows up in telemetry instead of passing silently. +_unparsed_diff_headers = set() + + +def _section_is_reviewable(file_path, old_path): + """Whether a diff section is sent for review: a source file, and not one + renamed away from a secret-shaped name, which still carries the secret's + content.""" + return _is_reviewable_source(file_path) and not _skip_secret_shaped(old_path) + + +def _iter_diff_sections(diff_output): + """Yield (new_path, old_path, lines after the header line) per file section.""" + for section in _DIFF_SECTION_RE.split(diff_output)[1:]: + lines = section.split('\n') + paths = _parse_diff_header(lines[0]) + if paths: + yield paths[1] or paths[0], paths[0], lines[1:] + else: + _unparsed_diff_headers.add(lines[0]) + + def extract_file_paths_from_diff(diff_output): """ Extract file paths from unified diff output (without content). Only includes files with source code extensions. - Returns a list of file paths. """ if not diff_output or not diff_output.strip(): return [] - paths = [] - file_diffs = diff_output.split("diff --git ") - - for file_diff in file_diffs: - if not file_diff.strip(): - continue - lines = file_diff.split('\n') - header_match = re.match(r'^a/(.+?) b/(.+)$', lines[0]) - if not header_match: - continue - file_path = header_match.group(2) or header_match.group(1) or '' - if not _is_reviewable_source(file_path): - continue - paths.append(file_path) - - return paths - + return [ + file_path for file_path, old_path, _ in _iter_diff_sections(diff_output) + if _section_is_reviewable(file_path, old_path) + ] def parse_diff_into_files(diff_output): @@ -714,28 +2030,16 @@ def parse_diff_into_files(diff_output): return [] files = [] - file_diffs = diff_output.split("diff --git ") - - for file_diff in file_diffs: - if not file_diff.strip(): - continue - - # Extract filename from first line: "a/path/to/file b/path/to/file" - lines = file_diff.split('\n') - header_match = re.match(r'^a/(.+?) b/(.+)$', lines[0]) - if not header_match: - continue - - file_path = header_match.group(2) or header_match.group(1) or '' - - # Filter to source code files only - if not _is_reviewable_source(file_path): + for file_path, old_path, lines in _iter_diff_sections(diff_output): + # Filter to source code files only. A rename away from a secret-shaped + # name still carries the secret's content, so the old name counts too. + if not _section_is_reviewable(file_path, old_path): continue # Extract the diff content (from first @@ onwards) diff_lines = [] in_hunks = False - for line in lines[1:]: + for line in lines: if line.startswith('@@'): in_hunks = True if in_hunks: |
