summaryrefslogtreecommitdiff
path: root/plugins/security-guidance/hooks/gitutil.py
diff options
context:
space:
mode:
Diffstat (limited to 'plugins/security-guidance/hooks/gitutil.py')
-rw-r--r--plugins/security-guidance/hooks/gitutil.py1450
1 files changed, 1377 insertions, 73 deletions
diff --git a/plugins/security-guidance/hooks/gitutil.py b/plugins/security-guidance/hooks/gitutil.py
index b6e8cd54..c80d0600 100644
--- a/plugins/security-guidance/hooks/gitutil.py
+++ b/plugins/security-guidance/hooks/gitutil.py
@@ -15,15 +15,51 @@ Functions that DO compose patched leaves (``compute_v2_review_set``,
``security_reminder_hook.py`` for that reason.
"""
import contextlib
+import errno
import os
import re
import subprocess
+import sys
+import tempfile
+import threading
+import time
from _base import debug_log
+def _git_program(windows=os.name == "nt"):
+ """How to start git. On Windows a bare name is looked up in the process's
+ working directory before PATH, and the hooks run in the project, so a
+ `git.exe` there would start in place of the installed one. Name the first
+ git.exe on PATH instead, leaving out relative entries, the working
+ directory, and folders inside the project (unless the project holds the
+ home folder, and with it the user's own tools). With none, a path that
+ does not exist: git is reported missing, as it would be anywhere else."""
+ if not windows:
+ return "git"
+ def norm(p):
+ return os.path.normcase(os.path.normpath(p))
+ try:
+ cwd = norm(os.getcwd())
+ except OSError:
+ cwd = ""
+ home = norm(os.path.expanduser("~")) + os.sep
+ roots = [norm(r) for r in (cwd, os.environ.get("CLAUDE_PROJECT_DIR")) if r]
+ roots = [r.rstrip(os.sep) + os.sep for r in roots if not home.startswith(r.rstrip(os.sep) + os.sep)]
+ for d in os.environ.get("PATH", "").split(os.pathsep):
+ d = d.strip().strip('"')
+ if not d or not os.path.isabs(d) or norm(d) == cwd:
+ continue
+ if any((norm(d) + os.sep).startswith(r) for r in roots):
+ continue
+ path = os.path.join(d, "git.exe")
+ if os.path.isfile(path):
+ return path
+ return os.path.join(os.path.dirname(os.path.abspath(__file__)), "git.exe")
+
+
GIT_CMD = [
- "git",
+ _git_program(),
"-c", "core.fsmonitor=false",
"-c", "core.hooksPath=/dev/null",
# core.quotePath=false: emit raw UTF-8 in path-emitting commands instead
@@ -37,13 +73,52 @@ GIT_CMD = [
# reflog %gs subjects, ls-files, status, etc. — without per-site
# flag duplication. See #2082, #2099.
"-c", "core.quotePath=false",
+ # A work-tree `git diff` that meets stat-stale entries (mtime changed,
+ # content same) rewrites the index under index.lock, GIT_OPTIONAL_LOCKS or
+ # not. Off, every diff here only reads the index, so none needs a copy.
+ "-c", "diff.autoRefreshIndex=false",
+ # The diff parsers read `diff --git a/<old> b/<new>` headers. A user's
+ # diff.mnemonicPrefix (c/ w/ i/), diff.noprefix or diff.srcPrefix /
+ # diff.dstPrefix changes that shape, and every file would be dropped.
+ "-c", "diff.mnemonicPrefix=false",
+ "-c", "diff.noprefix=false",
+ "-c", "diff.srcPrefix=a/",
+ "-c", "diff.dstPrefix=b/",
+ # A directory with its own `.git` inside the work tree (a submodule, a
+ # vendored checkout, an unpacked archive) is another repository with its
+ # own config. To report it as modified, or to show its changes inline
+ # (diff.submodule=diff), git starts a second git inside it, which reads
+ # that config and runs the commands it names. Such a directory is left
+ # out of the review set, and so is a commit that moves one. See also
+ # _drop_gitlinks for the commands that take no such setting.
+ "-c", "diff.ignoreSubmodules=all",
+ "-c", "diff.submodule=short",
+ # `git log` and `git show` print commit metadata for the hooks to parse.
+ # With log.showSignature set they would also verify each signed commit,
+ # adding lines to the output and starting the program gpg.program names.
+ "-c", "log.showSignature=false",
]
+# diff.ignoreSubmodules is a default that the repository's own settings for
+# one submodule (.gitmodules, submodule.<name>.ignore) override. The option
+# cannot be overridden. Pass it to every `git status` and work-tree `git diff`.
+IGNORE_SUBMODULES = "--ignore-submodules=all"
+
SAFE_GIT_CONFIG = (
("core.fsmonitor", "false"),
("core.hooksPath", "/dev/null"),
+ ("diff.ignoreSubmodules", "all"),
+ ("diff.submodule", "short"),
+ ("log.showSignature", "false"),
)
+# The hook's git runs in the background beside the session's own git. Without
+# this, `git status` holds .git/index.lock while it writes back a refreshed
+# index: a concurrent `git add` fails, and a timeout kill leaves the lock.
+SAFE_GIT_ENV = {
+ "GIT_OPTIONAL_LOCKS": "0",
+}
+
def git_config_env(pairs, base=None):
base = os.environ if base is None else base
@@ -61,6 +136,51 @@ def git_config_env(pairs, base=None):
def apply_safe_git_env():
os.environ.update(git_config_env(SAFE_GIT_CONFIG))
+ os.environ.update(SAFE_GIT_ENV)
+
+
+def index_entry_count(path):
+ """The entry count in an index file's header (signature, version, count),
+ or None if it cannot be read. A split or sparse index holds fewer entries
+ than the repository has tracked files."""
+ try:
+ with open(path, "rb") as f:
+ header = f.read(12)
+ except OSError:
+ return None
+ if header[:4] == b"DIRC" and len(header) == 12:
+ return int.from_bytes(header[8:12], "big")
+ return None
+
+
+def windows_drive_mount(path, mountinfo):
+ """Whether `path` is on a Windows drive mounted into WSL, given the text
+ of /proc/self/mountinfo. Every file status there is a round trip to the
+ Windows host."""
+ best, best_type = "", ""
+ for line in mountinfo.splitlines():
+ # id parent dev root mountpoint options [tags] - fstype source options
+ before, _, after = line.partition(" - ")
+ fields, tail = before.split(), after.split()
+ if len(fields) < 5 or not tail:
+ continue
+ point = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4])
+ inside = path == point or path.startswith(point.rstrip("/") + "/")
+ if inside and len(point) >= len(best):
+ best, best_type = point, tail[0]
+ return best_type in ("9p", "drvfs", "virtiofs")
+
+
+def on_windows_drive(path):
+ """windows_drive_mount for this machine. False anywhere but WSL."""
+ try:
+ with open("/proc/sys/kernel/osrelease") as f:
+ if "microsoft" not in f.read().lower():
+ return False
+ with open("/proc/self/mountinfo") as f:
+ return windows_drive_mount(os.path.realpath(path), f.read())
+ except (OSError, ValueError):
+ return False
def _git_rev_parse_head(cwd):
@@ -110,6 +230,35 @@ def _find_git_index(cwd):
return None
+def _find_git_index_and_top(cwd):
+ """_find_git_index plus the work tree's top directory, from one git
+ call. (None, None) when either is missing: no index file, a bare
+ repository, or a `cwd` inside the git directory."""
+ try:
+ result = subprocess.run(
+ [*GIT_CMD, "rev-parse", "--git-dir", "--show-toplevel"],
+ cwd=cwd, capture_output=True, timeout=5
+ )
+ if result.returncode != 0:
+ return None, None
+ # Paths as bytes from git: decode by hand (see _find_git_index).
+ out = result.stdout.decode("utf-8", errors="replace")
+ parts = out[:-1].split("\n") if out.endswith("\n") else []
+ if len(parts) != 2 or not all(parts):
+ # A newline in one of the paths: ask for them one at a time.
+ index_path, top = _find_git_index(cwd), _git_toplevel(cwd)
+ return (index_path, top) if index_path and top else (None, None)
+ git_dir, top = parts
+ if not os.path.isabs(git_dir):
+ git_dir = os.path.join(cwd, git_dir)
+ index_path = os.path.join(git_dir, "index")
+ if not os.path.isfile(index_path):
+ return None, None
+ return index_path, top
+ except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
+ return None, None
+
+
def _diff_pathspec(cwd, paths):
"""Convert absolute touched-paths to repo-relative pathspec args for
git diff. Paths outside cwd (e.g. ~/.claude/…) are dropped. Returns the
@@ -131,32 +280,528 @@ def _diff_pathspec(cwd, paths):
return ["--"] + rel if rel else []
+# Bytes of file paths per git call once a path list has proved too long for
+# one command. Windows caps a whole command line at 32,767 characters;
+# elsewhere the limit is about 1 MB (macOS) or 2 MB (Linux), environment
+# included.
+_MAX_PATH_ARGS_BYTES = (
+ 20000 if sys.platform in ("win32", "cygwin", "msys") else 500000)
+# More commands than this and the path list is not worth diffing: each one
+# matches its paths against the whole index, and a set that large is thrown
+# away as pathological after the diff anyway.
+_MAX_PATH_BATCHES = 4
+
+
+def _arg_cost(path):
+ # The exec limit also counts a terminator and a pointer per argument.
+ return len(os.fsencode(path)) + 9
+
+
+def _path_args_budget(env=None):
+ """_MAX_PATH_ARGS_BYTES, or less where this system's own limit is lower
+ (a small stack limit, a BSD, a very large environment). Half of what the
+ system leaves after the environment, so the rest of the command fits."""
+ try:
+ arg_max = os.sysconf("SC_ARG_MAX")
+ except (AttributeError, ValueError, OSError):
+ return _MAX_PATH_ARGS_BYTES
+ env_bytes = sum(len(k) + len(v) + 2 for k, v in (env or os.environ).items())
+ return min(_MAX_PATH_ARGS_BYTES, max(4096, (arg_max - env_bytes) // 2))
+
+
+def _path_batches(paths, budget=None):
+ if budget is None:
+ budget = _MAX_PATH_ARGS_BYTES
+ batch, size = [], 0
+ for p in paths:
+ n = _arg_cost(p)
+ if batch and size + n > budget:
+ yield batch
+ batch, size = [], 0
+ batch.append(p)
+ size += n
+ if batch:
+ yield batch
+
+
+# Why a path list too long for one command left the last get_git_diff call
+# without a diff, for the Stop review's skip reason: "gave_up" (still too
+# long once cut down) or "none_reviewable".
+_path_list_too_long = set()
+
+# A failed `git add --intent-to-add` during this hook run, for the Stop hook's
+# metrics: kind * 100000 + the paths the add was given. One number because a
+# review row is near CC's key cap. kind: 1 git could not write the empty blob
+# to the object store and the retry failed too, 2 any other git error, 3
+# timeout, 4 the retry with a private object directory worked. A git that
+# speaks another language reports a failed retry as 2. With 1 to 3 the new
+# files are not in the diff, so a turn that only added files looks like a turn
+# with no changes. `left_out` says so for the last get_git_diff alone: the code
+# stays set through a fallback diff that worked.
+_add_failed = {"code": 0, "left_out": False}
+
+# Next to an index copy: where a retried `git add -N` stores the empty blob.
+_OBJECTS_DIR_SUFFIX = ".objects"
+
+
+def _too_many_args(e):
+ """The OS refused to start the command because its arguments are too
+ long. Windows reports ERROR_FILENAME_EXCED_RANGE, which Python turns into
+ a FileNotFoundError."""
+ return e.errno == errno.E2BIG or getattr(e, "winerror", None) == 206
+
+
+def _run_captured(cmd, timeout, **kwargs):
+ return subprocess.run(cmd, capture_output=True, timeout=timeout, **kwargs)
+
+
+def _run_git_on_review_paths(args, paths, timeout, run=_run_captured, **kwargs):
+ """Run `git <args> -- <paths>` for the Stop review; return the results.
+ `run(cmd, timeout, **kwargs)` starts each command (_run_diff_bounded for
+ the diff, so its output is never held whole).
+
+ One command with every path whenever the OS takes it. When the OS refuses
+ the argument list as too long, only reviewable source paths are kept (the
+ review drops the rest from the diff anyway) and they go out in as few
+ commands as they need, all within `timeout` seconds together. If even
+ those need more than _MAX_PATH_BATCHES commands, the refusal is raised
+ again.
+
+ Secret-shaped paths that are gone from the work tree go into every one
+ of those commands: `git diff` must still pair a rename away from such a
+ name, or the secret's content would show as a new source file. A new file
+ that only resembles the secret may be paired with it too and left out of
+ the review. Any other rename is paired only when both of its paths are
+ reviewable source and in the same command; otherwise it shows as a delete
+ or an add.
+ """
+ cmd = [*GIT_CMD, *args, "--"]
+ paths = list(paths)
+ deadline = time.monotonic() + timeout
+ try:
+ return [run(cmd + paths, timeout, **kwargs)]
+ except OSError as e:
+ if not _too_many_args(e):
+ raise
+ too_long = e
+ budget = _path_args_budget(kwargs.get("env"))
+ batches = list(_path_batches(
+ [p for p in paths if _is_reviewable_source(p)], budget))
+ root = kwargs.get("cwd") or os.getcwd()
+ secret = [p for p in paths if _is_secret_shaped(p)
+ and not os.path.lexists(os.path.join(root, p))]
+ debug_log(f"git {args[0]}: {len(paths)} paths do not fit one command; "
+ f"{len(batches)} command(s) for the reviewable ones")
+ # The secret-shaped paths ride along with every batch, so they get a
+ # small share of a command of their own.
+ if (len(batches) > _MAX_PATH_BATCHES
+ or sum(map(_arg_cost, secret)) > budget // 4):
+ raise too_long
+ cmd += secret
+ results = []
+ for batch in batches:
+ left = deadline - time.monotonic()
+ if left <= 0:
+ raise subprocess.TimeoutExpired(cmd, timeout)
+ results.append(run(cmd + batch, left, **kwargs))
+ return results
+
+
+_TEMP_INDEX_PREFIX = "security_hook_idx_"
+# A copy is named <prefix>pid<hook pid>_<8 random chars>, so a later run can
+# tell that the hook which made it is gone. Names from earlier releases have
+# only the 8 random characters and never match.
+_OWNER_PID = re.compile(re.escape(_TEMP_INDEX_PREFIX) + r"pid(\d+)_[^.]{8}(?:\.|$)")
+# A live copy is in use for under a minute (a couple of bounded git calls), so
+# anything this old belongs to a run that was killed.
+_TEMP_INDEX_STALE_S = 1800
+# Unlinking index-sized files is slow; a large backlog drains over several
+# calls instead of stalling one hook past its timeout.
+_SWEEP_BUDGET_S = 2
+
+
+def _owner_is_gone(name):
+ """True when `name` records the pid of the hook that made it and no such
+ process exists. A reused pid reads as alive: the file then waits for the
+ age limit. Never probes on Windows, where os.kill would end the process."""
+ if not hasattr(os, "getuid"):
+ return False
+ m = _OWNER_PID.match(name)
+ if not m:
+ return False
+ try:
+ os.kill(int(m.group(1)), 0)
+ except ProcessLookupError:
+ return True
+ except (OSError, OverflowError, ValueError):
+ return False
+ return False
+
+
+def _sweep_stale_indexes(dirpath, uid):
+ """Delete index copies in `dirpath` that a killed run left behind, and
+ their private object directories. Only regular files and directories owned
+ by `uid` (None: no ownership check) are touched.
+
+ A copy goes as soon as the hook that made it is gone: a killed hook can
+ leave one per run, and index-sized files fill a disk long before the age
+ limit. The age limit covers names without a pid and reused pids.
+
+ Age is the newer of mtime and ctime: a copy keeps .git/index's mtime,
+ which may be days old, so mtime alone would delete a file that another
+ running hook is still using."""
+ import stat
+ import time
+ cutoff = time.time() - _TEMP_INDEX_STALE_S
+ deadline = time.monotonic() + _SWEEP_BUDGET_S
+ try:
+ names = os.listdir(dirpath)
+ except OSError:
+ return
+ for name in names:
+ if not name.startswith(_TEMP_INDEX_PREFIX):
+ continue
+ p = os.path.join(dirpath, name)
+ try:
+ st = os.lstat(p)
+ is_dir = stat.S_ISDIR(st.st_mode) and name.endswith(_OBJECTS_DIR_SUFFIX)
+ if not (is_dir or stat.S_ISREG(st.st_mode)):
+ continue
+ if uid is not None and st.st_uid != uid:
+ continue
+ if max(st.st_mtime, st.st_ctime) < cutoff or _owner_is_gone(name):
+ if is_dir:
+ _remove_objects_dir(p)
+ else:
+ os.unlink(p)
+ if time.monotonic() > deadline:
+ return
+ except OSError:
+ pass
+
+
+def _sweep_stale_stash_indexes(real_index):
+ """Delete `<index>.stash.<pid>` files a killed `git stash create` left
+ next to the real index. Releases before 2.0.10 ran it there, so a clone
+ can hold hundreds, each the size of the index, and nothing else removes
+ them. Only our own regular files past the age limit go: git keeps one for
+ seconds. `index.lock` is never touched; a live git may hold it."""
+ import stat
+ import time
+ uid = os.getuid() if hasattr(os, "getuid") else None
+ cutoff = time.time() - _TEMP_INDEX_STALE_S
+ deadline = time.monotonic() + _SWEEP_BUDGET_S
+ dirpath = os.path.dirname(real_index)
+ stale = re.compile(re.escape(os.path.basename(real_index)) + r"\.stash\.\d+$")
+ try:
+ names = os.listdir(dirpath)
+ except OSError:
+ return
+ for name in names:
+ if not stale.match(name):
+ continue
+ p = os.path.join(dirpath, name)
+ try:
+ st = os.lstat(p)
+ if not stat.S_ISREG(st.st_mode):
+ continue
+ if uid is not None and st.st_uid != uid:
+ continue
+ if max(st.st_mtime, st.st_ctime) < cutoff:
+ os.unlink(p)
+ if time.monotonic() > deadline:
+ return
+ except OSError:
+ pass
+
+
+# Index copies this process has made and not yet removed.
+_live_copies = set()
+
+# A hook working on a copy needs up to three times the index's size for a
+# moment (the copy, and git's rewrite and stash index next to it). With many
+# agents in one clone that alone can fill a disk, so only so many hooks per
+# user work on a copy at once; the others wait, then go without.
+_MAX_LIVE_COPIES = 8
+_SLOT_WAIT_S = 5
+
+
+def _copy_slots(tmpdir, index_bytes):
+ """How many hooks may work on a copy at once: what fits into a quarter of
+ the free space at three times the index each, from 1 to
+ _MAX_LIVE_COPIES."""
+ import shutil
+ try:
+ free = shutil.disk_usage(tmpdir).free
+ except OSError:
+ return 1
+ return max(1, min(_MAX_LIVE_COPIES, free // (12 * max(index_bytes, 1))))
+
+
+@contextlib.contextmanager
+def _copy_slot(tmpdir, index_bytes=0):
+ """Hold one of the copy slots; yields False when none came free in
+ _SLOT_WAIT_S. A slot is an flock, so the kernel frees it when its holder
+ dies, however it dies. No cap without flock or a private dir."""
+ import time
+ try:
+ import fcntl
+ except ImportError:
+ fcntl = None
+ if fcntl is None or tmpdir is None:
+ yield True
+ return
+ deadline = time.monotonic() + _SLOT_WAIT_S
+ held = None
+ while held is None:
+ for i in range(_copy_slots(tmpdir, index_bytes)):
+ try:
+ fd = os.open(os.path.join(tmpdir, f"slot{i}.lock"),
+ os.O_RDWR | os.O_CREAT, 0o600)
+ except OSError:
+ continue
+ try:
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ held = fd
+ break
+ except OSError:
+ os.close(fd)
+ if held is None:
+ if time.monotonic() >= deadline:
+ break
+ time.sleep(0.1)
+ try:
+ yield held is not None
+ finally:
+ if held is not None:
+ os.close(held)
+
+
+def _remove_objects_dir(path):
+ """Remove a private object directory. git writes objects read-only, which
+ stops Windows from deleting them."""
+ import shutil
+ import stat
+
+ def writable_then_retry(func, p, _exc):
+ # Not a link: chmod would follow it to whatever it points at.
+ if func in (os.unlink, os.rmdir) and not os.path.islink(p):
+ os.chmod(p, stat.S_IWRITE)
+ func(p)
+
+ try:
+ shutil.rmtree(path, onerror=writable_then_retry)
+ except OSError:
+ pass
+
+
+def _remove_copy(tmp_index):
+ """Remove an index copy and the files git wrote next to it."""
+ import glob
+ for p in glob.glob(glob.escape(tmp_index) + "*"):
+ try:
+ os.unlink(p)
+ except OSError:
+ pass
+ _remove_objects_dir(tmp_index + _OBJECTS_DIR_SUFFIX)
+
+
+def remove_copies_on_termination():
+ """Remove this process's index copies when it is told to terminate.
+ Python's default SIGTERM action ends the process without running
+ `finally`, which left an index-sized file behind each time."""
+ import signal
+
+ def _on_term(signum, _frame):
+ for tmp_index in list(_live_copies):
+ _remove_copy(tmp_index)
+ os._exit(128 + signum)
+
+ try:
+ signal.signal(signal.SIGTERM, _on_term)
+ except (ValueError, OSError, AttributeError):
+ pass # not the main thread, or no SIGTERM here
+
+
+def _hook_tmpdir():
+ """Private per-user directory for throwaway index copies, or None to use
+ the bare temp dir.
+
+ An index copy can be hundreds of MB and a killed hook cannot clean up, so
+ copies live in one 0700 directory and anything stale is swept on entry,
+ here and in the bare temp dir. The name is predictable and the temp dir
+ may be shared, so anything that is not a real directory we own is
+ refused. Windows has no uid and its temp dir is already per-user."""
+ import stat
+ import tempfile
+ base = tempfile.gettempdir()
+ uid = os.getuid() if hasattr(os, "getuid") else None
+ _sweep_stale_indexes(base, uid)
+ if uid is None:
+ return None
+ d = os.path.join(base, f"claude-security-guidance-{uid}")
+ try:
+ os.mkdir(d, 0o700)
+ except FileExistsError:
+ pass
+ except OSError:
+ return None
+ try:
+ st = os.lstat(d)
+ except OSError:
+ return None
+ if not stat.S_ISDIR(st.st_mode) or st.st_uid != uid:
+ return None
+ try:
+ os.chmod(d, 0o700)
+ except OSError:
+ pass
+ _sweep_stale_indexes(d, uid)
+ return d
+
+
+def _note_add_failure(kind, paths):
+ _add_failed["code"] = kind * 100000 + min(len(paths), 99999)
+ _add_failed["left_out"] = kind != 4
+
+
+def _env_with_private_objects(env, cwd, objects_dir):
+ """`env` for a git that writes objects to `objects_dir` and still reads
+ the repository's own, wherever git says they are (a linked worktree shares
+ them). A path holding the list separator is left out, since git would
+ split it; the add then fails only if it has to read an object."""
+ out = {**env, "GIT_OBJECT_DIRECTORY": objects_dir}
+ try:
+ r = subprocess.run([*GIT_CMD, "rev-parse", "--git-path", "objects"],
+ cwd=cwd, capture_output=True, timeout=5, env=env)
+ except (subprocess.TimeoutExpired, OSError):
+ return out
+ own = os.path.join(cwd, os.fsdecode(r.stdout).strip())
+ if r.returncode == 0 and os.pathsep not in own:
+ out["GIT_ALTERNATE_OBJECT_DIRECTORIES"] = os.pathsep.join(
+ p for p in (own, env.get("GIT_ALTERNATE_OBJECT_DIRECTORIES")) if p)
+ return out
+
+
+def _add_new_files(cwd, paths, env, tmp_index):
+ """`git add --intent-to-add` the untracked `paths` to the index copy `env`
+ names, so the diff shows them. The add has to store the empty blob, so a
+ user who cannot write the object store gets a second try that stores it in
+ a private directory next to the copy. Nothing needs that blob afterwards:
+ `git diff` reads these files from the work tree. `git stash create` would
+ fail on such an index, and only runs on a copy with nothing added.
+
+ No stdout used here (only returncode matters), but text=True still spawns
+ reader threads that decode stderr — git error messages can reference
+ non-ASCII filenames and crash on cp1252. See #2099. Drop text=True so bytes
+ stay raw."""
+ # The paths are file names, not patterns, for this git add only: an
+ # untracked file named `:(glob)**` must add only itself. git refuses that
+ # together with a glob or icase setting from the user's environment.
+ env = {k: v for k, v in env.items()
+ if k not in ("GIT_GLOB_PATHSPECS", "GIT_ICASE_PATHSPECS")}
+ env["GIT_LITERAL_PATHSPECS"] = "1"
+
+ def add(add_env):
+ results = _run_git_on_review_paths(
+ ["add", "--intent-to-add"], paths, timeout=10, cwd=cwd, env=add_env)
+ return next((r for r in results if r.returncode != 0), None)
+
+ try:
+ failed = add(env)
+ except subprocess.TimeoutExpired:
+ # No retry: git is slow here, not refused. The diff goes on for the
+ # tracked files, so a timeout on every turn cannot stop their reviews.
+ _note_add_failure(3, paths)
+ return
+ if not failed:
+ return
+ debug_log(f"git add -N failed: {failed.stderr[:200].decode('utf-8', errors='replace')}")
+ # Whatever happens to the retry, the diff goes on for the tracked files.
+ try:
+ objects_dir = tmp_index + _OBJECTS_DIR_SUFFIX
+ os.mkdir(objects_dir, 0o700)
+ retry_failed = add(_env_with_private_objects(env, cwd, objects_dir))
+ if retry_failed:
+ debug_log(f"git add -N retry failed: {retry_failed.stderr[:200].decode('utf-8', errors='replace')}")
+ except (subprocess.TimeoutExpired, OSError) as e:
+ debug_log(f"git add -N retry failed: {e}")
+ retry_failed = True
+ if retry_failed:
+ _note_add_failure(1 if b"empty blob" in failed.stderr else 2, paths)
+ else:
+ _note_add_failure(4, paths)
+
+
@contextlib.contextmanager
def _temp_index(cwd, untracked_paths=None):
"""Yield an env dict pointing GIT_INDEX_FILE at a throwaway copy of the
repo's index with `git add --intent-to-add` applied, so untracked files
show up in subsequent `git diff` calls without touching the user's real
- index. Yields None if no index can be found (bare repo / not a repo); the
- caller should fall back to a plain diff. Always cleans up the temp file.
+ index. The copy has no gitlink entry for a nested repository (see
+ _drop_gitlinks). Yields None if no index can be found (bare repo / not a
+ repo), no copy slot came free, or those entries could not be removed.
+ Callers then go on without the copy: the diff reads the real index, and
+ the baseline falls back to HEAD. Always cleans up the temp file.
Perf: when `untracked_paths` is given, only those paths are added (O(n)
in untracked count). The default `add -N .` stats every file in the
worktree — slow in large repos vs fast targeted scan. v2 callers
already know the untracked set from `git status --porcelain`, so they
- pass it; v1 keeps the whole-tree scan since it has no prior list."""
- import shutil
- import tempfile
+ pass it; v1 keeps the whole-tree scan since it has no prior list.
- real_index = _find_git_index(cwd)
+ Cleanup removes `<tmp_index>*`, not just the copy: git writes
+ `<index>.lock` and `<index>.stash.<pid>` next to the index it is given and
+ cannot remove them itself when a subprocess timeout kills it."""
+ real_index, top = _find_git_index_and_top(cwd)
if not real_index:
yield None
return
- tmp_fd, tmp_index = tempfile.mkstemp(prefix="security_hook_idx_")
+ _sweep_stale_stash_indexes(real_index)
+ tmpdir = _hook_tmpdir()
+ try:
+ index_bytes = os.path.getsize(real_index)
+ except OSError:
+ index_bytes = 0
+ with _copy_slot(tmpdir, index_bytes) as got_slot:
+ if not got_slot:
+ debug_log("no index copy slot came free; going without a copy")
+ yield None
+ return
+ with _copied_index(cwd, top, real_index, tmpdir, untracked_paths) as env:
+ yield env
+
+
+@contextlib.contextmanager
+def _copied_index(cwd, top, real_index, tmpdir, untracked_paths):
+ import shutil
+ import tempfile
+
+ tmp_fd, tmp_index = tempfile.mkstemp(
+ prefix=f"{_TEMP_INDEX_PREFIX}pid{os.getpid()}_", dir=tmpdir)
os.close(tmp_fd)
+ _live_copies.add(tmp_index)
try:
- shutil.copy2(real_index, tmp_index)
+ # A hard link where the filesystem allows it: no bytes are copied, and
+ # git never writes an index in place (it renames a new file over it),
+ # so the real index cannot change through the link. Either way the
+ # copy must keep .git/index's mtime (copy2, not copyfile): git
+ # re-reads any file whose mtime is not older than the index file's,
+ # and with a fresh mtime it would miss a same-size edit made in the
+ # same second as the last index write.
+ try:
+ os.unlink(tmp_index)
+ os.link(real_index, tmp_index)
+ except (OSError, NotImplementedError, AttributeError):
+ shutil.copy2(real_index, tmp_index)
env = {**os.environ, "GIT_INDEX_FILE": tmp_index}
+ # Before `git add` runs, since add reads the gitlinks its paths cover.
+ if not _drop_gitlinks(top, env, tmp_index):
+ debug_log("could not remove gitlinks from the index copy; going on "
+ "without the copy")
+ yield None
+ return
if untracked_paths is None:
add_args = ["."]
elif untracked_paths:
@@ -165,27 +810,157 @@ def _temp_index(cwd, untracked_paths=None):
# `git status` and here would silently drop ALL untracked files
# from the diff. --ignore-missing only works with --dry-run, so
# filter to surviving paths (lexists so dangling symlinks count).
+ # A nested repository (`dir/`) would be added as a submodule
+ # entry; see diff.ignoreSubmodules in GIT_CMD.
surviving = [p for p in untracked_paths
- if os.path.lexists(os.path.join(cwd, p))]
- add_args = ["--"] + surviving if surviving else None
+ if not p.endswith("/")
+ and os.path.lexists(os.path.join(cwd, p))]
+ add_args = surviving or None
else:
add_args = None
if add_args:
- # No stdout used here (only returncode matters), but text=True
- # still spawns reader threads that decode stderr — git error
- # messages can reference non-ASCII filenames and crash on
- # cp1252. See #2099. Drop text=True so bytes stay raw.
- subprocess.run(
- [*GIT_CMD, "add", "--intent-to-add"] + add_args,
- cwd=cwd, capture_output=True, timeout=10,
- env=env,
- )
+ _add_new_files(cwd, add_args, env, tmp_index)
+ # A whole-tree add puts a nested repository back.
+ if untracked_paths is None and not _drop_gitlinks(top, env, tmp_index):
+ yield None
+ return
yield env
finally:
- try:
- os.unlink(tmp_index)
- except OSError:
- pass
+ _remove_copy(tmp_index)
+ _live_copies.discard(tmp_index)
+
+
+# An index entry stores its mode as four bytes; this is a gitlink's.
+_GITLINK_MODE = b"\x00\x00\xe0\x00"
+# Signature of the index extension that says most entries are in a second,
+# shared file (core.splitIndex).
+_SPLIT_INDEX_EXT = b"link"
+
+
+def _may_have_gitlinks(index_path):
+ """False only when the index file at `index_path` cannot hold a gitlink
+ entry: neither a gitlink's mode bytes nor the split-index signature occur
+ anywhere in it. Either can occur by chance (in an object id, a file
+ size, a path), so True means "list the entries and see". Most
+ repositories have no gitlink, and for them this saves a git process on
+ every index copy."""
+ keep = max(len(_GITLINK_MODE), len(_SPLIT_INDEX_EXT)) - 1
+ try:
+ with open(index_path, "rb") as f:
+ tail = b""
+ while True:
+ chunk = f.read(1 << 20)
+ if not chunk:
+ return False
+ data = tail + chunk
+ if _GITLINK_MODE in data or _SPLIT_INDEX_EXT in data:
+ return True
+ tail = data[-keep:]
+ except OSError:
+ return True
+
+
+_GITLINK_LIST_TIMEOUT = 10
+# `git ls-files --sparse` (git 2.35) lists a sparse index as it is stored;
+# without it git first expands the index to every path in the repository.
+# Set to False once this git has refused the option.
+_ls_files_sparse = True
+
+
+def _gitlink_paths(top, env):
+ """Paths (bytes, relative to `top`) of the gitlink entries, at any depth,
+ in the index `env` names. None if they could not be listed. The listing
+ is read as it comes, so a large index costs time but not memory."""
+ global _ls_files_sparse
+ cmd = [*GIT_CMD, "ls-files", "-s", "-z"] + (["--sparse"] if _ls_files_sparse else [])
+ p = subprocess.Popen(cmd, cwd=top, stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL, env=env)
+ killer = threading.Timer(_GITLINK_LIST_TIMEOUT, p.kill)
+ killer.daemon = True
+ found = set()
+ tail = b""
+ try:
+ killer.start()
+ while True:
+ chunk = p.stdout.read1(65536)
+ if not chunk:
+ break
+ data = tail + chunk
+ end = data.rfind(b"\0") + 1
+ # "<mode> <object> <stage>\t<path>", NUL-terminated. Most chunks
+ # have no gitlink and are not split up.
+ if b"160000 " in data:
+ for rec in data[:end].split(b"\0"):
+ if rec.startswith(b"160000 "):
+ found.add(rec.split(b"\t", 1)[1])
+ tail = data[end:]
+ rc = p.wait()
+ finally:
+ killer.cancel()
+ if p.poll() is None:
+ p.kill()
+ p.wait()
+ p.stdout.close()
+ if rc == 129 and _ls_files_sparse:
+ # Usage error: a git without --sparse.
+ _ls_files_sparse = False
+ return _gitlink_paths(top, env)
+ if rc != 0:
+ debug_log(f"git ls-files on the index copy failed (rc={rc})")
+ return None
+ return found
+
+
+# True when the last _drop_gitlinks call removed at least one gitlink, so the
+# index copy and HEAD can no longer have the same tree even on a clean tree.
+last_dropped_gitlinks = False
+
+
+def _drop_gitlinks(top, env, tmp_index):
+ """Remove the gitlink (mode 160000) entry of every nested repository from
+ the index copy that `env` names, never from the real index, before
+ anything else runs on the copy. A gitlink marks a directory that is, or
+ can be, another repository (see diff.ignoreSubmodules in GIT_CMD):
+ `git stash create` takes no flag like IGNORE_SUBMODULES, and `git add`
+ reads the gitlinks its paths cover. An entry whose directory has no
+ `.git` stays: git starts nothing there, and without the entry an older
+ `git stash create` fails on the empty directory. `top` is the work
+ tree's top directory. Returns True on success, False on any failure; the
+ copy must not be used then."""
+ global last_dropped_gitlinks
+ last_dropped_gitlinks = False
+ try:
+ if not _may_have_gitlinks(tmp_index):
+ return True
+ gitlinks = _gitlink_paths(top, env)
+ if gitlinks is None:
+ return False
+ top_b = os.fsencode(top)
+ nested = sorted(p for p in gitlinks
+ if os.path.lexists(os.path.join(top_b, p, b".git")))
+ if not nested:
+ return True
+ mtime_ns = os.stat(tmp_index).st_mtime_ns
+ # git writes a new file and renames it over the copy, so a copy that
+ # is a hard link to the real index leaves the real one as it was.
+ rm = subprocess.run(
+ [*GIT_CMD, "update-index", "--force-remove", "-z", "--stdin"],
+ cwd=top, input=b"".join(p + b"\0" for p in nested),
+ capture_output=True, timeout=10, env=env)
+ if rm.returncode != 0:
+ debug_log(f"git update-index on the index copy failed (rc={rm.returncode}): "
+ f"{rm.stderr[:200].decode('utf-8', errors='replace')}")
+ return False
+ # Keep the mtime the copy came with; see _copied_index.
+ os.utime(tmp_index, ns=(mtime_ns, mtime_ns))
+ last_dropped_gitlinks = True
+ debug_log(f"gitlink entries (nested repositories) removed from the "
+ f"index copy: {len(nested)}; their contents are not reviewed")
+ return True
+ except (subprocess.TimeoutExpired, OSError, RuntimeError, ValueError) as e:
+ # RuntimeError: the timer thread could not start.
+ debug_log(f"removing gitlinks from the index copy failed: {e}")
+ return False
def _git_toplevel(cwd):
@@ -206,6 +981,41 @@ def _git_toplevel(cwd):
return None
+def _git_toplevel_and_head(cwd):
+ """(repo root, HEAD SHA) for `cwd` from one git call, or (None, None)
+ when git cannot give both (not a work tree, no commits)."""
+ try:
+ r = subprocess.run(
+ [*GIT_CMD, "rev-parse", "--show-toplevel", "HEAD"],
+ cwd=cwd, capture_output=True, timeout=5,
+ )
+ except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
+ return None, None
+ if r.returncode != 0:
+ return None, None
+ # See #2099: the first line is a PATH, so decode the bytes manually.
+ top, _, head = r.stdout.decode("utf-8", errors="replace").strip().rpartition("\n")
+ return (top.strip() or None), (head.strip() or None)
+
+
+def _git_head_at(cwd, epoch):
+ """The SHA this checkout's HEAD pointed at just before `epoch` (seconds),
+ read from its own reflog. A checkout created after `epoch` answers with
+ the commit it was created at. None when git cannot say (no reflog)."""
+ try:
+ # Reflog times are whole seconds: ask for the second before, so a
+ # commit made in the same second as `epoch` counts as after it.
+ r = subprocess.run(
+ [*GIT_CMD, "rev-parse", "--verify", "-q", f"HEAD@{{@{int(epoch) - 1}}}"],
+ cwd=cwd, capture_output=True, timeout=5,
+ )
+ except (subprocess.TimeoutExpired, FileNotFoundError, OSError,
+ TypeError, ValueError, OverflowError):
+ return None
+ sha = r.stdout.decode("utf-8", errors="replace").strip()
+ return sha if r.returncode == 0 and sha else None
+
+
def _git_dir(repo_root):
"""Absolute shared `.git` directory for repo_root.
@@ -373,12 +1183,28 @@ def _git_reflog_recent_commits(repo_root, max_age_s=120, max_n=5):
return fresh, stale
+# New name -> secret-shaped old name, for the renames and copies git reported
+# during this hook run. The Stop review limits its diff to a list of paths:
+# with only the new name in it, git shows the secret's content as a new file,
+# and the diff parsers cannot tell it came from a name they leave out.
+_secret_rename_origins = {}
+
+
+def _note_rename(old_path, new_path):
+ if _is_secret_shaped(old_path):
+ _secret_rename_origins[new_path] = old_path
+
+
def _git_name_only(cwd, base, include_untracked=False):
"""Return the set of repo-root-relative paths that differ from `base`,
or None if git failed (unresolvable ref, not a repo, timeout). Callers
must distinguish None (error → don't trust as a filter) from set()
(genuinely nothing changed). `-c core.quotePath=false -z` keeps non-ASCII
- and space-containing paths intact."""
+ and space-containing paths intact.
+
+ Against the work tree the set can also hold a file whose stat changed and
+ whose content did not: the diff does not refresh the index. Callers
+ intersect with `git status`, which compares content."""
# Decode stdout/stderr as UTF-8 with errors="replace" instead of using
# text=True. core.quotePath=false makes git emit raw UTF-8 for non-ASCII
# paths, and text=True on Windows decodes via cp1252 strict — a non-ASCII
@@ -388,7 +1214,7 @@ def _git_name_only(cwd, base, include_untracked=False):
def _run(env):
# core.quotePath=false comes from GIT_CMD globally (see definition).
result = subprocess.run(
- [*GIT_CMD, "diff", "--name-only", "-z", base],
+ [*GIT_CMD, "diff", IGNORE_SUBMODULES, "--name-status", "-z", base],
cwd=cwd, capture_output=True, timeout=30,
env=env,
)
@@ -397,9 +1223,22 @@ def _git_name_only(cwd, base, include_untracked=False):
debug_log(f"_git_name_only({base!r}) rc={result.returncode}: {stderr_str[:200]}")
return None
stdout = (result.stdout or b"").decode("utf-8", errors="replace")
- return {p for p in stdout.split("\0") if p}
+ # Fields are `status\0path\0`, or `R<score>\0old\0new\0` for a rename
+ # or copy, which is listed by its new name only.
+ fields = stdout.split("\0")
+ names = set()
+ i = 0
+ while i + 1 < len(fields):
+ if fields[i][:1] in ("R", "C") and i + 2 < len(fields):
+ _note_rename(fields[i + 1], fields[i + 2])
+ i += 1
+ names.add(fields[i + 1])
+ i += 2
+ return names
try:
+ # The diff only reads the index (diff.autoRefreshIndex is off in
+ # GIT_CMD); a copy is for `add -N` alone.
if not include_untracked:
return _run(None)
with _temp_index(cwd) as env:
@@ -427,7 +1266,7 @@ def _git_status_porcelain(cwd):
try:
# core.quotePath=false comes from GIT_CMD globally (see definition).
r = subprocess.run(
- [*GIT_CMD, "status", "--porcelain=v1", "-uall", "-z"],
+ [*GIT_CMD, "status", IGNORE_SUBMODULES, "--porcelain=v1", "-uall", "-z"],
cwd=cwd, capture_output=True, timeout=30,
)
if r.returncode != 0:
@@ -445,7 +1284,10 @@ def _git_status_porcelain(cwd):
continue
xy, path = e[:2], e[3:]
if xy == "??":
- untracked.add(path)
+ # With -uall only a nested repository is listed as `dir/`.
+ # It is not a file to review; see diff.ignoreSubmodules.
+ if not path.endswith("/"):
+ untracked.add(path)
else:
tracked.add(path)
# Rename/copy entries are XY old\0new\0 — second NUL field is
@@ -453,6 +1295,8 @@ def _git_status_porcelain(cwd):
# 2-char-status entry.
if "R" in xy or "C" in xy:
i += 1
+ if i < len(entries):
+ _note_rename(entries[i], path)
i += 1
return tracked, untracked
except (subprocess.TimeoutExpired, FileNotFoundError, OSError, ValueError) as e:
@@ -479,6 +1323,264 @@ def _is_ancestor(cwd, maybe_ancestor, descendant):
return False
+_DEFAULT_FILE_DIFF_BYTES = 4 * 1024 * 1024
+_LATE_FILE_DIFF_BYTES = 256 * 1024
+_LAST_FILE_DIFF_BYTES = 96 * 1024
+
+
+def _file_diff_cap():
+ """SG_MAX_FILE_DIFF_BYTES, or the default when it is unset, not a number
+ or not positive. Never below what a late file keeps, which is well above
+ what the review prompt takes of one file."""
+ try:
+ n = int(os.environ.get("SG_MAX_FILE_DIFF_BYTES", ""))
+ except ValueError:
+ return _DEFAULT_FILE_DIFF_BYTES
+ if n <= 0:
+ return _DEFAULT_FILE_DIFF_BYTES
+ return max(n, _LATE_FILE_DIFF_BYTES)
+
+
+# How much of one file's diff the Stop review keeps while it reads git's
+# output: this much from the start of the file's section, and as much again
+# of the added lines after that. The review prompt takes far less of a file
+# (DIFF_PER_FILE_BYTES); the rest is for filter_preexisting_from_diff, which
+# needs the removed and the added side of a rewritten file.
+MAX_FILE_DIFF_BYTES = _file_diff_cap()
+# Past this much kept in total, a further file keeps _LATE_FILE_DIFF_BYTES
+# the same two ways; past four times as much, _LAST_FILE_DIFF_BYTES from its
+# start, which is still more than the review prompt takes of one file.
+MAX_KEPT_DIFF_BYTES = 4 * MAX_FILE_DIFF_BYTES
+_SECTION_MARK = b"\ndiff --git "
+_MAX_HEADER_BYTES = 64 * 1024
+_CUT_MARK = b"\\ [cut by security-guidance: the rest of this file's diff was not read]\n"
+
+# What the last get_git_diff read, for the Stop hook's metrics: bytes of git
+# output, reviewable files whose diff was cut and the bytes cut from them,
+# and diffs that hit the timeout during this hook run.
+_diff_read = {"bytes": 0, "cut_files": 0, "cut_bytes": 0, "timeouts": 0}
+
+
+class _DiffKeeper:
+ """Takes `git diff` output as it arrives and keeps only what the review
+ can use: the header line of every file, and a bounded part of each file
+ parse_diff_into_files would keep. A turn that wrote large data files no
+ longer puts their whole diff in memory."""
+
+ def __init__(self):
+ self.parts = []
+ self.kept = 0
+ self.seen = 0
+ self.cut_files = 0
+ self.cut_bytes = 0
+ # Unhandled input. Starts as a newline that is not git's, so a header
+ # at the very start of the output is found like any other.
+ self._pending = b"\n"
+ self._at_start = True
+ self._reviewable = True
+ self._room = self._added_room = MAX_FILE_DIFF_BYTES
+ self._cut = False
+ self._ends_nl = True
+ # Past the cut: whether the next byte starts a line, and whether the
+ # line being read is an added one.
+ self._bol = True
+ self._in_added = False
+
+ def feed(self, data):
+ buf = self._pending + data
+ pos = 0
+ while True:
+ i = buf.find(_SECTION_MARK, pos)
+ if i < 0:
+ # Hold back a tail that could be the start of a header.
+ end = max(pos, len(buf) - len(_SECTION_MARK) + 1)
+ self._keep(buf, pos, end)
+ self._pending = buf[end:]
+ return
+ j = buf.find(b"\n", i + 1)
+ if j < 0 and len(buf) - i <= _MAX_HEADER_BYTES:
+ # The header line is not complete yet.
+ self._keep(buf, pos, i)
+ self._pending = buf[i:]
+ return
+ self._keep(buf, pos, i + 1)
+ if j >= 0:
+ self._begin(buf[i + 1:j])
+ pos = i + 1
+
+ def _begin(self, header):
+ """A new file's section starts with this `diff --git ` line."""
+ self._end_section()
+ text = header[len(_SECTION_MARK) - 1:].decode("utf-8", errors="replace")
+ paths = _parse_diff_header(text)
+ # A section the parser drops, or whose header it cannot read and only
+ # counts, keeps its header line.
+ self._reviewable = bool(paths) and _section_is_reviewable(
+ paths[1] or paths[0], paths[0])
+ if not self._reviewable:
+ self._room, self._added_room = len(header) + 1, 0
+ elif self.kept > 4 * MAX_KEPT_DIFF_BYTES:
+ self._room, self._added_room = max(len(header) + 1, _LAST_FILE_DIFF_BYTES), 0
+ elif self.kept > MAX_KEPT_DIFF_BYTES:
+ self._room = self._added_room = max(len(header) + 1, _LATE_FILE_DIFF_BYTES)
+ else:
+ self._room = self._added_room = MAX_FILE_DIFF_BYTES
+ self._cut = False
+
+ def _end_section(self):
+ if not self._ends_nl:
+ # Cut inside a line: end it, so what follows starts a line.
+ self._append(b"\n")
+ if self._cut:
+ self._append(_CUT_MARK)
+
+ def _append(self, piece):
+ self.parts.append(piece)
+ self.kept += len(piece)
+ self._ends_nl = piece.endswith(b"\n")
+
+ def _keep(self, buf, start, end):
+ if self._at_start and end > start:
+ self._at_start = False
+ start += 1
+ n = end - start
+ if n <= 0:
+ return
+ self.seen += n
+ take = min(n, self._room)
+ if take:
+ self._append(buf[start:start + take])
+ self._room -= take
+ if take == n or not self._reviewable:
+ return
+ if not self._cut:
+ self._cut = True
+ self.cut_files += 1
+ self._bol = self._ends_nl
+ self._in_added = False
+ self.cut_bytes += n - take - self._keep_added(buf, start + take, end)
+
+ def _keep_added(self, buf, pos, end):
+ """Past a file's cut, keep its added lines only, up to as much again.
+ A rewritten file lists every removed line before the first added one,
+ and the review is about the added ones. Returns the bytes kept."""
+ kept = 0
+ run = -1 # where a run of added lines not yet appended starts
+ while pos < end and self._added_room > 0:
+ if self._bol:
+ self._bol = False
+ self._in_added = buf.startswith(b"+", pos)
+ if self._in_added and run < 0 and not self._ends_nl:
+ self._append(b"\n")
+ if not self._in_added:
+ if run >= 0:
+ self._append(buf[run:pos])
+ run = -1
+ nxt = buf.find(b"\n+", pos, end)
+ if nxt < 0:
+ self._bol = buf.endswith(b"\n", pos, end)
+ break
+ pos = nxt + 1
+ self._bol = True
+ continue
+ nl = buf.find(b"\n", pos, end)
+ stop = end if nl < 0 else nl + 1
+ take = min(stop - pos, self._added_room)
+ if run < 0:
+ run = pos
+ self._added_room -= take
+ kept += take
+ self._bol = nl >= 0
+ if pos + take < stop:
+ self._append(buf[run:pos + take])
+ run = -1
+ pos = stop
+ if run >= 0:
+ self._append(buf[run:pos])
+ return kept
+
+ def result(self):
+ self._keep(self._pending, 0, len(self._pending))
+ self._pending = b""
+ self._end_section()
+ self._cut = False
+ return b"".join(self.parts)
+
+
+def _run_diff_bounded(cmd, timeout, **kwargs):
+ """subprocess.run(cmd, capture_output=True, timeout=timeout) for a
+ `git diff`, except that stdout goes through _DiffKeeper instead of being
+ held whole."""
+ keeper = _DiffKeeper()
+ deadline = time.monotonic() + timeout
+ try:
+ err = tempfile.TemporaryFile()
+ except OSError:
+ err = None
+ try:
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+ stderr=err if err is not None else subprocess.DEVNULL,
+ **kwargs)
+ failed = []
+ given_up = []
+
+ def _read():
+ try:
+ while not given_up:
+ chunk = p.stdout.read1(65536)
+ if not chunk or given_up:
+ break
+ keeper.feed(chunk)
+ except BaseException as e:
+ failed.append(e)
+
+ # Read on a thread, so the timeout holds even while something git
+ # started still has the pipe open after git itself is gone.
+ reader = threading.Thread(target=_read, daemon=True)
+ timed_out = False
+ try:
+ try:
+ reader.start()
+ except RuntimeError as e:
+ failed.append(e)
+ else:
+ reader.join(timeout)
+ timed_out = reader.is_alive()
+ if timed_out or failed:
+ p.kill()
+ try:
+ returncode = p.wait(max(deadline - time.monotonic(), 0.1))
+ except subprocess.TimeoutExpired:
+ timed_out = True
+ finally:
+ if timed_out or failed or p.returncode is None:
+ # The reader may be left waiting on a pipe something else
+ # holds: it keeps nothing more, and the pipe stays its.
+ given_up.append(True)
+ p.kill()
+ p.wait()
+ if reader.ident is not None:
+ reader.join(1)
+ if not reader.is_alive():
+ p.stdout.close()
+ _diff_read["bytes"] += keeper.seen
+ if timed_out:
+ _diff_read["timeouts"] += 1
+ raise subprocess.TimeoutExpired(cmd, timeout)
+ if failed:
+ raise OSError(f"reading the diff failed: {failed[0]!r}")
+ stdout = keeper.result()
+ _diff_read["cut_files"] += keeper.cut_files
+ _diff_read["cut_bytes"] += keeper.cut_bytes
+ stderr = b""
+ if err is not None:
+ err.seek(0)
+ stderr = err.read(4096)
+ return subprocess.CompletedProcess(cmd, returncode, stdout, stderr)
+ finally:
+ if err is not None:
+ err.close()
+
def get_git_diff(cwd, baseline_sha, full_context=False, paths=None, untracked_paths=None):
"""
@@ -492,9 +1594,16 @@ def get_git_diff(cwd, baseline_sha, full_context=False, paths=None, untracked_pa
If `paths` is given, the diff is restricted to those paths (relative to
cwd; absolute paths are converted, paths outside cwd are dropped).
+ A path list too long for one command is cut down to reviewable source
+ files: see _run_git_on_review_paths.
`untracked_paths` (repo-root-relative) is forwarded to _temp_index so it
can add only those files instead of scanning the whole worktree.
+ The output is read through _DiffKeeper: files the review would drop come
+ back as their header line only, and a file's diff past
+ MAX_FILE_DIFF_BYTES is cut down to its added lines.
"""
+ _path_list_too_long.clear()
+ _add_failed["left_out"] = False
pathspec = _diff_pathspec(cwd, paths)
if paths and not pathspec:
# Caller restricted to specific paths but none are inside this repo
@@ -504,19 +1613,36 @@ def get_git_diff(cwd, baseline_sha, full_context=False, paths=None, untracked_pa
return ""
# core.quotePath=false comes from GIT_CMD globally (see definition).
- cmd = [*GIT_CMD, "diff", "--no-color", "--no-ext-diff", "--no-textconv", baseline_sha] + (["--unified=99999"] if full_context else []) + pathspec
+ args = ["diff", IGNORE_SUBMODULES, "--no-color", "--no-ext-diff", "--no-textconv", baseline_sha] + (["--unified=99999"] if full_context else [])
+ _diff_read.update(bytes=0, cut_files=0, cut_bytes=0)
+
+ def _run(env):
+ if pathspec:
+ results = _run_git_on_review_paths(args, pathspec[1:], timeout=30, run=_run_diff_bounded, cwd=cwd, env=env)
+ else:
+ results = [_run_diff_bounded([*GIT_CMD, *args], 30, cwd=cwd, env=env)]
+ if not results:
+ _path_list_too_long.add("none_reviewable")
+ for result in results:
+ if result.returncode != 0:
+ debug_log(f"git diff failed: {result.stderr[:200].decode('utf-8', errors='replace')}")
+ return None
+ # Decode with errors='replace' so binary diffs don't crash
+ return b"".join(r.stdout for r in results).decode("utf-8", errors="replace")
+
try:
+ if untracked_paths is not None and not untracked_paths:
+ # Nothing to `add -N`, and the diff only reads the index.
+ return _run(None)
with _temp_index(cwd, untracked_paths) as env:
- # env is None when no index could be found (bare repo / not a
- # repo) — diff still runs, just without untracked-file support.
- result = subprocess.run(cmd, cwd=cwd, capture_output=True, timeout=30, env=env)
- if result.returncode != 0:
- debug_log(f"git diff failed: {result.stderr[:200].decode('utf-8', errors='replace')}")
- return None
- # Decode with errors='replace' so binary diffs don't crash
- return result.stdout.decode("utf-8", errors="replace")
+ # env is None when there is no index (bare repo / not a repo)
+ # or no copy slot — diff still runs, just without
+ # untracked-file support.
+ return _run(env)
except (subprocess.TimeoutExpired, FileNotFoundError, OSError) as e:
debug_log(f"git diff error: {e}")
+ if isinstance(e, OSError) and _too_many_args(e):
+ _path_list_too_long.add("gave_up")
return None
@@ -571,6 +1697,116 @@ SKIP_FILE_SUFFIXES = (
'.lock', '_pb2.py', '.pb.go',
)
+# Files that hold secrets by naming convention are never sent for review. The
+# review covers everything dirty in the work tree, including files the user
+# changed and Claude never opened, so a key file or an encrypted dotfile source
+# would otherwise leave the machine in the review request. The list is kept to
+# names that are secret stores by convention; ordinary source such as
+# `private_api.py` or `id_generator.py` stays reviewable.
+_SSH_PRIVATE_KEY_BASENAMES = frozenset((
+ 'id_rsa', 'id_dsa', 'id_ecdsa', 'id_ecdsa_sk', 'id_ed25519',
+ 'id_ed25519_sk', 'id_xmss',
+))
+_PRIVATE_KEY_NAME_SUFFIXES = ('_rsa', '_dsa', '_ecdsa', '_ed25519')
+# `.credentials.json` and `.claude.json` are Claude Code's own files: sign-in
+# tokens, a saved API key, MCP server secrets. Claude Code rewrites them as it
+# runs, so they show up as changed when its config directory is a git repo.
+_CREDENTIAL_BASENAMES = frozenset((
+ 'credentials', '.netrc', '.pgpass', '.npmrc', '.pypirc',
+ '.git-credentials', '.dockercfg', '.credentials.json', '.claude.json',
+))
+# chezmoi encodes a target file's attributes as stacked name prefixes
+# (`private_dot_netrc`, `encrypted_private_token`); `dot_` stands for a
+# leading dot.
+_CHEZMOI_ATTR_PREFIXES = (
+ 'after_', 'before_', 'create_', 'empty_', 'encrypted_', 'exact_',
+ 'executable_', 'external_', 'literal_', 'modify_', 'once_', 'onchange_',
+ 'private_', 'readonly_', 'remove_', 'run_', 'symlink_',
+)
+# An `encrypted_` source keeps its target's name, so `encrypted_store.py` could
+# be either ciphertext or code about encryption. Treat it as ciphertext only
+# when it has no extension or a data-file one.
+_ENCRYPTED_DATA_EXTENSIONS = frozenset((
+ '', '.json', '.yaml', '.yml', '.toml', '.properties', '.tfvars',
+))
+
+
+# Paths skipped as secret-shaped during this hook run. The Stop hook reports
+# the count (never the names) so the skip can be watched in telemetry.
+_secret_shaped_skipped = set()
+
+
+def _skip_secret_shaped(file_path):
+ if not _is_secret_shaped(file_path):
+ return False
+ _secret_shaped_skipped.add(file_path)
+ return True
+
+
+def _is_secret_shaped(file_path):
+ base = os.path.basename(file_path.replace("\\", "/")).lower()
+ target = base
+ encrypted = False
+ stripped = True
+ while stripped:
+ stripped = False
+ for prefix in _CHEZMOI_ATTR_PREFIXES:
+ if target.startswith(prefix) and len(target) > len(prefix):
+ target = target[len(prefix):]
+ encrypted = encrypted or prefix == 'encrypted_'
+ stripped = True
+ break
+ if target.startswith('dot_'):
+ target = '.' + target[4:]
+ ext = os.path.splitext(target)[1]
+ if encrypted and ext in _ENCRYPTED_DATA_EXTENSIONS:
+ return True
+ for name in {base, target}:
+ if name.startswith('.env'):
+ return True
+ if name in _SSH_PRIVATE_KEY_BASENAMES or name in _CREDENTIAL_BASENAMES:
+ return True
+ if '.' not in name and name.endswith(_PRIVATE_KEY_NAME_SUFFIXES):
+ return True
+ return False
+
+
+# Data files whose name says they hold secrets (`secrets.yml`,
+# `client_secret.json`). They are reviewed like any other data file; the Stop
+# hook only counts how many are in its review set (never the names), to show
+# how common they are.
+_SECRET_NAMED_STEMS = frozenset((
+ 'secret', 'secrets', 'credential', 'credentials',
+ 'client_secret', 'client_secrets', 'client-secret', 'client-secrets',
+ 'service-account', 'service_account', 'serviceaccount',
+ 'service-account-key', 'service_account_key', 'serviceaccountkey',
+))
+_SECRET_NAMED_DATA_EXTENSIONS = frozenset((
+ '.json', '.yaml', '.yml', '.toml', '.properties', '.tfvars',
+))
+# `secrets.example.yml` is a placeholder and `secrets.enc.yaml` is ciphertext.
+_NOT_SECRET_NAME_PARTS = frozenset((
+ 'example', 'sample', 'template', 'tmpl', 'dist', 'schema',
+ 'enc', 'encrypted', 'sops',
+))
+
+
+def _is_secret_named(file_path):
+ base = os.path.basename(file_path.replace("\\", "/")).lower()
+ parts = base.lstrip('.').split('.')
+ if len(parts) < 2 or '.' + parts[-1] not in _SECRET_NAMED_DATA_EXTENSIONS:
+ return False
+ if _NOT_SECRET_NAME_PARTS.intersection(parts[1:-1]):
+ return False
+ stem = parts[0]
+ # Google's OAuth download is `client_secret_<client id>.json`.
+ return stem in _SECRET_NAMED_STEMS or stem.startswith('client_secret_')
+
+
+def count_secret_named(diff_files):
+ return sum(1 for file_path, _ in diff_files if _is_secret_named(file_path))
+
+
# Path tokens that bump a file's review priority when a commit exceeds
# MAX_DIFF_FILES and we have to pick a subset. These are exactly the surfaces
# single-shot and agentic reviews disagree on most (auth, routing, IPC,
@@ -648,6 +1884,8 @@ def _is_reviewable_source(file_path):
norm = "/" + file_path.replace("\\", "/")
if any(("/" + p) in norm for p in SKIP_PATH_PATTERNS):
return False
+ if _skip_secret_shaped(file_path):
+ return False
if file_path.endswith(SKIP_FILE_SUFFIXES):
return False
ext = os.path.splitext(file_path)[1].lower()
@@ -677,32 +1915,110 @@ def _is_reviewable_source(file_path):
return False
+# A file's section starts at a `diff --git ` line. Inside a section every line
+# of file content carries a `+`, `-` or space prefix, so only a real header can
+# start a line with it; source text that merely contains the words cannot.
+_DIFF_SECTION_RE = re.compile(r'^diff --git ', re.MULTILINE)
+_UNQUOTED_HEADER_RE = re.compile(r'^a/(.+?) b/(.+)$')
+_C_ESCAPES = {
+ 'a': b'\a', 'b': b'\b', 'f': b'\f', 'n': b'\n', 'r': b'\r', 't': b'\t',
+ 'v': b'\v', '\\': b'\\', '"': b'"',
+}
+
+
+def _read_quoted_path(text, start):
+ """Decode the C-quoted path whose opening quote is at text[start].
+ Returns (path, index after the closing quote), or None if unterminated."""
+ out = bytearray()
+ i = start + 1
+ while i < len(text):
+ c = text[i]
+ if c == '"':
+ return out.decode("utf-8", errors="replace"), i + 1
+ if c == '\\' and i + 1 < len(text):
+ n = text[i + 1]
+ if n in _C_ESCAPES:
+ out += _C_ESCAPES[n]
+ i += 2
+ continue
+ octal = re.match(r'[0-7]{1,3}', text[i + 1:i + 4])
+ if octal:
+ out.append(int(octal.group(0), 8) & 0xFF)
+ i += 1 + len(octal.group(0))
+ continue
+ out += c.encode("utf-8")
+ i += 1
+ return None
+
+
+def _parse_diff_header(header):
+ """(old_path, new_path) from the text after `diff --git `, or None.
+
+ core.quotePath=false keeps non-ASCII names raw, but git still C-quotes a
+ name that contains a double quote, a backslash or a control character:
+ `"a/we\\"ird.py" "b/we\\"ird.py"`. Either side can be quoted on its own.
+ """
+ old = new = None
+ if header.startswith('"'):
+ first = _read_quoted_path(header, 0)
+ if not first:
+ return None
+ old, end = first
+ rest = header[end:].lstrip(' ')
+ if rest.startswith('"'):
+ second = _read_quoted_path(rest, 0)
+ new = second[0] if second else None
+ else:
+ new = rest
+ elif header.endswith('"') and ' "b/' in header:
+ split_at = header.index(' "b/')
+ old = header[:split_at]
+ second = _read_quoted_path(header, split_at + 1)
+ new = second[0] if second else None
+ else:
+ m = _UNQUOTED_HEADER_RE.match(header)
+ return (m.group(1), m.group(2)) if m else None
+ if not old or not new or not old.startswith('a/') or not new.startswith('b/'):
+ return None
+ return old[2:], new[2:]
+
+
+# Section headers that could not be parsed during this hook run; that file is
+# left out of the review. The Stop hook reports the count so a header shape
+# this parser misses shows up in telemetry instead of passing silently.
+_unparsed_diff_headers = set()
+
+
+def _section_is_reviewable(file_path, old_path):
+ """Whether a diff section is sent for review: a source file, and not one
+ renamed away from a secret-shaped name, which still carries the secret's
+ content."""
+ return _is_reviewable_source(file_path) and not _skip_secret_shaped(old_path)
+
+
+def _iter_diff_sections(diff_output):
+ """Yield (new_path, old_path, lines after the header line) per file section."""
+ for section in _DIFF_SECTION_RE.split(diff_output)[1:]:
+ lines = section.split('\n')
+ paths = _parse_diff_header(lines[0])
+ if paths:
+ yield paths[1] or paths[0], paths[0], lines[1:]
+ else:
+ _unparsed_diff_headers.add(lines[0])
+
+
def extract_file_paths_from_diff(diff_output):
"""
Extract file paths from unified diff output (without content).
Only includes files with source code extensions.
- Returns a list of file paths.
"""
if not diff_output or not diff_output.strip():
return []
- paths = []
- file_diffs = diff_output.split("diff --git ")
-
- for file_diff in file_diffs:
- if not file_diff.strip():
- continue
- lines = file_diff.split('\n')
- header_match = re.match(r'^a/(.+?) b/(.+)$', lines[0])
- if not header_match:
- continue
- file_path = header_match.group(2) or header_match.group(1) or ''
- if not _is_reviewable_source(file_path):
- continue
- paths.append(file_path)
-
- return paths
-
+ return [
+ file_path for file_path, old_path, _ in _iter_diff_sections(diff_output)
+ if _section_is_reviewable(file_path, old_path)
+ ]
def parse_diff_into_files(diff_output):
@@ -714,28 +2030,16 @@ def parse_diff_into_files(diff_output):
return []
files = []
- file_diffs = diff_output.split("diff --git ")
-
- for file_diff in file_diffs:
- if not file_diff.strip():
- continue
-
- # Extract filename from first line: "a/path/to/file b/path/to/file"
- lines = file_diff.split('\n')
- header_match = re.match(r'^a/(.+?) b/(.+)$', lines[0])
- if not header_match:
- continue
-
- file_path = header_match.group(2) or header_match.group(1) or ''
-
- # Filter to source code files only
- if not _is_reviewable_source(file_path):
+ for file_path, old_path, lines in _iter_diff_sections(diff_output):
+ # Filter to source code files only. A rename away from a secret-shaped
+ # name still carries the secret's content, so the old name counts too.
+ if not _section_is_reviewable(file_path, old_path):
continue
# Extract the diff content (from first @@ onwards)
diff_lines = []
in_hunks = False
- for line in lines[1:]:
+ for line in lines:
if line.startswith('@@'):
in_hunks = True
if in_hunks: