diff options
| author | Claude <noreply@anthropic.com> | 2026-11-06 00:54:48 +0000 |
|---|---|---|
| committer | Claude <noreply@anthropic.com> | 2026-11-06 00:54:48 +0000 |
| commit | 7aba3deae3ff17d032c37e07d2929d1426b34278 (patch) | |
| tree | 9c482f9acda1599bd2f9bed95a6a46b75e634e80 /plugins/security-guidance/hooks/diffstate.py | |
| parent | 5472f5f56e0070feeeb464318b316714fe3e9105 (diff) | |
| parent | b8331dd69ab3e132bbf8cd6e89b5d8c1747e21b1 (diff) | |
Merge remote-tracking branch 'origin/main' into add-plugin/xcodeadd-plugin/xcode
Diffstat (limited to 'plugins/security-guidance/hooks/diffstate.py')
| -rw-r--r-- | plugins/security-guidance/hooks/diffstate.py | 808 |
1 files changed, 756 insertions, 52 deletions
diff --git a/plugins/security-guidance/hooks/diffstate.py b/plugins/security-guidance/hooks/diffstate.py index 8d0665dd..34655335 100644 --- a/plugins/security-guidance/hooks/diffstate.py +++ b/plugins/security-guidance/hooks/diffstate.py @@ -6,16 +6,22 @@ there so callers keep resolving bare names through the hook module's globals — tests that ``monkeypatch.setattr(hook, "<fn>", …)`` continue to work without retargeting. """ +import hashlib import os import subprocess +import sys +import time -from _base import debug_log, _PV +from _base import debug_log, portable_output, _PV +import gitutil from gitutil import ( GIT_CMD, _git_dir, _git_toplevel, _git_status_porcelain, - _git_rev_parse_head, _is_ancestor, _git_name_only, + _git_rev_parse_head, _is_ancestor, _git_name_only, _temp_index, + _secret_rename_origins, index_entry_count, ) from session_state import with_locked_state +from extensibility import _same_dir # ===================================================================== @@ -95,6 +101,10 @@ def consume_stop_state(session_id, clear=True): "touched_paths": list(state.get("touched_paths", [])), "baseline_sha": state.get("baseline_sha"), "head_at_capture": state.get("head_at_capture"), + # Where and when the prompt hook captured it; see baseline_origin. + "baseline_root": state.get("baseline_root"), + "prompt_at": state.get("prompt_at"), + "baseline_kept": state.get("baseline_kept") or 0, "untracked_at_baseline": ( dict(state["untracked_at_baseline"]) if isinstance(state.get("untracked_at_baseline"), dict) else {} @@ -103,20 +113,92 @@ def consume_stop_state(session_id, clear=True): "fire_count_expired": expired and state.get("stop_hook_fire_count", 0) > 0, "previous_findings": [] if findings_expired else list(state.get("previous_findings", [])), "reviewed_diff_hash": state.get("reviewed_diff_hash"), + # State lost or overwritten since the last Stop (session_state). + "state_resets": state.get("state_resets", 0), + "state_overwrites": state.get("state_overwrites", 0), + "baseline_fail": state.get("baseline_fail") or 0, } + # touched_paths is cleared per Stop, but one baseline can be + # reviewed by several Stops (rewakes). turn_touched keeps every path + # recorded since the baseline; only telemetry reads it. + turn = list(dict.fromkeys( + [p for p in state.get("turn_touched", []) if isinstance(p, str)] + + snap["touched_paths"]))[-200:] + snap["turn_touched"] = turn if clear: + state["turn_touched"] = turn state["touched_paths"] = [] state.pop("reviewed_diff_hash", None) + state.pop("state_resets", None) + state.pop("state_overwrites", None) return snap return with_locked_state(session_id, _snap) or { "touched_paths": [], "baseline_sha": None, "head_at_capture": None, - "untracked_at_baseline": {}, + "untracked_at_baseline": {}, "baseline_root": None, "prompt_at": None, + "baseline_kept": 0, "fire_count": 0, "fire_count_expired": False, "previous_findings": [], - "reviewed_diff_hash": None, + "reviewed_diff_hash": None, "state_resets": 0, "state_overwrites": 0, + "baseline_fail": 0, "turn_touched": [], } +# Path names compare without case where the filesystem usually does. +_FOLD_CASE = os.name == "nt" or sys.platform == "darwin" + + +def _rel_in_repo(real_root, path): + """`path` relative to the repo root (a real path), or None when it lies + outside the repo.""" + real = os.path.realpath(path) + try: + rel = os.path.relpath(real, real_root) + except ValueError: # another drive + return None + if rel == os.pardir or rel.startswith(os.pardir + os.sep): + return None + return rel.casefold() if _FOLD_CASE else rel + + +def touched_scope(repo_root, cwd, touched_paths): + """Where the paths the session's edit tools wrote since the baseline lie: + (scope, repo-relative paths inside the repo). scope: 0 none recorded, + 1 all inside the reviewed repo, 2 some outside, 3 all outside. Relative + paths are read against `cwd`. The list holds the newest 200 paths, so + this is approximate past that. Telemetry only.""" + root = os.path.realpath(repo_root) + inside, outside = set(), 0 + for p in touched_paths: + rel = _rel_in_repo(root, os.path.join(cwd or root, p)) + if rel is None: + outside += 1 + else: + inside.add(rel) + if not touched_paths: + return 0, inside + if not outside: + return 1, inside + return (2 if inside else 3), inside + + +def review_scope_code(repo_root, scope, inside, files, base_fail): + """One number, for telemetry only, saying how a Stop review relates to + what the session's edit tools wrote since the baseline: + + base_fail * 100000 + scope * 10000 + min(untouched, 9999) + + scope and inside: from touched_scope. untouched: how many of `files` + (repo-root-relative or absolute) no edit tool wrote. base_fail: 0 the + review diffed against a real turn-start snapshot, 1 and 2 see + last_baseline_fail, 3 there was no usable snapshot. Packed into one key + and sent even when 0, so a row without it was cut by the metrics key cap + or had no repo.""" + root = os.path.realpath(repo_root) + untouched = sum(1 for f in files + if _rel_in_repo(root, os.path.join(root, f)) not in inside) + return int(base_fail or 0) * 100000 + scope * 10000 + min(untouched, 9999) + + def record_reviewed_diff(session_id, diff_hash): def _save(state): state["reviewed_diff_hash"] = diff_hash @@ -147,6 +229,48 @@ def restore_unreviewed_stop_state(session_id, paths, baseline_sha): with_locked_state(session_id, _restore) +# state["baseline_kept"]: why a prompt left an earlier turn's baseline in +# place. Only telemetry reads it. +KEPT_NO_CAPTURE = 1 # it could not capture one (not a repo, or git failed) +KEPT_UNREVIEWED = 2 # the previous turn's edits were never reviewed + +# baseline_origin() values that mean "captured in another checkout". +FOREIGN_ORIGINS = (1, 2, 3) + + +def baseline_origin(repo_root, baseline_sha, baseline_root): + """Where the session's baseline was captured, relative to the checkout a + Stop review is about to read (`repo_root`): + + 0 that checkout (or the origin was not recorded) + 1 another worktree of the same repository + 2 another repository + 3 another folder, which is gone or cannot be read as a checkout + 4 there is no baseline + + `baseline_root` is the repo root the prompt hook stored beside the SHA.""" + if not baseline_sha: + return 4 + if not baseline_root or not isinstance(baseline_root, str): + return 0 + if _same_dir(baseline_root, repo_root): + return 0 + if not os.path.isdir(baseline_root): + return 3 + here, there = _git_dir(repo_root), _git_dir(baseline_root) + if not here or not there: + return 3 + return 1 if _same_dir(here, there) else 2 + + +def baseline_origin_metric(origin, kept): + """The `bl_origin` number on Stop rows: baseline_origin, plus 10 or 20 + (KEPT_*) when the baseline is an earlier turn's.""" + if origin == 4 or kept not in (KEPT_NO_CAPTURE, KEPT_UNREVIEWED): + return origin + return origin + 10 * kept + + def get_baseline_file_content(session_id, file_path, cwd): """Get the content of a file at the baseline SHA. Returns None if unavailable. @@ -165,65 +289,186 @@ def get_baseline_file_content(session_id, file_path, cwd): try: abs_path = os.path.abspath(file_path) cwd_abs = os.path.abspath(cwd) if cwd else os.getcwd() - try: - rel_path = os.path.relpath(abs_path, cwd_abs) - except ValueError: - return None - result = subprocess.run( - [*GIT_CMD, "show", f"{baseline_sha}:{rel_path}"], - cwd=cwd, capture_output=True, timeout=5 - ) - if result.returncode == 0: - return (result.stdout or b"").decode("utf-8", errors="replace") + attempts = [(abs_path, cwd_abs, cwd)] + # The tool's path and the session directory can name the same place + # through different symlinks (/tmp and /var on macOS). The relative + # path between the two spellings then leaves the repo and git finds + # nothing, so the lookup is tried again with both resolved. + real_path, real_cwd = os.path.realpath(abs_path), os.path.realpath(cwd_abs) + if (real_path, real_cwd) != (abs_path, cwd_abs): + attempts.append((real_path, real_cwd, real_cwd)) + for path, start, git_cwd in attempts: + try: + rel_path = os.path.relpath(path, start) + except ValueError: + continue + # `<rev>:<path>` is read from the repo root unless the path starts + # with `./`, and takes forward slashes only. Without both, the + # lookup fails for a session started below the repo root, and on + # Windows for any file that is not directly in the session + # directory. + rel_path = rel_path.replace(os.sep, "/") + result = subprocess.run( + [*GIT_CMD, "show", f"{baseline_sha}:./{rel_path}"], + cwd=git_cwd, capture_output=True, timeout=5 + ) + if result.returncode == 0: + return (result.stdout or b"").decode("utf-8", errors="replace") return None except (subprocess.TimeoutExpired, FileNotFoundError, OSError, ValueError): return None +# How the last capture_git_baseline call got its SHA: 0 a real snapshot, +# 1 the snapshot failed and HEAD was returned, 2 no index copy and HEAD was +# returned. With 1 or 2 the next Stop review cannot tell the turn's changes +# from uncommitted work that was already there. Stored with baseline_sha. +# Telemetry only. +last_baseline_fail = 0 +# Entries in the index the last capture copied, None if unknown. Telemetry only. +last_index_entries = None + + +# Who the snapshot commit is made by. Set on every commit-tree call, so the +# user's own name never appears on a commit they did not make and a repo with +# no identity configured still works. The commit object does land in +# `.git/objects`, but nothing refers to it, like the one `git stash create` +# used to make here: it is prunable from the moment it is written. +_SNAPSHOT_IDENT = { + "GIT_AUTHOR_NAME": "security-guidance", "GIT_AUTHOR_EMAIL": "security-guidance@localhost", + "GIT_COMMITTER_NAME": "security-guidance", "GIT_COMMITTER_EMAIL": "security-guidance@localhost", +} + +# The whole snapshot, not one command: `stash create` was capped at 15 s and a +# per-command cap would let a slow filesystem hold the prompt for far longer. +_SNAPSHOT_BUDGET_S = 15 + + +def _snapshot_commit(cwd, env, head_tree): + """A commit of every tracked change, on the index copy `env` names. + + Returns its SHA, "" when nothing tracked has changed (so the caller uses + HEAD, `head_tree` being HEAD's tree), or None when git refused or the + time budget ran out. + + No `--missing-ok`: git counts a partial clone's promisor blobs as + present, so write-tree neither fetches nor fails on one, and the + existence check it would turn off is worth keeping. + """ + deadline = time.monotonic() + _SNAPSHOT_BUDGET_S + + def run(args, extra=None): + left = deadline - time.monotonic() + if left <= 0: + raise subprocess.TimeoutExpired(args, _SNAPSHOT_BUDGET_S) + return subprocess.run( + [*GIT_CMD, *args], cwd=cwd, capture_output=True, timeout=left, + env={**env, **(extra or {})}, + ) + + # No pathspec: `add -u` has staged the whole work tree, whatever directory + # of it `cwd` is, since git 2.0. `-- :/` would say the same but fails + # outright under a user's GIT_LITERAL_PATHSPECS. + add = run(["add", "-u"]) + if add.returncode != 0: + debug_log(f"Snapshot: add -u failed: {_stderr_of(add)}") + return None + tree = run(["write-tree"]) + sha = (tree.stdout or b"").decode("utf-8", errors="replace").strip() + if tree.returncode != 0 or not sha: + debug_log(f"Snapshot: write-tree failed: {_stderr_of(tree)}") + return None + if sha == head_tree: + return "" + if gitutil.last_dropped_gitlinks: + # A nested repository's entry is gone from the index copy, so the + # tree cannot equal HEAD's even on a clean work tree. Ask whether + # anything but those entries differs, rather than writing a commit + # object on every prompt in a repo with submodules. + same = run(["diff-tree", "--quiet", "--ignore-submodules=all", head_tree, sha]) + if same.returncode == 0: + return "" + commit = run(["-c", "commit.gpgsign=false", "commit-tree", sha, "-p", "HEAD", + "-m", "security-guidance turn baseline"], _SNAPSHOT_IDENT) + out = (commit.stdout or b"").decode("utf-8", errors="replace").strip() + if commit.returncode != 0 or not out: + debug_log(f"Snapshot: commit-tree failed: {_stderr_of(commit)}") + return None + return out + + +def _stderr_of(result): + return (result.stderr or b"").decode("utf-8", errors="replace").strip()[:200] + + def capture_git_baseline(cwd): """ Capture a git ref representing the current working tree state. - Uses `git stash create` which creates a commit object for the current state - (HEAD + uncommitted changes) without modifying the stash list or working tree. + Builds a commit object for the current state (HEAD + uncommitted changes) + without touching the stash list or the working tree: `add -u` stages every + tracked change into a throwaway index copy, `write-tree` turns that into a + tree, and `commit-tree` hangs it off HEAD. Falls back to HEAD if the working tree is clean. Returns the SHA string, or None if not in a git repo or if the repo has no commits. - NOTE: `git stash create` does NOT capture untracked files. UPS pairs this + This was `git stash create`, which does the same thing in one command but + checks the work tree on a single thread, while `add -u` uses git's index + preload (one thread per 500 entries, up to 20). On a filesystem where each + file status is slow — a WSL repository on a Windows drive, a network mount + — that is the whole cost of the prompt: see #6854, where `stash create` + took 4 s on 1,424 files and never finished on 10,000. + + The commands run on a throwaway copy of the index (GIT_INDEX_FILE), so + their `index.lock` lands next to the copy, not in `.git`: they cannot + collide with a concurrent git command, and a timeout kill cannot leave a + lock that blocks the user's next one. GIT_OPTIONAL_LOCKS does not cover + these. Copying the real index without a lock is safe: git replaces it by + atomic rename. + + NOTE: the snapshot does NOT capture untracked files. UPS pairs this SHA with a `_list_untracked()` snapshot stored as `untracked_at_baseline`, and `compute_v2_review_set` subtracts that set so pre-existing untracked - files are not reviewed as Claude-authored. + files are not reviewed as Claude-authored. UPS takes no snapshot while the + Stop review, its only reader, is switched off. """ # stdout is a SHA so text=True is safe on stdout, but a non-ASCII - # filename in `git stash create`'s STDERR warning (e.g. a worktree + # filename in git's STDERR warning (e.g. a worktree # with `Ávila_report.txt` triggers a quotePath/locale warning) would # trip the stderr reader thread on Windows cp1252. Decode both streams # leniently for symmetry with _list_untracked. See #2056. + global last_baseline_fail, last_index_entries + last_baseline_fail = 0 + last_index_entries = None try: - # Check if HEAD exists (i.e., repo has at least one commit) + # HEAD's commit and its tree in one call: the commit is the answer + # for a clean tree, the tree is what the snapshot is compared with. + # A failure here means the repo has no commits — skip the review + # rather than create commits in the user's repo. head_check = subprocess.run( - [*GIT_CMD, "rev-parse", "HEAD"], + [*GIT_CMD, "rev-parse", "HEAD", "HEAD^{tree}"], cwd=cwd, capture_output=True, timeout=5 ) - if head_check.returncode != 0: - # No commits yet — skip review rather than creating commits in the user's repo + head_out = (head_check.stdout or b"").decode("utf-8", errors="replace").split() + if head_check.returncode != 0 or len(head_out) != 2: debug_log("No commits in repo, skipping baseline capture") return None - - result = subprocess.run( - [*GIT_CMD, "stash", "create"], - cwd=cwd, capture_output=True, timeout=15 - ) - sha = (result.stdout or b"").decode("utf-8", errors="replace").strip() - if sha: - return sha - - # Working tree is clean — stash create returns empty. Use HEAD. - result = subprocess.run( - [*GIT_CMD, "rev-parse", "HEAD"], - cwd=cwd, capture_output=True, timeout=5 - ) - sha = (result.stdout or b"").decode("utf-8", errors="replace").strip() - return sha if sha else None + head_sha, head_tree = head_out + + sha = "" + # untracked_paths=[]: a plain copy of the index, no `add -N` pass. + with _temp_index(cwd, untracked_paths=[]) as env: + # env is None when no usable index copy could be made: fall + # back to HEAD. + if env is not None: + last_index_entries = index_entry_count(env["GIT_INDEX_FILE"]) + sha = _snapshot_commit(cwd, env, head_tree) + if sha is None: + last_baseline_fail = 1 + sha = "" + else: + last_baseline_fail = 2 + # No snapshot: nothing tracked has changed, or it could not be taken. + return sha or head_sha except (subprocess.TimeoutExpired, FileNotFoundError, OSError, ValueError) as e: debug_log(f"Failed to capture git baseline: {e}") return None @@ -267,14 +512,19 @@ def capture_git_baseline(cwd): _REVIEWED_SHAS_BASENAME = "sg-reviewed-shas" _REVIEWED_SHAS_CAP = 500 +def _is_full_sha(sha): + return len(sha) == 40 and all(c in "0123456789abcdef" for c in sha) + + def _reviewed_shas_path(repo_root): gd = _git_dir(repo_root) return os.path.join(gd, _REVIEWED_SHAS_BASENAME) if gd else None -def _load_reviewed_shas(repo_root): +def _load_reviewed_shas(repo_root, _gd=None): """Set of full 40-hex shas previously reviewed in this clone.""" - p = _reviewed_shas_path(repo_root) + p = (os.path.join(_gd, _REVIEWED_SHAS_BASENAME) if _gd + else _reviewed_shas_path(repo_root)) if not p or not os.path.exists(p): return set() out = set() @@ -282,7 +532,7 @@ def _load_reviewed_shas(repo_root): with open(p, "r") as f: for line in f: sha = line.split("\t", 1)[0].strip() - if len(sha) == 40 and all(c in "0123456789abcdef" for c in sha): + if _is_full_sha(sha): out.add(sha) except OSError: pass @@ -334,11 +584,429 @@ def _append_reviewed_shas(repo_root, shas, vulns_found=0): pass +# ─── repeated-diff count ──────────────────────────────────────────────────── +# +# Count only. A change that comes back under a new commit (undo and re-commit, +# rebase before the first push) is unknown to the reviewed-shas record and is +# reviewed again. These helpers say how often that happens, by remembering a +# hash of each reviewed hunk next to the reviewed-shas file. Nothing here +# decides whether a review runs, and nothing here waits on another process +# for more than a moment. + +_SEEN_DIFFS_BASENAME = "sg-reviewed-diffs" +_SEEN_DIFFS_TTL_S = 7 * 24 * 3600 +# Past this size the record is cut back to its newest half. +_SEEN_DIFFS_TRIM_BYTES = 1024 * 1024 +_SEEN_DIFFS_MAX_KEYS = 500 +_SEEN_DIFFS_LOCK_TRIES = 10 + +# `diff_seen` values, weakest first. A review reports the weakest of its +# hunks: 3 means every hunk already had a finished review in this clone. +DIFF_SEEN_ERROR = -1 # the record could not be read or written +DIFF_SEEN_NO = 0 # some hunk was never reviewed +DIFF_SEEN_UNFINISHED = 1 # a review of it started and has no verdict (yet) +DIFF_SEEN_OTHER_TREE = 2 # finished, on different repo contents +DIFF_SEEN_SAME_TREE = 3 # finished, on the same tree + + +def _hunk_keys(diff_files): + """One hash per hunk: the path plus the hunk's lines, context included. + + The `@@` line is left out, so a hunk keeps its key when a rebase only + moves it; the same edit next to different code gets a different key. + """ + keys = {} + for path, content in diff_files: + hunks, cur = [], [] + for ln in content.split("\n"): + if ln.startswith("@@"): + hunks.append(cur) + cur = [] + else: + cur.append(ln) + hunks.append(cur) + for body in hunks: + # A file's last hunk ends with the diff's trailing blank line. + while body and not body[-1]: + body.pop() + if not body: + continue + h = hashlib.sha256(path.encode("utf-8", "replace")) + h.update(b"\0") + h.update("\n".join(body).encode("utf-8", "replace")) + keys[h.hexdigest()[:16]] = None + keys = list(keys) + if len(keys) > _SEEN_DIFFS_MAX_KEYS: + # Keep the record small: the hunks past the cap share one key, so + # they only match the same set of hunks again. + rest = hashlib.sha256(" ".join(keys[_SEEN_DIFFS_MAX_KEYS - 1:]).encode("ascii")) + keys = keys[:_SEEN_DIFFS_MAX_KEYS - 1] + [rest.hexdigest()[:16]] + return keys + + +def _seen_lock(fd): + """True: locked. False: another process kept it. None: no flock here.""" + for _ in range(_SEEN_DIFFS_LOCK_TRIES): + try: + if _flock_nb(fd, exclusive=True): + return True + except (ImportError, OSError): + return None + time.sleep(0.05) + return False + + +def _append_diff_row(f, locked, token, done): + """Append one record for the review to `f`, the record opened "a+b".""" + _, keys, tree = token + row = f"{int(time.time())}\t{tree}\t{int(done)}\t{' '.join(keys)}\n".encode("ascii") + size = f.seek(0, os.SEEK_END) + if size: + f.seek(size - 1) + if f.read(1) != b"\n": + row = b"\n" + row # a torn last record must not swallow this one + # Cutting the record back rewrites it: not while another process holds + # the lock. Without flock at all, bounded beats exact. + if locked is not False and size + len(row) > _SEEN_DIFFS_TRIM_BYTES: + f.seek(0) + keep, room = [], _SEEN_DIFFS_TRIM_BYTES // 2 + for ln in reversed(f.read().splitlines(keepends=True)): + room -= len(ln) + if room < 0: + break + keep.append(ln) + f.seek(0) + f.truncate() + row = b"".join(reversed(keep)) + row + f.write(row) + f.flush() + + +def _diff_review_started(repo_root, diff_files, tip="HEAD"): + """Note that a review of `diff_files`, a diff ending at `tip`, is starting. + + Returns (token, metrics). `metrics` is {"diff_seen": <DIFF_SEEN_* code>}: + whether this clone already reviewed every hunk of this diff. `token` goes + to `_diff_review_done` once the review has a verdict. Never raises. + """ + if portable_output() or not diff_files: + # No metrics leave the hook in portable mode, so nothing to count. + return None, {} + try: + gd = _git_dir(repo_root) + keys = _hunk_keys(diff_files) + if not gd or not keys: + raise ValueError("no git dir" if not gd else "no hunks") + token = (os.path.join(gd, _SEEN_DIFFS_BASENAME), keys, + "".join(_full_shas(repo_root, [tip + "^{tree}"])[:1])) + tree = token[2] + now = time.time() + best = dict.fromkeys(keys, DIFF_SEEN_NO) + # Read and note under one lock, so two reviews of one diff that start + # together cannot both miss each other. + with open(token[0], "a+b") as f: + locked = _seen_lock(f.fileno()) + if locked is False: + raise OSError("record is locked") + f.seek(0) + for ln in f.read().decode("ascii", "replace").splitlines(): + row = ln.split("\t") + if len(row) != 4 or not row[0].isdigit() or len(row[0]) > 12: + continue + if now - int(row[0]) > _SEEN_DIFFS_TTL_S: + continue + if row[2] != "1": + seen = DIFF_SEEN_UNFINISHED + elif tree and row[1] == tree: + seen = DIFF_SEEN_SAME_TREE + else: + seen = DIFF_SEEN_OTHER_TREE + for k in row[3].split(" "): + if best.get(k, seen) < seen: + best[k] = seen + _append_diff_row(f, locked, token, done=False) + return token, {"diff_seen": min(best.values())} + except Exception as e: + debug_log(f"Repeated-diff count: not counted ({type(e).__name__}: {e})") + return None, {"diff_seen": DIFF_SEEN_ERROR} + + +def _diff_review_done(token): + """Note that the review `token` came from got a verdict. Never raises.""" + if not token: + return + try: + with open(token[0], "a+b") as f: + _append_diff_row(f, _seen_lock(f.fileno()), token, done=True) + except Exception as e: + debug_log(f"Repeated-diff count: verdict not noted ({type(e).__name__}: {e})") + + +# ─── in-flight review markers ─────────────────────────────────────────────── +# +# A commit or push review records its shas only once it has a verdict, which +# is minutes after it starts. A review that starts in between (the push right +# after a commit, the next commit's reflog scan) would send the same commits +# through a second full review. So a review first takes an exclusive flock on +# `sg-inflight-<sha>`, next to the reviewed-shas file, and holds it until its +# process ends; other reviews leave a locked commit to it. The kernel drops +# the lock however the holder dies, so a killed review blocks nobody. Native +# Windows Python has no flock; there the same lock is taken with LockFileEx. +# Where files cannot be locked (some network filesystems) there are no +# markers and nothing changes. + +_INFLIGHT_PREFIX = "sg-inflight-" +# A marker file nobody holds (a killed review's) is removed once this old. +_INFLIGHT_GC_AGE_S = 3600 +_inflight_held = {} +# Windows refuses to delete a file that is open, so a marker is closed first +# there. Elsewhere it is deleted while still locked, so nobody can lock a +# file that is about to go. +_UNLINK_BEFORE_CLOSE = os.name != "nt" + + +def _full_shas(repo_root, shas): + """Resolve abbreviated commit refs to full 40-hex shas; drops unresolved.""" + out = [] + for s in shas: + try: + r = subprocess.run( + [*GIT_CMD, "rev-parse", "--verify", "-q", s], + cwd=repo_root, capture_output=True, timeout=5, + ) + except (subprocess.TimeoutExpired, FileNotFoundError, OSError): + continue + if r.returncode == 0: + out.append(r.stdout.decode("utf-8", errors="replace").strip()) + return out + + +_win_lock_file_ex = None + + +def _win_lock_nb(fd, exclusive): + """`_flock_nb` for native Windows Python, with LockFileEx. + + Shared and exclusive as with flock, never waits, and Windows drops the + lock when the handle is closed or the holder's process ends. Windows + locks stop other processes reading the locked bytes, so the byte locked + is one far past the end of the (empty) marker: copying a repository while + a review runs still reads the marker. + """ + global _win_lock_file_ex + import errno + try: + import ctypes + import msvcrt + if _win_lock_file_ex is None: + from ctypes import wintypes + + class _Overlapped(ctypes.Structure): + _fields_ = [("Internal", ctypes.c_void_p), ("InternalHigh", ctypes.c_void_p), + ("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD), + ("hEvent", wintypes.HANDLE)] + + fn = ctypes.WinDLL("kernel32", use_last_error=True).LockFileEx + fn.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.DWORD, wintypes.DWORD, + wintypes.DWORD, ctypes.POINTER(_Overlapped)] + fn.restype = wintypes.BOOL + _win_lock_file_ex = (fn, _Overlapped) + except (ImportError, AttributeError, OSError) as e: + raise OSError(errno.ENOLCK, f"no file locking in this Python ({e})") + fn, _Overlapped = _win_lock_file_ex + fail_immediately, exclusive_lock = 0x1, 0x2 + where = _Overlapped(Offset=0, OffsetHigh=0x7FFFFFFF) + if fn(msvcrt.get_osfhandle(fd), fail_immediately | (exclusive_lock if exclusive else 0), + 0, 1, 0, ctypes.byref(where)): + return True + err = ctypes.get_last_error() + if err == 33: # ERROR_LOCK_VIOLATION: another handle holds it + return False + raise OSError(errno.ENOLCK, f"LockFileEx failed (WinError {err})") + + +def _flock_nb(fd, exclusive): + """True when the lock was taken, False when another process holds it. + Raises OSError when the file cannot be locked at all.""" + import errno + if os.name == "nt": + return _win_lock_nb(fd, exclusive) + try: + import fcntl + except ImportError as e: + raise OSError(errno.ENOLCK, f"no file locking in this Python ({e})") + try: + fcntl.flock(fd, (fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) | fcntl.LOCK_NB) + return True + except OSError as e: + if e.errno in (errno.EWOULDBLOCK, errno.EAGAIN): + return False + raise + + +def _again_if_refused(op, *args): + """`op(*args)`, tried a few times where another process having the file + open for a moment refuses it (Windows). Raises the last OSError.""" + import time as _time + tries = 1 if _UNLINK_BEFORE_CLOSE else 3 + for attempt in range(tries): + try: + return op(*args) + except PermissionError: + if attempt == tries - 1: + raise + _time.sleep(0.01) + + +def _drop_marker(p, fd, unlink=True): + """Close a marker this process has open and, with `unlink`, delete it, in + the order this platform needs. Never raises. A delete that stays refused + leaves an unlocked file, which is swept once old.""" + steps = [(_again_if_refused, os.unlink, p)] if unlink else [] + steps.insert(len(steps) if _UNLINK_BEFORE_CLOSE else 0, (os.close, fd)) + for step, *args in steps: + try: + step(*args) + except OSError: + pass + + +def _release_inflight(): + for p, fd in list(_inflight_held.items()): + _drop_marker(p, fd) + _inflight_held.clear() + + +def _sweep_inflight(gd): + """Remove old marker files no review holds.""" + import time as _time + cutoff = _time.time() - _INFLIGHT_GC_AGE_S + for name in os.listdir(gd): + p = os.path.join(gd, name) + if not name.startswith(_INFLIGHT_PREFIX) or p in _inflight_held: + continue + try: + if os.path.getmtime(p) > cutoff: + continue + fd = os.open(p, os.O_RDWR) + except OSError: + continue + try: + unheld = _flock_nb(fd, exclusive=True) + except OSError: + unheld = False + _drop_marker(p, fd, unlink=unheld) + + +def _mark_inflight(repo_root, shas): + """Claim `shas` (full 40-hex) for the review this process is about to + run, until the process ends. Best-effort; never raises. A commit another + live review holds stays with that review.""" + try: + gd = _git_dir(repo_root) + if not gd: + return + _sweep_inflight(gd) + for sha in shas: + p = os.path.join(gd, _INFLIGHT_PREFIX + sha) + if not _is_full_sha(sha) or p in _inflight_held: + continue + try: + fd = _again_if_refused(os.open, p, os.O_RDWR | os.O_CREAT, 0o600) + except OSError: + # One marker that cannot be opened does not cost the others + # their claim. + continue + try: + held = _flock_nb(fd, exclusive=True) + except OSError: + # Cannot lock here at all: leave no file that reads as held. + _drop_marker(p, fd) + return + if held and _UNLINK_BEFORE_CLOSE: + # The inode check: a holder unlinks its marker when it ends, + # and a lock on that unlinked file would claim nothing. Not + # where an open file cannot be unlinked in the first place. + try: + held = os.fstat(fd).st_ino == os.stat(p).st_ino + except OSError: + held = False + if not held: + os.close(fd) + continue + if not _inflight_held: + import atexit + atexit.register(_release_inflight) + _inflight_held[p] = fd + os.utime(p) + except Exception: + pass + + +def _load_inflight_shas(repo_root, shas, _gd=None): + """Which of `shas` another live review in this clone holds right now.""" + out = set() + gd = _gd or _git_dir(repo_root) + if not gd: + return out + for sha in shas: + p = os.path.join(gd, _INFLIGHT_PREFIX + sha) + if not _is_full_sha(sha) or p in _inflight_held: + continue + try: + fd = os.open(p, os.O_RDONLY) + except OSError: + continue + try: + # Shared, so two reviews asking at once do not look like a + # holder to each other. + if not _flock_nb(fd, exclusive=False): + out.add(sha) + except OSError: + pass + os.close(fd) + return out + + +def _reviewed_after_inflight(repo_root, push_range, max_wait_s, poll_s=1.0): + """The reviewed-sha set for a push of `push_range` (oldest first), read + once no other review is still running on the commit the push review + would start from, or after `max_wait_s`. Returns (reviewed, waited_s).""" + import time as _time + gd = _git_dir(repo_root) + start = _time.monotonic() + waited = False + reviewed = _load_reviewed_shas(repo_root, _gd=gd) + while gd: + # Only the first unreviewed commit matters: if nobody is reviewing + # it, the push review covers everything from there on anyway. + first = next((s for s in push_range if s not in reviewed), None) + if first is None: + break + # In-flight before reviewed: a review that ends between the two + # reads then shows up as reviewed rather than as neither. + busy = _load_inflight_shas(repo_root, [first], _gd=gd) + reviewed = _load_reviewed_shas(repo_root, _gd=gd) + if first in reviewed: + continue + if not busy or _time.monotonic() - start >= max_wait_s: + break + waited = True + _time.sleep(poll_s) + return reviewed, (_time.monotonic() - start) if waited else 0.0 + + # ===================================================================== # v2 review-set computation (Stop hook) # ===================================================================== UNTRACKED_BASELINE_CAP = 2000 +# Set in a snapshot that hit the cap, to the time the listing started. NUL +# cannot occur in a path, so the key never collides with a file. +UNTRACKED_CAPPED_AT_KEY = "\0capped_at_ns" +# Coarse filesystem timestamps (1 s on ext3, 2 s on FAT) can stamp a file +# created just after the snapshot as slightly older than it. +_CAPPED_BASELINE_SLACK_NS = 2_000_000_000 def _list_untracked(cwd): @@ -365,6 +1033,7 @@ def _list_untracked(cwd): the holdouts.""" try: repo = _git_toplevel(cwd) or cwd + started_ns = time.time_ns() # core.quotePath=false comes from GIT_CMD globally (see gitutil.py). r = subprocess.run( [*GIT_CMD, "ls-files", "--others", "--exclude-standard", "-z"], @@ -385,6 +1054,10 @@ def _list_untracked(cwd): out[p] = 0 if len(out) >= UNTRACKED_BASELINE_CAP: debug_log(f"_list_untracked: capped at {UNTRACKED_BASELINE_CAP}") + # The files past the cap are not recorded one by one. Record + # when the snapshot was taken so the Stop hook can still tell + # them from files created or edited during the turn. + out[UNTRACKED_CAPPED_AT_KEY] = started_ns break return out except (subprocess.TimeoutExpired, FileNotFoundError, OSError, ValueError) as e: @@ -393,21 +1066,26 @@ def _list_untracked(cwd): debug_log(f"_list_untracked error: {e}") return {} -def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_baseline=None): +def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_baseline=None, + since_ns=None): """v2 diff strategy: derive the review set from git state alone. review_set = (files dirty vs current HEAD, plus files committed this turn when HEAD advanced linearly) ∩ (files whose content differs from the - pre-turn stash baseline). The first term is immune to checkout/pull + pre-turn baseline). The first term is immune to checkout/pull ballooning; the second filters out the user's untouched pre-turn WIP. Falls back to dirty_now alone when no baseline is available. untracked_at_baseline: {repo-root-relative path: mtime_ns} captured at - UPS. `git stash create` doesn't include untracked files, so without this + UPS. The baseline snapshot doesn't include untracked files, so without this snapshot a pre-existing untracked file looks "new since baseline" forever. A file is excluded only if it was untracked at baseline AND its mtime is unchanged — an in-place edit during the turn is still reviewed. + since_ns: for a checkout the prompt hook took no baseline or snapshot of + (the turn moved into it), the time the turn began. A dirty or untracked + file last written before then was already that way and is left out. + Known limitation: a Bash-only turn that's interrupted before Stop fires leaves touched_paths empty, so the next UPS re-baselines past those edits. v1 never reviews Bash-only turns at all, so v2 is no worse there. @@ -430,16 +1108,36 @@ def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_basel if tracked_dirty is None: return [], "HEAD", repo, [], {"dirty_now_count": -1, "changed_since_count": -1, "review_set_count": 0} + capped_at_ns = untracked_at_baseline.get(UNTRACKED_CAPPED_AT_KEY) + def _unchanged_since_baseline(p): base_mtime = untracked_at_baseline.get(p) - if base_mtime is None: - return False try: - return os.stat(os.path.join(repo, p)).st_mtime_ns == base_mtime + st = os.stat(os.path.join(repo, p)) except OSError: return False + if base_mtime is not None: + return st.st_mtime_ns == base_mtime + if not capped_at_ns: + return False + # The snapshot stopped at the cap, so this file may simply be one it + # never reached. Without this, every pre-existing file past the cap + # looks new on every turn and the same files are reviewed again and + # again. A file created or edited during the turn has a later mtime + # or ctime than the snapshot; anything older was already there. + return max(st.st_mtime_ns, st.st_ctime_ns) < capped_at_ns - _CAPPED_BASELINE_SLACK_NS preexisting_unchanged = {p for p in untracked if _unchanged_since_baseline(p)} + if since_ns: + def _written_before_turn(p): + try: + st = os.stat(os.path.join(repo, p)) + except OSError: + return False + return max(st.st_mtime_ns, st.st_ctime_ns) < since_ns - _CAPPED_BASELINE_SLACK_NS + + preexisting_unchanged |= {p for p in untracked if _written_before_turn(p)} + tracked_dirty = {p for p in tracked_dirty if not _written_before_turn(p)} new_untracked = untracked - preexisting_unchanged dirty_now = tracked_dirty | new_untracked @@ -450,11 +1148,11 @@ def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_basel dirty_now |= _git_name_only(repo, f"{head_at_capture}..HEAD") or set() diff_base = head_at_capture - # changed_since: tracked files vs the stash baseline (no temp index — the - # stash never contained untracked files anyway), then union with + # changed_since: tracked files vs the turn baseline (no `add -N` — the + # baseline never contained untracked files anyway), then union with # currently-untracked. The previous `include_untracked=True` arm cost a # full `git add -N .` (slow in large repos) per call to surface - # untracked files in the diff output — but `git diff <stash>` already + # untracked files in the diff output — but `git diff <baseline>` already # lists them as "only in worktree" without that, and we have the explicit # set from status regardless. if baseline_sha: @@ -464,9 +1162,13 @@ def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_basel else: changed_since = None # changed_since is None on missing baseline OR on git error (e.g. the - # dangling stash SHA was pruned). Either way, don't intersect with ∅ — + # dangling baseline SHA was pruned). Either way, don't intersect with ∅ — # that would silently zero the review set. Fall back to dirty_now. review_set = (dirty_now & changed_since) if changed_since is not None else dirty_now + # Kept beside its new name so that the content diff pairs the two. + secret_origins = { + old for new, old in _secret_rename_origins.items() if new in review_set} + review_set = review_set | secret_origins review_paths = [os.path.join(repo, p) for p in sorted(review_set)] untracked_in_review = sorted(new_untracked & review_set) @@ -478,4 +1180,6 @@ def compute_v2_review_set(cwd, baseline_sha, head_at_capture, untracked_at_basel # Only emit when nonzero to stay under the 10-key telemetry cap. if preexisting_unchanged: metrics["preexisting_untracked_excluded"] = len(preexisting_unchanged) + if secret_origins: + metrics["secret_renamed"] = len(secret_origins) return review_paths, diff_base, repo, untracked_in_review, metrics |
