summaryrefslogtreecommitdiff
path: root/plugins/claude-security/scripts/write_scan_meta.py
diff options
context:
space:
mode:
Diffstat (limited to 'plugins/claude-security/scripts/write_scan_meta.py')
-rwxr-xr-xplugins/claude-security/scripts/write_scan_meta.py299
1 files changed, 223 insertions, 76 deletions
diff --git a/plugins/claude-security/scripts/write_scan_meta.py b/plugins/claude-security/scripts/write_scan_meta.py
index 0ff642bf..f5e77f2e 100755
--- a/plugins/claude-security/scripts/write_scan_meta.py
+++ b/plugins/claude-security/scripts/write_scan_meta.py
@@ -9,13 +9,25 @@ the tree's top-level directories, printed as a JSON array on a
symbolic links left out of them. For a codebase scan it also lists the scan
target's tracked files into target-files.json beside the meta file and prints
their count on a `file_count:` line and, for a whole-repository scan, the
-count under each top-level directory on a `dir_file_counts:` line.
+count under each top-level directory on a `dir_file_counts:` line. For a
+changes or commit scan it writes the files the change touches into numbered
+changed-files.<n>.json chunks beside the meta file and prints the change as its
+two commit ids on a `range:` line, the count of those files on a
+`changed_file_count:` line, their added-plus-deleted line count on a
+`diff_line_count:` line and, last, the list itself on a `changed_files:`
+line when it is short enough to hand over (null when it is not).
+
+`--effort max` is recorded as `high`, and a `relay:` line carries the note
+that tells the user so.
Usage:
write_scan_meta.py <run_dir> <scan_root> --mode scan|changes|commit
--effort low|medium|high|max [--scope a,b] [--base <ref>]
[--merge-base <sha>] [--commit <sha>]
+A changes scan names its `--merge-base`; a commit scan names its `--commit`,
+which must be the checked-out commit.
+
Exits 0 on success, 1 on a refusal naming what is wrong (a run directory that
already holds a scan-meta.json is one), 2 on a usage error; the file is
written only on success.
@@ -28,35 +40,30 @@ import argparse
import os
import re
import stat
-import subprocess
import sys
import uuid
from collections import Counter
from datetime import datetime, timezone
+from enum import Enum
from pathlib import Path
-from typing import Literal, NamedTuple, TypedDict
+from typing import NamedTuple, TypedDict
from urllib.parse import quote, unquote, urlsplit
# The lib/ package lives next to this script. Python normally adds a script's own
# directory to the import path, but not under -P or PYTHONSAFEPATH, so we add it here.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from lib import absolute, console, plugin, strictjson
-
-GIT_ENV = dict(os.environ, GIT_TERMINAL_PROMPT="0")
+from lib.git import run as git
+from lib.revision import Revision, change_range
-class Revision(TypedDict, total=False):
- """What was scanned. `versioned` is always present; the rest when in git."""
+class Effort(Enum):
+ """An effort tier, as `--effort` spells it."""
- versioned: bool
- commit: str | None
- parent: str | None
- branch: str | None
- dirty: bool | None
- sparse: Literal[True]
- not_checked_out_dirs: list[str]
- base: str | None
- merge_base: str | None
+ LOW = "low"
+ MEDIUM = "medium"
+ HIGH = "high"
+ MAX = "max"
class Args(argparse.Namespace):
@@ -65,7 +72,7 @@ class Args(argparse.Namespace):
run_dir: str = ""
scan_root: str = ""
mode: str = ""
- effort: str = ""
+ effort: Effort = Effort.MEDIUM
scope: str = ""
base: str | None = None
merge_base: str | None = None
@@ -76,22 +83,27 @@ class MetaError(Exception):
"""A refusal: the command line was well-formed but the run cannot be recorded."""
-def git(cwd: str, *args: str) -> str | None:
- """One read-only git call, prompts suppressed. None on any failure."""
- try:
- out = subprocess.run(
- ["git", "-C", cwd, *args],
- env=GIT_ENV,
- stdout=subprocess.PIPE,
- stderr=subprocess.DEVNULL,
- timeout=30,
- check=False,
- )
- except (OSError, subprocess.SubprocessError):
- return None
- if out.returncode != 0:
- return None
- return out.stdout.decode("utf-8", "surrogateescape").rstrip("\r\n")
+# Paths per changed-files.<n>.json; workflows/scan.js derives the chunk count from the same size.
+CHANGED_FILES_CHUNK = 100
+CHANGED_FILES_INLINE_CHARS = 25_000
+
+
+class ChangedFiles(TypedDict):
+ """One changed-files.<n>.json chunk: its number, the whole change's file count, its files."""
+
+ chunk: int
+ total_files: int
+ files: list[str]
+
+
+class Change(NamedTuple):
+ """A change: its two commits as the `<base>..<commit>` range the scan diffs,
+ the files it touches and its added-plus-deleted line count, None when a
+ binary file leaves the count unknown."""
+
+ range: str
+ files: list[str]
+ lines: int | None
class Extent(NamedTuple):
@@ -170,6 +182,48 @@ def target_files(scan_root: str, scope: list[str]) -> list[str] | None:
})
+def changed_files(scan_root: str, scope: list[str], *, commits: str) -> Change:
+ """The change `commits` names (`<base>..<commit>`) within `scope`, from `git diff-tree`."""
+ numstat = ("diff-tree", "-r", "--no-commit-id", "--numstat", "-z", "-M", "--relative")
+ listing = git(scan_root, *numstat, commits, "--", *scope)
+ if listing is None:
+ msg = f"git could not diff {commits} in {scan_root!r}"
+ raise MetaError(msg)
+ lines: dict[str, int | None] = {}
+ records = iter(listing.split("\0"))
+ for record in records:
+ if not record:
+ continue
+ added, deleted, path = record.split("\t", 2)
+ if not path:
+ next(records, "")
+ path = next(records, "")
+ if path and not path.startswith(plugin.REPORT_DIR_PREFIX):
+ lines[path] = None if "-" in {added, deleted} else int(added) + int(deleted)
+ known = [count for count in lines.values() if count is not None]
+ total = sum(known) if len(known) == len(lines) else None
+ return Change(commits, sorted(lines), total)
+
+
+def change_lines(change: Change | None) -> list[str]:
+ """The range, changed_file_count, diff_line_count and changed_files lines record() prints."""
+ if change is None:
+ return [
+ "range: null",
+ "changed_file_count: null",
+ "diff_line_count: null",
+ "changed_files: null",
+ ]
+ listed = strictjson.text(change.files)
+ handed = listed if len(listed) <= CHANGED_FILES_INLINE_CHARS else "null"
+ return [
+ f"range: {strictjson.text(change.range)}",
+ f"changed_file_count: {len(change.files)}",
+ f"diff_line_count: {strictjson.text(change.lines)}",
+ f"changed_files: {handed}",
+ ]
+
+
def regular_file(path: str) -> bool:
"""Whether `path` is a regular file, judged without following a symbolic link."""
try:
@@ -237,43 +291,89 @@ def worktree_dirty(scan_root: str) -> bool | None:
def capture_revision(scan_root: str, opts: Args) -> Revision:
versioned = git(scan_root, "rev-parse", "--is-inside-work-tree") == "true"
+ if opts.mode != "scan" and not versioned:
+ msg = f"--mode {opts.mode} needs a git repository; {scan_root!r} is not one"
+ raise MetaError(msg)
if opts.mode == "commit":
- if not versioned:
- msg = f"--mode commit needs a git repository; {scan_root!r} is not one"
- raise MetaError(msg)
commit_arg = opts.commit or ""
sha = git(scan_root, "rev-parse", "--verify", "--quiet", commit_arg + "^{commit}")
if not sha:
msg = f"--commit {commit_arg!r} does not resolve to a commit"
raise MetaError(msg)
+ short = sha[: plugin.SHORT_ID_CHARS]
+ parent = git(scan_root, "rev-parse", "--verify", "--quiet", sha + "^") or None
+ if parent is None:
+ msg = (
+ f"the parent of commit {short} is not in this clone (a shallow checkout); "
+ "fetch more history before scanning the commit"
+ if names_a_parent(scan_root, sha)
+ else f"commit {short} is the repository's first and has no parent to "
+ "diff against; scan the codebase at that commit instead"
+ )
+ raise MetaError(msg)
+ if sha != git(scan_root, "rev-parse", "--verify", "--quiet", "HEAD"):
+ msg = (
+ f"commit {short} is not checked out, and a commit scan runs only on the "
+ f"checked-out commit (HEAD); tell the user to check {short} out and ask again, "
+ "and stop: never check it out or add a worktree yourself"
+ )
+ raise MetaError(msg)
return {
"versioned": True,
"commit": sha,
- "parent": git(scan_root, "rev-parse", "--verify", "--quiet", sha + "^") or None,
+ "parent": parent,
"branch": git(scan_root, "rev-parse", "--abbrev-ref", "HEAD"),
"dirty": False,
}
if not versioned:
return {"versioned": False}
+ head = git(scan_root, "rev-parse", "--verify", "--quiet", "HEAD")
revision: Revision = {
"versioned": True,
- "commit": git(scan_root, "rev-parse", "HEAD"),
+ "commit": head,
"branch": git(scan_root, "rev-parse", "--abbrev-ref", "HEAD"),
"dirty": worktree_dirty(scan_root),
}
if opts.mode == "changes":
+ if not head:
+ msg = "HEAD names no commit yet, so there is no change to scan"
+ raise MetaError(msg)
+ base_arg = opts.merge_base or ""
+ merge_base = git(scan_root, "rev-parse", "--verify", "--quiet", base_arg + "^{commit}")
+ if not merge_base:
+ msg = (
+ f"--merge-base {opts.merge_base!r} does not resolve to a commit "
+ "(a shallow clone may not hold it: fetch more history)"
+ )
+ raise MetaError(msg)
+ if git(scan_root, "merge-base", "--is-ancestor", merge_base, head) is None:
+ msg = (
+ f"--merge-base {opts.merge_base!r} is not an ancestor of HEAD, so it is not "
+ "this branch's merge base (a shallow clone cannot compute one: fetch more history)"
+ )
+ raise MetaError(msg)
revision["base"] = opts.base
- revision["merge_base"] = opts.merge_base
+ revision["merge_base"] = merge_base
return revision
+def names_a_parent(scan_root: str, sha: str) -> bool:
+ """Whether the commit object itself records a parent, whatever the clone holds of it."""
+ body = git(scan_root, "cat-file", "commit", sha)
+ if body is None:
+ msg = f"git could not read commit {sha[: plugin.SHORT_ID_CHARS]}, so its parent is unknown"
+ raise MetaError(msg)
+ return any(line.startswith("parent ") for line in body.split("\n\n", 1)[0].splitlines())
+
+
def scoped(entry: str, scan_root: str) -> str:
"""A scope entry relative to the scan root; an absolute spelling of anything else is refused."""
if not absolute.spelled(entry):
return entry
- msg = f"--scope entry {entry!r} is not inside the scan root {scan_root!r}"
+ outside = f"--scope entry {entry!r} is not inside the scan root {scan_root!r}"
if not os.path.isabs(entry):
- raise MetaError(f"{msg}; write ./{entry} to name a directory in the tree")
+ msg = f"{outside}; write ./{entry} to name a directory in the tree"
+ raise MetaError(msg)
literal = os.path.abspath(entry)
parent, name = os.path.split(literal)
try:
@@ -287,30 +387,45 @@ def scoped(entry: str, scan_root: str) -> str:
for resolved in resolutions:
if (relative := absolute.relative(resolved, scan_root)) is not None:
return relative
- raise MetaError(msg)
+ raise MetaError(outside)
+
+
+def tier(word: str) -> Effort:
+ """The tier a word names, whatever its case or the spaces around it; any other is refused."""
+ try:
+ return Effort(word.strip().lower())
+ except ValueError:
+ offered = ", ".join(effort.value for effort in Effort if effort is not Effort.MAX)
+ msg = (
+ f"{word!r} is not an effort tier; the tiers are {offered}. "
+ "Tell the user so and start no scan; do not choose a tier for them"
+ )
+ raise argparse.ArgumentTypeError(msg) from None
def parse_options(argv: list[str]) -> Args:
"""The parsed command line; anything wrong with it is argparse's exit 2."""
- ap = argparse.ArgumentParser(prog="write_scan_meta", allow_abbrev=False)
- ap.add_argument("run_dir")
- ap.add_argument("scan_root")
- ap.add_argument("--mode", required=True, choices=plugin.MODES)
- ap.add_argument("--effort", required=True, choices=["low", "medium", "high", "max"])
- ap.add_argument("--scope")
- ap.add_argument("--base")
- ap.add_argument("--merge-base", dest="merge_base")
- ap.add_argument("--commit")
- opts = ap.parse_args(argv, namespace=Args())
+ parser = argparse.ArgumentParser(prog="write_scan_meta.py", allow_abbrev=False)
+ parser.add_argument("run_dir")
+ parser.add_argument("scan_root")
+ parser.add_argument("--mode", required=True, choices=plugin.MODES)
+ parser.add_argument("--effort", required=True, type=tier)
+ parser.add_argument("--scope")
+ parser.add_argument("--base")
+ parser.add_argument("--merge-base", dest="merge_base")
+ parser.add_argument("--commit")
+ opts = parser.parse_args(argv, namespace=Args())
if opts.mode == "commit" and not opts.commit:
- ap.error("--mode commit requires --commit <sha>")
+ parser.error("--mode commit requires --commit <sha>")
+ if opts.mode == "changes" and not opts.merge_base:
+ parser.error("--mode changes requires --merge-base <sha>")
if not os.path.isdir(opts.run_dir):
- ap.error(f"run directory does not exist: {opts.run_dir}")
+ parser.error(f"run directory does not exist: {opts.run_dir}")
return opts
-def main(argv: list[str]) -> int:
- opts = parse_options(argv)
+def record(opts: Args) -> None:
+ """Record what is scanned into the run directory's scan-meta.json and print its summary."""
# abspath first: "x/.." is x's parent as typed, where realpath alone would follow a symlink x.
run_dir = Path(os.path.realpath(os.path.abspath(opts.run_dir)))
scan_root = os.path.realpath(os.path.abspath(opts.scan_root))
@@ -335,9 +450,18 @@ def main(argv: list[str]) -> int:
tracked = opts.mode == "scan" and extent is not None and extent.tracked
files = target_files(scan_root, scope) if tracked else None
if tracked and files is None:
- sys.stderr.write(f"write_scan_meta: could not list {scan_root}; file_count unknown\n")
+ sys.stderr.write(f"write_scan_meta.py: could not list {scan_root}; file_count unknown\n")
+ change = None
+ if opts.mode != "scan":
+ commits = change_range(revision)
+ if commits is None:
+ msg = "the change's endpoints are unknown"
+ raise MetaError(msg)
+ change = changed_files(scan_root, scope, commits=commits)
if whole_repo and extent is None:
- sys.stderr.write(f"write_scan_meta: could not list {scan_root}; top_level_dirs unknown\n")
+ sys.stderr.write(
+ f"write_scan_meta.py: could not list {scan_root}; top_level_dirs unknown\n"
+ )
top_level, symlinks = (extent.dirs, extent.symlinks) if whole_repo and extent else (None, None)
dir_file_counts = None
if top_level is not None and files is not None:
@@ -345,9 +469,11 @@ def main(argv: list[str]) -> int:
dir_file_counts = {name: per_dir[name] for name in top_level}
if symlinks:
sys.stderr.write(
- "write_scan_meta: root-level symbolic links not followed, "
+ "write_scan_meta.py: root-level symbolic links not followed, "
f"left out of top_level_dirs: {', '.join(symlinks)}\n"
)
+ retired = opts.effort is Effort.MAX
+ effort = Effort.HIGH if retired else opts.effort
meta: dict[str, object] = {
"scan_id": str(uuid.uuid4()),
"started_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(),
@@ -359,7 +485,8 @@ def main(argv: list[str]) -> int:
"agent": f"{plugin.NAME}:{plugin.NAME}",
"mode": opts.mode,
"scope": scope,
- "effort": opts.effort,
+ "effort": effort.value,
+ "asked_for_max": retired,
"model": None,
"revision": revision,
"revision_source": "self-reported",
@@ -371,7 +498,9 @@ def main(argv: list[str]) -> int:
try:
with path.open("x", encoding="utf-8", newline="\n") as out:
out.write(strictjson.text(meta, indent=2) + "\n")
- except FileExistsError as error:
+ except (FileExistsError, PermissionError) as error:
+ if isinstance(error, PermissionError) and not path.is_dir():
+ raise
msg = (
f"{run_dir} already holds a scan's scan-meta.json, so another scan is using this "
"report directory; make a new report directory named for the current time and "
@@ -380,24 +509,42 @@ def main(argv: list[str]) -> int:
raise MetaError(msg) from error
if files is not None:
(run_dir / plugin.TARGET_FILES_NAME).write_bytes((strictjson.text(files) + "\n").encode())
- sys.stdout.write(f"scan-meta.json written: {path}\n")
- sys.stdout.write(f"revision: {revision.get('commit') or 'UNVERSIONED'}\n")
+ if change is not None:
+ for number, start in enumerate(range(0, len(change.files), CHANGED_FILES_CHUNK), start=1):
+ chunk: ChangedFiles = {
+ "chunk": number,
+ "total_files": len(change.files),
+ "files": change.files[start : start + CHANGED_FILES_CHUNK],
+ }
+ (run_dir / f"changed-files.{number}.json").write_bytes(
+ (strictjson.text(chunk, indent=1) + "\n").encode()
+ )
+ print(f"scan-meta.json written: {path}")
+ print(f"revision: {revision.get('commit') or 'UNVERSIONED'}")
if absent is not None:
- listed = strictjson.text(absent)
- sys.stdout.write(f"sparse checkout: top-level directories not checked out: {listed}\n")
- sys.stdout.write(f"top_level_dirs: {strictjson.text(top_level)}\n")
- sys.stdout.write(f"file_count: {strictjson.text(None if files is None else len(files))}\n")
- sys.stdout.write(f"dir_file_counts: {strictjson.text(dir_file_counts)}\n")
- return 0
+ print(f"sparse checkout: top-level directories not checked out: {strictjson.text(absent)}")
+ if retired:
+ print(plugin.RETIRED_MAX_RELAY)
+ print(f"top_level_dirs: {strictjson.text(top_level)}")
+ print(f"file_count: {strictjson.text(None if files is None else len(files))}")
+ print(f"dir_file_counts: {strictjson.text(dir_file_counts)}")
+ for line in change_lines(change):
+ print(line)
-if __name__ == "__main__":
- console.tolerate_undecodable_names()
+def main(argv: list[str]) -> int:
+ opts = parse_options(argv)
try:
- sys.exit(main(sys.argv[1:]))
+ record(opts)
except MetaError as error:
- sys.stderr.write(f"write_scan_meta: {error}\n")
- sys.exit(1)
+ sys.stderr.write(f"write_scan_meta.py: {error}\n")
+ return 1
except OSError as error:
- sys.stderr.write(f"write_scan_meta: could not write the run's output: {error}\n")
- sys.exit(1)
+ sys.stderr.write(f"write_scan_meta.py: could not write the run's output: {error}\n")
+ return 1
+ return 0
+
+
+if __name__ == "__main__":
+ console.tolerate_undecodable_names()
+ sys.exit(main(sys.argv[1:]))