summaryrefslogtreecommitdiff
path: root/plugins/claude-security/scripts/save_result.py
blob: 553142551c1305b27754f78b4ed54048bd337991 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
#!/usr/bin/env python3
"""Record a scan workflow's result in its run directory and say what to do next.

Reads the JSON file the Claude Code runtime writes when a workflow task
completes, folds the result's findings, votes and coverage into
findings.json, votes.json and coverage.json in the run directory (appending
to an earlier run's when this result continues one), writes the candidate
files a further verification run loads, and prints a summary line with the
count at each severity, a `votes:` line with each finding's panel tally, and
a `next:` line: the Workflow call that continues the verification, or the
instruction to write the report. Each finding is recorded at the line of its
file that its quoted code is on, with the line the researcher declared kept
beside it.

Usage:
  save_result.py <output_file> <run_dir>

Exits 0 when the result is recorded or already was, 1 when it could not be
(the `next:` line still says what to do), 2 on a usage error.
Python 3.9-compatible, stdlib only.
"""

from __future__ import annotations

import argparse
import os
import re
import sys
from collections import Counter
from pathlib import Path
from typing import NamedTuple

# The lib/ package lives next to this script. Python normally adds a script's own
# directory to the import path, but not under -P or PYTHONSAFEPATH, so we add it here.
sys.path.insert(0, str(Path(__file__).resolve().parent))
from lib import console, plugin, source, strictjson
from lib.chain import Chain, chain_of, pending_ranks
from lib.finding import (
    Confidence,
    FindingError,
    Panel,
    Severity,
    file_field,
    finding_panels,
    line_number,
    link_field,
    scan_prefix_fits,
    scan_prefix_shaped,
    vote_rounds,
)
from lib.revision import change_range
from lib.strictjson import JsonMap, is_int, is_list, is_map, is_str

# Rows per candidate file; workflows/scan.js derives each file's ranks from the same number.
CHUNK = 25
UNACCOUNTED_ECHO_CAP = 40
CID_RE = re.compile(r"^C([1-9][0-9]*)\Z")
WRITE_REPORT = (
    "next: write CLAUDE-SECURITY-RESULTS.md from findings.json and coverage.json per the "
    "report spec, then run render_report.py"
)


class Records(NamedTuple):
    """A scan's three records and the chain its vote record carries."""

    findings: list[JsonMap]
    votes: JsonMap
    coverage: JsonMap
    chain: Chain

    @property
    def rounds(self) -> JsonMap:
        return vote_rounds(self.votes)

    def severity_counts(self) -> Counter[Severity]:
        """How many findings carry each severity; a word that names no Severity is not counted."""
        return Counter(s for s in map(Severity.of, self.findings) if s is not None)

    def panels(self) -> list[tuple[str, Panel | None]]:
        return finding_panels(self.rounds, [str(f["id"]) for f in self.findings])


class Args(argparse.Namespace):
    """The parsed command line."""

    output_file: str = ""
    run_dir: str = ""


class NotAResultError(Exception):
    """The output file holds no scan result this script can fold; the message names why."""

    fallback: str = (
        "Write the result's findings, votes and coverage from the completion notice to "
        "findings.json, votes.json and coverage.json in the run directory yourself, then "
        + WRITE_REPORT[len("next: ") :]
    )


class CannotContinueError(Exception):
    """The run directory cannot take this result; the message names why."""

    fallback: str = "stop; nothing was recorded, so say so and name the report directory"


def count(record: JsonMap, key: str) -> int:
    """A count the workflow wrote; anything but a non-negative integer cannot be folded."""
    value = record.get(key)
    if not is_int(value) or value < 0:
        msg = f"{key!r} is {value!r}, not a count"
        raise CannotContinueError(msg)
    return value


def texts(record: JsonMap, key: str) -> list[object]:
    """A list the workflow wrote, empty when absent."""
    value = record.get(key)
    return list(value) if is_list(value) else []


def rank_of(row: JsonMap) -> int:
    """A candidate row's rank, from its `cid`."""
    cid = row.get("cid")
    matched = CID_RE.match(cid) if is_str(cid) else None
    if not matched:
        msg = f"a pending row's cid is {cid!r}, not C<rank>"
        raise NotAResultError(msg)
    return int(matched[1])


def scan_settings(run_dir: Path) -> JsonMap:
    """scan-meta.json from the run directory: the settings every run of this scan shares."""
    if run_dir.name != plugin.RUN_DIR_NAME:
        msg = f"{run_dir} is not a {plugin.RUN_DIR_NAME} run directory"
        raise CannotContinueError(msg)
    try:
        meta = strictjson.load(run_dir / "scan-meta.json")
    except (OSError, ValueError) as error:
        msg = f"scan-meta.json cannot be read from {run_dir}: {error}"
        raise CannotContinueError(msg) from error
    if not is_map(meta) or not is_str(meta.get("scan_root")):
        msg = "scan-meta.json names no scan_root"
        raise CannotContinueError(msg)
    return meta


def recorded(run_dir: Path) -> Records | None:
    """The scan's records as an earlier save left them; None before the first."""
    if not os.path.isfile(run_dir / "votes.json"):
        return None
    findings, votes, coverage = (
        strictjson.load(run_dir / name) for name in ("findings.json", "votes.json", "coverage.json")
    )
    if not is_list(findings) or not is_map(votes) or not is_map(coverage):
        msg = "the run directory's records are not the shapes this script wrote"
        raise CannotContinueError(msg)
    kept = [f for f in findings if is_map(f) and is_str(f.get("id"))]
    if len(kept) != len(findings):
        msg = "findings.json holds a finding without an id"
        raise CannotContinueError(msg)
    try:
        chain = chain_of(votes.get("chain"))
    except ValueError as error:
        msg = f"votes.json chain {error}"
        raise CannotContinueError(msg) from error
    return Records(kept, votes, coverage, chain)


def result_in(output_file: Path) -> JsonMap:
    """The `result` object the runtime wrote to the workflow's output file."""
    try:
        output = strictjson.load(output_file)
    except (OSError, ValueError) as error:
        msg = f"the output file cannot be read: {error}"
        raise NotAResultError(msg) from error
    result = output.get("result") if is_map(output) else None
    if not is_map(result):
        msg = "the output file holds no result object"
        raise NotAResultError(msg)
    return result


def records_in(result: JsonMap) -> tuple[Records, dict[int, JsonMap]]:
    """A run's records and its pending rows by rank, narrowed from its result."""
    raw, votes, coverage, pending = (
        result.get(key) for key in ("findings", "votes", "coverage", "pending")
    )
    findings = [f for f in raw if is_map(f) and is_str(f.get("id"))] if is_list(raw) else []
    if not is_list(raw) or len(findings) != len(raw):
        msg = "'findings' is not a list of findings with ids"
        raise NotAResultError(msg)
    if not is_map(votes) or votes.get("provenance") != plugin.VOTES_PROVENANCE:
        msg = "'votes' is not the scan workflow's vote record"
        raise NotAResultError(msg)
    if not is_map(coverage) or not is_str(coverage.get("effort")):
        msg = "'coverage' does not name the effort"
        raise NotAResultError(msg)
    if not is_list(pending):
        msg = "'pending' is not a list of candidate rows"
        raise NotAResultError(msg)
    try:
        chain = chain_of(votes.get("chain"))
    except ValueError as error:
        msg = f"'votes' chain {error}"
        raise NotAResultError(msg) from error

    rows = {rank_of(row): row for row in pending if is_map(row)}
    if len(rows) != len(pending) or sorted(rows) != pending_ranks(chain):
        msg = "'pending' rows are not the ranks the chain hands on"
        raise NotAResultError(msg)
    return Records(findings, votes, coverage, chain), rows


def ancestry(path: str) -> list[str]:
    """`path` and each directory above it, nearest last: a/b/c gives a, a/b, a/b/c."""
    parts = path.split("/")
    return ["/".join(parts[:depth]) for depth in range(1, len(parts) + 1)]


def checked(coverage: JsonMap, meta: JsonMap, run_dir: Path) -> JsonMap:
    """`coverage` carrying, under `research.tree`, its coverage accounts counted against the
    scan target's tracked files; `coverage` as given when the accounts are not checkable."""
    research = coverage.get("research")
    if not is_map(research) or research.get("checkable") is not True:
        return coverage
    listing = run_dir / plugin.TARGET_FILES_NAME
    try:
        listed = strictjson.load(listing)
    except (OSError, ValueError) as error:
        sys.stderr.write(f"save_result.py: coverage account not checked: {error}\n")
        return coverage
    if not is_list(listed):
        sys.stderr.write(f"save_result.py: coverage account not checked: {listing} is not a list\n")
        return coverage
    files = [path for path in listed if is_str(path)]
    names = {name for path in files for name in ancestry(path)}
    scan_prefix = str(meta.get("scan_prefix") or "")
    scan_root = str(meta.get("scan_root") or "").replace("\\", "/").strip("/") + "/"
    names.add("")

    def named(path: str) -> str:
        slashed = path.replace("\\", "/")
        folded = (
            (slashed + "/").removeprefix(root).strip("/") for root in (scan_prefix, scan_root)
        )
        return next((s for s in (path, slashed, *folded) if s in names), path)

    accounts = [account for account in texts(research, "components") if is_map(account)]
    inside = {named(p) for account in accounts for p in texts(account, "paths") if is_str(p)}
    read = {named(p) for account in accounts for p in texts(account, "filesRead") if is_str(p)}
    declared = (e.get("path") for a in accounts for e in texts(a, "notReached") if is_map(e))
    missed = {named(p) for p in declared if is_str(p)}
    assigned = [f for f in files if not inside.isdisjoint(["", *ancestry(f)])]
    unread = [f for f in assigned if f not in read]
    unaccounted = [f for f in unread if missed.isdisjoint(["", *ancestry(f)])]
    tree = {
        "files": len(assigned),
        "read": len(assigned) - len(unread),
        "notReached": len(unread) - len(unaccounted),
        "unaccounted": len(unaccounted),
        "unaccountedPaths": unaccounted[:UNACCOUNTED_ECHO_CAP],
        "outsideComponents": len(files) - len(assigned),
    }
    return {**coverage, "research": {**research, "tree": tree}}


def placed(finding: JsonMap, meta: JsonMap) -> JsonMap:
    """`finding` with its file, change link and line as the render will carry them.

    `line` becomes the line of the file its quoted snippet is on, and the line as it arrived
    moves to `declared_line`. Fields the render would refuse are left as they are.
    """
    finding_id, line, snippet = (
        finding["id"],
        line_number(finding.get("line")),
        finding.get("snippet"),
    )
    scan_root, prefix = meta.get("scan_root"), meta.get("scan_prefix") or ""
    if (
        line is None
        or not (snippet is None or is_str(snippet))
        or not is_str(finding_id)
        or not is_str(scan_root)
        or not is_str(prefix)
        or not scan_prefix_shaped(prefix)
        or not scan_prefix_fits(prefix, scan_root=scan_root)
    ):
        return finding
    try:
        file = file_field(finding, finding_id, scan_root, prefix, meta.get("mode") == "scan")
        via_change = link_field(finding, finding_id, scan_root, prefix)
    except FindingError:
        return {**finding, "line": line, "declared_line": line}
    return {
        **finding,
        **({"file": file} if file == file.strip() else {}),
        **({"via_change": via_change} if via_change else {}),
        "line": source.placed_line(source.read(scan_root, file), line, snippet or ""),
        "declared_line": line,
    }


def report_order(finding: JsonMap) -> tuple[int, int]:
    """Sort key: severity, then confidence, strongest first; unknown values last."""
    severity = Severity.of(finding)
    confidence = str(finding.get("confidence", "")).lower()
    return -(severity.value if severity else 0), -Confidence.__members__.get(confidence, 0)


def merged(scan: Records, run: Records) -> Records:
    """The scan's records with one further verification run folded in."""
    handed = pending_ranks(scan.chain)
    if run.chain["shard"] != scan.chain["shard"] + 1:
        msg = (
            f"this is run {run.chain['shard']} but run {scan.chain['shard']} was the last recorded"
        )
        raise CannotContinueError(msg)
    if run.coverage.get("received") != len(handed):
        received = run.coverage.get("received")
        msg = f"the run was handed {received!r} candidates, not the {len(handed)} pending"
        raise CannotContinueError(msg)
    if run.coverage.get("effort") != scan.coverage.get("effort"):
        msg = "the run's effort is not the scan's"
        raise CannotContinueError(msg)

    findings = scan.findings + [f for f in run.findings if f not in scan.findings]
    ids = [f["id"] for f in findings]
    if len(ids) != len(set(ids)):
        msg = "the run reuses a finding id the scan already has"
        raise CannotContinueError(msg)

    scan_rounds, run_rounds = scan.votes.get("rounds"), run.votes.get("rounds")
    if not is_map(scan_rounds) or not is_map(run_rounds):
        msg = "'rounds' is not an object"
        raise CannotContinueError(msg)
    repanelled = {r.get("candidate") for r in run_rounds.values() if is_map(r)}
    rounds = {
        rid: r
        for rid, r in scan_rounds.items()
        if not (is_map(r) and r.get("continued") is True and r.get("candidate") in repanelled)
    }
    if rounds.keys() & run_rounds.keys():
        msg = "the run reuses a round id the scan already has"
        raise CannotContinueError(msg)

    votes = {
        **scan.votes,
        "panel_votes": count(scan.votes, "panel_votes") + count(run.votes, "panel_votes"),
        "unreviewed_candidate_sites": count(scan.votes, "unreviewed_candidate_sites")
        - len(handed)
        + count(run.votes, "unreviewed_candidate_sites"),
        "rounds": {**rounds, **run_rounds},
        "chain": run.chain,
    }
    coverage = (
        scan.coverage
        if scan.coverage.get("verificationRun") == run.chain["shard"]
        else {
            **scan.coverage,
            "adversarialCasualties": texts(scan.coverage, "adversarialCasualties")
            + texts(run.coverage, "adversarialCasualties"),
            "lostCandidates": texts(scan.coverage, "lostCandidates")
            + texts(run.coverage, "lostCandidates"),
            "severityLowered": texts(scan.coverage, "severityLowered")
            + texts(run.coverage, "severityLowered"),
            "preExisting": texts(scan.coverage, "preExisting") + texts(run.coverage, "preExisting"),
            "outsideScope": texts(scan.coverage, "outsideScope")
            + texts(run.coverage, "outsideScope"),
            "dispatchRefusals": count(scan.coverage, "dispatchRefusals")
            + count(run.coverage, "dispatchRefusals"),
            "continued": run.coverage.get("continued"),
            "verificationRun": run.chain["shard"],
        }
    )
    return Records(sorted(findings, key=report_order), votes, coverage, run.chain)


def write_json(run_dir: Path, name: str, value: object, indent: int = 2) -> None:
    """Write one record through a temporary name, so it is never seen half-written."""
    partial = run_dir / f"{name}.partial"
    partial.write_bytes((strictjson.text(value, indent=indent) + "\n").encode())
    partial.replace(run_dir / name)


def write_records(run_dir: Path, records: Records, rows: dict[int, JsonMap]) -> None:
    """The three records and the next run's candidate files; votes.json last."""
    write_json(run_dir, "findings.json", records.findings)
    write_json(run_dir, "coverage.json", records.coverage)
    shard = records.chain["shard"] + 1
    ranks = sorted(rows)
    for number, start in enumerate(range(0, len(ranks), CHUNK), start=1):
        chunk = {
            "runDir": str(run_dir),
            "shard": shard,
            "chunk": number,
            "candidates": [rows[rank] for rank in ranks[start : start + CHUNK]],
        }
        write_json(run_dir, f"candidates.{shard}.{number}.json", chunk, indent=1)
    write_json(run_dir, "votes.json", records.votes)


def summary(records: Records) -> str:
    """Two lines: where the scan stands, with the count at each severity, then each
    finding's tally, or "?" when its panel is not complete, in findings.json's order.
    """
    counts = records.severity_counts()
    by_severity = ", ".join(f"{counts[severity]} {severity.name}" for severity in Severity)
    tallies = [f"{i} {p['true']}/{p['voters']}" if p else f"{i} ?" for i, p in records.panels()]
    return (
        f"recorded verification run {records.chain['shard']}: {len(records.findings)} findings "
        f"({by_severity}), "
        f"{len(records.rounds)} rounds, "
        f"{len(pending_ranks(records.chain))} pending, "
        f"{len(texts(records.coverage, 'lostCandidates'))} lost\n"
        f"votes: {', '.join(tallies) or 'no findings'}"
    )


def next_step(run_dir: Path, meta: JsonMap, records: Records) -> str:
    """The `next:` line: the Workflow call that panels what is pending, or the report."""
    chain = records.chain
    if not pending_ranks(chain):
        return WRITE_REPORT
    scope, reviewed_range = meta.get("scope"), records.coverage.get("range")
    args = {
        "scanRoot": meta.get("scan_root"),
        "runDir": str(run_dir),
        "mode": meta.get("mode"),
        "effort": meta.get("effort"),
        "scope": scope if is_list(scope) and scope else None,
        "range": reviewed_range if is_str(reviewed_range) else None,
        "verify": {
            "shard": chain["shard"] + 1,
            "idBase": chain["next_id"],
            "pending": chain["pending"],
            "retry": chain["retry"],
        },
    }
    return (
        "next: make this Workflow call exactly as printed, wait for it with keep-waiting.sh "
        "as before, then run save_result.py on its output file\n"
        f'Workflow({{ name: "claude-security:scan", args: {strictjson.text(args)} }})'
    )


def otherwise(run_dir: Path, fallback: str) -> str:
    """The `next:` line when this result could not be recorded."""
    if os.path.isfile(run_dir / "votes.json"):
        return WRITE_REPORT + "; its stamp will name what was not verified"
    return f"next: {fallback}"


def standing(run_dir: Path, meta: JsonMap, scan: Records | None, result: JsonMap) -> Records:
    """The scan's records once `result` is taken into account, written if that changed them.

    `meta` is the scan's settings (scan_settings), which placing a finding reads.
    """
    if result.get("started") is False:
        if scan is None:
            msg = "the workflow did not accept its settings, so no scan ran"
            raise CannotContinueError(msg)
        return scan
    run, rows = records_in(result)
    made_for = result.get("runDir")
    if not is_str(made_for) or os.path.abspath(made_for) != str(run_dir):
        msg = f"the result names {made_for!r} as its run directory, not this one"
        raise CannotContinueError(msg)
    revision = meta.get("revision")
    reviewed_range = run.coverage.get("range")
    recorded_range = change_range(revision) if is_map(revision) else None
    if is_str(reviewed_range) and reviewed_range != recorded_range:
        msg = (
            f"the result reviewed {reviewed_range} but this scan recorded "
            f"{recorded_range or 'no change'}, so it is not a review of the change that was "
            "listed: run the scan again with the range: line write_scan_meta.py prints"
        )
        raise CannotContinueError(msg)
    if scan is None and run.chain["shard"] != 1:
        msg = f"this is run {run.chain['shard']} but nothing is recorded yet"
        raise CannotContinueError(msg)
    if scan is not None and run.chain["shard"] <= scan.chain["shard"]:
        return scan
    run = run._replace(
        findings=[placed(item, meta) for item in run.findings],
        coverage=checked(run.coverage, meta, run_dir),
    )
    records = run if scan is None else merged(scan, run)
    write_records(run_dir, records, rows)
    return records


def save(output_file: Path, run_dir: Path) -> None:
    """Fold the result into the run directory and print where the scan stands."""
    meta = scan_settings(run_dir)
    records = standing(run_dir, meta, recorded(run_dir), result_in(output_file))
    print(summary(records))
    print(next_step(run_dir, meta, records))


def argument_parser() -> argparse.ArgumentParser:
    """The command line: the workflow's output file and the scan's run directory."""
    parser = argparse.ArgumentParser(
        prog="save_result.py",
        description="Record a scan workflow's result in its run directory and say what to do next.",
        allow_abbrev=False,
    )
    parser.add_argument("output_file", help="the file the workflow's completion notice names")
    parser.add_argument("run_dir", help="the scan's run directory")
    return parser


def main(argv: list[str]) -> int:
    args = argument_parser().parse_args(argv, namespace=Args())
    output_file = Path(os.path.abspath(args.output_file))
    # abspath folds ".." without following symlinks, so later checks see this path's own name.
    run_dir = Path(os.path.abspath(args.run_dir))
    try:
        save(output_file, run_dir)
    except (NotAResultError, CannotContinueError) as error:
        sys.stderr.write(f"save_result.py: {error}\n")
        print(otherwise(run_dir, error.fallback))
        return 1
    except (OSError, ValueError) as error:
        sys.stderr.write(f"save_result.py: could not read or write the run's files: {error}\n")
        print(otherwise(run_dir, CannotContinueError.fallback))
        return 1
    return 0


if __name__ == "__main__":
    console.tolerate_undecodable_names()
    sys.exit(main(sys.argv[1:]))