summaryrefslogtreecommitdiff
path: root/plugins/code-modernization/scripts
diff options
context:
space:
mode:
Diffstat (limited to 'plugins/code-modernization/scripts')
-rw-r--r--plugins/code-modernization/scripts/baseline_diff.py809
-rw-r--r--plugins/code-modernization/scripts/build_report.py725
-rw-r--r--plugins/code-modernization/scripts/compare.py487
-rw-r--r--plugins/code-modernization/scripts/make_shards.py271
-rw-r--r--plugins/code-modernization/scripts/proof_pack.py1020
-rw-r--r--plugins/code-modernization/scripts/render_rules.py200
-rw-r--r--plugins/code-modernization/scripts/telemetry.py732
-rw-r--r--plugins/code-modernization/scripts/telemetry.sh129
-rw-r--r--plugins/code-modernization/scripts/tests/test_compare_tolerance.py151
-rw-r--r--plugins/code-modernization/scripts/tests/test_proof_rules_run.py506
-rw-r--r--plugins/code-modernization/scripts/tests/test_proof_tools.py1993
-rw-r--r--plugins/code-modernization/scripts/tests/test_report_next.py43
-rw-r--r--plugins/code-modernization/scripts/tests/test_report_tools.py791
-rw-r--r--plugins/code-modernization/scripts/tests/test_rules_tools.py196
-rw-r--r--plugins/code-modernization/scripts/tests/test_telemetry.py810
-rw-r--r--plugins/code-modernization/scripts/trace_rules.py520
-rw-r--r--plugins/code-modernization/scripts/uplift_checks.py356
17 files changed, 9739 insertions, 0 deletions
diff --git a/plugins/code-modernization/scripts/baseline_diff.py b/plugins/code-modernization/scripts/baseline_diff.py
new file mode 100644
index 00000000..93686737
--- /dev/null
+++ b/plugins/code-modernization/scripts/baseline_diff.py
@@ -0,0 +1,809 @@
+#!/usr/bin/env python3
+"""Compare a fresh test run with the baseline recorded before a same-stack uplift.
+
+ python3 baseline_diff.py <BASELINE.md> --junit DIR_OR_FILE [--junit ...] [--json] [--max-list N]
+
+BASELINE.md (analysis/<system>/BASELINE.md) records how the tests behaved on the OLD version. This
+script reads it and the fresh results, and reports what changed. Nothing is judged by a model.
+
+Baseline, any of these (the most detailed one found is used):
+ a per-test table | Test | Result | one row per test, PASS / FAIL / ERROR / SKIP
+ a per-module table | Module | Pass | Fail | Error | Skip | (the first column may be a module,
+ project or test class; column names are matched, their order does not matter)
+ a totals table | Executed | 946 |, | Passed | 945 |, | Failed | 1 | ...
+ a line target-only: <why the old version could not run here> (nothing to compare)
+
+Measured or typed: a baseline typed by hand proves nothing about the old version. The old version's own evidence is looked for in
+analysis/<system>/baseline/ and at the paths on a "Recorded:" or "Machine-readable:" line of BASELINE.md inside analysis/<system>/ (JUnit or .trx XML, a per-test JSON map of test ids to
+outcomes, or a raw runner log with a known summary line; a file that only holds counts is ignored, and sources that disagree are a conflict). When it exists, it is what the fresh results are compared with,
+and the typed table must agree with it; when it does not, the report says "typed table only".
+
+Fresh results: JUnit-style XML (Maven surefire, Gradle, pytest --junitxml, Ant, jest-junit ...) or
+Visual Studio .trx files. Give a file or a folder; a folder is searched for result files, links are
+never followed, and a file that declares a DTD or entity is refused. A test id is
+"<classname>#<name>"; a repeated id in one file gets a "~N" suffix in document order (the first
+stays plain); the module's own absolute path in a parameterized name is replaced by <MODULE>.
+
+Reported: regressions (passed before, fails now), new failures (a test the baseline never listed),
+tests that still fail as they did, tests that now pass, tests that were skipped and now run, tests
+that no longer run, missing modules, skipped growth and a drop in executed tests. When test names
+drifted inside a class but its counts did not get worse, that is counted as renamed, not missing.
+
+Exit 0 when nothing regressed and something ran; 1 on any regression or new failure, or when no test
+executed; 2 for input that cannot be used. Standard library only.
+"""
+import argparse
+import json
+import os
+import re
+import stat
+import sys
+from xml.parsers import expat
+
+BAD = ("FAIL", "ERROR")
+MAX_FILE, MAX_FILES, MAX_CASES, MAX_DEPTH, KEEP = 200 << 20, 20000, 2000000, 60, 200
+SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv"}
+STATUS = {"pass": "PASS", "passed": "PASS", "ok": "PASS", "success": "PASS", "green": "PASS", "fail": "FAIL", "failed": "FAIL", "failure": "FAIL",
+ "red": "FAIL", "error": "ERROR", "errors": "ERROR", "errored": "ERROR", "skip": "SKIP", "skipped": "SKIP", "ignored": "SKIP",
+ "pending": "SKIP", "disabled": "SKIP"}
+COLUMNS = (("pass", r"pass|passed|passing|ok|successes?"), ("fail", r"fail|failed|failing|failures?"), ("error", r"errors?|errored"),
+ ("skip", r"skip|skipped|ignored|pending|disabled"))
+TOTALS = {"executed": "executed", "test cases": "cases", "passed": "pass", "pass": "pass", "failed": "fail", "failures": "fail", "errors": "error",
+ "skipped": "skip", "ignored": "skip"}
+
+
+def clean(value, limit=200):
+ text = " ".join(re.sub(r"[\x00-\x1f\x7f-\x9f\u2028\u2029]", " ", str(value if value is not None else "")).split())
+ return text if len(text) <= limit else text[:limit - 1] + "…"
+
+
+class InputError(Exception):
+ pass
+
+
+def read_capped(path, cap):
+ """Bytes of a regular file inside the size cap; raises InputError with a plain reason for a link, a huge file or an unreadable one."""
+ try:
+ st = os.lstat(path)
+ if not stat.S_ISREG(st.st_mode):
+ raise InputError("%s is a link or not a regular file" % clean(os.path.basename(path), 80))
+ if st.st_size > cap:
+ raise InputError("%s is larger than %d MB" % (clean(os.path.basename(path), 80), cap >> 20))
+ with open(path, "rb") as fh:
+ return fh.read()
+ except OSError as err:
+ raise InputError("cannot read %s (%s)" % (clean(os.path.basename(path), 80), err.__class__.__name__))
+
+
+# ---------------------------------------------------------------- the baseline
+def cell(text):
+ text = text.replace("\\|", "|").strip()
+ if len(text) > 4 and text.startswith("**") and text.endswith("**"):
+ text = text[2:-2].strip()
+ if len(text) >= 2 and text.startswith("`") and text.endswith("`"):
+ text = text[1:-1]
+ return text
+
+
+def split_row(line):
+ s = line.strip()
+ s = s[1:] if s.startswith("|") else s
+ s = s[:-1] if s.endswith("|") and not s.endswith("\\|") else s
+ return [cell(c) for c in re.split(r"(?<!\\)\|", s[:20000])]
+
+
+def join_cut_rows(lines):
+ """A test name that contains a newline cuts its table row in two; put the row back together (at most 3 extra lines)."""
+ out, i = [], 0
+ while i < len(lines):
+ line, extra = lines[i], 0
+ while (line.lstrip().startswith("|") and not line.rstrip().endswith("|") and extra < 3 and i + 1 < len(lines)
+ and lines[i + 1].strip() and not lines[i + 1].lstrip().startswith(("|", "#"))):
+ i += 1
+ extra += 1
+ line += "\n" + lines[i]
+ out.append(line)
+ i += 1
+ return out
+
+
+def md_tables(text):
+ """(nearest heading, header cells, [row cells]) for every pipe table."""
+ lines, heading = join_cut_rows(text.split("\n")), ""
+ i = 0
+ while i < len(lines):
+ h = re.match(r"#{1,6}[ \t]+(.*)", lines[i][:2000])
+ heading = h.group(1) if h else heading
+ if "|" in lines[i] and i + 1 < len(lines) and re.fullmatch(r"\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*", lines[i + 1][:5000]):
+ rows, j = [], i + 2
+ while j < len(lines) and "|" in lines[j] and lines[j].strip():
+ rows.append(split_row(lines[j]))
+ j += 1
+ yield heading, split_row(lines[i]), rows
+ i = j
+ else:
+ i += 1
+
+
+def to_int(text):
+ t = text.strip().replace(",", "")
+ if t in ("", "-", "\u2013", "\u2014"):
+ return 0
+ return int(t) if re.fullmatch(r"\d{1,12}", t) else None
+
+
+def parse_baseline(text):
+ """-> {targetOnly, why, modules {name: counts}, tests {id: status}, totals {name: n}}."""
+ text = text.replace("\r\n", "\n").replace("\r", "\n")
+ m = re.search(r"(?im)^\W*target-only:[ \t]*(.*)$", text)
+ out = {"targetOnly": bool(m), "why": clean(m.group(1), 300) if m else "", "modules": {}, "tests": {}, "totals": {}, "flaky": [], "approved": {}}
+ for heading, header, rows in md_tables(text):
+ low = [h.lower() for h in header]
+ if re.search(r"(?i)flak", heading): # tests the baseline itself says flip on the same version
+ out["flaky"] += [clean(r[0], 400) for r in rows if r and r[0]]
+ continue
+ if re.search(r"(?i)approved", heading): # differences a person accepted: | Test | Reason |
+ for r in rows:
+ if r and r[0]:
+ out["approved"].setdefault(clean(r[0], 400), clean(r[1] if len(r) > 1 else "", 300))
+ continue
+ cols = {key: next((i for i, h in enumerate(low) if re.fullmatch(rx, h)), None) for key, rx in COLUMNS}
+ result = next((i for i, h in enumerate(low) if re.fullmatch(r"result|status|outcome|verdict", h)), None)
+ if cols["pass"] is not None and any(cols[k] is not None for k in ("fail", "error", "skip")):
+ for row in rows:
+ name, nums = row[0] if row else "", {k: to_int(row[i]) if i is not None and i < len(row) else 0 for k, i in cols.items()}
+ if name and not re.fullmatch(r"(?i)totals?|all|sum", name) and None not in nums.values():
+ out["modules"][name[:300]] = nums
+ elif result is not None:
+ name_col = next((i for i, h in enumerate(low) if re.fullmatch(r"tests?|test name|test id|id|name|cases?|test case", h)), 0)
+ for row in rows:
+ status = STATUS.get(row[result].lower()) if result < len(row) else None
+ if status and name_col < len(row) and row[name_col] and result != name_col:
+ out["tests"][row[name_col][:400]] = status
+ elif len(header) >= 2:
+ for row in rows:
+ label, n = (row[0].lower() if row else ""), to_int(row[1]) if len(row) > 1 else None
+ if label in TOTALS and n is not None:
+ out["totals"][TOTALS[label]] = n
+ if not (out["modules"] or out["tests"] or out["totals"]):
+ counts = {}
+ for word, key in (("pass(?:ed|ing)", "pass"), ("fail(?:ed|ing|ures?)", "fail"), ("skipped", "skip"), ("errors?", "error")):
+ hit = re.search(r"(?i)(\d{1,9})\s+(?:tests?\s+|cases?\s+)?" + word, text)
+ if hit:
+ counts[key] = int(hit.group(1))
+ out["totals"] = counts
+ return out
+
+
+def tally(statuses):
+ c = {"pass": 0, "fail": 0, "error": 0, "skip": 0}
+ for s in statuses:
+ c[s.lower()] += 1
+ c["executed"] = c["pass"] + c["fail"] + c["error"]
+ return c
+
+
+def baseline_counts(base):
+ """(counts, where they came from)."""
+ if base["tests"]:
+ return tally(base["tests"].values()), "per-test table"
+ if base["modules"]:
+ c = {k: sum(v[k] for v in base["modules"].values()) for k in ("pass", "fail", "error", "skip")}
+ c["executed"] = c["pass"] + c["fail"] + c["error"]
+ return c, "per-module table"
+ t = base["totals"]
+ if t:
+ c = {k: t.get(k, 0) for k in ("pass", "fail", "error", "skip")}
+ c["executed"] = t["executed"] if "executed" in t else c["pass"] + c["fail"] + c["error"]
+ return c, "totals"
+ return None, "nothing"
+
+
+# ---------------------------------------------------------------- the fresh results
+def refuse(*_):
+ raise ValueError("it declares a DTD or an entity, which test results never do")
+
+
+def parse_results(path):
+ """[(classname, name, status, skip message given, suite name)] from one JUnit or TRX file. Raises ValueError."""
+ cases, suites, cur, depth = [], [], [], [0]
+ trx = {"Passed": "PASS", "Failed": "FAIL", "Error": "ERROR", "Timeout": "ERROR", "Aborted": "ERROR"}
+
+ def start(tag, attrs):
+ depth[0] += 1
+ if depth[0] > MAX_DEPTH:
+ raise ValueError("the XML is nested more than %d levels deep" % MAX_DEPTH)
+ if tag == "testsuite":
+ suites.append(attrs.get("name", "")[:300])
+ elif tag == "testcase":
+ if len(cases) >= MAX_CASES:
+ raise ValueError("more than %d test cases" % MAX_CASES)
+ flag = (attrs.get("status") or attrs.get("result") or "").lower()
+ cur[:] = [(attrs.get("classname") or (suites[-1] if suites else ""))[:300], attrs.get("name", "")[:400],
+ "SKIP" if flag in ("skipped", "ignored", "notrun", "notexecuted", "disabled") else "PASS", False]
+ elif cur and tag in ("failure", "error", "skipped"):
+ if tag == "skipped":
+ cur[3] = bool(attrs.get("message", "").strip())
+ new = {"failure": "FAIL", "error": "ERROR", "skipped": "SKIP"}[tag]
+ if cur[2] == "PASS" or (cur[2] == "SKIP" and new != "SKIP"):
+ cur[2] = new
+ elif tag == "UnitTestResult":
+ name = attrs.get("testName", "")[:400]
+ head, sep, tail = name.partition("(")
+ klass, dot, method = head.rpartition(".")
+ outcome = attrs.get("outcome", "")
+ cases.append((klass[:300] if dot else "", (method + sep + tail) if dot else name, trx.get(outcome, "SKIP"), False, ""))
+
+ def end(tag):
+ depth[0] -= 1
+ if tag == "testcase" and cur:
+ cases.append((cur[0], cur[1], cur[2], cur[3], suites[-1] if suites else ""))
+ cur.clear()
+ elif tag == "testsuite" and suites:
+ suites.pop()
+
+ parser = expat.ParserCreate()
+ parser.StartDoctypeDeclHandler = refuse
+ parser.EntityDeclHandler = refuse
+ parser.StartElementHandler = start
+ parser.EndElementHandler = end
+ try:
+ with open(path, "rb") as fh:
+ parser.ParseFile(fh)
+ except expat.ExpatError as err:
+ raise ValueError("the XML is not well formed (%s)" % expat.ErrorString(err.code))
+ return cases
+
+
+def result_files(paths):
+ """([files], [problems]) from files and folders. Links are never followed."""
+ found, problems = [], []
+ for p in paths:
+ if os.path.isfile(p) and not os.path.islink(p):
+ found.append(p)
+ elif os.path.isdir(p) and not os.path.islink(p):
+ for base, dirs, names in os.walk(p):
+ dirs[:] = [d for d in sorted(dirs) if d not in SKIP_DIRS and not os.path.islink(os.path.join(base, d))]
+ for n in sorted(names):
+ full = os.path.join(base, n)
+ if os.path.islink(full) or not n.lower().endswith((".xml", ".trx")):
+ continue
+ try:
+ with open(full, "rb") as fh:
+ head = fh.read(4096)
+ except OSError:
+ continue
+ if b"<testsuite" in head or b"<TestRun" in head:
+ found.append(full)
+ if len(found) > MAX_FILES:
+ problems.append("More than %d result files were found; the rest were not read." % MAX_FILES)
+ return found[:MAX_FILES], problems
+ else:
+ problems.append("%s is not a file or folder that can be read." % clean(p, 120))
+ return found, problems
+
+
+def module_label(path):
+ """The module folder a Maven or Gradle report belongs to (its parent of target/ or build/), or ''."""
+ parts = os.path.abspath(path).split(os.sep)
+ for i in range(len(parts) - 2, 1, -1):
+ if parts[i] in ("target", "build") and parts[i + 1] in ("surefire-reports", "failsafe-reports", "test-results"):
+ return os.sep.join(parts[:i]) or os.sep
+ return ""
+
+
+def read_results(paths):
+ """-> {tests {id: status}, skipReasons {id: bool}, classes, modules, suites, files, unreadable [str], problems [str], oldest, newest}."""
+ files, problems = result_files(paths)
+ out = {"tests": {}, "skipReasons": {}, "classes": {}, "modules": {}, "suites": {}, "files": 0, "unreadable": [], "problems": problems, "oldest": None, "newest": None}
+ for path in files:
+ label = clean(os.path.basename(path), 100)
+ try:
+ if os.path.getsize(path) > MAX_FILE:
+ raise ValueError("the file is larger than %d MB" % (MAX_FILE >> 20))
+ cases = parse_results(path)
+ except (OSError, ValueError) as err:
+ out["unreadable"].append("%s: %s" % (label, clean(err, 120)))
+ continue
+ root = module_label(path)
+ roots = sorted({root, os.path.realpath(root), root[len("/private"):] if root.startswith("/private/") else root} - {os.sep}, key=len, reverse=True) if root else []
+ seen, mtime = {}, os.path.getmtime(path)
+ out["files"] += 1
+ out["oldest"] = mtime if out["oldest"] is None else min(out["oldest"], mtime)
+ out["newest"] = mtime if out["newest"] is None else max(out["newest"], mtime)
+ for klass, name, status, has_reason, suite in cases:
+ for r in roots:
+ name = name.replace(r, "<MODULE>")
+ name = re.sub(r"(test)\d+(\.tmp)", r"\1<N>\2", name)
+ base = "%s#%s" % (klass, name) if klass else name
+ n = seen.get(base, 0)
+ seen[base] = n + 1
+ tid = base if n == 0 else "%s~%d" % (base, n)
+ out["tests"][tid] = status
+ out["skipReasons"][tid] = has_reason
+ for key, store in ((klass, out["classes"]), (root and os.path.basename(root), out["modules"]), (suite, out["suites"])):
+ if key:
+ store.setdefault(key, []).append(status)
+ for store in ("classes", "modules", "suites"):
+ out[store] = {k: tally(v) for k, v in out[store].items()}
+ return out
+
+
+# ---------------------------------------------------------------- raw runner logs (a saved run's own summary lines)
+LOG_CAP, LOG_LINE = 16 << 20, 2000
+NUM_WORD = re.compile(r"(\d+)\s+([A-Za-z]+)")
+
+
+def words(text):
+ """{'passed': 10, 'failed': 2} from '2 failed, 10 passed' style text."""
+ out = {}
+ for n, w in NUM_WORD.findall(text[:LOG_LINE]):
+ out[w.lower()] = out.get(w.lower(), 0) + int(n)
+ return out
+
+
+def _maven(m):
+ run, f, e, sk = (int(g) for g in m.groups())
+ return max(0, run - f - e - sk), f + e, sk
+
+
+def _pytest(m):
+ w = words(m.group(1))
+ return w.get("passed", 0) + w.get("xpassed", 0) + w.get("xfailed", 0), w.get("failed", 0) + w.get("error", 0) + w.get("errors", 0), w.get("skipped", 0)
+
+
+def _jest(m):
+ w = words(m.group(1))
+ return w.get("passed", 0), w.get("failed", 0), w.get("skipped", 0)
+
+
+def _php(m):
+ w = {k.lower(): int(v) for k, v in re.findall(r"([A-Za-z]+):\s*(\d+)", m.group(1)[:LOG_LINE])}
+ failed, skipped = w.get("failures", 0) + w.get("errors", 0), w.get("skipped", 0) + w.get("incomplete", 0)
+ return max(0, w.get("tests", 0) - failed - skipped), failed, skipped
+
+
+def _ctest(m):
+ failed, total = int(m.group(1)), int(m.group(2))
+ return max(0, total - failed), failed, 0
+
+
+def _gradle(m):
+ total, failed, skipped = int(m.group(1)), int(m.group(2) or 0), int(m.group(3) or 0)
+ return max(0, total - failed - skipped), failed, skipped
+
+
+# runner, a line pattern, and how to read (passed, failed, skipped) from a match. Every pattern is anchored and bounded.
+LOG_LINES = (
+ ("maven", re.compile(r"^(?:\[\w+\]\s*)?Tests run: (\d+), Failures: (\d+), Errors: (\d+), Skipped: (\d+)\s*$"), _maven),
+ ("gradle", re.compile(r"^(\d+) tests? completed(?:, (\d+) failed)?(?:, (\d+) skipped)?\s*$"), _gradle),
+ ("cargo", re.compile(r"^test result: (?:ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored;"), lambda m: (int(m.group(1)), int(m.group(2)), int(m.group(3)))),
+ ("pytest", re.compile(r"^(?:=+ )?((?:\d+ [a-z]+(?:, )?)+) in [\d.]+s(?: \([\d:]+\))?(?: =+)?\s*$"), _pytest),
+ ("pytest", re.compile(r"^(?:=+ )?no tests ran in [\d.]+s"), lambda m: (0, 0, 0)),
+ ("go test", re.compile(r"^\s*--- (PASS|FAIL|SKIP): \S"), lambda m: {"PASS": (1, 0, 0), "FAIL": (0, 1, 0), "SKIP": (0, 0, 1)}[m.group(1)]),
+ ("go test -json", re.compile(r'^\{.*"Action":"(pass|fail|skip)".*"Test":"[^"]+"'), lambda m: {"pass": (1, 0, 0), "fail": (0, 1, 0), "skip": (0, 0, 1)}[m.group(1)]),
+ ("dotnet test", re.compile(r"^(?:Passed|Failed)!\s+-\s+Failed:\s+(\d+),\s+Passed:\s+(\d+),\s+Skipped:\s+(\d+)"), lambda m: (int(m.group(2)), int(m.group(1)), int(m.group(3)))),
+ ("jest", re.compile(r"^\s*Tests:\s+(.*\b\d+ total)\s*$"), _jest),
+ ("vitest", re.compile(r"^\s*Tests\s{2,}(.*)\(\d+\)\s*$"), _jest),
+ ("ctest", re.compile(r"^\d+% tests passed, (\d+) tests? failed out of (\d+)"), _ctest),
+ ("phpunit", re.compile(r"^OK \((\d+) tests?, \d+ assertions?\)"), lambda m: (int(m.group(1)), 0, 0)),
+ ("phpunit", re.compile(r"^(Tests: \d+, Assertions: \d+.*?)\.?\s*$"), _php),
+)
+UNITTEST_RAN = re.compile(r"^Ran (\d+) tests? in [\d.]+s\s*$")
+UNITTEST_END = re.compile(r"^(?:OK|FAILED)(?: \(([^)]*)\))?\s*$")
+
+
+def parse_runner_log(text):
+ """(executed, failed, skipped, runners) from the runner's own summary lines, summed over the log; None when no line is recognised.
+ Recognised: Maven/Gradle, cargo, pytest, unittest, go test (-v or -json), dotnet test, jest, vitest, ctest, phpunit."""
+ passed = failed = skipped = 0
+ seen, ran = [], None
+ for line in text.split("\n")[:3000000]:
+ line = line.rstrip("\r")[:LOG_LINE]
+ if ran is not None and line.strip():
+ m = UNITTEST_END.match(line)
+ if m:
+ w = {k: int(v) for k, v in re.findall(r"(\w+)=(\d+)", (m.group(1) or "")[:LOG_LINE])}
+ f, sk = w.get("failures", 0) + w.get("errors", 0), w.get("skipped", 0)
+ passed, failed, skipped = passed + max(0, ran - f - sk), failed + f, skipped + sk
+ seen.append("unittest")
+ ran = None
+ m = UNITTEST_RAN.match(line)
+ if m:
+ ran = int(m.group(1))
+ continue
+ for name, rx, read in LOG_LINES:
+ m = rx.match(line)
+ if m:
+ p, f, sk = read(m)
+ passed, failed, skipped = passed + p, failed + f, skipped + sk
+ seen.append(name)
+ break
+ return (passed + failed, failed, skipped, sorted(set(seen))) if seen else None
+
+
+def read_logs(paths):
+ """Saved runner logs -> {"files", "executed", "failed", "skipped", "runners", "oldest", "newest", "unrecognised"}. A log without a summary line of a known runner is no evidence."""
+ out = {"files": 0, "executed": 0, "failed": 0, "skipped": 0, "runners": [], "oldest": None, "newest": None, "unrecognised": []}
+ for path in paths:
+ try:
+ st = os.lstat(path)
+ if not stat.S_ISREG(st.st_mode) or st.st_size > LOG_CAP:
+ out["unrecognised"].append("%s (a link, not a file, or larger than %d MB)" % (clean(os.path.basename(path), 80), LOG_CAP >> 20))
+ continue
+ with open(path, "rb") as fh:
+ got = parse_runner_log(fh.read(LOG_CAP).decode("utf-8", "replace"))
+ except OSError:
+ out["unrecognised"].append("%s (could not be read)" % clean(os.path.basename(path), 80))
+ continue
+ if got is None:
+ out["unrecognised"].append("%s (no summary line of a runner this script knows)" % clean(os.path.basename(path), 80))
+ continue
+ out["files"] += 1
+ out["executed"], out["failed"], out["skipped"] = out["executed"] + got[0], out["failed"] + got[1], out["skipped"] + got[2]
+ out["runners"] = sorted(set(out["runners"]) | set(got[3]))
+ out["oldest"] = st.st_mtime if out["oldest"] is None else min(out["oldest"], st.st_mtime)
+ out["newest"] = st.st_mtime if out["newest"] is None else max(out["newest"], st.st_mtime)
+ return out
+
+
+# ---------------------------------------------------------------- measured baselines: result files, raw logs, machine-readable results
+EVIDENCE_EXT = (".xml", ".trx", ".json", ".txt", ".log", ".out")
+PATH_TOKEN = re.compile(r"[A-Za-z0-9_.@+~-]+(?:/[A-Za-z0-9_.@+~-]+)*/?")
+
+
+RECORDED = re.compile(r"^[ \t>*-]*\**[ \t]*(?:recorded|machine-readable)\b[^:\n]{0,40}:", re.I)
+
+
+def evidence_paths(baseline_path, text):
+ """Where the old version's measured results are kept: the folder analysis/<system>/baseline/, and the paths on a line of BASELINE.md
+ that starts with "Recorded:" or "Machine-readable:" (a folder, or a file ending in .xml .trx .json .txt .log .out). Paths mentioned
+ anywhere else in the file are prose, not evidence. A path with a `..`, a link anywhere below the analysis folder, or a place outside
+ analysis/<system>/ is never used. -> the existing paths, at most 50."""
+ adir = os.path.dirname(os.path.abspath(baseline_path))
+ system, real_adir, found, seen = os.path.basename(adir), os.path.realpath(adir), [], set()
+
+ def add(comps):
+ cur = adir
+ for c in comps:
+ cur = os.path.join(cur, c)
+ if os.path.islink(cur):
+ return
+ if not os.path.lexists(cur):
+ return
+ real = os.path.realpath(cur)
+ try:
+ ok = os.path.commonpath([real_adir, real]) == real_adir
+ except ValueError:
+ ok = False
+ if ok and real not in seen and (os.path.isdir(cur) or os.path.isfile(cur)):
+ seen.add(real)
+ found.append(cur)
+
+ add(["baseline"])
+ for line in text.split("\n")[:20000]:
+ if not RECORDED.match(line[:4000]):
+ continue
+ for m in PATH_TOKEN.finditer(line[:4000]):
+ tok = m.group(0)
+ if not (tok.lower().endswith(EVIDENCE_EXT) or tok.endswith("/")):
+ continue
+ comps = [c for c in tok.split("/") if c]
+ if comps and comps[0] == "analysis":
+ comps = comps[2:] if len(comps) >= 3 and comps[1] == system else []
+ if comps and not any(c in (".", "..") for c in comps):
+ add(comps)
+ if len(found) >= 50:
+ return found
+ return found
+
+
+def parse_results_json(data):
+ """{test id: status} from a machine-readable per-test results document, or None. Recognised: {"test id": "PASS", ...}, {"tests": ...}
+ and [{"name": ..., "status": ...}, ...]. A document that only holds counts ({"PASS": 10, "FAIL": 1}) is not evidence of any test: anyone
+ can type one, so it is ignored."""
+ def status(v):
+ return STATUS.get(re.sub(r"[^a-z]", "", v.lower())) if isinstance(v, str) else None
+
+ obj = data["tests"] if isinstance(data, dict) and isinstance(data.get("tests"), (dict, list)) else data
+ tests = {}
+ if isinstance(obj, dict):
+ items = list(obj.items())[:MAX_CASES]
+ good = {str(k)[:400]: status(v) for k, v in items if status(v)}
+ if good and len(good) >= 0.9 * len(items):
+ tests = good
+ elif isinstance(obj, list):
+ for e in obj[:MAX_CASES]:
+ if isinstance(e, dict):
+ name = next((e[k] for k in ("test", "id", "name", "fullName", "fullname") if isinstance(e.get(k), str) and e[k]), None)
+ st = next((status(e[k]) for k in ("status", "result", "outcome") if status(e.get(k))), None)
+ if name and st:
+ tests.setdefault(name[:400], st)
+ return tests or None
+
+
+def read_baseline_evidence(paths):
+ """Read what evidence_paths found: JUnit/TRX XML, machine-readable per-test JSON, and raw runner logs with a known summary line.
+ Two sources that disagree are reported as conflicts, never resolved by taking the first.
+ -> {"tests": {id: status} or None, "counts": {...} or None (a log alone), "sources", "considered", "ignored", "unreadable", "conflicts", "measured"}."""
+ out = {"tests": None, "counts": None, "sources": [], "considered": [], "ignored": [], "unreadable": [], "conflicts": [], "measured": False}
+ files = []
+ for p in paths:
+ if os.path.islink(p):
+ continue
+ if os.path.isdir(p):
+ for base, dirs, names in os.walk(p, followlinks=False):
+ dirs[:] = [d for d in sorted(dirs) if d not in SKIP_DIRS and not os.path.islink(os.path.join(base, d))]
+ files += [os.path.join(base, n) for n in sorted(names) if n.lower().endswith(EVIDENCE_EXT) and not os.path.islink(os.path.join(base, n))]
+ if len(files) >= 2000:
+ break
+ elif p.lower().endswith(EVIDENCE_EXT):
+ files.append(p)
+ files = files[:2000]
+ out["considered"] = [clean(os.path.basename(f), 80) for f in files[:20]]
+ cat = lambda st: "BAD" if st in BAD else st # noqa: E731
+ tests, conflicts = {}, []
+
+ def merge(more, label):
+ for k, v in more.items():
+ if k not in tests:
+ tests[k] = v
+ elif cat(tests[k]) != cat(v):
+ conflicts.append("%s: %s in one source, %s in %s" % (clean(k, 100), tests[k], v, label))
+
+ xml = [f for f in files if f.lower().endswith((".xml", ".trx"))]
+ if xml:
+ got = read_results(xml)
+ out["unreadable"] += got["unreadable"][:5]
+ if got["files"]:
+ merge(got["tests"], "the result files")
+ out["sources"].append({"kind": "result files", "files": got["files"], "tests": len(got["tests"])})
+ for f in [f for f in files if f.lower().endswith(".json")]:
+ name = clean(os.path.basename(f), 80)
+ try:
+ t = parse_results_json(json.loads(read_capped(f, MAX_FILE).decode("utf-8-sig")))
+ except (InputError, ValueError, RecursionError) as err:
+ out["unreadable"].append("%s: %s" % (name, clean(err, 100)))
+ continue
+ if t:
+ merge(t, name)
+ out["sources"].append({"kind": "machine-readable results", "files": 1, "tests": len(t), "name": name})
+ else:
+ out["ignored"].append(name) # counts only, or no test entries: nothing that can be checked test by test
+ logs = read_logs([f for f in files if f.lower().endswith((".txt", ".log", ".out"))])
+ if logs["files"]:
+ out["sources"].append({"kind": "runner log", "files": logs["files"], "executed": logs["executed"], "runners": logs["runners"]})
+ if tests:
+ c = tally(tests.values())
+ if c["executed"] != logs["executed"] or c["fail"] + c["error"] != logs["failed"]:
+ conflicts.append("the runner log says %d executed and %d failed; the per-test results say %d and %d" % (logs["executed"], logs["failed"], c["executed"], c["fail"] + c["error"]))
+ out["tests"] = tests or None
+ out["counts"] = {"executed": logs["executed"], "failed": logs["failed"], "skipped": logs["skipped"]} if logs["files"] and not tests else None
+ out["conflicts"] = conflicts[:20]
+ out["measured"] = bool(tests or out["counts"])
+ return out
+
+
+def assess_baseline(base, measured):
+ """Is BASELINE.md measured or only typed? -> {"kind": "target-only" | "measured" | "typed" | "none", "sources", "disagree", "examples", ...}.
+ A typed table is measured only when a result file, a machine-readable results file or a raw log that this script parsed backs it, and
+ agrees with it (the same tests with the same outcomes, or the same executed and failed counts)."""
+ typed = bool(base["tests"] or base["modules"] or base["totals"])
+ info = {"kind": "typed" if typed else "none", "sources": [], "disagree": 0, "examples": [], "conflict": 0, "conflictExamples": [], "considered": list((measured or {}).get("considered", []))[:20],
+ "ignored": list((measured or {}).get("ignored", []))[:10], "unreadable": list((measured or {}).get("unreadable", []))[:5], "measuredTests": 0,
+ "measuredExecuted": None, "measuredFailed": None}
+ if base["targetOnly"]:
+ info["kind"] = "target-only"
+ return info
+ if not measured or not measured["measured"]:
+ return info
+ info["kind"], info["sources"] = "measured", list(measured["sources"])[:10]
+ info["conflict"] = len(measured.get("conflicts", []))
+ info["conflictExamples"] = [clean(c, 200) for c in measured.get("conflicts", [])[:3]]
+ cat = lambda st: "BAD" if st in BAD else st # noqa: E731
+ typed_counts, _ = baseline_counts(base)
+ if measured["tests"] is not None:
+ c = tally(measured["tests"].values())
+ info.update(measuredTests=len(measured["tests"]), measuredExecuted=c["executed"], measuredFailed=c["fail"] + c["error"])
+ if base["tests"]:
+ bad = [t for t, st in base["tests"].items() if cat(measured["tests"].get(t, "")) != cat(st)]
+ info["disagree"], info["examples"] = len(bad), [clean(t, 120) for t in bad[:5]]
+ elif typed_counts and (typed_counts["executed"] != c["executed"] or typed_counts["fail"] + typed_counts["error"] != c["fail"] + c["error"]):
+ info["disagree"], info["examples"] = 1, ["typed: %d executed, %d failed; measured: %d executed, %d failed" % (
+ typed_counts["executed"], typed_counts["fail"] + typed_counts["error"], c["executed"], c["fail"] + c["error"])]
+ else:
+ m = measured["counts"]
+ info.update(measuredExecuted=m["executed"], measuredFailed=m["failed"])
+ if typed_counts and (typed_counts["executed"] != m["executed"] or typed_counts["fail"] + typed_counts["error"] != m["failed"]):
+ info["disagree"], info["examples"] = 1, ["typed: %d executed, %d failed; measured: %d executed, %d failed" % (
+ typed_counts["executed"], typed_counts["fail"] + typed_counts["error"], m["executed"], m["failed"])]
+ totals = base["totals"] # a Totals table typed beside the rows must say what the evidence says
+ ex, fl = info["measuredExecuted"], info["measuredFailed"]
+ if totals and ex is not None and (("executed" in totals and totals["executed"] != ex) or (("fail" in totals or "error" in totals) and totals.get("fail", 0) + totals.get("error", 0) != fl)):
+ info["disagree"] += 1
+ info["examples"].append("the Totals table says %s executed and %s failed; measured: %d and %d" % (totals.get("executed", "?"), totals.get("fail", 0) + totals.get("error", 0), ex, fl))
+ return info
+
+
+# ---------------------------------------------------------------- the comparison
+def keep(items):
+ return items[:KEEP]
+
+
+def compare(base, fresh):
+ """-> the report dict (see the module docstring)."""
+ counts_b, source = baseline_counts(base)
+ counts_f = tally(fresh["tests"].values())
+ if base.get("measuredTests"):
+ source = "measured results"
+ rep = {"baseline": {"targetOnly": base["targetOnly"], "why": base["why"], "source": source, "counts": counts_b, "assessment": base.get("assessment")},
+ "fresh": {"files": fresh["files"], "counts": counts_f, "unreadable": fresh["unreadable"][:20], "unreadableCount": len(fresh["unreadable"]),
+ "oldest": fresh["oldest"], "newest": fresh["newest"]},
+ "regressions": [], "flaky": [], "newFailures": [], "stillFailing": [], "fixed": [], "newlySkipped": [], "missing": [], "renamed": 0,
+ "missingModules": [], "moduleDiffs": [], "executedDrop": None, "skippedGrowth": None, "problems": list(fresh["problems"])}
+ bt, ft, flaky = base["tests"], fresh["tests"], set(base.get("flaky", []))
+ if bt:
+ missing = [t for t in bt if t not in ft]
+ added = [t for t in ft if t not in bt]
+ for t, b in bt.items():
+ f = ft.get(t)
+ if f is None:
+ continue
+ if b == "PASS" and f in BAD:
+ (rep["flaky"] if t in flaky else rep["regressions"]).append({"id": t, "before": b, "now": f})
+ elif b in BAD and f in BAD:
+ rep["stillFailing"].append(t)
+ elif b in BAD and f == "PASS":
+ rep["fixed"].append(t)
+ elif b == "PASS" and f == "SKIP":
+ rep["newlySkipped"].append(t)
+ elif b == "SKIP" and f in BAD:
+ rep["newFailures"].append(t)
+ by_class = {}
+ for t in missing:
+ by_class.setdefault(t.split("#", 1)[0], [[], []])[0].append(t)
+ for t in added:
+ by_class.setdefault(t.split("#", 1)[0], [[], []])[1].append(t)
+ drifting = {k for k, (gone, new) in by_class.items() if gone and new}
+ grouped_b, grouped_f = {}, {}
+ for src, dst in ((bt, grouped_b), (ft, grouped_f)):
+ for t, v in src.items():
+ if t.split("#", 1)[0] in drifting:
+ dst.setdefault(t.split("#", 1)[0], []).append(v)
+ for klass, (gone, new) in by_class.items():
+ b_counts, f_counts = tally(grouped_b.get(klass, [])), tally(grouped_f.get(klass, []))
+ if gone and new and f_counts["executed"] >= b_counts["executed"] and f_counts["fail"] + f_counts["error"] <= b_counts["fail"] + b_counts["error"]:
+ rep["renamed"] += len(gone)
+ else:
+ rep["missing"] += gone
+ rep["newFailures"] += [t for t in new if ft[t] in BAD]
+ for name, b in base["modules"].items():
+ f = fresh["classes"].get(name) or fresh["modules"].get(name) or fresh["suites"].get(name)
+ if f is None:
+ rep["missingModules"].append(name)
+ continue
+ problems = []
+ if f["executed"] < b["pass"] + b["fail"] + b["error"]:
+ problems.append("%d test(s) executed, %d before" % (f["executed"], b["pass"] + b["fail"] + b["error"]))
+ if f["skip"] > b["skip"]:
+ problems.append("%d skipped, %d before" % (f["skip"], b["skip"]))
+ if not bt and f["fail"] + f["error"] > b["fail"] + b["error"]:
+ rep["regressions"].append({"id": name + " (all its tests)", "before": "%d failing" % (b["fail"] + b["error"]), "now": "%d failing" % (f["fail"] + f["error"])})
+ if problems:
+ rep["moduleDiffs"].append({"module": name, "problems": problems})
+ if counts_b:
+ if counts_f["executed"] < counts_b["executed"]:
+ rep["executedDrop"] = {"before": counts_b["executed"], "now": counts_f["executed"]}
+ if counts_f["skip"] > counts_b["skip"]:
+ rep["skippedGrowth"] = {"before": counts_b["skip"], "now": counts_f["skip"]}
+ rep["approved"] = []
+ for key, label in (("regressions", "passed before, fails now"), ("newFailures", "not in the baseline, fails now")):
+ kept = []
+ for item in rep[key]:
+ tid = item["id"] if isinstance(item, dict) else item
+ if tid in base.get("approved", {}):
+ rep["approved"].append({"id": tid, "reason": base["approved"][tid], "was": label})
+ else:
+ kept.append(item)
+ rep[key] = kept
+ for key in ("regressions", "flaky", "newFailures", "stillFailing", "fixed", "newlySkipped", "missing", "missingModules", "moduleDiffs", "approved"):
+ rep[key + "Count"], rep[key] = len(rep[key]), keep(rep[key])
+ rep["ok"] = counts_f["executed"] > 0 and not rep["regressionsCount"] and not rep["newFailuresCount"]
+ return rep
+
+
+def render(rep, limit=10):
+ b, f = rep["baseline"]["counts"], rep["fresh"]["counts"]
+ show = lambda c: "%d executed (%d pass, %d fail, %d error, %d skip)" % (c["executed"], c["pass"], c["fail"], c["error"], c["skip"]) # noqa: E731
+ lines = ["baseline (%s): %s" % (rep["baseline"]["source"], show(b) if b else "nothing readable"),
+ "this run (%d result file(s)): %s" % (rep["fresh"]["files"], show(f))]
+ if rep["baseline"]["targetOnly"]:
+ lines.append("BASELINE.md says target-only (%s): there is no old-version result to compare with." % (rep["baseline"]["why"] or "no reason given"))
+ a = rep["baseline"].get("assessment")
+ if a and a["kind"] == "measured":
+ lines.append("baseline evidence: measured (%s)" % "; ".join("%s%s" % (s["kind"], " %s" % s["name"] if s.get("name") else "") for s in a["sources"][:4]))
+ if a["conflict"]:
+ lines.append(" but the sources disagree with each other: %s" % "; ".join(a["conflictExamples"][:3]))
+ elif a["disagree"]:
+ lines.append(" but the typed table disagrees with it on %d test(s): %s" % (a["disagree"], "; ".join(a["examples"][:3])))
+ elif a and a["kind"] == "typed":
+ lines.append("baseline evidence: typed table only. No result file, per-test results file or raw log that this script can read was found in analysis/<system>/baseline/ or on a Recorded: or Machine-readable: line of BASELINE.md.%s" % (
+ " Ignored (counts only): " + ", ".join(a["ignored"][:3]) if a["ignored"] else ""))
+
+ def block(title, key, fmt=lambda x: clean(x, 160)):
+ if rep[key + "Count"]:
+ lines.append("%s: %d" % (title, rep[key + "Count"]))
+ lines.extend(" " + fmt(x) for x in rep[key][:limit])
+ if rep[key + "Count"] > limit:
+ lines.append(" ... and %d more (use --json)" % (rep[key + "Count"] - limit))
+ else:
+ lines.append("%s: 0" % title)
+ block("regressions (passed before, fail now)", "regressions", lambda x: "%s: %s -> %s" % (clean(x["id"], 140), x["before"], x["now"]))
+ block("differences a person approved in BASELINE.md (not counted)", "approved", lambda x: "%s: %s (%s)" % (clean(x["id"], 120), x["was"], clean(x["reason"], 100)))
+ block("flaky flips the baseline lists (not counted)", "flaky", lambda x: "%s: %s -> %s" % (clean(x["id"], 140), x["before"], x["now"]))
+ block("new failures (not in the baseline)", "newFailures")
+ block("still failing, as in the baseline", "stillFailing")
+ block("now passing (failed before)", "fixed")
+ block("skipped now that ran before", "newlySkipped")
+ block("baseline tests that did not run", "missing")
+ block("modules missing from this run", "missingModules")
+ block("modules with fewer tests or more skips", "moduleDiffs", lambda x: "%s: %s" % (clean(x["module"], 100), "; ".join(x["problems"])))
+ if rep["renamed"]:
+ lines.append("renamed: %d test name(s) changed but their class did not get worse" % rep["renamed"])
+ if rep["executedDrop"]:
+ lines.append("executed tests dropped from %(before)d to %(now)d" % rep["executedDrop"])
+ if rep["skippedGrowth"]:
+ lines.append("skipped tests grew from %(before)d to %(now)d" % rep["skippedGrowth"])
+ lines += ["%d result file(s) could not be read: %s" % (rep["fresh"]["unreadableCount"], "; ".join(rep["fresh"]["unreadable"][:3])) if rep["fresh"]["unreadableCount"] else "all result files were read"]
+ lines += rep["problems"]
+ lines.append("OK: nothing regressed and %d test(s) executed" % f["executed"] if rep["ok"] else "NOT OK: %s" % (
+ "no test executed" if not f["executed"] else "%d regression(s), %d new failure(s)" % (rep["regressionsCount"], rep["newFailuresCount"])))
+ return "\n".join(lines)
+
+
+def run(baseline_path, paths, measured="auto"):
+ """-> the report. Raises InputError for a baseline or results that cannot be used. `measured` is the old version's own evidence
+ (from read_baseline_evidence); by default it is looked for beside BASELINE.md (see evidence_paths). Measured per-test results,
+ when there are any, are what the fresh results are compared with; the typed table is only checked against them."""
+ if os.path.islink(baseline_path):
+ raise InputError("%s is a symbolic link" % baseline_path)
+ try:
+ with open(baseline_path, "rb") as fh:
+ text = fh.read(16 << 20).decode("utf-8", "replace")
+ except OSError as err:
+ raise InputError("cannot read %s: %s" % (baseline_path, err.__class__.__name__))
+ base = parse_baseline(text)
+ if not (base["targetOnly"] or base["modules"] or base["tests"] or base["totals"]):
+ raise InputError("%s has no per-test table, per-module table, totals or target-only line that can be read" % baseline_path)
+ if measured == "auto":
+ found = evidence_paths(baseline_path, text)
+ measured = read_baseline_evidence(found) if found else None
+ base["assessment"] = assess_baseline(base, measured)
+ if base["assessment"]["kind"] == "measured" and measured["tests"]:
+ base["tests"], base["measuredTests"] = measured["tests"], True
+ return compare(base, read_results(paths))
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description="Compare a fresh test run with BASELINE.md.")
+ ap.add_argument("baseline", help="analysis/<system>/BASELINE.md")
+ ap.add_argument("--junit", "--trx", action="append", default=[], metavar="PATH", help="a results file or a folder to search (repeat for several)")
+ ap.add_argument("--baseline-evidence", action="append", default=[], metavar="PATH",
+ help="a file or folder with the old version's own results (XML, JSON map, raw log); default: analysis/<system>/baseline/ and the paths BASELINE.md names")
+ ap.add_argument("--json", action="store_true", help="print the full report as JSON")
+ ap.add_argument("--max-list", type=int, default=10, help="how many entries to print per list (default 10)")
+ args = ap.parse_args(argv)
+ if not args.junit:
+ print("baseline_diff.py: give at least one --junit file or folder", file=sys.stderr)
+ return 2
+ try:
+ rep = run(args.baseline, args.junit, read_baseline_evidence(args.baseline_evidence) if args.baseline_evidence else "auto")
+ except InputError as err:
+ print("baseline_diff.py: %s" % err, file=sys.stderr)
+ return 2
+ print(json.dumps(rep, indent=1, ensure_ascii=True) if args.json else render(rep, max(1, args.max_list)))
+ return 0 if rep["ok"] else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/plugins/code-modernization/scripts/build_report.py b/plugins/code-modernization/scripts/build_report.py
new file mode 100644
index 00000000..b690eb3f
--- /dev/null
+++ b/plugins/code-modernization/scripts/build_report.py
@@ -0,0 +1,725 @@
+#!/usr/bin/env python3
+"""Build one self-contained HTML report from a system's modernization artifacts.
+
+ python3 build_report.py <system> [--workspace DIR] [--out FILE]
+
+Reads whatever exists under <workspace>/analysis/<system>/ and <workspace>/modernized/ (every file is
+optional; a missing or malformed one is reported in the page, never an error) and writes
+<workspace>/analysis/<system>/REPORT.html by default. Standard library only. Exit 0; 1 when nothing
+at all was found; 2 for a system name that is not a folder name.
+
+The artifacts come from untrusted code, so every byte is treated as hostile: text travels only as
+escaped JSON inside <script type="application/json"> blocks, the page builds its DOM with
+textContent, a Content-Security-Policy pins the only scripts that may run (by hash) and blocks every
+external request, and files named *.local.*, SECRETS* or reached through a symlink are never read.
+"""
+import argparse
+import base64
+import collections
+import datetime
+import hashlib
+import json
+import os
+import re
+import stat
+import sys
+import tempfile
+from urllib.parse import quote
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+ASSETS = os.path.join(os.path.dirname(HERE), "assets")
+FILE_CAP, TOTAL_CAP, LINE_CAP, MAX_CASES, MAX_NOTES = 3 << 20, 6 << 20, 20000, 2000, 100
+NOTE_NAMES = ("TRANSFORMATION_NOTES.md", "UPLIFT_NOTES.md", "README.md")
+STEPS = [("preflight", "Preflight"), ("assess", "Assess"), ("map", "Map"), ("rules", "Rules"),
+ ("brief", "Brief"), ("build", "Build"), ("harden", "Harden")]
+TITLES = [("overview", "Overview"), ("assessment", "Assessment"), ("diagrams", "Diagrams"),
+ ("rules", "Business rules"), ("brief", "Brief"), ("security", "Security"),
+ ("delta", "Delta catalog & baseline"), ("spec", "Spec & architecture"),
+ ("build", "Build notes"), ("proof", "Proof"), ("equivalence", "Equivalence")]
+DOCS = [("overview", ["PREFLIGHT.md"]), ("assessment", ["ASSESSMENT.md"]),
+ ("rules", ["BUSINESS_RULES.md", "RULE_REVIEWS.md", "DATA_OBJECTS.md"]), ("brief", ["MODERNIZATION_BRIEF.md"]),
+ ("security", ["SECURITY_FINDINGS.md"]), ("delta", ["DELTA_CATALOG.md", "BASELINE.md", "PLAYBOOK.md"]),
+ ("spec", ["AI_NATIVE_SPEC.md", "REIMAGINED_ARCHITECTURE.md"])]
+MERMAID_FENCE = re.compile(r"(?m)^ {0,3}(?:`{3,}|~{3,})[ \t]*mermaid\b")
+ID = r"[A-Z][A-Z0-9]*(?:-[A-Z][A-Z0-9]*)*-\d+[a-z]?"
+CARD = re.compile(r"^(#{3,4})[ \t]+[*`]*(" + ID + r")[*`]*(?:[ \t]*[·:\-–—][ \t]*(.*))?$") # the title's trailing spaces and #s are cut in Python: no nested quantifiers here
+FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})")
+CITE = re.compile(r"[A-Za-z0-9_][\w./+-]*\.[A-Za-z][A-Za-z0-9]{0,7}:\d+(?:-\d+)?(?:,\s*\d+(?:-\d+)?)*")
+DAY = "%Y-%m-%d"
+
+
+def num(n):
+ return "{:,}".format(n)
+
+
+def utc_day(ts):
+ return datetime.datetime.fromtimestamp(ts, datetime.timezone.utc).strftime(DAY)
+
+
+# ---------------------------------------------------------------- reading files safely
+class Source:
+ """Everything read from the workspace goes through here: caps, notes, and refusals."""
+
+ def __init__(self, workspace, system):
+ self.root, self.system, self.used = os.path.realpath(workspace), system, 0
+ self.adir, self.mdir = os.path.join(workspace, "analysis", system), os.path.join(workspace, "modernized")
+ self.notes, self.found = [], []
+
+ def read(self, path, label):
+ """Cleaned text of a regular file inside the workspace; None when absent or refused."""
+ name = os.path.basename(path).lower()
+ if ".local." in name or name.startswith("secrets"):
+ return None
+ try:
+ st = os.lstat(path)
+ except OSError:
+ return None
+ if not stat.S_ISREG(st.st_mode) or os.path.commonpath([self.root, os.path.realpath(path)]) != self.root:
+ self.notes.append("%s is a link or not a regular file inside the workspace, so it was not read." % label)
+ return None
+ self.found.append((label, st.st_size, utc_day(st.st_mtime)))
+ try:
+ with open(path, "rb") as fh:
+ raw = fh.read(FILE_CAP + 1)
+ except OSError as err:
+ self.notes.append("%s could not be read (%s)." % (label, err.__class__.__name__))
+ return ""
+ text = raw[:FILE_CAP].decode("utf-8", "replace").lstrip("\N{ZERO WIDTH NO-BREAK SPACE}").replace("\r\n", "\n").replace("\r", "\n")
+ lines, shortened = re.sub(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]", "\N{REPLACEMENT CHARACTER}", text).split("\n"), 0
+ for i, line in enumerate(lines):
+ if len(line) > LINE_CAP:
+ lines[i] = "%s ... [%s more characters not shown]" % (line[:LINE_CAP], num(len(line) - LINE_CAP))
+ shortened += 1
+ text = "\n".join(lines)
+ if shortened:
+ self.notes.append("%s: %d line(s) longer than %s characters were shortened." % (label, shortened, num(LINE_CAP)))
+ room = TOTAL_CAP - self.used
+ if len(raw) > FILE_CAP or len(text.encode("utf-8")) > room:
+ text = text.encode("utf-8")[:max(0, min(FILE_CAP, room))].decode("utf-8", "ignore")
+ text = text[:text.rfind("\n")] if "\n" in text else ""
+ self.notes.append("%s: %s (report size limit; the file is %s KB)." % (
+ label, "only the first %s KB is included" % num(len(text.encode("utf-8")) >> 10) if text else "left out",
+ num(max(1, st.st_size >> 10))))
+ self.used += len(text.encode("utf-8"))
+ return text
+
+ def json(self, path, label):
+ """-> (parsed value or None, whether the file exists)."""
+ text = self.read(path, label)
+ if text is None:
+ return None, False
+ try:
+ return json.loads(text), True
+ except ValueError as err:
+ self.notes.append("%s could not be parsed as JSON (%s)." % (label, str(err)[:80]))
+ return None, True
+
+
+# ---------------------------------------------------------------- what the markdown says
+def cells(line):
+ s = line.strip()
+ s = s[1:] if s.startswith("|") else s
+ s = s[:-1] if s.endswith("|") and not s.endswith("\\|") else s
+ return [re.sub(r"[*`]", "", c).strip() for c in re.split(r"(?<!\\)\|", s)]
+
+
+def tables(text):
+ """(nearest heading, header cells, rows) for every pipe table in a markdown text."""
+ lines, head = text.split("\n"), ""
+ for i, line in enumerate(lines):
+ m = re.match(r"#{1,6}\s+(.*)", line)
+ head = m.group(1) if m else head
+ if "|" in line and i + 1 < len(lines) and "|" in lines[i + 1] and re.fullmatch(r"\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*", lines[i + 1]):
+ rows, j = [], i + 2
+ while j < len(lines) and "|" in lines[j] and lines[j].strip():
+ rows.append(cells(lines[j]))
+ j += 1
+ yield head, cells(line), rows
+
+
+def field(body, name):
+ for m in re.finditer(r"(?im)^[ \t>*-]*\**" + name + r"\**[ \t]*:[ \t]*\**[ \t]*(.*)$", body):
+ if m.group(1).strip():
+ return m.group(1).strip()
+ return ""
+
+
+def parse_rules(text):
+ """Split a rules file into markdown chunks and rule cards. With no card headings it is one chunk."""
+ blocks, buf, card, fence = [], [], None, ""
+
+ def flush():
+ nonlocal buf, card
+ if card is not None:
+ blocks.append({"k": "rule", **finish(card, "\n".join(buf))})
+ elif "".join(buf).strip():
+ blocks.append({"k": "md", "text": "\n".join(buf)})
+ buf, card = [], None
+
+ for line in text.split("\n"):
+ f = FENCE.match(line)
+ if fence:
+ if f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence) and not line.strip().strip(fence[0]):
+ fence = ""
+ elif f:
+ fence = f.group(1)
+ else:
+ m, h = CARD.match(line.rstrip()), re.match(r"(#{1,6})\s", line)
+ if m or (h and card is not None and len(h.group(1)) <= len(card[0])):
+ flush()
+ if m:
+ card = (m.group(1), m.group(2), (m.group(3) or "").rstrip(" \t#").strip(), re.sub(r"^#+\s*", "", line).strip())
+ continue
+ buf.append(line)
+ flush()
+ return blocks
+
+
+def finish(card, body):
+ _, rid, title, heading = card
+ head = next((ln for ln in body.split("\n") if ln.strip()), "")
+ prio = re.search(r"\b(P\d)\b", field(body, "Priority")) or re.match(r"(P\d)-", rid) or re.match(r"[ `*]*(P\d)[ `*]*(?:·|$)", head)
+ conf = re.search(r"(?i)\b(high|medium|low)\b", field(body, "Confidence")) or re.search(r"(?i)confidence:?\s*(high|medium|low)", head)
+ source, defect, cat = field(body, "Source") or head, field(body, "Suspected defect"), field(body, "Category")
+ body = re.sub(r"(?im)^[ \t>*-]*\**(?:Priority|Category|Source)\**[ \t]*:.*\n?", "", body).strip("\n")
+ return {"id": rid, "heading": heading[:300], "title": title[:300], "p": prio.group(1).upper() if prio else "",
+ "cat": re.sub(r"\s+", " ", cat)[:60], "conf": conf.group(1).capitalize() if conf else "",
+ "cites": list(dict.fromkeys(CITE.findall(source)))[:12],
+ "defect": bool(defect) and not re.match(r"(?i)\W*(none|n/?a|no|nil)\W*$", defect), "body": body}
+
+
+def rule_facts(text, cards, reviews):
+ """Priority tally: from each card, else from the index-table row that carries its id."""
+ rated = {}
+ for _, _, rows in tables(text):
+ for row in rows:
+ m, p = re.search(ID, row[0]) if row else None, next((c for c in row if re.fullmatch(r"P\d", c)), None)
+ if m and p:
+ rated.setdefault(m.group(0), p)
+ counts = collections.Counter(c["p"] or rated.get(c["id"], "") for c in cards)
+ unrated = counts.pop("", 0)
+ out = {"total": len(cards), "byPriority": dict(sorted(counts.items())), "unrated": unrated, "defects": sum(1 for c in cards if c["defect"])}
+ if reviews:
+ out["reviews"] = dict(collections.Counter(str(v.get("verdict", "?"))[:20] for v in reviews.values() if isinstance(v, dict)))
+ return out
+
+
+def security_facts(text):
+ """Severity tally from the findings table; a summary scorecard only when there is no findings table."""
+ finds, score = collections.Counter(), collections.Counter()
+ for head, header, rows in tables(text):
+ low = [h.lower() for h in header]
+ sev = next((i for i, h in enumerate(low) if "severity" in h), None)
+ if sev is None or any(re.search(r"package|installed|fixed", h) for h in low) or \
+ re.search(r"coverage|not judged|unverified|dependenc|cve|patch review|remediation", head, re.I):
+ continue
+ count = next((i for i, h in enumerate(low) if h in ("count", "total", "n", "#", "findings")), None)
+ for row in rows:
+ m = re.match(r"(?i)(critical|high|medium|low)\b", row[sev] if sev < len(row) else "")
+ if m and count is not None and len(header) <= 3 and count < len(row) and row[count].isdigit():
+ score[m.group(1).capitalize()] += int(row[count])
+ elif m:
+ finds[m.group(1).capitalize()] += 1
+ return dict(finds or score) or None
+
+
+def baseline_facts(text):
+ """Pass/fail/skip counts from a per-test result column, or from a sentence like '12 passed, 1 failed'."""
+ words, counts = {"pass": "pass", "passed": "pass", "passing": "pass", "ok": "pass", "fail": "fail", "failed": "fail",
+ "failing": "fail", "skip": "skip", "skipped": "skip", "ignored": "skip"}, collections.Counter()
+ for _, header, rows in tables(text):
+ col = next((i for i, h in enumerate(header) if re.search(r"result|status|outcome|verdict|passed", h, re.I)), None)
+ for row in rows if col is not None else []:
+ if col < len(row) and row[col].lower() in words:
+ counts[words[row[col].lower()]] += 1
+ for key, pat in (() if counts else (("pass", r"(\d+)\s+(?:tests?\s+|cases?\s+)?(?:passed|passing)|(?:passed|passing)\s*[:=]\s*(\d+)"),
+ ("fail", r"(\d+)\s+(?:tests?\s+|cases?\s+)?(?:failed|failing)|(?:failed|failing)\s*[:=]\s*(\d+)"))):
+ m = re.search(r"(?i)" + pat, text)
+ if m:
+ counts[key] = int(m.group(1) or m.group(2))
+ target_only = bool(re.search(r"(?im)^\W*target-only:", text))
+ return {**counts, "targetOnly": target_only} if counts or target_only else None
+
+
+def topology_facts(obj):
+ nodes = []
+
+ def walk(n, depth):
+ if isinstance(n, dict) and depth < 12:
+ nodes.append(n)
+ for c in n.get("children") or []:
+ walk(c, depth + 1)
+
+ if isinstance(obj, dict):
+ walk(obj.get("root"), 0)
+ mods = [n for n in nodes if n.get("kind") == "module"] or [n for n in nodes if not n.get("children") and n.get("file") and isinstance(n.get("loc"), int)]
+ if not mods:
+ return None
+ seen = obj.get("observations")
+ return {"modules": len(mods), "loc": sum(n["loc"] for n in mods if isinstance(n.get("loc"), int) and n["loc"] > 0),
+ "languages": sorted({str(n["language"])[:20] for n in mods if isinstance(n.get("language"), str)})[:6],
+ "edges": len(obj["edges"]) if isinstance(obj.get("edges"), list) else None,
+ "observations": [str(o)[:600] for o in seen[:12]] if isinstance(seen, list) else []}
+
+
+VERDICTS = ("same", "differs", "differs-approved", "missing")
+
+
+def endpoint(o):
+ return {k: str(o.get(k) or "")[:300] for k in ("label", "command")} if isinstance(o, dict) else {"label": "", "command": ""}
+
+
+def equivalence_view(obj):
+ """Tally the verdicts from the case list itself: the file's own totals and wording are never trusted."""
+ if not isinstance(obj, dict) or not isinstance(obj.get("cases"), list):
+ return None
+ problems, tally, rows = [], collections.Counter(), []
+ for c in (c for c in obj["cases"] if isinstance(c, dict)):
+ s = lambda k, n=300: str(c.get(k) or "")[:n] # noqa: E731
+ masked = [str(m)[:200] for m in c["masked"][:20]] if isinstance(c.get("masked"), list) else []
+ ld, nd = (d if re.fullmatch(r"[0-9a-f]{64}", d) else "" for d in (s("legacySha256", 80), s("newSha256", 80)))
+ v = s("verdict", 30) if s("verdict", 30) in VERDICTS else "unknown"
+ if v == "same" and (c.get("firstDiff") or (not masked and ld and nd and ld != nd)) or (v == "differs-approved" and not s("approvedDifference").strip()):
+ v = "inconsistent"
+ tally[v] += 1
+ row = {"id": s("id", 80), "title": s("title"), "verdict": v, "reason": s("reason", 500), "legacyPath": s("legacyPath"), "newPath": s("newPath"),
+ "legacySha256": s("legacySha256", 80), "newSha256": s("newSha256", 80), "masked": masked,
+ "approvedDifference": s("approvedDifference", 500), "note": s("note", 500), "empty": c.get("empty") is True}
+ if isinstance(c.get("firstDiff"), dict):
+ fd = c["firstDiff"]
+ row["firstDiff"] = {k: fd[k] if isinstance(fd.get(k), int) else 0 for k in ("offset", "line", "at")}
+ row["firstDiff"].update({k: str(fd.get(k) or "")[:120] for k in ("legacy", "new")})
+ if isinstance(c.get("bytes"), dict) and isinstance(c.get("maskedBytes"), dict):
+ row["sizes"] = {k: [c["bytes"].get(k), c["maskedBytes"].get(k)] for k in ("legacy", "new") if isinstance(c["bytes"].get(k), int) and isinstance(c["maskedBytes"].get(k), int)}
+ if v == "inconsistent":
+ problems.append("Case %s: the recorded verdict contradicts its own evidence, so it counts as a failure." % row["id"])
+ rows.append(row)
+ ran = ("same", "differs", "differs-approved", "inconsistent")
+ executed, bad = sum(tally[k] for k in ran), tally["differs"] + tally["missing"] + tally["inconsistent"] + tally["unknown"]
+ sc = obj["selfCheck"] if isinstance(obj.get("selfCheck"), dict) else None
+ check = {"passed": sc["passed"] if sc and isinstance(sc.get("passed"), bool) else None, "detail": str(sc.get("detail") or "")[:300] if sc else "", "recorded": sc is not None}
+ if not sc:
+ problems.append("No self-check is recorded, so this file may not have been written by scripts/compare.py.")
+ declared = obj["totals"] if isinstance(obj.get("totals"), dict) else {}
+ if declared and (declared.get("cases") != len(rows) or declared.get("executed") != executed):
+ problems.append("The totals stored in the file disagree with its case list; the case list was used.")
+ empty = executed > 0 and all(r["empty"] for r in rows if r["verdict"] in ran)
+ if executed == 0:
+ headline = "Equivalence not proven: no case was executed."
+ elif empty:
+ headline = "Equivalence not proven: every compared output was empty."
+ elif bad or check["passed"] is False:
+ why = ["%d %s" % (tally[k], w) for k, w in (("differs", "differing"), ("missing", "missing"), ("inconsistent", "contradictory"), ("unknown", "unrecognised")) if tally[k]]
+ headline = "Equivalence not proven: %s%s (%d of %d cases executed)." % (
+ ", ".join(why) or "no case failed", "; the comparator's self-check failed" if check["passed"] is False else "", executed, len(rows))
+ else:
+ approved = tally["differs-approved"]
+ headline = "Outputs match on all %d executed cases%s." % (executed, " (%d with a difference a person approved)" % approved if approved else "")
+ return {"state": "red" if (executed == 0 or empty or bad or check["passed"] is False) else "green", "headline": headline, "problems": problems,
+ "selfCheck": check, "generated": str(obj.get("generated") or "")[:40], "tally": {"cases": len(rows), "executed": executed, **dict(tally)},
+ "legacy": endpoint(obj.get("legacy")), "new": endpoint(obj.get("new")), "cases": rows[:MAX_CASES], "cut": max(0, len(rows) - MAX_CASES)}
+
+
+# ---------------------------------------------------------------- the proof pack (VERIFICATION.json)
+PROOF_VERDICTS = ("PROVEN", "PARTLY PROVEN", "NOT PROVEN")
+PROOF_CHECKS = (("tests", "Tests ran"), ("rules", "Rules traced"), ("same", "Same behavior"), ("fresh", "Fresh inputs"), ("canary", "Canary"), ("source", "Source untouched"))
+UPLIFT_CHECKS = (("baseline", "Baseline measured"), ("kept", "Tests kept"), ("deltas", "Deltas covered")) # an uplift has these three as well
+KEPT_SHARE = 0.25
+RULE_STATES = ("tested", "named, not run", "claimed only", "code only", "none")
+FRESH_NEEDED = 10
+
+
+def text_of(value, limit=300):
+ return value[:limit] if isinstance(value, str) else ""
+
+
+def count_of(value):
+ return value if isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= 10 ** 9 else 0
+
+
+def texts_of(value, limit=300, cap=40):
+ return [v[:limit] for v in value[:cap] if isinstance(v, str)] if isinstance(value, list) else []
+
+
+def num_of(value):
+ return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) and value == value and abs(value) < 1e300 else 0.0
+
+
+def tolerances_of(obj):
+ return [{"why": text_of(t.get("why"), 200), "rel": num_of(t.get("rel")), "abs": num_of(t.get("abs")), "cases": count_of(t.get("cases"))}
+ for t in (obj.get("tolerances") if isinstance(obj, dict) and isinstance(obj.get("tolerances"), list) else [])[:10] if isinstance(t, dict)]
+
+
+def counts_of(obj, keys):
+ obj = obj if isinstance(obj, dict) else {}
+ return {k: count_of(obj.get(k)) for k in keys}
+
+
+def uplift_view(ev):
+ """The uplift-only evidence of a pack, made safe: the baseline's assessment, the test files that changed, and the deltas no test names."""
+ bev, tc, dc = ev.get("baselineEvidence"), ev.get("testsChanged"), ev.get("deltas")
+ if not any(isinstance(x, dict) for x in (bev, tc, dc)):
+ return {"baseline": None, "testsChanged": None, "deltas": None}
+ files = lambda xs, keys: [dict({"path": text_of(x.get("path"), 300)}, **{k: count_of(x.get(k)) for k in keys}, note=text_of(x.get("note"), 80)) for x in (xs if isinstance(xs, list) else [])[:40] if isinstance(x, dict)] # noqa: E731
+ listed = lambda xs: [{"id": text_of(x.get("id"), 40), "title": text_of(x.get("title"), 200), "why": text_of(x.get("why"), 200), "sites": texts_of(x.get("sites"), 200, 5)} for x in (xs if isinstance(xs, list) else [])[:40] if isinstance(x, dict)] # noqa: E731
+ out = {"baseline": None, "testsChanged": None, "deltas": None}
+ if isinstance(bev, dict):
+ out["baseline"] = {"kind": bev.get("kind") if bev.get("kind") in ("measured", "typed", "target-only", "none") else "none", "disagree": count_of(bev.get("disagree")), "conflict": count_of(bev.get("conflict")),
+ "sources": [{"kind": text_of(x.get("kind"), 40), "name": text_of(x.get("name"), 80), "tests": count_of(x.get("tests")), "executed": count_of(x.get("executed"))}
+ for x in (bev.get("sources") if isinstance(bev.get("sources"), list) else [])[:10] if isinstance(x, dict)],
+ "examples": texts_of(bev.get("examples"), 200, 5), "conflictExamples": texts_of(bev.get("conflictExamples"), 200, 5), "considered": texts_of(bev.get("considered"), 80, 10), "ignored": texts_of(bev.get("ignored"), 80, 10)}
+ if isinstance(tc, dict):
+ counts = counts_of(tc.get("counts"), ("added", "removed", "changed"))
+ out["testsChanged"] = {"checked": tc.get("checked") is True, "why": text_of(tc.get("why"), 300), "legacyTests": count_of(tc.get("legacyTests")), "workTests": count_of(tc.get("workTests")),
+ "counts": counts, "share": num_of(tc.get("share")), "removed": files(tc.get("removed"), ("lines",)), "added": files(tc.get("added"), ("lines",)),
+ "changed": files(tc.get("changed"), ("added", "removed"))}
+ if isinstance(dc, dict):
+ unc = listed(dc.get("uncovered"))
+ out["deltas"] = {"parsed": count_of(dc.get("parsed")), "silent": count_of(dc.get("silent")), "coveredCount": len(dc["covered"]) if isinstance(dc.get("covered"), list) else 0,
+ "uncoveredCount": max(len(unc), len(dc["uncovered"]) if isinstance(dc.get("uncovered"), list) else 0), "uncovered": unc, "config": listed(dc.get("config")), "why": text_of(dc.get("why"), 300)}
+ return out
+
+
+def proof_module(m, problems):
+ """One module of VERIFICATION.json, made safe, with its verdict recomputed from its own checks: the file's word is never taken."""
+ name = text_of(m.get("name"), 120) or "(unnamed)"
+ ev = m["evidence"] if isinstance(m.get("evidence"), dict) else {}
+ track = text_of(m.get("track"), 20)
+ required = PROOF_CHECKS + (UPLIFT_CHECKS if track == "uplift" else ())
+ known = dict(PROOF_CHECKS + UPLIFT_CHECKS)
+ by_id = {}
+ for c in (m["checks"] if isinstance(m.get("checks"), list) else [])[:20]:
+ if isinstance(c, dict) and c.get("id") in known and c.get("state") in ("pass", "gap", "fail", "na") and c["id"] not in by_id:
+ by_id[c["id"]] = {"id": c["id"], "label": known[c["id"]], "state": c["state"], "detail": text_of(c.get("detail"), 700)}
+ tests = counts_of(ev.get("tests"), ("executed", "failed", "skipped"))
+ rules = ev["rules"] if isinstance(ev.get("rules"), dict) else None
+ p0 = [{"id": text_of(r.get("id"), 40), "name": text_of(r.get("name"), 200), "confidence": text_of(r.get("confidence"), 10),
+ "status": r.get("status") if r.get("status") in RULE_STATES else "none", "tests": count_of(r.get("tests")), "main": count_of(r.get("main")), "where": text_of(r.get("where"), 200)}
+ for r in (rules["p0"] if rules and isinstance(rules.get("p0"), list) else [])[:200] if isinstance(r, dict)]
+ fresh = dict(counts_of(ev.get("fresh"), ("inputs", "executed", "same", "differs", "missing", "withinTolerance")), tolerances=tolerances_of(ev.get("fresh"))) if isinstance(ev.get("fresh"), dict) else None
+ eq = ev.get("equivalence") if isinstance(ev.get("equivalence"), dict) else None
+ notes = []
+ for cid, label in required: # a pack that leaves a check out cannot be PROVEN
+ if cid not in by_id:
+ by_id[cid] = {"id": cid, "label": label, "state": "gap", "detail": "This check is missing from VERIFICATION.json."}
+ notes.append("the %s check is missing" % label.lower())
+ if by_id["tests"]["state"] == "pass" and tests["executed"] == 0:
+ by_id["tests"].update(state="fail", detail="No test executed, whatever the file says.")
+ notes.append("it says tests passed but none executed")
+ if by_id["rules"]["state"] == "pass" and any(r["status"] != "tested" for r in p0):
+ by_id["rules"].update(state="gap", detail="Some P0 rules are not named by a test, whatever the file says.")
+ notes.append("it says every P0 rule is tested but the table shows otherwise")
+ if by_id["fresh"]["state"] == "pass" and (fresh is None or fresh["inputs"] < FRESH_NEEDED):
+ by_id["fresh"].update(state="gap", detail="Fewer than %d fresh inputs are recorded, whatever the file says." % FRESH_NEEDED)
+ notes.append("it says the fresh-input check passed with fewer than %d inputs" % FRESH_NEEDED)
+ for cid, bad in (("same", eq), ("fresh", fresh)): # a pass cannot stand next to differing or missing cases
+ if by_id[cid]["state"] == "pass" and isinstance(bad, dict) and (count_of(bad.get("differs")) or count_of(bad.get("missing"))):
+ by_id[cid].update(state="fail", detail="Some cases differ or are missing, whatever the file says.")
+ notes.append("it says %s passed but cases differ or are missing" % by_id[cid]["label"].lower())
+ if by_id["tests"]["state"] == "pass" and tests["failed"] and track != "uplift":
+ by_id["tests"].update(state="fail", detail="Some tests failed, whatever the file says.")
+ notes.append("it says tests passed but some failed")
+ up = uplift_view(ev) if track == "uplift" else None
+ if up is not None: # the three uplift checks must match the evidence beside them
+ bev, tc, dc = up["baseline"], up["testsChanged"], up["deltas"]
+ expected = "na" if bev is None or bev["kind"] == "target-only" else "pass" if bev["kind"] == "measured" and not bev["disagree"] and not bev["conflict"] else "gap"
+ if by_id["baseline"]["state"] in ("pass", "na") and expected == "gap":
+ by_id["baseline"].update(state="gap", detail="The baseline is not shown to be measured, whatever the file says.")
+ notes.append("it says the baseline is measured but the evidence shows otherwise")
+ kept_ok = tc is not None and tc["checked"] and tc["legacyTests"] > 0 and not tc["counts"]["removed"] and tc["counts"]["changed"] <= KEPT_SHARE * tc["legacyTests"]
+ if by_id["kept"]["state"] in ("pass", "na") and not kept_ok:
+ by_id["kept"].update(state="gap", detail="The test files are not shown to be kept, whatever the file says.")
+ notes.append("it says the tests were kept but the evidence shows otherwise")
+ deltas_ok = dc is not None and dc["parsed"] > 0 and not dc["uncoveredCount"]
+ if by_id["deltas"]["state"] in ("pass", "na") and not deltas_ok:
+ by_id["deltas"].update(state="gap", detail="The deltas are not shown to be covered, whatever the file says.")
+ notes.append("it says the deltas are covered but the evidence shows otherwise")
+ checks = [by_id[cid] for cid, _ in required]
+ states = {c["state"] for c in checks}
+ verdict = "NOT PROVEN" if "fail" in states else "PARTLY PROVEN" if "gap" in states else "PROVEN"
+ if m.get("verdict") in PROOF_VERDICTS and m.get("verdict") != verdict or notes:
+ problems.append("%s: the file recorded %s; its own evidence gives %s, so %s is shown%s." % (
+ name, text_of(m.get("verdict"), 20) or "no verdict", verdict, verdict, " (" + "; ".join(notes) + ")" if notes else ""))
+ base = ev.get("baseline") if isinstance(ev.get("baseline"), dict) else None
+ brief = m["brief"] if isinstance(m.get("brief"), dict) else {}
+ items = [{"text": text_of(i.get("text"), 300), "where": text_of(i.get("where"), 80)} for i in (brief.get("items") if isinstance(brief.get("items"), list) else [])[:40] if isinstance(i, dict)]
+ return {"name": name, "track": text_of(m.get("track"), 20), "verdict": verdict, "verifiedAt": text_of(m.get("verifiedAt"), 40), "checks": checks,
+ "reasons": ["%s: %s" % (c["label"], c["detail"]) for c in checks if c["state"] in ("fail", "gap")], "passed": ["%s: %s" % (c["label"], c["detail"]) for c in checks if c["state"] == "pass"],
+ "tests": tests, "hasRules": rules is not None, "p0": p0,
+ "p0Counts": {"tested": sum(1 for r in p0 if r["status"] == "tested"), "claimedOnly": sum(1 for r in p0 if r["status"] == "claimed only"),
+ "notRun": sum(1 for r in p0 if r["status"] == "named, not run"), "none": sum(1 for r in p0 if r["status"] not in ("tested", "claimed only", "named, not run"))},
+ "equivalence": dict(counts_of(eq, ("cases", "executed", "same", "differs", "missing", "withinTolerance")), tolerances=tolerances_of(eq), approved=[{"id": text_of(a.get("id"), 80), "why": text_of(a.get("why"), 300)} for a in (eq.get("approved") if isinstance(eq.get("approved"), list) else [])[:20] if isinstance(a, dict)],
+ masks=[{"why": text_of(k.get("why"), 200), "cases": count_of(k.get("cases"))} for k in (eq.get("masks") if isinstance(eq.get("masks"), list) else [])[:10] if isinstance(k, dict)]) if eq else None,
+ "fresh": fresh, "baseline": counts_of(base, ("regressionsCount", "newFailuresCount", "fixedCount", "missingCount", "renamed")) if base else None,
+ "canary": [{"change": text_of(c.get("change"), 200), "testsFailed": count_of(c.get("testsFailed")), "shown": c.get("shown") is True} for c in (ev.get("canary") if isinstance(ev.get("canary"), list) else [])[:10] if isinstance(c, dict)],
+ "uplift": up, "notProven": texts_of(m.get("notProven"), 400, 16), "needsPerson": texts_of(m.get("needsPerson"), 300, 10), "caveats": texts_of(m.get("caveats"), 400, 20),
+ "people": items, "peopleCount": max(count_of(brief.get("count")), len(items)), "briefFound": brief.get("found") is True}
+
+
+def proof_view(obj):
+ """VERIFICATION.json as the page uses it, or None when it is not a proof pack. Every module's verdict is recomputed from its checks."""
+ if not isinstance(obj, dict) or not isinstance(obj.get("modules"), list):
+ return None
+ problems = texts_of(obj.get("problems"), 300, 10)
+ mods = [proof_module(m, problems) for m in obj["modules"][:100] if isinstance(m, dict)]
+ counts = {v: sum(1 for m in mods if m["verdict"] == v) for v in PROOF_VERDICTS}
+ verdict = "NOT PROVEN" if counts["NOT PROVEN"] or not mods else "PARTLY PROVEN" if counts["PARTLY PROVEN"] else "PROVEN"
+ lg = obj["legacy"] if isinstance(obj.get("legacy"), dict) else {}
+ so = obj["signoff"] if isinstance(obj.get("signoff"), dict) else {}
+ headline = "Proof: %s (%d module%s)." % (", ".join("%d %s" % (counts[v], v) for v in PROOF_VERDICTS if counts[v]) or "no module was verified", len(mods), "" if len(mods) == 1 else "s")
+ tooling = [{"name": text_of(t.get("name"), 120), "path": text_of(t.get("path"), 200)} for t in (obj["toolingOnly"] if isinstance(obj.get("toolingOnly"), list) else [])[:20] if isinstance(t, dict)]
+ return {"generated": text_of(obj.get("generated"), 40), "verdict": verdict, "counts": counts, "headline": headline, "problems": problems, "modules": mods, "tooling": tooling,
+ "legacy": {"path": text_of(lg.get("path"), 200), "target": text_of(lg.get("target"), 300), "ran": lg.get("ran") if isinstance(lg.get("ran"), bool) else None,
+ "state": text_of(lg.get("state"), 20), "detail": text_of(lg.get("detail"), 300)},
+ "rules": texts_of(obj.get("rules"), 700, 12), "signoff": {k: text_of(so.get(k), 120) for k in ("name", "role", "date", "decision")}}
+
+
+# ---------------------------------------------------------------- assembling the report
+def nonempty_dir(p):
+ try:
+ return os.path.isdir(p) and bool(os.listdir(p))
+ except OSError:
+ return False
+
+
+def newest(paths):
+ """(mtime, path) of the newest existing file, or non-empty folder, among `paths`."""
+ best = None
+ for p in paths:
+ try:
+ if os.path.islink(p) or (os.path.isdir(p) and not nonempty_dir(p)):
+ continue
+ times = [os.stat(p).st_mtime] + ([e.stat().st_mtime for e in os.scandir(p)] if os.path.isdir(p) else [])
+ except OSError:
+ continue
+ best = max(best, (max(times), p)) if best else (max(times), p)
+ return best
+
+
+def modernized_docs(src):
+ """(label, text) for the notes files of modernized/<system>{,-uplifted,-reimagined}, one folder level down."""
+ out, skipped = [], 0
+ for suffix in ("", "-uplifted", "-reimagined"):
+ base = os.path.join(src.mdir, src.system + suffix)
+ try:
+ folders = [base] + sorted(e.path for e in os.scandir(base) if e.is_dir(follow_symlinks=False))[:200]
+ except OSError:
+ continue
+ for folder in folders:
+ for name in NOTE_NAMES:
+ path = os.path.join(folder, name)
+ label = os.path.relpath(path, src.mdir).replace(os.sep, "/")
+ if len(out) >= MAX_NOTES and os.path.lexists(path):
+ skipped += 1
+ continue
+ text = src.read(path, "modernized/" + label)
+ if text is not None:
+ out.append((label, text))
+ if skipped:
+ src.notes.append("%d more notes files under modernized/ are not included (limit %d)." % (skipped, MAX_NOTES))
+ return out
+
+
+def load_mermaid(src):
+ try:
+ with open(os.path.join(ASSETS, "vendor", "mermaid.min.js"), encoding="utf-8") as fh:
+ return fh.read().replace("</script", "<\\/script").replace("<!--", "<\\!--")
+ except OSError:
+ src.notes.append("The Mermaid library was not found next to the template, so diagrams are shown as source.")
+ return ""
+
+
+def next_step(system, done, glance):
+ """The one command to run next, from which artifacts exist. The status command decides it the same way and also
+ reads the brief's phases; this page only knows what is on disk."""
+ def go(command, why):
+ return {"command": "/code-modernization:%s %s" % (command, system), "why": why}
+ if not done["preflight"]:
+ return go("modernize-preflight", "checks the environment and records where the code lives")
+ if not done["assess"]:
+ return go("modernize-assess", "says what you are dealing with and recommends a pattern")
+ if not done["map"]:
+ return go("modernize-map", "draws the structure of the system")
+ if not done["rules"]:
+ return go("modernize-extract-rules", "mines the business rules, each with a citation")
+ rules = glance.get("rules") or {}
+ if not done["brief"] and rules.get("defects") and not rules.get("reviews"):
+ return go("modernize-review", "a person confirms or corrects the rules that look wrong")
+ if not done["brief"]:
+ return go("modernize-brief", "writes the plan a person approves")
+ if not done["build"]:
+ return go("modernize-status", "names the first build command from the approved plan")
+ proof = glance.get("proof")
+ if not proof:
+ return go("modernize-verify", "re-checks what was built and gives each module a verdict")
+ if proof.get("verdict") != "PROVEN":
+ return go("modernize-verify", "the last check was %s: fix what its reasons name, then run it again" % str(proof.get("verdict"))[:20])
+ if not done["harden"]:
+ return go("modernize-harden", "scans the legacy system for security problems")
+ return go("modernize-status", "says whether modules or phases are left")
+
+
+def build(system, workspace, out=None):
+ """-> (html, section count, output path), or None when nothing was found."""
+ src = Source(workspace, system)
+ out = os.path.abspath(out or os.path.join(src.adir, "REPORT.html"))
+ a = lambda name: os.path.join(src.adir, name) # noqa: E731
+ parts, texts, docs = {k: [] for k, _ in TITLES}, [], {}
+ reviews, _ = src.json(a("RULE_REVIEWS.json"), "RULE_REVIEWS.json")
+ reviews = {k: v for k, v in reviews["reviews"].items() if isinstance(v, dict)} if isinstance(reviews, dict) and isinstance(reviews.get("reviews"), dict) else {}
+ glance = {"rules": None, "security": None, "baseline": None, "topology": None, "equivalence": None, "proof": None}
+
+ for key, names in DOCS:
+ for name in names:
+ text = src.read(a(name), name)
+ if text is None:
+ continue
+ docs[name] = text
+ blocks = parse_rules(text) if name == "BUSINESS_RULES.md" else []
+ cards = [b for b in blocks if b["k"] == "rule"]
+ if cards:
+ glance["rules"] = rule_facts(text, cards, reviews)
+ parts[key].append({"t": "rules", "name": name, "blocks": blocks, "reviews": {k: str(v.get("verdict", ""))[:20] for k, v in reviews.items()}})
+ else:
+ parts[key].append({"t": "md", "name": name, "text": text})
+ texts += docs.values()
+ glance["security"] = security_facts(docs["SECURITY_FINDINGS.md"]) if "SECURITY_FINDINGS.md" in docs else None
+ glance["baseline"] = baseline_facts(docs["BASELINE.md"]) if "BASELINE.md" in docs else None
+
+ topo, topo_seen = src.json(a("topology.json"), "topology.json")
+ t = glance["topology"] = topology_facts(topo)
+ try:
+ names = sorted(e.name for e in os.scandir(src.adir) if e.name.lower().endswith(".mmd"))
+ except OSError:
+ names = []
+ figures = [{"name": n, "src": text} for n in names for text in [src.read(a(n), n)] if text is not None]
+ texts += [f["src"] for f in figures]
+ if t:
+ parts["diagrams"].append({"t": "facts", "items": [["Modules", num(t["modules"])], ["Lines of code", num(t["loc"])]] + (
+ [["Dependencies", num(t["edges"])]] if t["edges"] is not None else []) + ([["Languages", ", ".join(t["languages"])]] if t["languages"] else [])})
+ html_map = a("TOPOLOGY.html")
+ has_map = os.path.isfile(html_map) and not os.path.islink(html_map)
+ if has_map:
+ src.found.append(("TOPOLOGY.html", os.path.getsize(html_map), utc_day(os.path.getmtime(html_map))))
+ try:
+ rel = quote(os.path.relpath(html_map, os.path.dirname(out)).replace(os.sep, "/"), safe="/")
+ parts["diagrams"].append({"t": "link", "href": rel, "text": "Open the interactive map (TOPOLOGY.html, kept as a separate file)"})
+ except ValueError:
+ pass
+ if t and t["observations"]:
+ parts["diagrams"].append({"t": "list", "title": "What the map found", "items": t["observations"]})
+ if figures:
+ parts["diagrams"].append({"t": "figures", "items": figures})
+
+ for label, text in modernized_docs(src):
+ texts.append(text)
+ parts["build"].append({"t": "md", "name": "modernized/" + label, "text": text})
+
+ eq, eq_seen = src.json(a("EQUIVALENCE.json"), "EQUIVALENCE.json")
+ if eq_seen:
+ view = equivalence_view(eq)
+ if view:
+ parts["equivalence"].append({"t": "equiv", "eq": view})
+ glance["equivalence"] = {k: view[k] for k in ("state", "headline", "problems", "tally")}
+ else:
+ glance["equivalence"] = {"state": "red", "headline": "EQUIVALENCE.json could not be used, so equivalence is not proven.", "problems": [], "tally": None}
+ parts["equivalence"].append({"t": "p", "text": glance["equivalence"]["headline"]})
+
+ ver, ver_seen = src.json(a("VERIFICATION.json"), "VERIFICATION.json")
+ if ver_seen:
+ pv = proof_view(ver)
+ if pv:
+ parts["proof"].append({"t": "proof", "proof": pv})
+ glance["proof"] = {"verdict": pv["verdict"], "headline": pv["headline"], "counts": pv["counts"], "problems": pv["problems"][:3], "modules": [{"name": m["name"], "verdict": m["verdict"]} for m in pv["modules"][:8]]}
+ else:
+ glance["proof"] = {"verdict": "NOT PROVEN", "headline": "VERIFICATION.json could not be used, so nothing is proven.", "counts": {}, "problems": [], "modules": []}
+ parts["proof"].append({"t": "p", "text": glance["proof"]["headline"]})
+ mods = [os.path.join(src.mdir, system + s) for s in ("", "-uplifted", "-reimagined")]
+ done = {"preflight": "PREFLIGHT.md" in docs, "assess": "ASSESSMENT.md" in docs, "rules": "BUSINESS_RULES.md" in docs, "brief": "MODERNIZATION_BRIEF.md" in docs,
+ "map": topo is not None or has_map or any(f["name"] != "ARCHITECTURE.mmd" for f in figures), "build": any(nonempty_dir(p) for p in mods), "harden": "SECURITY_FINDINGS.md" in docs}
+ glance["steps"] = [{"key": k, "label": label, "done": bool(done[k])} for k, label in STEPS]
+ glance["next"] = next_step(system, done, glance)
+ tracks = {"rewrite": [mods[0]], "same-stack uplift": [a("DELTA_CATALOG.md"), a("PLAYBOOK.md"), mods[1]], "reimagine": [a("AI_NATIVE_SPEC.md"), a("REIMAGINED_ARCHITECTURE.md"), mods[2]]}
+ ranked = sorted(((newest(p), k) for k, p in tracks.items() if newest(p)), reverse=True)
+ glance["track"] = {"label": ranked[0][1], "why": "newest artifact: " + os.path.relpath(ranked[0][0][1], workspace).replace(os.sep, "/")} if ranked else None
+ if not src.found:
+ return None
+
+ lib = load_mermaid(src) if figures or any(MERMAID_FENCE.search(x) for x in texts) else ""
+ parts["overview"].insert(0, {"t": "table", "head": ["Artifact", "Size", "Modified"], "rows": [[n, "%s KB" % num(s >> 10) if s >= 1024 else "%d B" % s, m] for n, s, m in sorted(src.found)]})
+ if src.notes:
+ parts["overview"].insert(0, {"t": "list", "title": "Notes about this report", "items": src.notes})
+ sections = [{"id": k, "title": title, "parts": parts[k]} for k, title in TITLES if parts[k]]
+ data = {"v": 1, "system": system[:200], "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC"), "glance": glance, "sections": sections}
+ return render(data, lib), len(sections), out
+
+
+def script_json(data):
+ """JSON that can sit inside a <script> element: no <, > or & appears in it, so </script> and <!-- cannot."""
+ return json.dumps(data, ensure_ascii=True, separators=(",", ":")).replace("<", "\\u003c").replace(">", "\\u003e").replace("&", "\\u0026")
+
+
+def render(data, lib):
+ with open(os.path.join(ASSETS, "report-template.html"), encoding="utf-8") as fh:
+ template = fh.read().replace("\r\n", "\n")
+ app = re.search(r'<script id="app">(.*?)</script>', template, re.S).group(1)
+ sha = lambda text: "'sha256-%s'" % base64.b64encode(hashlib.sha256(text.encode("utf-8")).digest()).decode() # noqa: E731
+ policy = "default-src 'none'; script-src %s; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'" % " ".join([sha(app)] + ([sha(lib)] if lib else []))
+ fill = {"CSP": policy, "DATA": script_json(data), "MERMAID": '<script id="mermaid-lib">%s</script>' % lib if lib else ""}
+ return re.sub(r"@@(CSP|DATA|MERMAID)@@", lambda m: fill[m.group(1)], template)
+
+
+def write_atomic(path, data):
+ """Write `data` to a temp file beside `path`, then rename it over `path`.
+ Returns why it refused (a symlinked path or folder), or None once written."""
+ folder = os.path.dirname(path)
+ for p in (path, folder):
+ if os.path.islink(p):
+ return "refusing to write through a symbolic link: %s" % p
+ os.makedirs(folder, exist_ok=True)
+ fd, tmp = tempfile.mkstemp(dir=folder, prefix=".report-", suffix=".tmp")
+ try:
+ with os.fdopen(fd, "wb") as fh:
+ fh.write(data)
+ umask = os.umask(0)
+ os.umask(umask)
+ os.chmod(tmp, 0o666 & ~umask)
+ os.replace(tmp, path)
+ except BaseException:
+ if os.path.lexists(tmp):
+ os.unlink(tmp)
+ raise
+ return None
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description="Build a self-contained HTML report from a system's modernization artifacts.")
+ ap.add_argument("system", help="the system's folder name under analysis/")
+ ap.add_argument("--workspace", default=".", help="the project root holding analysis/ and modernized/ (default: current folder)")
+ ap.add_argument("--out", help="where to write the report (default: <workspace>/analysis/<system>/REPORT.html)")
+ args = ap.parse_args(argv)
+ if not args.system or args.system in (".", "..") or re.search(r"[\\/\x00]", args.system):
+ print("build_report.py: the system must be a folder name under analysis/, not a path", file=sys.stderr)
+ return 2
+ try:
+ result = build(args.system, args.workspace, args.out)
+ except OSError as err:
+ print("build_report.py: %s" % err, file=sys.stderr)
+ return 1
+ if result is None:
+ print("build_report.py: nothing found for %r under %s (analysis/ or modernized/)" % (args.system, os.path.abspath(args.workspace)), file=sys.stderr)
+ return 1
+ html, count, path = result
+ try:
+ refused = write_atomic(path, html.encode("utf-8"))
+ except OSError as err:
+ refused = "could not write %s (%s)" % (path, err.__class__.__name__)
+ if refused:
+ print("build_report.py: %s" % refused, file=sys.stderr)
+ return 1
+ print("%s: %d section%s, %s KB" % (path, count, "" if count == 1 else "s", num(os.path.getsize(path) >> 10)))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/plugins/code-modernization/scripts/compare.py b/plugins/code-modernization/scripts/compare.py
new file mode 100644
index 00000000..4a1e4429
--- /dev/null
+++ b/plugins/code-modernization/scripts/compare.py
@@ -0,0 +1,487 @@
+#!/usr/bin/env python3
+"""Deterministic judge of "does the new system give the same output as the legacy one".
+
+ python3 compare.py <cases.json> [--out EQUIVALENCE.json] [--quiet] [--allow-outside]
+
+The verdict comes from comparing bytes, never from a model's opinion. Input:
+
+ {"system": "carddemo",
+ "legacy": {"label": "COBOL, GnuCOBOL 3.2", "command": "optional free text"},
+ "new": {"label": "Java 17", "command": "optional free text"},
+ "cases": [{"id": "C01", "title": "Interest posting, normal account",
+ "legacy": "out/legacy/C01.out", "new": "out/new/C01.out",
+ "mask": [{"bytes": "278-330", "why": "run timestamp"},
+ {"regex": "\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}"}],
+ "approvedDifference": "optional: why a person accepted a difference",
+ "note": "optional"}]}
+
+Paths are relative to the cases file's folder and may not leave it unless --allow-outside is
+given. `bytes` ranges are 0-based and inclusive; `regex` runs on the bytes decoded as latin-1.
+Every masked span, on both sides, is replaced by one fixed marker, so masks never hide the
+position of a difference and a variable-length match still compares equal. A case is:
+ same identical after masking
+ differs different; the first difference is recorded (a person may approve it: differs-approved)
+ missing a file is absent, unreadable or outside the cases folder. Never a pass.
+
+A person who accepts a difference records it on the case (`"approvedDifference": "why"`), or once for every
+case that shares an `input` label with a top-level `"approvedInputs": {"F23": "why"}`. A reason is required either way, and an approved difference is still listed as a
+difference in the result.
+
+Numbers that a run legitimately prints a little differently (floating-point results from another
+compiler or math library) can be compared within a declared tolerance, per case or for all cases:
+
+ "tolerance": {"rel": 1e-9, "abs": 0, "why": "last-digit rounding of the exp() in the interest formula"}
+
+The reason is required and is recorded. Only numbers written with a decimal point or an exponent
+may differ, and only within the tolerance (compared as exact decimals); every other byte, every integer and every
+dotted run like 1.2.3 must match exactly. The case is `same`, with the largest relative difference
+recorded; anything outside the tolerance is `differs`. A relative tolerance above 1% or an absolute one
+above 1e-6 is refused: that is a different result, not rounding.
+
+Writes EQUIVALENCE.json (default: next to the cases file). Exit 0 only when at least one case
+executed, none is missing or differs (approved differences are allowed), the self-check did not
+fail and some compared output was not empty; 1 otherwise; 2 for unusable input.
+"""
+import argparse
+import bisect
+import datetime
+import hashlib
+import decimal
+import json
+import math
+import os
+import re
+import sys
+import tempfile
+
+MARK = b"\x00<MASK>\x00"
+MAX_BYTES = 256 * 1024 * 1024
+MAX_TOLERANT_BYTES = 32 * 1024 * 1024
+# A dotted run such as 1.2.3 or 10.0.0.1 is an identifier, not a number: it is left inside the text around it.
+NUMBER = re.compile(rb"(?P<dotted>\d+(?:\.\d+){2,})|(?P<num>[-+]?(?:\d+\.\d*|\.\d+|\d+)(?:[eE][-+]?\d+)?)")
+MAX_ABS_TOLERANCE = 1e-6
+MAX_NUMBER_CHARS = 64
+
+
+class InputError(Exception):
+ pass
+
+
+def clip(value, limit):
+ return value[:limit] if isinstance(value, str) else ""
+
+
+def parse_masks(specs, cid):
+ """[{"bytes": "0-3,10"} | {"regex": "..."}] -> [(kind, ranges-or-pattern, label)]."""
+ masks = []
+ for spec in specs if isinstance(specs, list) else []:
+ if not isinstance(spec, dict) or ("bytes" in spec) == ("regex" in spec):
+ raise InputError("case %s: each mask needs exactly one of 'bytes' or 'regex'" % cid)
+ why = " (%s)" % clip(spec.get("why"), 120) if spec.get("why") else ""
+ if "bytes" in spec:
+ ranges = []
+ for part in str(spec["bytes"]).split(","):
+ m = re.fullmatch(r"\s*(\d+)\s*(?:-\s*(\d+))?\s*", part)
+ if not m or (m.group(2) and int(m.group(2)) < int(m.group(1))):
+ raise InputError("case %s: bad byte range %r" % (cid, part))
+ ranges.append((int(m.group(1)), int(m.group(2) or m.group(1))))
+ masks.append(("bytes", ranges, "bytes %s%s" % (spec["bytes"], why)))
+ else:
+ try:
+ masks.append(("regex", re.compile(str(spec["regex"])), "regex %s%s" % (clip(str(spec["regex"]), 120), why)))
+ except re.error as err:
+ raise InputError("case %s: bad regex %r: %s" % (cid, spec["regex"], err))
+ return masks
+
+
+def parse_tolerance(spec, cid):
+ """{"rel": 1e-9, "abs": 0, "why": "..."} -> a validated dict, or None when there is none."""
+ if spec is None:
+ return None
+ if not isinstance(spec, dict):
+ raise InputError("case %s: 'tolerance' must be an object with rel, abs and why" % cid)
+ try:
+ rel, ab = float(spec.get("rel", 0)), float(spec.get("abs", 0))
+ except (TypeError, ValueError):
+ raise InputError("case %s: the tolerance's rel and abs must be numbers" % cid)
+ if not (math.isfinite(rel) and math.isfinite(ab)) or rel < 0 or ab < 0 or (rel == 0 and ab == 0):
+ raise InputError("case %s: give a positive rel or abs tolerance" % cid)
+ if rel > 0.01:
+ raise InputError("case %s: a relative tolerance above 1%% is a different result, not rounding" % cid)
+ if ab > MAX_ABS_TOLERANCE:
+ raise InputError("case %s: an absolute tolerance above %g is a different result, not rounding" % (cid, MAX_ABS_TOLERANCE))
+ why = clip(spec.get("why"), 300).strip()
+ if not why:
+ raise InputError("case %s: a tolerance needs a 'why' saying what legitimately differs" % cid)
+ return {"rel": rel, "abs": ab, "why": why}
+
+
+def split_numbers(data):
+ """-> [(is_number, bytes, offset)] covering `data` end to end."""
+ parts, pos = [], 0
+ for m in NUMBER.finditer(data):
+ if m.group("dotted") is not None:
+ continue
+ if m.start() > pos:
+ parts.append((False, data[pos:m.start()], pos))
+ parts.append((True, m.group(), m.start()))
+ pos = m.end()
+ if pos < len(data):
+ parts.append((False, data[pos:], pos))
+ return parts
+
+
+def is_float(token):
+ return any(c in token for c in b".eE")
+
+
+def within_tolerance(a, b, tol):
+ """Compare two masked outputs number by number. -> (True, stats) or (False, offset of the first real difference)."""
+ if len(a) > MAX_TOLERANT_BYTES or len(b) > MAX_TOLERANT_BYTES:
+ return False, first_diff(a, b) or 0
+ ta, tb = split_numbers(a), split_numbers(b)
+ if len(ta) != len(tb):
+ return False, first_diff(a, b) or 0
+ allow_abs, allow_rel = decimal.Decimal(repr(tol["abs"])), decimal.Decimal(repr(tol["rel"]))
+ differing, worst = 0, 0.0
+ for (na, xa, sa), (nb, xb, _) in zip(ta, tb):
+ if xa == xb:
+ continue
+ if na != nb:
+ return False, sa
+ if not na:
+ return False, sa + (first_diff(xa, xb) or 0)
+ if not (is_float(xa) and is_float(xb)) or max(len(xa), len(xb)) > MAX_NUMBER_CHARS:
+ return False, sa # an integer, or a very long number, must match exactly
+ try: # exact decimal arithmetic: no double-precision blind spot
+ x, y = decimal.Decimal(xa.decode("ascii")), decimal.Decimal(xb.decode("ascii"))
+ if not (x.is_finite() and y.is_finite()):
+ return False, sa
+ scale, diff = max(abs(x), abs(y)), abs(x - y)
+ if diff > max(allow_abs, allow_rel * scale):
+ return False, sa
+ if scale:
+ worst = max(worst, float(diff / scale))
+ except (ArithmeticError, ValueError):
+ return False, sa
+ differing += 1
+ return True, {"differing": differing, "maxRelativeDifference": worst}
+
+
+def mask_spans(data, masks):
+ """Merged (start, end) spans of `data` that the masks hide."""
+ spans, text = [], None
+ for kind, spec, _ in masks:
+ if kind == "bytes":
+ spans += [(s, min(e + 1, len(data))) for s, e in spec if s < len(data)]
+ else:
+ text = data.decode("latin-1") if text is None else text
+ spans += [m.span() for m in spec.finditer(text) if m.end() > m.start()]
+ merged = []
+ for s, e in sorted(spans):
+ if merged and s <= merged[-1][1]:
+ merged[-1] = (merged[-1][0], max(e, merged[-1][1]))
+ else:
+ merged.append((s, e))
+ return merged
+
+
+def apply_masks(data, masks):
+ """-> (masked bytes, cuts, hidden byte count); cuts map masked offsets back to file offsets."""
+ out, pos, cuts, hidden, at = [], 0, [], 0, 0
+ for s, e in mask_spans(data, masks):
+ out.append(data[pos:s])
+ at += s - pos
+ cuts.append((at, s, e))
+ out.append(MARK)
+ at += len(MARK)
+ hidden += e - s
+ pos = e
+ out.append(data[pos:])
+ return b"".join(out), cuts, hidden
+
+
+def original_offset(cuts, masked_offset):
+ shift = 0
+ for at, s, e in cuts:
+ if masked_offset < at:
+ break
+ if masked_offset < at + len(MARK):
+ return s
+ shift += (e - s) - len(MARK)
+ return masked_offset + shift
+
+
+def first_diff(a, b):
+ n, step, i = min(len(a), len(b)), 1 << 16, 0
+ while i < n:
+ j = min(i + step, n)
+ if a[i:j] != b[i:j]:
+ return next(k for k in range(i, j) if a[k] != b[k])
+ i = j
+ return n if len(a) != len(b) else None
+
+
+def escape(chunk):
+ out = []
+ for byte in chunk:
+ if byte == 0x5C:
+ out.append("\\\\")
+ elif 0x20 <= byte < 0x7F:
+ out.append(chr(byte))
+ else:
+ out.append({10: "\\n", 13: "\\r", 9: "\\t"}.get(byte, "\\x%02x" % byte))
+ return "".join(out).replace("\\x00<MASK>\\x00", "[masked]")
+
+
+def context(masked, off):
+ before = escape(masked[max(0, off - 12):off])[-12:]
+ after = escape(masked[off:off + 28])[:28] if off < len(masked) else "<end of output>"
+ return before + after, len(before)
+
+
+def read_file(rel, base, allow_outside):
+ """-> (bytes, None) or (None, plain-English problem)."""
+ if not isinstance(rel, str) or not rel:
+ return None, "no path was given"
+ path = os.path.join(base, rel)
+ try:
+ real, root = os.path.realpath(path), os.path.realpath(base)
+ if not allow_outside and os.path.commonpath([root, real]) != root:
+ return None, "the path leaves the cases folder (pass --allow-outside if that is intended)"
+ if not os.path.exists(path):
+ return None, "the file does not exist"
+ if os.path.getsize(path) > MAX_BYTES:
+ return None, "the file is larger than %d MB" % (MAX_BYTES >> 20)
+ with open(path, "rb") as fh:
+ return fh.read(), None
+ except (OSError, ValueError) as err:
+ return None, "the file could not be read (%s)" % err.__class__.__name__
+
+
+def rel_to(path, folder):
+ try:
+ return os.path.relpath(path, folder).replace(os.sep, "/")
+ except ValueError:
+ return path
+
+
+def judge(case, index, base, out_dir, allow_outside, default_tolerance=None, approved_inputs=None):
+ """-> (case record, legacy bytes or None, masks)."""
+ cid = clip(str(case.get("id") or "C%02d" % (index + 1)), 80)
+ masks = parse_masks(case.get("mask"), cid)
+ tol = parse_tolerance(case["tolerance"] if "tolerance" in case else default_tolerance, cid)
+ rec = {"id": cid, "title": clip(case.get("title"), 300), "verdict": "missing", "reason": "",
+ "legacyPath": "", "newPath": "", "legacySha256": "", "newSha256": "",
+ "masked": [m[2] for m in masks],
+ "approvedDifference": clip(case.get("approvedDifference") or (approved_inputs or {}).get(str(case.get("input", "")), ""), 500),
+ "note": clip(case.get("note"), 500)}
+ data, problems = {}, []
+ for side in ("legacy", "new"):
+ given = case.get(side)
+ rec[side + "Path"] = rel_to(os.path.join(base, given), out_dir) if isinstance(given, str) and given else ""
+ data[side], why = read_file(given, base, allow_outside)
+ if why:
+ problems.append("The %s output was not compared: %s (%s)." % (side, why, clip(str(given), 200)))
+ if problems:
+ rec["reason"] = " ".join(problems)
+ return rec, None, masks, tol
+ legacy, new = data["legacy"], data["new"]
+ rec["legacySha256"], rec["newSha256"] = hashlib.sha256(legacy).hexdigest(), hashlib.sha256(new).hexdigest()
+ a, cuts, hid_a = apply_masks(legacy, masks)
+ b, _, hid_b = apply_masks(new, masks)
+ rec["bytes"], rec["maskedBytes"] = {"legacy": len(legacy), "new": len(new)}, {"legacy": hid_a, "new": hid_b}
+ rec["empty"] = not legacy and not new
+ at = first_diff(a, b)
+ if at is None:
+ rec["verdict"] = "same"
+ rec["reason"] = ("Both outputs are empty, so nothing was compared." if rec["empty"] else
+ "Identical after masking: %d of %d bytes were hidden by %d mask(s)." % (hid_a, len(legacy), len(masks)) if masks else
+ "Identical, %d bytes." % len(legacy))
+ return rec, legacy, masks, tol
+ seen, seen_at = context(a, at)
+ got, _ = context(b, at)
+ line = a.count(b"\n", 0, at) + 1
+ rec["firstDiff"] = {"offset": original_offset(cuts, at), "line": line, "legacy": seen, "new": got, "at": seen_at}
+ where = "line %d, byte %d" % (line, rec["firstDiff"]["offset"])
+ if at >= len(a) or at >= len(b):
+ rec["reason"] = "The %s output ends at %s while the %s output continues." % (
+ "legacy" if at >= len(a) else "new", where, "new" if at >= len(a) else "legacy")
+ else:
+ rec["reason"] = "The outputs first differ at %s." % where
+ if tol:
+ rec["tolerance"] = dict(tol)
+ ok, info = within_tolerance(a, b, tol)
+ if ok:
+ rec["verdict"] = "same"
+ rec["withinTolerance"] = info
+ rec["reason"] = ("Identical apart from %d number(s) that differ within the declared tolerance "
+ "(relative %g, absolute %g; largest relative difference %.3g). Declared reason: %s"
+ % (info["differing"], tol["rel"], tol["abs"], info["maxRelativeDifference"], clip(tol["why"], 200)))
+ return rec, legacy, masks, tol
+ line = a.count(b"\n", 0, info) + 1
+ rec["reason"] = "A number or text differs beyond the declared tolerance at line %d, byte %d." % (line, original_offset(cuts, info))
+ if rec["approvedDifference"].strip():
+ rec["verdict"] = "differs-approved"
+ rec["reason"] += " A person approved this difference: %s" % clip(rec["approvedDifference"], 200)
+ else:
+ rec["verdict"] = "differs"
+ return rec, legacy, masks, tol
+
+
+def differs_under(changed, data, masks, tol):
+ """Would the comparator report `changed` as different from `data`?"""
+ a, b = apply_masks(data, masks)[0], apply_masks(changed, masks)[0]
+ return a != b and not (tol and within_tolerance(a, b, tol)[0])
+
+
+def self_check(items):
+ """Prove the comparator can fail: change one unmasked byte of each legacy output; it must differ."""
+ tested = 0
+ for cid, data, masks, tol in items:
+ if not data:
+ continue
+ tested += 1
+ free, pos = [], 0
+ for s, e in mask_spans(data, masks) + [(len(data), len(data))]:
+ if s > pos:
+ free.append((pos, s))
+ pos = e
+ if not free:
+ return False, "case %s: the masks hide every byte, so no difference could ever be seen" % cid
+ biggest = max(free, key=lambda f: f[1] - f[0])
+ flip = lambda p: data[:p] + bytes([data[p] ^ 1]) + data[p + 1:]
+ spots = [free[0][0], free[-1][1] - 1, (biggest[0] + biggest[1]) // 2]
+ caught = any(differs_under(flip(p), data, masks, tol) for p in spots)
+ if tol and caught:
+ # A tolerance must never hide a change to the numbers it covers: changing the leading digit of every
+ # sampled float that is not masked has to be reported, or the tolerance is too loose to prove anything.
+ for m in list(NUMBER.finditer(data[:1 << 20]))[:200]:
+ token = m.group("num")
+ if token is None or not is_float(token) or not any(a <= m.start() < b for a, b in free):
+ continue
+ at = m.start() + next(i for i, c in enumerate(token) if 48 <= c <= 57)
+ if not differs_under(flip(at), data, masks, tol):
+ caught = False
+ break
+ if not caught:
+ return False, "case %s: a one-byte change to the legacy output was not reported as different%s" % (
+ cid, " (the tolerance may be too loose)" if tol else "")
+ if not tested:
+ return None, "not applicable: no executed case had bytes to test"
+ return True, "each legacy output was compared with a copy changed by one byte and reported as different"
+
+
+def totals(records, executed, count):
+ t = {"cases": len(records), "executed": executed, "same": count("same"), "differs": count("differs"),
+ "differsApproved": count("differs-approved"), "missing": count("missing")}
+ tolerant = sum(1 for r in records if r.get("withinTolerance"))
+ if tolerant: # additive: absent unless a declared tolerance was used
+ t["sameWithinTolerance"] = tolerant
+ return t
+
+
+def run(cases_path, out_path=None, allow_outside=False):
+ """-> the EQUIVALENCE record. Raises InputError for a cases file that cannot be used."""
+ try:
+ with open(cases_path, "rb") as fh:
+ spec = json.loads(fh.read().decode("utf-8-sig"))
+ except (OSError, ValueError) as err:
+ raise InputError("cannot read %s: %s" % (cases_path, err))
+ if not isinstance(spec, dict) or not isinstance(spec.get("cases"), list):
+ raise InputError("%s must be a JSON object with a 'cases' list" % cases_path)
+ base = os.path.dirname(os.path.abspath(cases_path))
+ out_dir = os.path.dirname(os.path.abspath(out_path)) if out_path else base
+ records, items, ids = [], [], set()
+ approved = {str(k): v for k, v in spec["approvedInputs"].items() if isinstance(v, str) and v.strip()} if isinstance(spec.get("approvedInputs"), dict) else {}
+ for i, case in enumerate(spec["cases"]):
+ if not isinstance(case, dict):
+ raise InputError("case %d is not an object" % (i + 1))
+ rec, data, masks, tol = judge(case, i, base, out_dir, allow_outside, spec.get("tolerance"), approved)
+ if rec["id"] in ids:
+ raise InputError("duplicate case id %r" % rec["id"])
+ ids.add(rec["id"])
+ records.append(rec)
+ if data is not None:
+ items.append((rec["id"], data, masks, tol))
+ passed, detail = self_check(items)
+ count = lambda v: sum(1 for r in records if r["verdict"] == v)
+ ends = lambda key: {k: clip(spec.get(key, {}).get(k), 300) for k in ("label", "command")} if isinstance(spec.get(key), dict) else {"label": "", "command": ""}
+ return {"system": clip(spec.get("system"), 200), "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+ "legacy": ends("legacy"), "new": ends("new"), "selfCheck": {"passed": passed, "detail": detail},
+ "totals": totals(records, len(items), count), "cases": records}
+
+
+def verdict_of(result):
+ """-> None when proven, else the reason it is not."""
+ t = result["totals"]
+ if t["executed"] == 0:
+ return "no case executed"
+ if t["missing"] or t["differs"]:
+ return " and ".join(filter(None, ["%d case(s) differ" % t["differs"] if t["differs"] else "",
+ "%d case(s) are missing" % t["missing"] if t["missing"] else ""]))
+ if result["selfCheck"]["passed"] is False:
+ return "the self-check failed: " + result["selfCheck"]["detail"]
+ if all(r.get("empty") for r in result["cases"] if r["verdict"] != "missing"):
+ return "every compared output was empty"
+ return None
+
+
+def write_atomic(path, text):
+ path = os.path.abspath(path)
+ folder = os.path.dirname(path)
+ for p in (path, folder):
+ if os.path.islink(p):
+ raise InputError("refusing to write through a symbolic link: %s" % p)
+ os.makedirs(folder, exist_ok=True)
+ fd, tmp = tempfile.mkstemp(dir=folder, prefix=".equivalence-", suffix=".tmp")
+ try:
+ with os.fdopen(fd, "wb") as fh:
+ fh.write(text.encode("utf-8"))
+ umask = os.umask(0)
+ os.umask(umask)
+ os.chmod(tmp, 0o666 & ~umask)
+ os.replace(tmp, path)
+ except BaseException:
+ if os.path.lexists(tmp):
+ os.unlink(tmp)
+ raise
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description="Compare legacy and new outputs byte for byte.")
+ ap.add_argument("cases", help="cases.json (see the module docstring for the schema)")
+ ap.add_argument("--out", help="where to write EQUIVALENCE.json (default: next to the cases file)")
+ ap.add_argument("--quiet", action="store_true", help="write the file, print nothing")
+ ap.add_argument("--allow-outside", action="store_true", help="allow case paths that leave the cases folder")
+ args = ap.parse_args(argv)
+ out = args.out or os.path.join(os.path.dirname(os.path.abspath(args.cases)), "EQUIVALENCE.json")
+ try:
+ result = run(args.cases, out, args.allow_outside)
+ except InputError as err:
+ print("compare.py: %s" % err, file=sys.stderr)
+ return 2
+ try:
+ write_atomic(out, json.dumps(result, indent=2, ensure_ascii=True) + "\n")
+ except InputError as err:
+ print("compare.py: %s" % err, file=sys.stderr)
+ return 2
+ problem = verdict_of(result)
+ if not args.quiet:
+ t, plain = result["totals"], lambda s: re.sub(r"[\x00-\x1f\x7f-\x9f]", "?", s)
+ print("equivalence cases executed: %d" % t["executed"])
+ print("same %d%s | differs %d | differs but approved %d | missing %d | of %d cases" %
+ (t["same"], " (%d within a declared tolerance)" % t["sameWithinTolerance"] if t.get("sameWithinTolerance") else "",
+ t["differs"], t["differsApproved"], t["missing"], t["cases"]))
+ sc = result["selfCheck"]
+ print("self-check: %s - %s" % ({True: "passed", False: "FAILED", None: "not applicable"}[sc["passed"]], plain(sc["detail"])))
+ for r in result["cases"]:
+ if r["verdict"] != "same" or r.get("withinTolerance"):
+ print(" %s %s: %s" % (plain(r["id"]), r["verdict"], plain(r["reason"])[:200]))
+ print(("NOT PROVEN: %s" % problem) if problem else "PROVEN: %d case(s) executed, none differ" % t["executed"])
+ print("wrote %s" % out)
+ return 1 if problem else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/plugins/code-modernization/scripts/make_shards.py b/plugins/code-modernization/scripts/make_shards.py
new file mode 100644
index 00000000..175d6980
--- /dev/null
+++ b/plugins/code-modernization/scripts/make_shards.py
@@ -0,0 +1,271 @@
+#!/usr/bin/env python3
+"""Split a system's source into shards for per-module rule extraction.
+
+ python3 make_shards.py <system> [module-pattern] [--workspace DIR] [--all]
+
+The source directory is legacy/<system> (a copy, or a symlink to where the code lives). With analysis/<system>/topology.json, one shard per map
+module (small modules of a domain merged); without it, one shard per directory of source
+files. The list is written to analysis/<system>/extract-rules.modules.json as
+[{"name", "domain", "files": [relative to the source dir], "loc"}], the form the
+extract-rules workflow takes. Standard library only.
+"""
+import fnmatch
+import json
+import os
+import sys
+
+SOURCE_EXT = {
+ # mainframe and legacy business languages
+ '.cbl', '.cob', '.cobol', '.cpy', '.pli', '.pl1', '.jcl', '.proc', '.rpg', '.rpgle', '.sqlrpgle', '.clle', '.nat', '.ada', '.adb', '.ads',
+ '.asm', '.bms', '.map', '.pas', '.dpr', '.prg', '.f', '.for', '.f90', '.f77',
+ # general purpose
+ '.java', '.kt', '.scala', '.groovy', '.cs', '.vb', '.vbs', '.bas', '.cls', '.frm', '.fs', '.c', '.h', '.cc', '.cpp', '.cxx', '.hpp', '.m', '.mm',
+ '.go', '.rs', '.py', '.rb', '.php', '.pl', '.pm', '.js', '.jsx', '.ts', '.tsx', '.swift', '.lua', '.tcl', '.erl', '.ex', '.exs', '.clj',
+ '.hs', '.ml', '.r', '.sh', '.bat', '.ps1',
+ # web and data
+ '.asp', '.aspx', '.ascx', '.cshtml', '.jsp', '.jspx', '.sql', '.pks', '.pkb', '.plsql', '.tf',
+}
+SKIP_DIRS = {
+ '.git', '.svn', '.hg', 'node_modules', 'vendor', 'third_party', 'thirdparty', 'dist', 'build', 'target', 'bin', 'obj', 'out',
+ '__pycache__', '.venv', 'venv', '.tox', '.gradle', '.idea', 'coverage', 'fixtures', 'testdata', 'test-data', 'generated',
+}
+TEST_DIRS = {'test', 'tests', '__tests__', 'spec', 'specs'}
+MAX_FILES = 25
+MAX_LOC = 5000
+SMALL = 300
+TINY_ESTATE = 30
+MAX_BYTES = 5_000_000
+
+
+def too_broad(path):
+ """The filesystem root, the home directory or one of its parents, or a top-level system directory: never a system's code."""
+ real = os.path.realpath(path)
+ home = os.path.realpath(os.path.expanduser('~'))
+ parts = [p for p in real.split(os.sep) if p]
+ return real == os.sep or len(parts) < 2 or home == real or home.startswith(real + os.sep)
+
+
+def source_dir(workspace, system):
+ """The code is legacy/<system>: a copy, or a symlink to where it really lives."""
+ return os.path.join(workspace, 'legacy', system)
+
+
+def count_lines(path):
+ try:
+ if os.path.getsize(path) > MAX_BYTES:
+ return None
+ with open(path, 'rb') as handle:
+ data = handle.read()
+ except OSError:
+ return None
+ if b'\x00' in data[:4096]:
+ return None
+ return data.count(b'\n') + (1 if data and not data.endswith(b'\n') else 0)
+
+
+def matcher(pattern):
+ """A glob matches a name or a path (or its file name); a plain name such as `jetty-util` also matches what lies beneath it."""
+ plain_name = not any(ch in pattern for ch in '*?[')
+ wanted = pattern.strip('/')
+
+ def match(text):
+ if fnmatch.fnmatch(text, pattern) or fnmatch.fnmatch(os.path.basename(text), pattern):
+ return True
+ if plain_name:
+ padded = '/' + text.strip('/') + '/'
+ return padded.startswith('/' + wanted + '/') or ('/' + wanted + '/') in padded
+ return False
+
+ return match
+
+
+def contained(source, rel):
+ """True when `rel` under the source directory really resolves inside it (no `..`, no symlink out)."""
+ root = os.path.realpath(source)
+ real = os.path.realpath(os.path.join(source, rel))
+ return real == root or real.startswith(root + os.sep)
+
+
+def relative_to_source(path, source, system):
+ """A topology `file` may be absolute, workspace-relative or source-relative: return it source-relative."""
+ path = path.replace('\\', '/')
+ if os.path.isabs(path):
+ rel = os.path.relpath(path, source).replace(os.sep, '/')
+ return None if rel.startswith('..') else rel
+ for prefix in (f'legacy/{system}/', os.path.normpath(source).replace(os.sep, '/') + '/'):
+ if path.startswith(prefix):
+ return path[len(prefix):]
+ return path
+
+
+def shared_locations(modules, source):
+ """Locations several map modules share, or that are directories: modules without files of their own cannot be sharded."""
+ seen, shared = {}, []
+ for module in modules:
+ for rel in module['files']:
+ seen.setdefault(rel, []).append(module['name'])
+ for rel, owners in seen.items():
+ if len(owners) > 1 or os.path.isdir(os.path.join(source, rel)):
+ shared.append((rel, len(owners)))
+ return shared
+
+
+def from_topology(topology, source, system, pattern):
+ modules, names, escaped = [], set(), []
+
+ def walk(node, domain):
+ kind = node.get('kind')
+ if kind == 'domain':
+ domain = node.get('name') or node.get('id', '')
+ if kind == 'module' and node.get('file'):
+ rel = relative_to_source(str(node['file']), source, system)
+ if rel and not contained(source, rel):
+ escaped.append(rel)
+ elif rel:
+ name = str(node.get('name') or node.get('id'))
+ if name in names:
+ name = str(node.get('id') or name) # names can repeat across domains; ids are unique
+ names.add(name)
+ modules.append({'name': name, 'domain': domain, 'files': [rel], 'loc': node.get('loc') or None})
+ for child in node.get('children', []):
+ walk(child, domain)
+
+ walk(topology['root'], '')
+ total = len(modules)
+ unusable = shared_locations(modules, source)
+ if unusable:
+ raise ValueError('%d map module(s) share a location or name a directory instead of a file (first: %s)' % (len(unusable), unusable[0][0]))
+ if pattern:
+ match = matcher(pattern)
+ modules = [m for m in modules if match(m['name']) or any(match(f) for f in m['files'])]
+ # merge small modules of the same domain; never split one
+ shards, pool, counter = [], {}, {}
+ for module in modules:
+ if module['loc'] and module['loc'] < SMALL:
+ current = pool.get(module['domain'])
+ if current is None or current['loc'] >= SMALL or len(current['files']) >= MAX_FILES:
+ counter[module['domain']] = counter.get(module['domain'], 0) + 1
+ current = pool[module['domain']] = {
+ 'name': f"{module['domain'] or 'misc'}:small-{counter[module['domain']]}",
+ 'domain': module['domain'], 'files': [], 'loc': 0,
+ }
+ shards.append(current)
+ current['files'] += module['files']
+ current['loc'] += module['loc']
+ else:
+ shards.append(module)
+ return shards, len(modules), escaped, total
+
+
+def from_tree(source, pattern, include_all):
+ by_dir, skipped = {}, {'vendored or generated': 0, 'tests': 0, 'unreadable or binary': 0, 'symbolic links': 0}
+ for root, dirs, files in os.walk(source):
+ kept = []
+ for d in sorted(dirs):
+ if d in SKIP_DIRS or (d.startswith('.') and d not in ('.',)):
+ skipped['vendored or generated'] += 1
+ elif d.lower() in TEST_DIRS and not include_all:
+ skipped['tests'] += 1
+ else:
+ kept.append(d)
+ dirs[:] = kept
+ for name in sorted(files):
+ if os.path.splitext(name)[1].lower() not in SOURCE_EXT or name.endswith('.min.js'):
+ continue
+ path = os.path.join(root, name)
+ if os.path.islink(path):
+ skipped['symbolic links'] += 1 # a link can point outside the source directory
+ continue
+ lines = count_lines(path)
+ if lines is None:
+ skipped['unreadable or binary'] += 1
+ continue
+ rel = os.path.relpath(path, source).replace(os.sep, '/')
+ by_dir.setdefault(os.path.dirname(rel), []).append((rel, lines))
+ whole = sum(len(fs) for fs in by_dir.values())
+ if pattern:
+ match = matcher(pattern)
+ by_dir = {d: [f for f in fs if match(f[0])] for d, fs in by_dir.items()}
+ shards, total = [], 0
+ for directory in sorted(by_dir):
+ entries = by_dir[directory]
+ total += len(entries)
+ chunk, chunk_loc, part = [], 0, 1
+ label = directory or '.'
+ for rel, lines in entries + [(None, 0)]:
+ if rel is None or (chunk and (len(chunk) >= MAX_FILES or chunk_loc + lines > MAX_LOC)):
+ if chunk:
+ shards.append({'name': label if part == 1 else f'{label}#{part}', 'domain': directory,
+ 'files': [f for f, _ in chunk], 'loc': chunk_loc or None})
+ part += 1
+ chunk, chunk_loc = [], 0
+ if rel is not None:
+ chunk.append((rel, lines))
+ chunk_loc += lines
+ return shards, total, skipped, whole
+
+
+def main(argv):
+ args = [a for a in argv[1:] if not a.startswith('--')]
+ flags = [a for a in argv[1:] if a.startswith('--')]
+ workspace = '.'
+ if '--workspace' in argv:
+ workspace = argv[argv.index('--workspace') + 1]
+ args = [a for a in args if a != workspace]
+ if not args:
+ print('usage: make_shards.py <system> [module-pattern] [--workspace DIR] [--all]', file=sys.stderr)
+ return 2
+ system, pattern = args[0], (args[1] if len(args) > 1 else '')
+ source = source_dir(workspace, system)
+ print(f'source: {source}')
+ if os.path.isdir(source) and too_broad(source):
+ print(f'{source} is too broad to be one system (the filesystem root, your home directory or a top-level directory). '
+ 'Point --source at the directory that holds the code.', file=sys.stderr)
+ return 1
+ if not os.path.isdir(source):
+ print(f'No code at {source}. Run preflight first (with --source <path> if the code lives elsewhere).', file=sys.stderr)
+ return 1
+ out = os.path.join(workspace, 'analysis', system, 'extract-rules.modules.json')
+ topo_path = os.path.join(workspace, 'analysis', system, 'topology.json')
+ notes = []
+ if os.path.isfile(topo_path):
+ try:
+ with open(topo_path, encoding='utf-8') as handle:
+ topology = json.load(handle)
+ shards, modules, escaped, whole = from_topology(topology, source, system, pattern)
+ origin = f'{modules} map modules'
+ if escaped:
+ notes.append(f'{len(escaped)} map file(s) resolve outside the source directory and were left out (first: {escaped[0]})')
+ missing = [f for s in shards for f in s['files'] if not os.path.exists(os.path.join(source, f))]
+ if missing:
+ notes.append(f'{len(missing)} file(s) named by the map do not exist under the source directory (first: {missing[0]})')
+ except (ValueError, KeyError, OSError) as error:
+ print(f'note: the map cannot be sharded by module ({error}); reading the directory tree instead.')
+ shards, files, skipped, whole = from_tree(source, pattern, '--all' in flags)
+ origin = f'{files} source files (tree)'
+ else:
+ shards, files, skipped, whole = from_tree(source, pattern, '--all' in flags)
+ origin = f'{files} source files (no topology.json: read from the directory tree)'
+ left_out = ', '.join(f'{n} {k}' for k, n in skipped.items() if n)
+ if left_out:
+ notes.append(f'left out: {left_out}')
+ if not shards:
+ print('0 shards: the pattern matched nothing, or no source files were found. Nothing was written.', file=sys.stderr)
+ return 1
+ os.makedirs(os.path.dirname(out), exist_ok=True)
+ tmp = out + '.tmp'
+ with open(tmp, 'w', encoding='utf-8') as handle:
+ json.dump(shards, handle, indent=1)
+ os.replace(tmp, out)
+ file_count = sum(len(s['files']) for s in shards)
+ loc = sum(s['loc'] or 0 for s in shards)
+ print(f'{len(shards)} shards from {origin}, {file_count} files, {loc} lines -> {os.path.relpath(out, workspace)}')
+ if whole < TINY_ESTATE:
+ print(f'tiny estate: the system has fewer than {TINY_ESTATE} source files, so sharding adds little (run the workflow in lens mode).')
+ for note in notes:
+ print(f'note: {note}')
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
diff --git a/plugins/code-modernization/scripts/proof_pack.py b/plugins/code-modernization/scripts/proof_pack.py
new file mode 100644
index 00000000..260326a7
--- /dev/null
+++ b/plugins/code-modernization/scripts/proof_pack.py
@@ -0,0 +1,1020 @@
+#!/usr/bin/env python3
+"""Independent proof pack: one plain verdict per built module, computed from the evidence files.
+
+ python3 proof_pack.py <system> [module] [--workspace DIR]
+
+Writes analysis/<system>/VERIFICATION.md (for people) and VERIFICATION.json (for the report). The
+verdict is computed here by fixed rules, never by a model's opinion, and every rule is written into
+the output so an auditor can follow it:
+
+ PROVEN all six checks below pass.
+ NOT PROVEN any check fails: nothing executed, a test failed, an equivalence case differs or is
+ missing, a baseline regression or new failure, a deliberate break that no test noticed.
+ PARTLY PROVEN nothing failed, but at least one check could not be passed; each is listed.
+
+ 1 Tests ran at least one test executed in a fresh run, none failed, none was skipped
+ without a reason, and every result file is newer than the code. The counts must
+ come from files this script parsed itself: JUnit-style XML, or a saved raw runner
+ log (Maven/Gradle, cargo, pytest, unittest, go test, dotnet test, jest, vitest,
+ ctest, phpunit). Counts only typed into test-runs.json cap the check at "gap".
+ 2 Rules traced rewrite and reimagine only: every P0 rule the module answers for is backed by a
+ test that ran and passed here: the rule id is in the name (test or class) of a test
+ result this script parsed, or a test file names it on a line not marked skipped or
+ pending and its class or file is one of a test that passed. A rule named only by
+ skipped, pending or failing tests is "named, not run"; one only the notes name is
+ "claimed": neither is tested. Counts and logs name no test, so they back no rule.
+ Uplifts keep the code, so they trace no rules.
+ 3 Same behavior the development cases, judged again here by compare.py, executed at least one
+ case with none differing or missing (a case a person approved is allowed and
+ listed); for an uplift, baseline_diff.py may stand in: no regression, no new
+ failure, no missing test or module, no drop in executed tests.
+ 4 Fresh inputs at least 10 new inputs, none with the same output as a development case, ran on
+ the legacy and the new code and compare.py finds no difference. Needed whenever
+ the legacy could run here; when it could not, the proof rests on recorded
+ traces and the best possible verdict is PARTLY PROVEN.
+ 5 Canary a deliberate one-line break whose own result file or saved runner log, parsed
+ here, shows failing tests. A "Canary:" line in the notes is a claim, not evidence.
+ 6 Source untouched no file under legacy/<system> is newer than the analysis (PREFLIGHT.md); a plain
+ file walk, no version-control tool is run inside the untrusted tree.
+
+Open questions, unticked criteria and the sign-off are for a person: they never change the verdict.
+A built folder that holds only test code and the files that build it (a parity harness, for one) is not a
+module: it is listed as "Test tooling (not judged)", under "toolingOnly" in the JSON, and not counted.
+
+Evidence read (under analysis/<system>/, or as named): equivalence/test-runs.json (written by the
+modernize-verify command), equivalence/[<module>/]cases.json and fresh-cases.json (judged again here,
+not read from EQUIVALENCE.json), BASELINE.md, BUSINESS_RULES.md with the code under modernized/,
+MODERNIZATION_BRIEF.md, the notes files, and legacy/<system>. Everything read is untrusted text: it is
+never run, links out of the workspace are refused, sizes are capped. Exit 0 when every verdict is
+PROVEN, 1 when not, 2 for input that cannot be used.
+"""
+import argparse
+import datetime
+import json
+import os
+import re
+import stat
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.dont_write_bytecode = True # importing the sibling scripts must not leave __pycache__ in the plugin folder
+sys.path.insert(0, HERE)
+import baseline_diff # noqa: E402
+import compare # noqa: E402
+import trace_rules # noqa: E402
+import uplift_checks # noqa: E402
+
+FRESH_MIN, LIST_CAP, JSON_CAP = 10, 40, 8 << 20
+VERDICTS = ("PROVEN", "PARTLY PROVEN", "NOT PROVEN")
+UPLIFT_CHECKS = (("baseline", "Baseline measured"), ("kept", "Tests kept"), ("deltas", "Deltas covered"))
+CHECKS = (("tests", "Tests ran"), ("rules", "Rules traced"), ("same", "Same behavior"), ("fresh", "Fresh inputs"),
+ ("canary", "Canary"), ("source", "Source untouched")) + UPLIFT_CHECKS
+RULES_TEXT = (
+ "PROVEN needs every check to pass: the six below, and for an uplift three more (7 to 9). NOT PROVEN when any check fails. PARTLY PROVEN when nothing failed but a check could not pass.",
+ "1. Tests ran: at least one test executed in a fresh run, none failed, none was skipped without a reason, and every result file is newer than the code. "
+ "The counts must be read by this script from result files or a saved raw runner log; counts only typed in cannot reach PROVEN.",
+ "2. Rules traced (rewrite and reimagine): every P0 rule the module answers for is backed by a test that ran and passed: its id is in the test or class name of a result this script parsed, "
+ "or a test file names it on a line not marked skipped or pending and that file's class ran and passed. A rule named only by skipped, pending or failing tests is named, not run; one only the notes name is claimed. Neither is tested.",
+ "3. Same behavior: the development cases, judged again by compare.py, executed at least one case and none differs or is missing (a difference a person approved is allowed and listed); "
+ "for an uplift, no regression, no new failure, no missing test and no drop in executed tests against BASELINE.md.",
+ "4. Fresh inputs: at least 10 new inputs, none with the same output as a development case, ran on the legacy and the new code with no difference. Needed whenever the legacy could run here; "
+ "when it could not, the proof is trace-based and the best verdict is PARTLY PROVEN.",
+ "5. Canary: a deliberate one-line break whose own result file or saved runner log shows failing tests. A line in the notes is a claim.",
+ "6. Source untouched: no file under legacy/<system> is newer than the analysis (PREFLIGHT.md), by a plain file walk; no version-control tool is run.",
+ "7. Baseline measured (uplift): the old version's numbers in BASELINE.md come from per-test result files, a per-test results file or a raw runner log that this script parsed "
+ "(in analysis/<system>/baseline/, or on a Recorded: or Machine-readable: line of BASELINE.md), and the typed table agrees with them. A table typed by hand, or sources that disagree, cannot reach PROVEN.",
+ "8. Tests kept (uplift): no test file of the untouched legacy tree is missing from the working copy, no more than 25% of them changed, and the legacy tree has test files to compare against "
+ "(walked by file, no process is run). The list is for a person to review; weakened assertions cannot be detected, only that files changed.",
+ "9. Deltas covered (uplift): every Behavioral-silent delta in DELTA_CATALOG.md has its site's file named, as a whole word, by some test file of the working copy. A name is not proof that the test exercises the change.",
+ "Open questions, unticked criteria and the sign-off are for a person. They never change the verdict.",
+ "A folder that holds only test code and the files that build it is test tooling: it is listed, not judged, and not counted.",
+)
+WORDS = {"pass": "pass", "gap": "not proven", "fail": "FAIL", "na": "not applicable"}
+CANARY = re.compile(r"Canary[^:\n|]{0,20}:\s*(.{1,300}?)\s*(?:→|->|=>|,)\s*(\d[\d,]*)\s+tests?\s+(?:failed|went red|failing)", re.I)
+CHECKBOX = re.compile(r"^\s*[-*]\s*\[( |x|X)\]\s*(.*)$")
+NOTE_NAMES = trace_rules.NOTES_NAMES
+clean = trace_rules.clean
+
+
+class InputError(Exception):
+ pass
+
+
+# ---------------------------------------------------------------- reading evidence safely
+def read_json(path, cap=8 << 20):
+ """-> (value, problem). A missing file is (None, None); a link, a huge file or bad JSON is (None, reason)."""
+ try:
+ st = os.lstat(path)
+ except OSError:
+ return None, None
+ name = clean(os.path.basename(path), 80)
+ if not stat.S_ISREG(st.st_mode):
+ return None, "%s is a link or not a regular file, so it was not read" % name
+ if st.st_size > cap:
+ return None, "%s is larger than %d MB, so it was not read" % (name, cap >> 20)
+ try:
+ with open(path, "rb") as fh:
+ return json.loads(fh.read().decode("utf-8-sig")), None
+ except (OSError, ValueError, RecursionError) as err:
+ return None, "%s could not be read as JSON (%s)" % (name, clean(str(err), 80))
+
+
+def read_text(path, cap=8 << 20):
+ """Text of a regular, non-binary file, or None."""
+ return trace_rules.read_text(path, cap) if os.path.isfile(path) and not os.path.islink(path) else None
+
+
+def nint(value):
+ return value if isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= 10 ** 9 else None
+
+
+def inside(workspace, rel):
+ """The real path of `rel` when it stays inside the workspace, else None (absolute paths and links out are refused)."""
+ if not isinstance(rel, str) or not rel or "\x00" in rel:
+ return None
+ real = os.path.realpath(rel if os.path.isabs(rel) else os.path.join(workspace, rel))
+ root = os.path.realpath(workspace)
+ try:
+ return real if os.path.commonpath([root, real]) == root else None
+ except ValueError:
+ return None
+
+
+# ---------------------------------------------------------------- test-runs.json
+def load_test_runs(adir):
+ """-> the validated content of equivalence/test-runs.json (everything defaulted when it is absent or malformed)."""
+ obj, problem = read_json(os.path.join(adir, "equivalence", "test-runs.json"))
+ out = {"present": isinstance(obj, dict), "problems": [problem] if problem else [], "date": "", "legacy": {"ran": None, "how": "", "why": ""}, "suites": [], "canaries": [], "leftOut": []}
+ if not isinstance(obj, dict):
+ if obj is not None:
+ out["problems"].append("test-runs.json is not a JSON object, so it was not used.")
+ return out
+ out["date"] = clean(obj.get("date"), 40)
+ out["leftOut"] = [clean(v, 200) for v in (obj.get("leftOut") if isinstance(obj.get("leftOut"), list) else [])[:20] if isinstance(v, str) and v.strip()]
+ lg = obj.get("legacy") if isinstance(obj.get("legacy"), dict) else {}
+ out["legacy"] = {"ran": lg.get("ran") if isinstance(lg.get("ran"), bool) else None, "how": clean(lg.get("how"), 300), "why": clean(lg.get("why"), 300)}
+ for s in (obj.get("suites") if isinstance(obj.get("suites"), list) else [])[:200]:
+ if isinstance(s, dict):
+ out["suites"].append({"module": clean(s.get("module"), 100), "name": clean(s.get("name"), 120), "command": clean(s.get("command"), 300), "date": clean(s.get("date"), 40),
+ "executed": nint(s.get("executed")), "failed": nint(s.get("failed")), "skipped": nint(s.get("skipped")),
+ "skippedReason": clean(s.get("skippedReason"), 300), "note": clean(s.get("note"), 300), "junit": paths_of(s.get("junit")), "log": paths_of(s.get("log"))})
+ for c in (obj.get("canaries") if isinstance(obj.get("canaries"), list) else [])[:50]:
+ if isinstance(c, dict):
+ out["canaries"].append({"module": clean(c.get("module"), 100), "change": clean(c.get("change"), 300), "testsFailed": nint(c.get("testsFailed")),
+ "junit": paths_of(c.get("junit")), "log": paths_of(c.get("log"))})
+ return out
+
+
+def paths_of(value):
+ """A list of path strings from a string or a list, however malformed."""
+ return [v for v in (value if isinstance(value, list) else [value])[:20] if isinstance(v, str) and v]
+
+
+def suites_for(runs, subject, unit, subjects):
+ """The suites that belong to one subject. A suite that names no module belongs to the only subject."""
+ others = {s["name"].lower() for s in subjects if s["track"] != "uplift"}
+ out = []
+ for s in runs["suites"]:
+ mod = s["module"].lower()
+ if not mod:
+ ok = len(subjects) == 1
+ elif subject["track"] == "uplift": # one uplifted working copy holds many units; a unit argument narrows it
+ ok = mod not in others and (not unit or mod == unit.lower() or mod == subject["name"].lower())
+ else:
+ ok = mod == subject["name"].lower()
+ if ok:
+ out.append(s)
+ return out
+
+
+def code_mtime(root):
+ """Newest modification time among a module's code (no build output, hidden files or documents)."""
+ newest = 0.0
+ for base, dirs, names in os.walk(root):
+ dirs[:] = [d for d in dirs if not d.startswith(".") and d not in trace_rules.SKIP_DIRS and not os.path.islink(os.path.join(base, d))]
+ for n in names:
+ full = os.path.join(base, n)
+ if n.startswith(".") or os.path.splitext(n)[1].lower() in trace_rules.DOC_EXT:
+ continue
+ try:
+ st = os.lstat(full)
+ except OSError:
+ continue
+ if stat.S_ISREG(st.st_mode):
+ newest = max(newest, st.st_mtime)
+ return newest
+
+
+# ---------------------------------------------------------------- raw runner logs (read by baseline_diff.py, which the baseline check shares)
+LOG_CAP, parse_runner_log, read_logs = baseline_diff.LOG_CAP, baseline_diff.parse_runner_log, baseline_diff.read_logs
+
+
+def utc(ts):
+ return datetime.datetime.fromtimestamp(ts or 0, datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+
+
+def suite_evidence(suite, workspace):
+ """-> (counts for one suite, its parsed JUnit-style results or None). Evidence, best first: result files, a saved raw runner log,
+ and only then the counts the command typed, which are never enough for PROVEN."""
+ ev = {"name": suite["name"], "command": suite["command"], "date": suite["date"], "note": suite["note"], "notes": [], "source": "reported" if suite["executed"] is not None else "none",
+ "files": 0, "newest": None, "oldest": None, "written": "", "executed": suite["executed"] or 0, "failed": suite["failed"] or 0, "skipped": suite["skipped"] or 0,
+ "skippedNoReason": (suite["skipped"] or 0) if not suite["skippedReason"] else 0}
+ said = {"executed": suite["executed"], "failed": suite["failed"], "skipped": suite["skipped"]}
+ junit = [(j, inside(workspace, j)) for j in suite["junit"]]
+ logs = [(j, inside(workspace, j)) for j in suite["log"]]
+ ev["notes"] += ["%s is outside the workspace, so it was not read" % clean(j, 100) for j, p in junit + logs if p is None]
+
+ def mismatch(mine):
+ for label, value in mine.items():
+ if said[label] is not None and said[label] != value:
+ ev["notes"].append("the suite reported %d %s but the files show %d; the files were used" % (said[label], label, value))
+
+ good = [p for _, p in junit if p]
+ if good:
+ fresh = baseline_diff.read_results(good)
+ ev["notes"] += ["result file not used: " + u for u in fresh["unreadable"][:3]]
+ if fresh["files"]:
+ c = baseline_diff.tally(fresh["tests"].values())
+ ev.update({"source": "result files", "files": fresh["files"], "newest": fresh["newest"], "oldest": fresh["oldest"], "written": utc(fresh["newest"]), "executed": c["executed"],
+ "failed": c["fail"] + c["error"], "skipped": c["skip"],
+ "skippedNoReason": 0 if suite["skippedReason"] else sum(1 for t, st in fresh["tests"].items() if st == "SKIP" and not fresh["skipReasons"].get(t))})
+ mismatch({"executed": ev["executed"], "failed": ev["failed"], "skipped": ev["skipped"]})
+ return ev, fresh
+ ev["notes"].append("no readable result file was found under %s" % ", ".join(clean(j, 100) for j, p in junit if p))
+ good = [p for _, p in logs if p]
+ if good:
+ got = read_logs(good)
+ ev["notes"] += ["log not used: " + u for u in got["unrecognised"][:3]]
+ if got["files"]:
+ ev.update({"source": "runner log", "files": got["files"], "newest": got["newest"], "oldest": got["oldest"], "written": utc(got["newest"]), "executed": got["executed"],
+ "failed": got["failed"], "skipped": got["skipped"], "skippedNoReason": 0 if suite["skippedReason"] else got["skipped"]})
+ ev["notes"].append("counts read from the summary lines of %s" % ", ".join(got["runners"]))
+ mismatch({"executed": ev["executed"], "failed": ev["failed"], "skipped": ev["skipped"]})
+ return ev, None
+
+
+def suite_results(ctx, suite):
+ """suite_evidence once per suite: a result file is parsed a single time, however many times it is asked for."""
+ if id(suite) not in ctx["parsed"]:
+ ctx["parsed"][id(suite)] = suite_evidence(suite, ctx["workspace"])
+ return ctx["parsed"][id(suite)]
+
+
+def module_run(ctx, subject):
+ """What ran for one module (trace_rules.run_evidence): read from its suites' result files. Counts and logs name no test, so they back no rule."""
+ return trace_rules.run_evidence([suite_results(ctx, s)[1] for s in suites_for(ctx["runs"], subject, ctx["unit"], ctx["subjects"])])
+
+
+# ---------------------------------------------------------------- the source check
+SOURCE_CAP, EXAMPLES = 400000, 20
+NO_VCS = "by modification time, and no version-control tool was run"
+
+
+def reference_time(adir):
+ """When the analysis began: the modification time of PREFLIGHT.md, else that of the oldest file in analysis/<system>/."""
+ def mtime(path):
+ try:
+ st = os.lstat(path)
+ return st.st_mtime if stat.S_ISREG(st.st_mode) else None
+ except OSError:
+ return None
+ ref = mtime(os.path.join(adir, "PREFLIGHT.md"))
+ if ref is not None:
+ return ref
+ try:
+ times = [t for t in (mtime(os.path.join(adir, n)) for n in os.listdir(adir)) if t is not None]
+ except OSError:
+ return None
+ return min(times) if times else None
+
+
+def source_check(workspace, system, adir):
+ """-> {"state": clean|changed|unknown, "method", "detail", "files", "path", "target"} for legacy/<system>.
+ A plain file walk, never following a link and never running a program in the tree: any regular file newer than the
+ analysis is reported as changed. A walk that cannot finish is "not checked", never "untouched"."""
+ link = os.path.join(workspace, "legacy", system)
+ target = os.path.realpath(link) if os.path.lexists(link) else ""
+ out = {"state": "unknown", "method": NO_VCS, "detail": "", "files": [], "path": "legacy/" + system, "target": clean(target if os.path.islink(link) else "", 300)}
+ if not target or not os.path.isdir(target):
+ out["detail"] = "legacy/%s was not found, so it was not checked" % system
+ return out
+ ref = reference_time(adir)
+ if ref is None:
+ out["detail"] = "there is no PREFLIGHT.md or other analysis file to date the analysis by, so legacy/%s was not checked" % system
+ return out
+ newer, seen, unreadable = [], 0, [0]
+
+ def failed(_err):
+ unreadable[0] += 1
+
+ for base, dirs, names in os.walk(target, onerror=failed, followlinks=False):
+ dirs[:] = [d for d in dirs if d != ".git" and not os.path.islink(os.path.join(base, d))]
+ for n in names:
+ seen += 1
+ if seen > SOURCE_CAP:
+ break
+ full = os.path.join(base, n)
+ try:
+ st = os.lstat(full)
+ except OSError:
+ unreadable[0] += 1
+ continue
+ if stat.S_ISREG(st.st_mode) and st.st_mtime > ref:
+ newer.append(clean(os.path.relpath(full, target), 200))
+ if seen > SOURCE_CAP:
+ break
+ out["files"] = newer[:EXAMPLES]
+ if seen > SOURCE_CAP or unreadable[0]:
+ out["detail"] = "the walk could not finish (%s), so legacy/%s was not fully checked" % ("more than %d files" % SOURCE_CAP if seen > SOURCE_CAP else "%d path(s) could not be read" % unreadable[0], system)
+ out["state"] = "changed" if newer else "unknown"
+ if newer:
+ out["detail"] = "%d file(s) changed after the analysis started, and the walk could not finish" % len(newer)
+ return out
+ out["state"] = "changed" if newer else "clean"
+ out["detail"] = ("%d file(s) changed after the analysis started" % len(newer)) if newer else "no file changed after the analysis started (%d files compared)" % seen
+ return out
+
+
+# ---------------------------------------------------------------- the brief
+def brief_items(text, module, track):
+ """Unticked criteria a person decides: the exit criteria of the module's phase, and the open questions of section 7."""
+ if not text:
+ return {"found": False, "phase": "", "items": [], "count": 0}
+ lines = [ln[:2000] for ln in text.split("\n")]
+ phases, cur = [], None
+ for line in lines:
+ m = re.match(r"#{2,5}\s+Phase\s+(\d+)\b", line)
+ if m:
+ cur = {"n": int(m.group(1)), "title": clean(re.split(r"\s[—-]\s", line.lstrip("# "))[0], 60), "command": "", "modules": set(), "exit": [], "in_exit": False}
+ phases.append(cur)
+ elif re.match(r"#{1,3}\s", line):
+ cur = None
+ elif cur is not None:
+ k = re.match(r"\s*(Command|Modules)\s*:\s*(.*)$", line)
+ if k and k.group(1) == "Command":
+ cur["command"] = k.group(2).lower()
+ elif k:
+ cur["modules"] = {t.strip().lower() for t in re.split(r"[,;]", k.group(2)) if t.strip()}
+ elif re.match(r"\s*Exit criteria\s*:", line):
+ cur["in_exit"] = True
+ elif re.match(r"\s*(Entry criteria|Scale|Risk|Command|Modules)\s*:", line):
+ cur["in_exit"] = False
+ elif cur["in_exit"]:
+ c = CHECKBOX.match(line)
+ if c and c.group(1) == " ":
+ cur["exit"].append(clean(c.group(2), 300))
+ verb = {"rewrite": "transform", "reimagine": "reimagine", "uplift": "uplift"}[track]
+ by_name = [p for p in phases if module.lower() in p["modules"]] if track != "uplift" else []
+ mine = by_name or [p for p in phases if verb in p["command"]]
+ items = [{"text": t, "where": "%s exit criterion" % p["title"]} for p in mine for t in p["exit"]]
+ phase_n = min((p["n"] for p in mine), default=None)
+ level = None
+ for line in lines:
+ h = re.match(r"(#{1,3})\s+(.*)", line)
+ if h:
+ if re.match(r"(?:\d+[.)]?\s*)?open questions|7[.):\s]", h.group(2), re.I):
+ level = len(h.group(1))
+ elif level is not None and len(h.group(1)) <= level:
+ level = None
+ continue
+ c = CHECKBOX.match(line) if level else None
+ if c and c.group(1) == " ":
+ tag = re.search(r"\[before Phase (\d+)\]", c.group(2), re.I)
+ if not (tag and phase_n is not None and int(tag.group(1)) > phase_n):
+ items.append({"text": clean(c.group(2).replace("**", ""), 300), "where": "open question, section 7"})
+ return {"found": True, "phase": mine[0]["title"] if len(mine) == 1 else "", "items": items[:LIST_CAP], "count": len(items)}
+
+
+# ---------------------------------------------------------------- the six checks
+def row(cid, state, detail, **extra):
+ r = {"id": cid, "label": dict(CHECKS)[cid], "state": state, "detail": clean(detail, 700)}
+ r.update(extra)
+ return r
+
+
+def judge(path):
+ """A cases file judged again with compare.py: (record or None, problem)."""
+ if os.path.islink(path):
+ return None, "%s is a link, so it was not read" % clean(os.path.basename(path), 80)
+ try:
+ return compare.run(path, None, False), None
+ except compare.InputError as err:
+ return None, clean(str(err), 200)
+ except (OSError, ValueError, RecursionError, MemoryError) as err:
+ return None, "the cases file could not be judged (%s)" % err.__class__.__name__
+
+
+def mask_summary(records):
+ """Fields the comparison ignored, grouped by the reason given for them: [{"why", "cases"}]."""
+ groups = {}
+ for r in records:
+ for m in r["masked"]:
+ why = re.search(r"\(([^()]*)\)\s*$", m)
+ groups.setdefault(clean(why.group(1).strip() if why else "no reason given: " + m, 200), set()).add(r["id"])
+ return [{"why": k, "cases": len(v)} for k, v in sorted(groups.items())][:10]
+
+
+def tolerance_summary(records):
+ """The declared numeric tolerances that were used, grouped: [{"why", "rel", "abs", "cases"}]. A tolerance is never silent."""
+ groups = {}
+ for r in records:
+ if r.get("withinTolerance") and isinstance(r.get("tolerance"), dict):
+ t = r["tolerance"]
+ groups.setdefault((clean(t.get("why"), 200), t.get("rel"), t.get("abs")), set()).add(r["id"])
+ return [{"why": k[0], "rel": k[1], "abs": k[2], "cases": len(v)} for k, v in sorted(groups.items(), key=lambda kv: str(kv[0]))][:10]
+
+
+def summary_of(rec):
+ t = rec["totals"]
+ return {"cases": t["cases"], "executed": t["executed"], "same": t["same"], "differs": t["differs"], "missing": t["missing"], "selfCheck": rec["selfCheck"]["passed"],
+ "withinTolerance": t.get("sameWithinTolerance", 0), "tolerances": tolerance_summary(rec["cases"]),
+ "approved": [{"id": r["id"], "why": clean(r["approvedDifference"], 300)} for r in rec["cases"] if r["verdict"] == "differs-approved"][:LIST_CAP],
+ "masks": mask_summary(rec["cases"]),
+ "differing": [{"id": r["id"], "reason": clean(r["reason"], 300)} for r in rec["cases"] if r["verdict"] in ("differs", "missing")][:LIST_CAP]}
+
+
+def cases_paths(adir, module, n_subjects):
+ """The cases files of a module: its own folder equivalence/<module>/, else the shared equivalence/ folder."""
+ for base, shared in ((os.path.join(adir, "equivalence", module), False), (os.path.join(adir, "equivalence"), True)):
+ cases, fresh = os.path.join(base, "cases.json"), os.path.join(base, "fresh-cases.json")
+ if os.path.lexists(cases) or os.path.lexists(fresh):
+ return {"cases": cases if os.path.lexists(cases) else "", "fresh": fresh if os.path.lexists(fresh) else "", "shared": shared and n_subjects > 1}
+ return {"cases": "", "fresh": "", "shared": False}
+
+
+def recorded_matches(path, rec):
+ """Does EQUIVALENCE.json on disk agree with what the cases say now? None when there is no usable file."""
+ obj, _ = read_json(path)
+ if not isinstance(obj, dict) or not isinstance(obj.get("totals"), dict):
+ return None
+ mine = rec["totals"]
+ return all(obj["totals"].get(k) == mine[k] for k in ("cases", "executed", "same", "differs", "differsApproved", "missing"))
+
+
+def check_tests(g, uplift, comparable):
+ """Check 1. g: {executed, failed, skipped, skippedNoReason, suites, stale}. A suite's own note is added to the detail."""
+ r = check_tests_state(g, uplift, comparable)
+ notes = " ".join(e["note"] for e in g["suites"] if e.get("note"))
+ if notes:
+ r["detail"] = clean(r["detail"] + " Note: " + notes, 700)
+ return r
+
+
+def check_tests_state(g, uplift, comparable):
+ ex, fail, skip, nr = g["executed"], g["failed"], g["skipped"], g["skippedNoReason"]
+ if not g["suites"]:
+ return row("tests", "fail", "No test run is recorded for this module, so nothing executed. Run its tests again and record them in equivalence/test-runs.json.")
+ unread = [e["name"] or "a suite" for e in g["suites"] if e["source"] == "none"]
+ if len(unread) == len(g["suites"]):
+ return row("tests", "gap", "No test result could be read: no result file was parsed, no saved runner log has a summary line this script knows, and no counts were given (%s)." % "; ".join(n for e in g["suites"] for n in e["notes"][:2])[:300])
+ if ex == 0:
+ return row("tests", "fail", "Nothing executed: no test ran%s." % (" (%d were skipped)" % skip if skip else ""))
+ if fail and not (uplift and comparable):
+ return row("tests", "fail", "%d of %d executed test(s) failed." % (fail, ex))
+ if nr and not (uplift and comparable):
+ return row("tests", "gap", "%d test(s) were skipped with no reason given (%d executed, %d failed)." % (nr, ex, fail))
+ if g["stale"]:
+ return row("tests", "gap", "%d test(s) executed, but some result files are older than the code, so this was not a clean run of the current code: delete the build output and run the tests again." % ex)
+ typed = [e["name"] or "a suite" for e in g["suites"] if e["source"] == "reported"]
+ if unread or typed:
+ return row("tests", "gap", "%d test(s) executed, but the counts for %s were typed in by the command, not read from a result file or a saved runner log, so this cannot be PROVEN." % (ex, ", ".join(unread + typed)))
+ judged = " (failures are judged against the baseline below)" if fail and uplift else ""
+ return row("tests", "pass", "%d test(s) executed, %d failed%s, %d skipped." % (ex, fail, judged, skip))
+
+
+def check_rules(ctx, subject, evidence, caveats):
+ """Check 2."""
+ if subject["track"] == "uplift":
+ return row("rules", "na", "An uplift keeps the code, so no business rule is traced. The baseline comparison stands in.")
+ tr = ctx["trace"]
+ if tr is None:
+ return row("rules", "gap", ctx["trace_problem"])
+ if not tr["rules"]:
+ return row("rules", "gap", "BUSINESS_RULES.md has no rule cards in the form '### RULE-001: name', so no rule can be traced.")
+ view = trace_rules.module_view(tr, subject["rel"])
+ p0 = [r for r in view["rows"] if r["priority"] == "P0"]
+ missing = [r["id"] for r in p0 if r["status"] != "tested"]
+ unrun = [r["id"] for r in p0 if r["status"] == trace_rules.NOT_RUN]
+ known = bool((ctx["ran"].get(subject["rel"]) or trace_rules.NO_RUN).get("known"))
+ evidence["rules"] = {"tied": view["tied"], "counted": len(view["rows"]), "outOfScope": view["outOfScope"], "totals": view["totals"], "p0NoTests": missing, "namedNotRun": unrun[:100], "perTestResults": known,
+ "p0": [{"id": r["id"], "name": r["name"], "confidence": r["confidence"], "status": r["status"], "main": r["main"], "tests": r["tests"], "claimed": r["claimed"], "by": r["by"],
+ "where": clean(r["at"] or (r["samples"]["tests"] or r["samples"]["main"] or [""])[0], 200)} for r in p0][:100]}
+ if not view["tied"]:
+ caveats.append("Nothing ties a rule to this module by name, so every rule in BUSINESS_RULES.md was counted.")
+ if not p0:
+ return row("rules", "pass", "No rule this module answers for is rated P0 (%d rule(s) counted)." % len(view["rows"]))
+ if missing:
+ other = [r for r in p0 if r["status"] not in ("tested", trace_rules.NOT_RUN)]
+ say = []
+ if unrun:
+ listed = ", ".join(unrun[:12]) + (" and %d more" % (len(unrun) - 12) if len(unrun) > 12 else "")
+ say.append("P0 rules named only by tests that did not run: %s (skipped, pending, failing or missing from the results)." % listed if known else
+ "P0 rules named by tests, but no result file lists each test, so nothing shows one ran: %s." % listed)
+ if other:
+ claimed = sum(1 for r in other if r["status"] == "claimed only")
+ say.append("%d of %d P0 rule(s) are not named by any test: %s%s." % (len(other), len(p0), ", ".join(r["id"] for r in other[:12]), " (%d claimed only in the notes)" % claimed if claimed else ""))
+ return row("rules", "gap", " ".join(say))
+ return row("rules", "pass", "All %d P0 rule(s) this module answers for are backed by a test that ran and passed." % len(p0))
+
+
+def baseline_info(ctx):
+ """(assessment of BASELINE.md, the measured evidence) for an uplift: is the baseline measured or only typed? (None, None) without a BASELINE.md."""
+ path = os.path.join(ctx["adir"], "BASELINE.md")
+ text = read_text(path)
+ if text is None:
+ return None, None
+ found = baseline_diff.evidence_paths(path, text)
+ measured = baseline_diff.read_baseline_evidence(found) if found else None
+ return baseline_diff.assess_baseline(baseline_diff.parse_baseline(text), measured), measured
+
+
+def baseline_state(ctx, suites, xml_paths, evidence, caveats, person, measured=None):
+ """The uplift's baseline comparison: (state, detail, whether it could compare)."""
+ path = os.path.join(ctx["adir"], "BASELINE.md")
+ if not os.path.isfile(path):
+ return "gap", "BASELINE.md is missing: an uplift needs the old version's results recorded first (uplift, Step 4).", False
+ if not xml_paths:
+ return "gap", "BASELINE.md exists, but there are no result files to compare it with.", False
+ try:
+ b = baseline_diff.run(path, xml_paths, measured=measured)
+ except baseline_diff.InputError as err:
+ return "gap", clean(str(err), 300), False
+ keys = ("ok", "regressionsCount", "newFailuresCount", "fixedCount", "missingCount", "missingModulesCount", "moduleDiffsCount", "renamed", "executedDrop", "skippedGrowth",
+ "stillFailingCount", "newlySkippedCount", "flakyCount", "approvedCount")
+ evidence["baseline"] = dict({k: b[k] for k in keys}, source=b["baseline"]["source"], targetOnly=b["baseline"]["targetOnly"], why=b["baseline"]["why"], before=b["baseline"]["counts"],
+ now=b["fresh"]["counts"], approved=b["approved"][:LIST_CAP], regressions=b["regressions"][:LIST_CAP], newFailures=b["newFailures"][:LIST_CAP], fixed=b["fixed"][:LIST_CAP], missing=b["missing"][:LIST_CAP])
+ if b["fixedCount"]:
+ person.append("%d test(s) failed on the old version and pass now. A person decides whether each is an intended fix." % b["fixedCount"])
+ if b["flakyCount"]:
+ caveats.append("%d test(s) the baseline lists as flaky flipped; they are not counted as regressions." % b["flakyCount"])
+ if b["approvedCount"]:
+ caveats.append("%d test difference(s) were approved by a person in BASELINE.md and are not counted: %s." % (
+ b["approvedCount"], "; ".join("%s (%s)" % (clean(x["id"], 80), clean(x["reason"] or "no reason given", 80)) for x in b["approved"][:5])))
+ if b["baseline"]["targetOnly"] or not b["baseline"]["counts"]:
+ return "gap", "The baseline says target-only (%s): the old version did not run here, so there is nothing to compare with." % (b["baseline"]["why"] or "no reason given"), False
+ if b["renamed"]:
+ caveats.append("%d test name(s) changed while their class did not get worse (parameterized names that embed a path, for example); they were not counted as missing." % b["renamed"])
+ if b["regressionsCount"] or b["newFailuresCount"]:
+ hint = " Tests the baseline never listed were added after it was recorded: run them on the old version too and add them to BASELINE.md to tell a regression from a test that was never green." if b["newFailuresCount"] else ""
+ return "fail", "%d test(s) passed before and fail now; %d test(s) the baseline never listed (or skipped) fail now.%s" % (b["regressionsCount"], b["newFailuresCount"], hint), True
+ gaps = ["%d %s" % (n, what) for n, what in ((b["missingCount"], "baseline test(s) did not run now"), (b["missingModulesCount"], "baseline module(s) are missing from the results"),
+ (b["moduleDiffsCount"], "module(s) ran fewer tests or skipped more"), (b["newlySkippedCount"], "test(s) that passed before are skipped now")) if n]
+ if b["executedDrop"]:
+ gaps.append("executed tests dropped from %(before)d to %(now)d" % b["executedDrop"])
+ if gaps:
+ return "gap", "; ".join(gaps) + ".", True
+ return "pass", "No regression: %d test(s) executed against %d in the baseline (%s)." % (b["fresh"]["counts"]["executed"], b["baseline"]["counts"]["executed"], b["baseline"]["source"]), True
+
+
+def check_baseline_measured(info, evidence):
+ """Check 7 (uplift): is the old version's baseline measured, or only typed?"""
+ if info is None:
+ return row("baseline", "na", "There is no BASELINE.md to check (see Same behavior).")
+ evidence["baselineEvidence"] = info
+ if info["kind"] == "target-only":
+ return row("baseline", "na", "BASELINE.md says target-only: the old version was not measured here (see Same behavior).")
+ if info["kind"] in ("typed", "none"):
+ ignored = " Files that only hold counts were ignored: %s." % ", ".join(info["ignored"][:3]) if info["ignored"] else ""
+ return row("baseline", "gap", "BASELINE.md is a table typed by hand: no per-test result file, per-test results file or raw runner log that this script can read backs it, so nothing shows the old "
+ "version really produced those numbers. Save the old version's results in analysis/<system>/baseline/ (or name them on a Recorded: line of BASELINE.md) "
+ "and run this again.%s" % ignored)
+ names = ", ".join("%s%s" % (x["kind"], " " + x["name"] if x.get("name") else "") for x in info["sources"][:3])
+ if info["conflict"]:
+ return row("baseline", "gap", "The baseline's own sources disagree with each other (%s), so it cannot be trusted." % "; ".join(info["conflictExamples"][:2]))
+ if info["disagree"]:
+ return row("baseline", "gap", "The measured results and BASELINE.md's typed table disagree on %d test(s) or count(s) (%s): the measured results were used, but a typed table that does not match its evidence cannot be trusted." % (
+ info["disagree"], "; ".join(info["examples"][:2])))
+ what = "%d test(s) (%d executed, %d failed)" % (info["measuredTests"], info["measuredExecuted"], info["measuredFailed"]) if info["measuredTests"] else "%d executed, %d failed" % (
+ info["measuredExecuted"] or 0, info["measuredFailed"] or 0)
+ return row("baseline", "pass", "The baseline is measured: %s from %s, and BASELINE.md agrees with it." % (what, names))
+
+
+def check_tests_kept(tc, person):
+ """Check 8 (uplift): were the tests kept as they were?"""
+ if not tc["checked"]:
+ return row("kept", "gap", "The test files could not be compared: %s." % tc["why"])
+ c = tc["counts"]
+ if c["added"] or c["removed"] or c["changed"]:
+ person.append("Review the test files that changed during the uplift (%d changed, %d added, %d removed; the list is in VERIFICATION.md). Weakened assertions cannot be detected, only that files changed." % (
+ c["changed"], c["added"], c["removed"]))
+ if not tc["legacyTests"]:
+ return row("kept", "gap", "%s (%d test file(s) in the working copy)." % (tc["why"][:1].upper() + tc["why"][1:], tc["workTests"]))
+ reasons = []
+ if c["removed"]:
+ reasons.append("%d legacy test file(s) are missing from the working copy (%s)" % (c["removed"], ", ".join(x["path"].rsplit("/", 1)[-1] for x in tc["removed"][:3])))
+ if tc["legacyTests"] and c["changed"] > uplift_checks.SHARE_LIMIT * tc["legacyTests"]:
+ reasons.append("%d of %d legacy test files (%.0f%%) were changed, more than the 25%% allowed" % (c["changed"], tc["legacyTests"], 100 * tc["share"]))
+ if reasons:
+ return row("kept", "gap", "; ".join(reasons) + ": the tests were edited or deleted, so passing them proves less. A person should review the list.")
+ return row("kept", "pass", "No legacy test file was removed and %d of %d (%.0f%%) changed; %d test file(s) were added. The list is for a person to review." % (c["changed"], tc["legacyTests"], 100 * tc["share"], c["added"]))
+
+
+def check_deltas(dc, person):
+ """Check 9 (uplift): does some test name the site of every Behavioral-silent delta?"""
+ if dc["parsed"] == 0:
+ return row("deltas", "gap", "%s." % dc["why"])
+ if dc["config"]:
+ person.append("%d Behavioral-silent delta(s) sit at a build or configuration file that a test cannot name (%s): a person decides how each is checked." % (
+ len(dc["config"]), ", ".join(x["id"] for x in dc["config"][:6])))
+ extra = " %d more at a build or configuration file (a test cannot name it; for a person)." % len(dc["config"]) if dc["config"] else ""
+ if dc["uncovered"]:
+ listed = "; ".join("%s (%s)" % (x["id"], x.get("why", "")) for x in dc["uncovered"][:4])
+ return row("deltas", "gap", "%d of %d Behavioral-silent delta(s) have no test that names their site: %s.%s" % (len(dc["uncovered"]), dc["silent"], listed, extra))
+ if not dc["silent"]:
+ return row("deltas", "pass", "%d delta(s) are in DELTA_CATALOG.md and none is Behavioral-silent.%s" % (dc["parsed"], extra))
+ return row("deltas", "pass", "All %d Behavioral-silent delta(s) (of %d in the catalog) have their site's file named by a test in the working copy.%s" % (len(dc["covered"]), dc["parsed"], extra))
+
+
+def check_same(ctx, subject, paths, dev, dev_problem, base, evidence, caveats):
+ """Check 3. `base` is baseline_state's answer for an uplift, else None."""
+ uplift, dev_state, dev_detail = subject["track"] == "uplift", None, ""
+ if dev:
+ s = evidence["equivalence"] = summary_of(dev)
+ problem = compare.verdict_of(dev)
+ dev_state = "pass" if not problem else "fail"
+ dev_detail = "%d of %d development case(s) executed: %d same%s, %d differ, %d missing; comparator self-check %s." % (
+ s["executed"], s["cases"], s["same"], " (%d within a declared tolerance)" % s["withinTolerance"] if s["withinTolerance"] else "", s["differs"], s["missing"],
+ {True: "passed", False: "FAILED", None: "not applicable"}[s["selfCheck"]])
+ if problem and not (s["differs"] or s["missing"]):
+ dev_detail = "Not proven: %s." % clean(problem, 200)
+ if s["approved"]:
+ dev_detail += " %d difference(s) a person approved are listed." % len(s["approved"])
+ if recorded_matches(os.path.join(ctx["adir"], "EQUIVALENCE.json"), dev) is False:
+ caveats.append("EQUIVALENCE.json does not match a fresh comparison of the cases: run compare.py again so the report shows the current result.")
+ if paths["shared"]:
+ caveats.append("The equivalence cases are shared by every module, so they are not attributed to this module alone.")
+ elif dev_problem:
+ caveats.append(dev_problem)
+ if uplift:
+ b_state, b_detail, comparable = base
+ detail = " ".join(x for x in (b_detail, dev_detail) if x)
+ if "fail" in (dev_state, b_state):
+ return row("same", "fail", detail)
+ if comparable:
+ return row("same", b_state, detail)
+ return row("same", "pass" if dev_state == "pass" else "gap", detail)
+ if dev_state:
+ return row("same", dev_state, dev_detail)
+ if os.path.isfile(os.path.join(ctx["adir"], "EQUIVALENCE.json")):
+ return row("same", "gap", "EQUIVALENCE.json exists, but the cases behind it (equivalence/cases.json) were not found, so it could not be judged again.")
+ return row("same", "gap", "No equivalence cases were recorded: the new code was never compared with the legacy output.")
+
+
+def input_labels(path):
+ """{case id: input label} from the raw cases file. A case may carry "input": "F01" so that several output files of one
+ run count as one input; a case without a label counts as its own input."""
+ obj, _ = read_json(path)
+ cases = obj.get("cases") if isinstance(obj, dict) and isinstance(obj.get("cases"), list) else []
+ return {clean(c.get("id"), 80): clean(c.get("input"), 80) for c in cases[:100000] if isinstance(c, dict) and isinstance(c.get("input"), str) and c.get("input").strip()}
+
+
+def check_fresh(ctx, paths, dev, legacy_ran, frs, fr_problem, evidence):
+ """Check 4."""
+ legacy = ctx["runs"]["legacy"]
+ if ctx["runs"]["leftOut"]:
+ evidence["leftOut"] = ctx["runs"]["leftOut"]
+ if not legacy_ran:
+ return row("fresh", "gap", "The legacy could not run here%s, so the proof is trace-based: no fresh-input comparison was possible and the verdict cannot be PROVEN." % (" (%s)" % legacy["why"] if legacy["why"] else ""))
+ if frs is None:
+ return row("fresh", "gap", "The fresh-input check was not run: no equivalence/fresh-cases.json%s." % (" (%s)" % fr_problem if fr_problem else ""))
+ s = summary_of(frs)
+ dev_hashes = {r["legacySha256"] for r in dev["cases"] if r["legacySha256"]} if dev else set()
+ ran = ("same", "differs", "differs-approved")
+ reused = {r["id"] for r in frs["cases"] if r["verdict"] in ran and r["legacySha256"] in dev_hashes}
+ empty = {r["id"] for r in frs["cases"] if r["verdict"] in ran and r.get("empty")}
+ labels = input_labels(paths["fresh"])
+ counted = [r["id"] for r in frs["cases"] if r["verdict"] in ran and r["id"] not in reused and r["id"] not in empty]
+ inputs = len({labels.get(c) or c for c in counted})
+ s.update({"reused": sorted(reused)[:LIST_CAP], "comparisons": len(counted), "inputs": inputs, "needed": FRESH_MIN})
+ evidence["fresh"] = s
+ if ctx["runs"]["leftOut"]:
+ s["leftOut"] = ctx["runs"]["leftOut"]
+ if s["differs"] or s["missing"] or s["selfCheck"] is False:
+ shown = s["differing"][:5]
+ why = "; ".join("%s: %s" % (d["id"], d["reason"]) for d in shown) or "the comparator's self-check failed"
+ more = " ... and %d more (see VERIFICATION.json)" % (s["differs"] + s["missing"] - len(shown)) if s["differs"] + s["missing"] > len(shown) else ""
+ return row("fresh", "fail", "%d fresh comparison(s) differ and %d are missing (of %d): %s%s" % (s["differs"], s["missing"], s["cases"], why, more))
+ if inputs < FRESH_MIN:
+ return row("fresh", "gap", "Only %d new input(s) counted; at least %d are needed%s." % (inputs, FRESH_MIN, " (%d comparison(s) had the same legacy output as a development case, which is expected of status lines and does not count)" % len(reused) if reused else ""))
+ extra = ([" (%d approved by a person)" % len(s["approved"])] if s["approved"] else []) + ([" (%d within a declared tolerance)" % s["withinTolerance"]] if s["withinTolerance"] else [])
+ return row("fresh", "pass", "%d new input(s), %d comparison(s), ran on the legacy and the new code with no difference%s." % (inputs, len(counted), "".join(extra)))
+
+
+def canaries_in(text):
+ """Canary lines in a notes file: [(change, tests failed)]. Only lines that say Canary are looked at, each cut short."""
+ found = []
+ for line in [ln for ln in (text or "").split("\n") if "anary" in ln[:2000]][:500]:
+ for m in CANARY.finditer(line[:600]):
+ found.append((clean(m.group(1), 200), int(m.group(2).replace(",", ""))))
+ return found
+
+
+def canary_run(entry, workspace, clean_ids, clean_failed):
+ """Failures a canary run's own result file or saved runner log shows beyond the clean run's: (new failures or None, where).
+ With result files, a test that fails in both runs is not the canary's doing; with a log, only the count can be compared."""
+ junit, logs = [p for p in (inside(workspace, j) for j in entry["junit"]) if p], [p for p in (inside(workspace, j) for j in entry["log"]) if p]
+ if junit:
+ fresh = baseline_diff.read_results(junit)
+ if fresh["files"]:
+ bad = {t for t, st in fresh["tests"].items() if st in ("FAIL", "ERROR")}
+ return len(bad - clean_ids), [os.path.relpath(p, workspace).replace(os.sep, "/") for p in junit][:3]
+ if logs:
+ got = read_logs(logs)
+ if got["files"]:
+ return max(0, got["failed"] - clean_failed), [os.path.relpath(p, workspace).replace(os.sep, "/") for p in logs][:3]
+ return None, []
+
+
+def check_canary(subject, module, runs, evidence, workspace, others, clean_ids, clean_failed):
+ """Check 5. A canary counts only when its own result file or saved runner log, parsed here, shows tests that fail because of the break
+ (failures the clean run did not have). A line in the notes, or a count typed into test-runs.json, is a claim."""
+ shown, claimed = [], []
+ for c in runs["canaries"]:
+ mine = c["module"].lower() in ("", module.lower()) or (subject["track"] == "uplift" and c["module"].lower() not in others)
+ if not mine:
+ continue
+ failed, where = canary_run(c, workspace, clean_ids, clean_failed)
+ (shown if failed is not None else claimed).append({"change": c["change"], "testsFailed": failed if failed is not None else c["testsFailed"], "shown": failed is not None, "where": where})
+ for name in NOTE_NAMES:
+ claimed += [{"change": c, "testsFailed": n, "shown": False, "where": []} for c, n in canaries_in(read_text(os.path.join(subject["path"], name)))]
+ evidence["canary"] = (shown + claimed)[:10]
+ silent = [c for c in shown if c["testsFailed"] == 0]
+ if silent:
+ return row("canary", "fail", "A deliberate break was run and no test failed because of it: %s (%s)." % (silent[0]["change"] or "no description", ", ".join(silent[0]["where"])))
+ if shown:
+ first = shown[0]
+ return row("canary", "pass", "%d canary run(s) shown by a result file or log; the first (%s) made %d more test(s) fail than the clean run (%s).%s" % (
+ len(shown), first["change"] or "no description", first["testsFailed"], ", ".join(first["where"]), " %d more are only claimed." % len(claimed) if claimed else ""))
+ if claimed:
+ return row("canary", "gap", "A canary is claimed (%s) but no result file or saved runner log from that run is recorded, so it was not shown." % (claimed[0]["change"] or "no description"))
+ return row("canary", "gap", "No canary is recorded (a deliberate one-line break that makes tests fail), so the tests were never shown able to fail.")
+
+
+def check_source(ctx):
+ """Check 6."""
+ src, name = ctx["source"], "legacy/" + ctx["system"]
+ if src["state"] == "clean":
+ return row("source", "pass", "%s is untouched: %s. Checked %s." % (name, src["detail"], src["method"]))
+ if src["state"] == "changed":
+ return row("source", "gap", "%s has changed since the analysis began: %s. Checked %s.%s" % (name, src["detail"], src["method"], " First: %s." % ", ".join(src["files"][:3]) if src["files"] else ""))
+ return row("source", "gap", "%s: %s. Checked %s." % (name, src["detail"], src["method"]))
+
+
+def not_proven(subject, ev, legacy_ran):
+ """What even a PROVEN verdict would not tell you, specific to this module."""
+ out = ["It does not prove behavior on inputs nobody tried: the equivalence cases and the fresh inputs are samples, not the whole input space."]
+ if not legacy_ran:
+ out.append("The legacy could not run here, so the comparison rests on recorded outputs and traces. It is only as good as that recording.")
+ masks = (ev.get("equivalence") or {}).get("masks") or []
+ if masks:
+ out.append("Fields the comparison ignored could hide a difference there: %s." % "; ".join(m["why"] for m in masks[:5]))
+ for where, key in (("development cases", "equivalence"), ("fresh inputs", "fresh")):
+ for t in (ev.get(key) or {}).get("tolerances") or []:
+ out.append("Numbers may differ in the %s within a declared tolerance (relative %g, absolute %g) in %d case(s): %s. That is a person's rule, not an exact match." % (
+ where, t["rel"] or 0, t["abs"] or 0, t["cases"], t["why"]))
+ if (ev.get("equivalence") or {}).get("approved") or (ev.get("fresh") or {}).get("approved") or (ev.get("baseline") or {}).get("approvedCount"):
+ out.append("A difference a person approved is a decision, not evidence.")
+ out.append("It does not cover speed, capacity, security, concurrency, or anything the tests and cases do not exercise.")
+ if subject["track"] != "uplift":
+ out.append("It traces only the rules that were extracted: behavior nobody wrote down as a rule has no rule to trace.")
+ else:
+ out.append("Weakened assertions cannot be detected: only that test files changed. A test file that was edited may check less than it did.")
+ out.append("A silent version change counts as covered when some test names its site's file, not when a test exercises the changed behavior.")
+ out.append("A test that names a rule shows the rule is mentioned, not that the test is a good one. The canary shows only that some test can fail.")
+ out.append("It does not replace the review and sign-off of the people who own the system.")
+ return out
+
+
+def evaluate(subject, ctx):
+ """-> the module record: verdict, reasons, the six checks and the evidence behind each."""
+ ws, module, uplift = ctx["workspace"], subject["name"], subject["track"] == "uplift"
+ suites = suites_for(ctx["runs"], subject, ctx["unit"], ctx["subjects"])
+ evidence, caveats, person = {}, [], []
+ per, xml_paths, oldest, clean_ids = [], [], None, set()
+ for s in suites:
+ ev, fresh = suite_results(ctx, s)
+ per.append(ev)
+ caveats += ["%s: %s" % (ev["name"] or "a suite", n) for n in ev["notes"]]
+ if ev["oldest"] is not None:
+ oldest = ev["oldest"] if oldest is None else min(oldest, ev["oldest"])
+ if fresh:
+ xml_paths += [p for p in (inside(ws, j) for j in s["junit"]) if p]
+ clean_ids |= {t for t, st in fresh["tests"].items() if st in ("FAIL", "ERROR")}
+ g = {k: sum(e[k] for e in per) for k in ("executed", "failed", "skipped", "skippedNoReason")}
+ g.update({"suites": per, "stale": oldest is not None and code_mtime(subject["path"]) > oldest + 2}) # any result file older than the code: not a clean run
+ evidence["tests"] = dict({k: g[k] for k in ("executed", "failed", "skipped", "skippedNoReason", "stale")},
+ suites=[{k: e.get(k, "") for k in ("name", "command", "date", "note", "source", "files", "written", "executed", "failed", "skipped")} for e in per][:20])
+ binfo, bmeasured = baseline_info(ctx) if uplift else (None, None)
+ base = baseline_state(ctx, suites, xml_paths, evidence, caveats, person, bmeasured) if uplift else None
+ paths = cases_paths(ctx["adir"], module, len(ctx["subjects"]))
+ dev, dev_problem = judge(paths["cases"]) if paths["cases"] else (None, None)
+ frs, fr_problem = judge(paths["fresh"]) if paths["fresh"] else (None, None)
+ legacy_ran = ctx["runs"]["legacy"]["ran"]
+ if uplift and evidence.get("baseline", {}).get("targetOnly"):
+ legacy_ran = False
+ if legacy_ran is None:
+ legacy_ran = bool(frs and frs["totals"]["executed"])
+ evidence["legacy"] = dict(ctx["runs"]["legacy"], ran=legacy_ran)
+ checks = [check_tests(g, uplift, bool(base and base[2])), check_rules(ctx, subject, evidence, caveats), check_same(ctx, subject, paths, dev, dev_problem, base, evidence, caveats),
+ check_fresh(ctx, paths, dev, legacy_ran, frs, fr_problem, evidence), check_canary(subject, module, ctx["runs"], evidence, ws, {x["name"].lower() for x in ctx["subjects"] if x["track"] != "uplift"}, clean_ids, g["failed"]), check_source(ctx)]
+ if uplift:
+ legacy_root = os.path.realpath(os.path.join(ws, "legacy", ctx["system"])) if os.path.lexists(os.path.join(ws, "legacy", ctx["system"])) else ""
+ evidence["testsChanged"] = uplift_checks.tests_changed(legacy_root, subject["path"])
+ catalog = read_text(os.path.join(ctx["adir"], "DELTA_CATALOG.md"))
+ evidence["deltas"] = uplift_checks.delta_coverage(catalog, subject["path"])
+ checks += [check_baseline_measured(binfo, evidence), check_tests_kept(evidence["testsChanged"], person), check_deltas(evidence["deltas"], person)]
+ states = [c["state"] for c in checks]
+ label = dict(CHECKS)
+ return {"name": module, "track": subject["track"], "path": subject["rel"], "verdict": "NOT PROVEN" if "fail" in states else "PARTLY PROVEN" if "gap" in states else "PROVEN",
+ "reasons": ["%s: %s" % (label[c["id"]], c["detail"]) for c in checks if c["state"] in ("fail", "gap")],
+ "passed": ["%s: %s" % (label[c["id"]], c["detail"]) for c in checks if c["state"] == "pass"],
+ "checks": checks, "evidence": evidence, "caveats": caveats[:20], "needsPerson": person[:10], "brief": ctx["brief"](module, subject["track"]),
+ "notProven": not_proven(subject, evidence, legacy_ran), "verifiedAt": ctx["now"]}
+
+
+# ---------------------------------------------------------------- putting the pack together
+def build(system, workspace, module=None, now=None):
+ """-> the pack (a dict). Raises InputError when nothing is built to verify."""
+ workspace = os.path.abspath(workspace)
+ adir = os.path.join(workspace, "analysis", system)
+ now = now or datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+ everything = trace_rules.discover_modules(workspace, system)
+ if not everything:
+ raise InputError("nothing is built under modernized/ for %s: run transform, uplift or reimagine first" % system)
+ unit = None
+ if module and not any(m["name"].lower() == module.lower() for m in everything):
+ # not a module: it may be a unit inside an uplifted working copy, which narrows that copy's suites
+ if not any(m["track"] == "uplift" and os.path.isdir(os.path.join(m["path"], module)) and not os.path.islink(os.path.join(m["path"], module)) for m in everything):
+ raise InputError("nothing is built under modernized/ for %s module %s (built: %s)" % (system, module, ", ".join(clean(m["name"], 40) for m in everything[:10])))
+ unit = module
+ runs = load_test_runs(adir)
+ brief = read_text(os.path.join(adir, "MODERNIZATION_BRIEF.md"))
+ ctx = {"workspace": workspace, "adir": adir, "system": system, "runs": runs, "subjects": everything, "unit": unit, "trace": None, "trace_problem": "", "now": now, "parsed": {}, "ran": {},
+ "source": source_check(workspace, system, adir), "brief": lambda mod, track: brief_items(brief, mod, track)}
+ ctx["ran"] = {s["rel"]: module_run(ctx, s) for s in everything if s["track"] != "uplift"} # what ran, per module: a rule is traced against it
+ try:
+ ctx["trace"] = trace_rules.trace(workspace, system, ctx["ran"])
+ except OSError as err:
+ ctx["trace_problem"] = "%s, so no rule can be traced (run /code-modernization:modernize-extract-rules %s)." % (clean(str(err), 200), system)
+ # a folder of only test code and the files that build it (a parity harness) is not a module: listed, not judged, not counted
+ tooling = [s for s in everything if s["track"] != "uplift" and trace_rules.tooling_only(s["path"])]
+ mods = [evaluate(s, ctx) for s in sorted(everything, key=lambda s: s["name"].lower()) if s not in tooling]
+ problems = [clean(p, 300) for p in runs["problems"]] + ([] if runs["present"] else ["equivalence/test-runs.json was not found, so no test run is recorded and no module can be proven."])
+ if tooling and not mods:
+ problems.append("Every built folder holds only test code and the files that build it, so there is no module to judge.")
+ return {"v": 1, "system": clean(system, 100), "generated": now, "asked": clean(module or "", 100), "rules": list(RULES_TEXT), "legacy": dict(ctx["source"], ran=runs["legacy"]["ran"]),
+ "problems": problems, "modules": mods, "toolingOnly": [{"name": clean(s["name"], 100), "track": s["track"], "path": clean(s["rel"], 200)} for s in sorted(tooling, key=lambda s: s["name"].lower())],
+ "signoff": {"name": "", "role": "", "date": "", "decision": ""}}
+
+
+def overall(pack):
+ counts = {v: sum(1 for m in pack["modules"] if m["verdict"] == v) for v in VERDICTS}
+ pack["overall"] = {"verdict": "NOT PROVEN" if counts["NOT PROVEN"] or not pack["modules"] else "PARTLY PROVEN" if counts["PARTLY PROVEN"] else "PROVEN", "counts": counts}
+ return pack
+
+
+# ---------------------------------------------------------------- VERIFICATION.md
+def cell(text):
+ return clean(text, 400).replace("|", "\\|")
+
+
+def render_md(pack):
+ o = pack["overall"]
+ out = ["# Verification: %s" % pack["system"], "",
+ "Written by `scripts/proof_pack.py` on %s. Each module's verdict is computed from the evidence files by the fixed rules at the end of this page. No model's opinion is part of it." % pack["generated"], "",
+ "**Overall: %s** (%s)" % (o["verdict"], ", ".join("%d %s" % (o["counts"][v], v.lower()) for v in VERDICTS if o["counts"][v]) or "no module"), ""]
+ out += ["- " + p for p in pack["problems"]]
+ if pack.get("asked"):
+ out += ["", "You asked about `%s`. Every built module is judged from its current evidence files on every run, so no verdict here depends on an earlier run." % cell(pack["asked"])]
+ src = pack["legacy"]
+ out += ["", "Legacy source: `%s`%s. Could it run here: %s. The source was checked %s." % (src["path"], " (a link to `%s`)" % src["target"] if src["target"] else "", {True: "yes", False: "no", None: "not stated"}[src.get("ran")], src["method"]), "",
+ "| Module | Track | Verdict | Tests executed | Failed | P0 rules tested | Same behavior | Fresh inputs |", "|---|---|---|---|---|---|---|---|"]
+ for m in pack["modules"]:
+ ev, st = m["evidence"], {c["id"]: c["state"] for c in m["checks"]}
+ p0 = ev.get("rules", {}).get("p0", [])
+ tested = "%d of %d" % (sum(1 for r in p0 if r["status"] == "tested"), len(p0)) if "rules" in ev else "not applicable"
+ out.append("| %s | %s | **%s** | %d | %d | %s | %s | %s |" % (cell(m["name"]), m["track"], m["verdict"], ev["tests"]["executed"], ev["tests"]["failed"], tested, WORDS[st["same"]], WORDS[st["fresh"]]))
+ if pack.get("toolingOnly"):
+ out += ["", "## Test tooling (not judged)", "", "These folders hold only test code and the files that build it, so there is no behavior in them to prove. They are not in the verdict or its counts. A module whose code is not written yet is listed here too, until its code exists.", ""]
+ out += ["- `%s` (%s)" % (cell(t["path"]), t["track"]) for t in pack["toolingOnly"]]
+ for m in pack["modules"]:
+ ev, t, br = m["evidence"], m["evidence"]["tests"], m["brief"]
+ out += ["", "## %s: %s" % (cell(m["name"]), m["verdict"]), "", "Track: %s. Folder: `%s`. Checked %s." % (m["track"], cell(m["path"]), m["verifiedAt"]), ""]
+ if m["reasons"]:
+ out += ["**What is missing or wrong**", ""] + ["- " + cell(r) for r in m["reasons"]] + [""]
+ if m["passed"]:
+ out += ["**What passed**", ""] + ["- " + cell(r) for r in m["passed"]] + [""]
+ out += ["| Check | Result | Detail |", "|---|---|---|"]
+ out += ["| %s | %s | %s |" % (c["label"], WORDS[c["state"]], cell(c["detail"])) for c in m["checks"]]
+ out += ["", "tests executed: %d, failed %d, skipped %d" % (t["executed"], t["failed"], t["skipped"]), ""]
+ out += ["- %s: `%s` (from %s): %d executed, %d failed, %d skipped%s%s" % (cell(s["name"] or "suite"), cell(s["command"]), s["source"], s["executed"], s["failed"], s["skipped"],
+ ", %d result files written %s" % (s["files"], s["written"]) if s["files"] else "",
+ ". Note: " + cell(s["note"]) if s.get("note") else "") for s in t["suites"]]
+ if ev.get("rules", {}).get("p0"):
+ out += ["", "### P0 business rules", "", "| Rule | Name | Confidence | Result | Tests | Code | A test that names it |", "|---|---|---|---|---|---|---|"]
+ out += ["| %s | %s | %s | %s | %d | %d | %s |" % (r["id"], cell(r["name"]), r["confidence"] or "?", r["status"], r["tests"], r["main"], "`%s`" % cell(r["where"]) if r["where"] and r["tests"] else "") for r in ev["rules"]["p0"]]
+ if "equivalence" in ev:
+ e = ev["equivalence"]
+ out += ["", "### Development cases, judged again", "", "%d of %d executed: %d same%s, %d differ, %d missing." % (
+ e["executed"], e["cases"], e["same"], " (%d within a declared tolerance)" % e["withinTolerance"] if e["withinTolerance"] else "", e["differs"], e["missing"])]
+ out += ["- Declared tolerance (relative %g, absolute %g), %d case(s): %s" % (t["rel"] or 0, t["abs"] or 0, t["cases"], cell(t["why"])) for t in e["tolerances"]]
+ out += ["- Approved difference %s: %s" % (cell(a["id"]), cell(a["why"])) for a in e["approved"]] + ["- Ignored by the comparison: %s (%d case(s))" % (cell(x["why"]), x["cases"]) for x in e["masks"]]
+ out += ["- %s: %s" % (cell(d["id"]), cell(d["reason"])) for d in e["differing"]]
+ if "baseline" in ev:
+ b = ev["baseline"]
+ out += ["", "### Baseline comparison", "", "%d regression(s), %d new failure(s), %d fixed, %d missing, %d renamed." % (b["regressionsCount"], b["newFailuresCount"], b["fixedCount"], b["missingCount"], b["renamed"])]
+ out += ["- Regression: %s (%s, now %s)" % (cell(x["id"]), x["before"], x["now"]) for x in b["regressions"][:10]] + ["- New failure: " + cell(x) for x in b["newFailures"][:10]]
+ if ev.get("baseline", {}).get("approvedCount"):
+ out += ["- Difference a person approved in BASELINE.md: %s (%s)" % (cell(x["id"]), cell(x["reason"] or "no reason given")) for x in ev["baseline"]["approved"][:10]]
+ if ev.get("baselineEvidence"):
+ a = ev["baselineEvidence"]
+ out += ["", "### Baseline evidence", "", ("Measured: %s." % ", ".join("%s%s" % (x["kind"], " " + cell(x["name"]) if x.get("name") else "") for x in a["sources"][:5])) if a["kind"] == "measured" else
+ "Typed by hand: no per-test result file, per-test results file or raw runner log backs BASELINE.md."]
+ out += ["- The typed table disagrees on %d test(s) or count(s): %s" % (a["disagree"], "; ".join(cell(x) for x in a["examples"][:3]))] if a["disagree"] else []
+ out += ["- Sources disagree: " + "; ".join(cell(x) for x in a["conflictExamples"][:3])] if a["conflict"] else []
+ out += ["- Looked at: " + ", ".join(cell(x) for x in a["considered"][:8])] if a["considered"] else []
+ if ev.get("testsChanged"):
+ tc = ev["testsChanged"]
+ out += ["", "### Test files changed during the uplift", ""]
+ if tc["checked"]:
+ out += ["%d test file(s) in the legacy tree, %d in the working copy: %d removed, %d added, %d changed (%.0f%% of the legacy test files). Weakened assertions cannot be detected, only that files changed." % (
+ tc["legacyTests"], tc["workTests"], tc["counts"]["removed"], tc["counts"]["added"], tc["counts"]["changed"], 100 * tc["share"])]
+ out += ["- Removed: `%s` (%d lines)" % (cell(x["path"]), x["lines"]) for x in tc["removed"]]
+ out += ["- Changed: `%s` (+%d -%d lines%s)" % (cell(x["path"]), x["added"], x["removed"], "; " + cell(x["note"]) if x["note"] else "") for x in tc["changed"]]
+ out += ["- Added: `%s` (%d lines)" % (cell(x["path"]), x["lines"]) for x in tc["added"]]
+ shown = len(tc["removed"]) + len(tc["changed"]) + len(tc["added"])
+ if sum(tc["counts"].values()) > shown:
+ out.append("- ... and %d more (see VERIFICATION.json)." % (sum(tc["counts"].values()) - shown))
+ else:
+ out.append("Not compared: %s." % cell(tc["why"]))
+ if ev.get("deltas"):
+ dc = ev["deltas"]
+ out += ["", "### Version changes (deltas) and the tests that name them", ""]
+ if dc["parsed"]:
+ out += ["%d delta(s) in DELTA_CATALOG.md; %d are Behavioral-silent: %d have a test naming their site, %d do not; %d more at a build or configuration file." % (
+ dc["parsed"], dc["silent"], len(dc["covered"]), len(dc["uncovered"]), len(dc["config"]))]
+ out += ["- Not named by any test: %s %s (%s)" % (cell(x["id"]), cell(x["title"]), cell(x.get("why", ""))) for x in dc["uncovered"][:LIST_CAP]]
+ out += ["- At a build or configuration file (for a person): %s %s (%s)" % (cell(x["id"]), cell(x["title"]), ", ".join(cell(y) for y in x["sites"][:3])) for x in dc["config"][:LIST_CAP]]
+ else:
+ out.append(cell(dc["why"]) + ".")
+ if ev.get("leftOut"):
+ out += ["", "Inputs left out of the fresh check: " + "; ".join(cell(x) for x in ev["leftOut"])]
+ if "fresh" in ev:
+ f = ev["fresh"]
+ out += ["", "### Fresh inputs", "", "%d new input(s) counted from %d comparison(s) executed: %d same%s, %d differ, %d missing." % (
+ f["inputs"], f["executed"], f["same"], " (%d within a declared tolerance)" % f["withinTolerance"] if f["withinTolerance"] else "", f["differs"], f["missing"])]
+ out += ["- Declared tolerance (relative %g, absolute %g), %d case(s): %s" % (t["rel"] or 0, t["abs"] or 0, t["cases"], cell(t["why"])) for t in f["tolerances"]]
+ out += ["- Not counted, because the legacy output equals a development output (status lines repeat by nature): " + ", ".join(cell(x) for x in f["reused"][:20])] if f["reused"] else []
+ out += (["", "### Caveats", ""] + ["- " + cell(c) for c in m["caveats"]]) if m["caveats"] else []
+ out += ["", "### Waiting for a person", "", "Claude never ticks these. A person decides them.", ""]
+ out += ["- [ ] %s [%s]" % (cell(i["text"]), i["where"]) for i in br["items"]] + ["- [ ] " + cell(n) for n in m["needsPerson"]]
+ if not br["items"] and not m["needsPerson"]:
+ out.append("- The brief lists no unticked criterion for this module." if br["found"] else "- No MODERNIZATION_BRIEF.md was found, so no criterion could be listed.")
+ if br["count"] > len(br["items"]):
+ out.append("- ... and %d more in the brief." % (br["count"] - len(br["items"])))
+ out += ["", "### What this does not prove", ""] + ["- " + cell(n) for n in m["notProven"]]
+ s = pack["signoff"]
+ out += ["", "## Sign-off", "", "A person fills this in. Claude leaves it blank.", "", "| | |", "|---|---|", "| Name | %s |" % (s["name"] or "________________"), "| Role | %s |" % (s["role"] or "________________"),
+ "| Date | %s |" % (s["date"] or "__________"), "| Decision | %s |" % (s["decision"] or "accept / accept with conditions / reject"), "", "## How each verdict is computed", ""]
+ return "\n".join(out + [pack["rules"][0], ""] + pack["rules"][1:]) + "\n"
+
+
+def write_outputs(pack, adir):
+ for name, text in (("VERIFICATION.json", json.dumps(pack, indent=2, ensure_ascii=True) + "\n"), ("VERIFICATION.md", render_md(pack))):
+ try:
+ compare.write_atomic(os.path.join(adir, name), text)
+ except compare.InputError as err:
+ raise InputError(str(err))
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description="Compute a verdict per built module from the evidence files and write VERIFICATION.md and VERIFICATION.json.")
+ ap.add_argument("system", help="the system's folder name under analysis/")
+ ap.add_argument("module", nargs="?", help="the module you re-tested (or a unit of an uplifted copy); every built module is judged from its current evidence either way")
+ ap.add_argument("--workspace", default=".", help="the project root holding analysis/ and modernized/ (default: current folder)")
+ args = ap.parse_args(argv)
+ for name in (args.system, args.module or "x"):
+ if not name or name in (".", "..") or re.search(r"[\\/\x00]", name):
+ print("proof_pack.py: the system and module must be folder names, not paths", file=sys.stderr)
+ return 2
+ adir = os.path.join(os.path.abspath(args.workspace), "analysis", args.system)
+ try:
+ pack = overall(build(args.system, args.workspace, args.module))
+ os.makedirs(adir, exist_ok=True)
+ write_outputs(pack, adir)
+ except InputError as err:
+ print("proof_pack.py: %s" % err, file=sys.stderr)
+ return 2
+ print("%s: %s" % (pack["system"], pack["overall"]["verdict"]))
+ for m in pack["modules"]:
+ print(" %s (%s): %s, tests executed: %d, failed %d" % (clean(m["name"], 60), m["track"], m["verdict"], m["evidence"]["tests"]["executed"], m["evidence"]["tests"]["failed"]))
+ for r in m["reasons"]:
+ print(" - " + clean(r, 300))
+ for t in pack["toolingOnly"]:
+ print(" %s (%s): test tooling, not judged" % (clean(t["name"], 60), t["track"]))
+ print("wrote analysis/%s/VERIFICATION.md and VERIFICATION.json" % args.system)
+ return 0 if pack["overall"]["verdict"] == "PROVEN" else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/plugins/code-modernization/scripts/render_rules.py b/plugins/code-modernization/scripts/render_rules.py
new file mode 100644
index 00000000..fa609332
--- /dev/null
+++ b/plugins/code-modernization/scripts/render_rules.py
@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+"""Render the extract-rules workflow's result into BUSINESS_RULES.md and DATA_OBJECTS.md.
+
+ python3 render_rules.py <system> [--workspace DIR] [--result FILE]
+
+Reads analysis/<system>/rules_result.json (the object the workflow returned, saved by the
+command) and writes analysis/<system>/BUSINESS_RULES.md and DATA_OBJECTS.md. Rules are
+numbered RULE-001, RULE-002, ... in the order they appear (by category, then priority) and the heading is always
+`### RULE-NNN: <name>`, the pattern later commands and the pane look for. Every value comes
+from analysis of untrusted code, so it is written as plain text on one line where it is
+a heading or a table cell. Standard library only.
+"""
+import json
+import os
+import re
+import sys
+
+CATEGORIES = ['Calculation', 'Validation', 'Lifecycle', 'Policy']
+PRIORITY = {'P0': 0, 'P1': 1, 'P2': 2}
+
+
+def one_line(value, limit=300):
+ text = ' '.join(str(value if value is not None else '').split())
+ text = text.replace('|', '/').replace('`', "'")
+ return text if len(text) <= limit else text[: limit - 1].rstrip() + '…'
+
+
+CITATION = re.compile(r'^\s*`?([^\s`:,;()]+):(\d+)(?:\s*[-\u2013]\s*(\d+))?')
+
+
+def split_citation(value):
+ """One citation per rule: the first path:line[-line] in the text is it, and whatever follows
+ (a second file, a note) comes back separately so later tools can rely on the citation alone."""
+ text = ' '.join(str(value if value is not None else '').split())
+ m = CITATION.match(text)
+ if not m:
+ return one_line(text, 200), ''
+ primary = f'{m.group(1)}:{m.group(2)}' + (f'-{m.group(3)}' if m.group(3) else '')
+ rest = re.sub(r'\s*;\s*also\s+', '; ', text[m.end():], flags=re.I) # citations a consolidation appended
+ rest = re.sub(r'^[\s;,.\-\u2013\u2014`]*(?:also\s+)?', '', rest, flags=re.I)
+ return one_line(primary, 200), one_line(rest.strip(' \t;,.-\u2013\u2014`'), 300)
+
+
+def block(value):
+ """Multi-line text: keep the lines, but never let a value open a heading or a fence."""
+ lines = []
+ for raw in str(value if value is not None else '').splitlines():
+ line = raw.rstrip()
+ if line.lstrip().startswith(('#', '```', '~~~')):
+ line = '\\' + line.lstrip()
+ lines.append(line)
+ return '\n'.join(lines).strip()
+
+
+def render_rule(number, rule):
+ rid = f'RULE-{number:03d}'
+ source, also = split_citation(rule.get('source'))
+ out = [f"### {rid}: {one_line(rule.get('name'), 120)}",
+ f"**Category:** {one_line(rule.get('category'), 20)}",
+ f"**Priority:** {one_line(rule.get('priority'), 4)}",
+ f"**Source:** `{source.replace(chr(39), '')}`"]
+ if also:
+ out.append(f'**Also cited:** {also}')
+ out += [f"**Plain English:** {one_line(rule.get('plainEnglish'), 400)}",
+ '**Specification:**',
+ f" Given {block(rule.get('given'))}",
+ f" When {block(rule.get('when'))}",
+ f" Then {block(rule.get('then'))}"]
+ if rule.get('and'):
+ out.append(f" And {block(rule['and'])}")
+ if rule.get('parameters'):
+ out.append(f"**Parameters:** {block(rule['parameters'])}")
+ cases = rule.get('edgeCases') or []
+ if cases:
+ out.append('**Edge cases handled:** ' + '; '.join(one_line(c, 200) for c in cases))
+ if rule.get('suspectedDefect'):
+ out.append(f"**Suspected defect:** {block(rule['suspectedDefect'])}")
+ confidence = one_line(rule.get('confidence'), 10)
+ question = rule.get('smeQuestion')
+ out.append(f"**Confidence:** {confidence}" + (f" — {one_line(question, 400)}" if question and confidence != 'High' else ''))
+ return rid, '\n'.join(out)
+
+
+def main(argv):
+ args = [a for a in argv[1:] if not a.startswith('--')]
+ workspace, result_path = '.', None
+ if '--workspace' in argv:
+ workspace = argv[argv.index('--workspace') + 1]
+ args = [a for a in args if a != workspace]
+ if '--result' in argv:
+ result_path = argv[argv.index('--result') + 1]
+ args = [a for a in args if a != result_path]
+ if not args:
+ print('usage: render_rules.py <system> [--workspace DIR] [--result FILE]', file=sys.stderr)
+ return 2
+ system = args[0]
+ base = os.path.join(workspace, 'analysis', system)
+ result_path = result_path or os.path.join(base, 'rules_result.json')
+ try:
+ with open(result_path, encoding='utf-8') as handle:
+ result = json.load(handle)
+ except (OSError, ValueError) as error:
+ print(f'Cannot read {result_path}: {error}', file=sys.stderr)
+ return 1
+
+ # Numbered in the order they appear in the document (by category, then priority), so RULE-001.. read in sequence.
+ rules = sorted(result.get('confirmedRules') or [],
+ key=lambda r: (CATEGORIES.index(r['category']) if r.get('category') in CATEGORIES else 9,
+ PRIORITY.get(r.get('priority'), 1),
+ str(r.get('source'))))
+ numbered = [(i + 1, r) for i, r in enumerate(rules)]
+ rendered = {n: render_rule(n, r) for n, r in numbered}
+ stats = result.get('stats') or {}
+
+ doc = [f'# Business Rules — {one_line(system, 80)}', '',
+ f"At extraction: {len(rules)} confirmed rules ({sum(1 for r in rules if r.get('priority') == 'P0')} P0); "
+ f"later steps may add or correct rules below. "
+ f"Each citation was checked by a second agent that read the cited lines; "
+ f"{len(result.get('rejectedRules') or [])} candidate rules were refuted and left out.", '',
+ '| ID | Name | Category | Priority | Source | Confidence |', '|---|---|---|---|---|---|']
+ for n, r in numbered:
+ doc.append(f"| RULE-{n:03d} | {one_line(r.get('name'), 80)} | {one_line(r.get('category'), 20)} | "
+ f"{one_line(r.get('priority'), 4)} | `{split_citation(r.get('source'))[0][:80].replace(chr(39), '')}` | {one_line(r.get('confidence'), 10)} |")
+ doc.append('')
+ for category in CATEGORIES + [None]:
+ group = [n for n, r in numbered if (r.get('category') if r.get('category') in CATEGORIES else None) == category]
+ if not group:
+ continue
+ doc += [f"## {category or 'Other'}", '']
+ for n in group:
+ doc += [rendered[n][1], '']
+ sme = [(n, r) for n, r in numbered if r.get('confidence') != 'High']
+ doc += ['## Rules requiring SME confirmation', '']
+ if sme:
+ for n, r in sme:
+ doc.append(f"- **RULE-{n:03d}** ({one_line(r.get('confidence'), 10)}): "
+ f"{one_line(r.get('smeQuestion') or 'Confirm this rule with someone who knows the system.', 400)}")
+ else:
+ doc.append('None: every confirmed rule has High confidence.')
+ doc.append('')
+
+ flags = result.get('injectionFlags') or []
+ if flags:
+ doc += ['## ⚠ Instruction-shaped content found in source', '',
+ 'These lines of the source tried to steer automated analysis. A person should look at them.', '']
+ doc += [f'- {one_line(f, 300)}' for f in flags]
+ doc.append('')
+
+ folded = [f for f in (result.get('foldedRules') or []) if isinstance(f, dict)]
+ if folded:
+ doc += ['## Rules folded into another', '',
+ 'These rules described the same behavior as another rule in a different place, so they were merged into it '
+ '(the kept rule lists their locations under "Also cited"):', '']
+ doc += [f"- {one_line(f.get('name'), 100)} ({one_line(f.get('source'), 100)}) into {one_line(f.get('into'), 100)}" for f in folded[:100]]
+ if len(folded) > 100:
+ doc.append(f'- … and {len(folded) - 100} more')
+ doc.append('')
+
+ gaps = [('never attempted (token budget or agent cap)', stats.get('skippedModules')),
+ ('extractor returned nothing', stats.get('failedModules')),
+ ('malformed entries, fix by hand', stats.get('droppedModules')),
+ ('phases cut short', stats.get('skippedPhases')),
+ ('candidate rules no referee judged (not in the catalog)',
+ [f"{one_line(u.get('name'), 80)} ({one_line(u.get('source'), 80)})" for u in (result.get('unverifiedRules') or [])])]
+ gaps = [(label, items) for label, items in gaps if items]
+ if gaps:
+ doc += ['## Coverage gaps', '', 'These parts were NOT fully mined:', '']
+ for label, items in gaps:
+ doc.append(f"- **{len(items)} {label}:** " + ', '.join(one_line(i, 100) for i in items[:30]) + (' …' if len(items) > 30 else ''))
+ if result.get('rerunModules'):
+ doc.append(f"\nA follow-up run for the {len(result['rerunModules'])} affected shard(s) covers them.")
+ doc.append('')
+
+ with open(os.path.join(base, 'BUSINESS_RULES.md'), 'w', encoding='utf-8') as handle:
+ handle.write('\n'.join(doc))
+
+ objects = result.get('dataObjects') or []
+ dto = [f'# Data Objects — {one_line(system, 80)}', '']
+ if not objects:
+ dto.append('No data objects were cataloged.')
+ for obj in objects:
+ dto += [f"## {one_line(obj.get('name'), 100)}", f"Source: `{one_line(obj.get('source'), 160).replace(chr(39), '')}`", '',
+ '| Field | Type | Note |', '|---|---|---|']
+ for field in obj.get('fields') or []:
+ dto.append(f"| {one_line(field.get('name'), 80)} | {one_line(field.get('type'), 60)} | {one_line(field.get('note'), 160)} |")
+ used = obj.get('consumedBy') or []
+ if used:
+ dto += ['', 'Used by: ' + ', '.join(one_line(u, 80) for u in used)]
+ dto.append('')
+ with open(os.path.join(base, 'DATA_OBJECTS.md'), 'w', encoding='utf-8') as handle:
+ handle.write('\n'.join(dto))
+
+ p0 = sum(1 for r in rules if r.get('priority') == 'P0')
+ print(f"BUSINESS_RULES.md: {len(rules)} rules ({p0} P0, {len(sme)} need SME review); DATA_OBJECTS.md: {len(objects)} objects; "
+ f"rejected {len(result.get('rejectedRules') or [])}, unverified {len(result.get('unverifiedRules') or [])}")
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main(sys.argv))
diff --git a/plugins/code-modernization/scripts/telemetry.py b/plugins/code-modernization/scripts/telemetry.py
new file mode 100644
index 00000000..5079e684
--- /dev/null
+++ b/plugins/code-modernization/scripts/telemetry.py
@@ -0,0 +1,732 @@
+#!/usr/bin/env python3
+"""Usage counts for the code-modernization plugin. Whole numbers only, never text.
+
+ telemetry.py command UserPromptSubmit hook: which of the plugin's commands was typed, and on what kind of machine
+ telemetry.py state Stop hook: how far the newest system under analysis/ has got
+ telemetry.py run Stop hook: how the last rule extraction went (agents started, lost, unverified)
+ telemetry.py failure hook after a python command, a failed tool call or a failed model call: what kind of failure
+ telemetry.py show [DIR] [--prompt TEXT] [--json]
+ print what the hooks would send for the workspace DIR (default: this folder)
+
+The two hooks read the hook's JSON from stdin and print at most one line, {"metrics": {...}}, which Claude Code
+records as usage counts when its own telemetry is on. Every key is fixed below and every value is a whole number,
+capped, and rounded to two significant figures from 100 up, so a count is not a fingerprint of one system. Nothing is
+read except the plugin's own artifacts under analysis/ and modernized/, and no file name, path, system name, code or
+prompt text leaves this script. Nothing is sent when the plugin has not been used in the folder. It never raises,
+gives up after three seconds and always exits 0.
+
+Off switches: CODE_MODERNIZATION_TELEMETRY=0, the plugin's "Usage counts" option, or Claude Code's own
+DISABLE_TELEMETRY / CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC. Anything but an explicit "on" in the first two
+switches its off. To find out why nothing was sent, set CODE_MODERNIZATION_TELEMETRY_DEBUG=1: each hook run then
+appends one line saying what it decided to telemetry-debug.log in the plugin's data folder.
+"""
+import collections
+import hashlib
+import itertools
+import json
+import os
+import platform
+import re
+import signal
+import stat
+import sys
+import tempfile
+import threading
+import time
+
+sys.dont_write_bytecode = True # a hook must not leave files in the plugin's own folder
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, HERE)
+import build_report as br # noqa: E402 (reads the artifacts safely and counts them the way the report does)
+
+MAX_KEYS, CAP = 20, 10 ** 9
+TRUE = {"1", "true", "on", "yes"}
+# The most a key can carry: a hostile artifact cannot make a count bigger than this. Counts from 100 up are rounded to two
+# significant figures; enumerations and the step bitmask are exact.
+LIMITS = {"pv": 999999, "cmd": 99, "perm": 6, "has_source": 1, "fresh": 1, "systems": 99, "goal": 4, "lang": 99, "done": 65535, "phases": 99,
+ "os": 9, "py": 9, "pyv": 999, "pathf": 7, "tool": 99, "kind": 99, "api": 99, "err": 99, "err_at": 99999}
+COUNT_LIMIT = 10 ** 7
+EXACT = {"pv", "cmd", "perm", "has_source", "fresh", "goal", "lang", "done", "os", "py", "pyv", "pathf", "tool", "kind", "api", "err", "err_at"}
+NAME = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}")
+LINE_WIDTH = 600 # longer lines are cut before anything is parsed, so no artifact can make a pattern slow
+
+# One number per command: 1 is the front door, 20-22 are the pane's own commands (panel, review-pane, sign), typed without the plugin's name.
+COMMANDS = {"": 1, "preflight": 2, "assess": 3, "map": 4, "extract-rules": 5, "review": 6, "brief": 7, "transform": 8,
+ "uplift": 9, "reimagine": 10, "verify": 11, "harden": 12, "status": 13}
+PANE = {"panel": 20, "review-pane": 21, "sign": 22}
+OTHER_COMMAND = 99
+PERMISSION_MODES = {"default": 1, "acceptEdits": 2, "plan": 3, "auto": 4, "bypassPermissions": 5, "dontAsk": 6}
+TOOLS = {"Bash": 1, "Read": 2, "Edit": 3, "Write": 3, "MultiEdit": 3, "NotebookEdit": 3, "Glob": 4, "Grep": 4, "Agent": 5, "Task": 5, "Workflow": 6,
+ "AskUserQuestion": 7, "Skill": 8, "WebFetch": 9, "WebSearch": 9}
+# What went wrong, decided here from the text of a failure and never sent: first match wins. 1-5 are about the interpreter and the plugin's own scripts.
+FAILURE_KINDS = [
+ (2, re.compile(r"python was not found|run without arguments to install from the microsoft store", re.I)),
+ (3, re.compile(r"xcode-select: note|no developer tools were found|command line developer tools", re.I)),
+ (1, re.compile(r"\b(?:python3?|py)(?:\.exe)?: (?:command )?not found|command not found: (?:python3?|py)\b|'(?:python3?|py)' is not recognized|python3?: no such file|env: .?python3?.?: no such file", re.I)),
+ (4, re.compile(r"syntaxerror|requires python 3|invalid syntax", re.I)),
+ (6, re.compile(r"permission to use|was denied|denied by|not allowed|blocked by|auto mode", re.I)),
+ (7, re.compile(r"eacces|permission denied|access is denied", re.I)),
+ (8, re.compile(r"timed out|timeout|etimedout", re.I)),
+ (11, re.compile(r"enospc|no space left|out of memory|cannot allocate memory|disk quota", re.I)),
+ (10, re.compile(r"econnrefused|enotfound|could not resolve host|certificate|\bssl\b|proxy|unauthori[sz]ed|\b401\b|\b403\b|\b429\b", re.I)),
+ (9, re.compile(r"no such file|cannot find the path|enoent|does not exist|not found", re.I)),
+ (12, re.compile(r"scriptpath|unknown tool|workflow|subagent", re.I)),
+]
+API_ERRORS = [
+ (1, re.compile(r"rate.?limit|\b429\b", re.I)),
+ (2, re.compile(r"authenticat|unauthori[sz]ed|invalid.?api.?key|\b40[13]\b|oauth", re.I)),
+ (3, re.compile(r"billing|credit|payment|quota", re.I)),
+ (6, re.compile(r"max.?output|max_tokens|too long", re.I)),
+ (7, re.compile(r"timeout|timed out|network|econn|connection", re.I)),
+ (5, re.compile(r"server.?error|overloaded|\b5\d\d\b|internal", re.I)),
+ (4, re.compile(r"invalid.?request|\b400\b|unrecognized_model|not_found", re.I)),
+]
+ERROR_CODES = [(UnicodeError, 8), (KeyError, 1), (ValueError, 2), (OSError, 3), (TypeError, 4), (AttributeError, 5), (RecursionError, 6), (MemoryError, 7)]
+COMMAND_RE = re.compile(r"^/(?P<ns>code-modernization:)?modernize(?:-(?P<verb>[a-z][a-z-]*))?(?=\s|$)")
+
+GOALS = {"understand": 1, "uplift": 2, "transform": 3, "reimagine": 4}
+GOAL_LABEL = re.compile(r"(?i)^goal\b\s*[:\u2013\u2014]?\s*(.*)$")
+GOAL_WORDS = re.compile(r"(?i)(understand|uplift|transform|reimagine|rewrite|rebuild)\b")
+GOAL_ALIASES = {"rewrite": "transform", "rebuild": "reimagine"}
+# the pop-up's own wording, when that is what was written down
+GOAL_PHRASES = [(re.compile(r"(?i)newer version|same technology"), 2), (re.compile(r"(?i)different technology|one piece at a time"), 3),
+ (re.compile(r"(?i)from scratch|new architecture"), 4), (re.compile(r"(?i)understand it|map it"), 1)]
+
+# What the code is written in: the dominant language of the map, as one of these numbers (0 unknown, 99 other).
+LANGS = {"cobol": 1, "jcl": 1, "java": 2, "jsp": 2, "c#": 3, "csharp": 3, "vb": 3, "vb.net": 3, "f#": 3, "python": 4, "php": 5,
+ "perl": 6, "c": 7, "c++": 8, "cpp": 8, "javascript": 9, "typescript": 9, "js": 9, "ts": 9, "node": 9, "angularjs": 9,
+ "ruby": 10, "go": 11, "golang": 11, "rust": 12, "kotlin": 13, "scala": 13, "groovy": 13, "sql": 14, "plsql": 14,
+ "pl/sql": 14, "tsql": 14, "fortran": 15, "rpg": 16, "rpgle": 16, "pascal": 17, "delphi": 17, "shell": 18, "bash": 18,
+ "assembler": 19, "assembly": 19, "hlasm": 19, "asp": 3, "vb6": 3, "nodejs": 9, "c/c++": 7}
+# labels in a map that are not a language somebody modernizes
+NOT_CODE = {"artifact", "make", "makefile", "cmake", "yaml", "yml", "json", "xml", "toml", "ini", "markdown", "md", "text", "txt", "html", "css",
+ "pkgconfig", "config", "properties", "docker", "dockerfile", "csv", "svg", "data", "binary", "license", "gradle", "maven", "unknown", "other"}
+
+# `done` adds up one of these for every step that has left its file: 15 = preflight, assess, map and rules.
+STEPS = [("preflight", 1), ("assess", 2), ("map", 4), ("rules", 8), ("reviewed", 16), ("brief", 32), ("approved", 64),
+ ("built", 128), ("verified", 256), ("signed", 512), ("hardened", 1024), ("report", 2048), ("deltas", 4096),
+ ("baseline", 8192), ("playbook", 16384), ("spec", 32768)]
+# Files that only this plugin writes: a folder with none of them is not something the plugin has worked on.
+GATE = ("INTENT.md", "PREFLIGHT.md", "MODERNIZATION_BRIEF.md", "BUSINESS_RULES.md", "DELTA_CATALOG.md", "VERIFICATION.json", "RULE_REVIEWS.json", "AI_NATIVE_SPEC.md")
+FILES = {"preflight": "PREFLIGHT.md", "assess": "ASSESSMENT.md", "map": "topology.json", "brief": "MODERNIZATION_BRIEF.md",
+ "verified": "VERIFICATION.json", "hardened": "SECURITY_FINDINGS.md", "report": "REPORT.html",
+ "deltas": "DELTA_CATALOG.md", "baseline": "BASELINE.md", "playbook": "PLAYBOOK.md", "spec": "AI_NATIVE_SPEC.md"}
+
+# Every key that can be sent, and what it counts. Order is send order; the version comes first so it always survives.
+MEANING = collections.OrderedDict([
+ ("pv", "plugin version as major*10000 + minor*100 + patch"),
+ ("cmd", "command typed: 1 front door, 2 preflight, 3 assess, 4 map, 5 extract-rules, 6 review, 7 brief, 8 transform, "
+ "9 uplift, 10 reimagine, 11 verify, 12 harden, 13 status, 20-22 pane commands, 99 other"),
+ ("perm", "permission mode the session ran in: 0 unknown, 1 default, 2 accept edits, 3 plan, 4 auto, 5 bypass, 6 don't ask"),
+ ("has_source", "1 when the command carried --source"),
+ ("fresh", "1 when the system has no artifacts yet"),
+ ("systems", "systems under analysis/"),
+ ("goal", "0 unknown, 1 understand, 2 uplift, 3 transform, 4 reimagine"),
+ ("lang", "dominant language of the map: 1 COBOL, 2 Java, 3 C#/.NET, 4 Python, 5 PHP, 6 Perl, 7 C, 8 C++, "
+ "9 JavaScript/TypeScript, 10 Ruby, 11 Go, 12 Rust, 13 Kotlin/Scala, 14 SQL, 15 Fortran, 16 RPG, 17 Pascal/Delphi, "
+ "18 shell, 19 assembler, 99 other, 0 unknown"),
+ ("done", "steps that have left their file, added up: " + ", ".join("%s=%d" % s for s in STEPS)),
+ ("map_kloc", "thousand lines of code in the map"),
+ ("rules", "business rules found"),
+ ("p0", "of those, the critical (P0) ones"),
+ ("rev_ok", "rules a person confirmed"),
+ ("rev_wrong", "rules a person marked wrong"),
+ ("phases", "phases in the plan"),
+ ("built", "modules or services built (an uplift's working copy counts as one once it exists)"),
+ ("eq_cases", "old-versus-new comparison cases run"),
+ ("eq_diff", "of those, differences or missing outputs nobody approved"),
+ ("eq_appr", "differences a person approved"),
+ ("v_proven", "modules judged PROVEN"),
+ ("v_partly", "modules judged PARTLY PROVEN"),
+ ("v_not", "modules judged NOT PROVEN"),
+ ("sec_crit", "critical security findings"),
+ ("sec_high", "high security findings"),
+ ("os", "system: 1 macOS, 2 Linux, 3 Windows, 4 Windows subsystem for Linux, 0 other"),
+ ("py", "how python worked: 0 python3 ran, 1 none found, 2 macOS developer-tools stub, 3 python3 present but broken (the Windows Store stub), "
+ "4 python3 unusable but python or py ran, 5 too old, 6 the script crashed"),
+ ("pyv", "python version as major*100 + minor (311 is 3.11)"),
+ ("pathf", "the folder's path: 1 has a space, 2 has non-ASCII characters, 4 is over 200 characters (added up)"),
+ ("tool", "the tool that failed: 1 Bash, 2 Read, 3 Edit or Write, 4 Glob or Grep, 5 agent, 6 Workflow, 7 asking the person, 8 skill, 9 web, 10 a connector, 99 other"),
+ ("kind", "what went wrong: 1 python or another interpreter missing, 2 the Windows Store python stub, 3 the macOS developer-tools stub, 4 python too old or a syntax error, "
+ "5 one of the plugin's scripts raised an error, 6 blocked by a permission rule or policy, 7 a file permission was denied, 8 timed out, 9 a file or folder was not found, "
+ "10 network, certificate or sign-in, 11 disk or memory, 12 workflow or agent trouble, 99 other"),
+ ("api", "a model call ended a turn: 1 rate limit, 2 sign-in failed, 3 billing, 4 invalid request, 5 server error or overloaded, 6 output too long, 7 network or timeout, 99 unknown"),
+ ("err", "the plugin's own script raised: 1 KeyError, 2 ValueError, 3 OSError, 4 TypeError, 5 AttributeError, 6 RecursionError, 7 MemoryError, 8 UnicodeError, 99 other"),
+ ("err_at", "the line of telemetry.py where it raised"),
+ ("agents", "agents the last rule extraction started"),
+ ("wf_failed", "modules or agents that extraction lost"),
+ ("wf_skip", "modules that extraction skipped"),
+ ("wf_unver", "rules that extraction could not verify"),
+])
+COMMAND_KEYS = ("pv", "cmd", "perm", "has_source", "fresh", "systems", "goal", "done", "os", "py", "pyv", "pathf")
+RUN_KEYS = ("pv", "agents", "wf_failed", "wf_skip", "wf_unver")
+FAILURE_KEYS = ("pv", "os", "py", "pyv", "pathf", "tool", "kind", "api")
+ERROR_KEYS = ("pv", "os", "py", "pyv", "err", "err_at")
+STATE_KEYS = ("pv", "goal", "lang", "done", "map_kloc", "rules", "p0", "rev_ok", "rev_wrong", "phases", "built", "eq_cases",
+ "eq_diff", "eq_appr", "v_proven", "v_partly", "v_not", "sec_crit", "sec_high")
+
+
+def off_reason(env=None):
+ """Why nothing is sent, or "" when counts may be sent. Sending needs the plugin's switches to be unset or an explicit
+ "on": any other word (off, no, 0, disabled) turns it off."""
+ env = os.environ if env is None else env
+ val = lambda key: str(env.get(key, "")).strip().lower() # noqa: E731
+ if val("CODE_MODERNIZATION_TELEMETRY") and val("CODE_MODERNIZATION_TELEMETRY") not in TRUE:
+ return "CODE_MODERNIZATION_TELEMETRY is off"
+ if val("CLAUDE_PLUGIN_OPTION_TELEMETRY") and val("CLAUDE_PLUGIN_OPTION_TELEMETRY") not in TRUE:
+ return "the plugin's Usage counts option is off"
+ for key in ("DISABLE_TELEMETRY", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"):
+ if val(key) in TRUE:
+ return "%s is set" % key
+ return ""
+
+
+def whole(value):
+ """A whole number from 0 to CAP, whatever the artifact held."""
+ if isinstance(value, bool):
+ return int(value)
+ try:
+ return max(0, min(CAP, int(value)))
+ except (TypeError, ValueError, OverflowError):
+ return 0
+
+
+def plugin_version():
+ try:
+ with open(os.path.join(HERE, "..", ".claude-plugin", "plugin.json"), encoding="utf-8") as fh:
+ major, minor, patch = (int(x) for x in str(json.load(fh)["version"]).split(".")[:3])
+ return major * 10000 + minor * 100 + patch
+ except (OSError, ValueError, KeyError, TypeError):
+ return 0
+
+
+def plain(path):
+ return os.path.isfile(path) and not os.path.islink(path)
+
+
+def inside(cwd, path):
+ """True when path really lives under cwd (links that lead elsewhere are not followed)."""
+ try:
+ root = os.path.realpath(cwd)
+ return os.path.commonpath([root, os.path.realpath(path)]) == root
+ except ValueError:
+ return False
+
+
+def systems_in(cwd):
+ """[(newest modification time, name)] of the folders under analysis/, newest first."""
+ out, folder = [], os.path.join(cwd, "analysis")
+ if os.path.islink(folder) or not inside(cwd, folder):
+ return out
+ try:
+ entries = list(itertools.islice(os.scandir(folder), 200))
+ except OSError:
+ return out
+ for e in entries:
+ if e.name.startswith(".") or e.is_symlink() or not e.is_dir():
+ continue
+ try:
+ times = [e.stat().st_mtime]
+ kids = list(itertools.islice(os.scandir(e.path), 200))
+ except OSError:
+ continue
+ for x in kids:
+ try:
+ times.append(x.stat().st_mtime)
+ except OSError: # a dangling link or a file that just went away
+ pass
+ out.append((max(times), e.name))
+ return sorted(out, reverse=True)
+
+
+def language_code(topo):
+ """The language with the most lines in the map, as its number."""
+ weight, nodes = collections.Counter(), []
+
+ def walk(node, depth):
+ if isinstance(node, dict) and depth < 12:
+ nodes.append(node)
+ for child in node.get("children") or []:
+ walk(child, depth + 1)
+
+ if isinstance(topo, dict):
+ walk(topo.get("root"), 0)
+ for n in nodes:
+ lang = n.get("language")
+ if not n.get("children") and isinstance(lang, str) and isinstance(n.get("loc"), int) and n["loc"] > 0:
+ low = lang.strip().lower()[:40]
+ first = re.match(r"[a-z0-9#+]+", low)
+ first = first.group(0) if first else ""
+ if low in NOT_CODE or first in NOT_CODE:
+ continue # build files, data and the like are not what is being modernized
+ weight[LANGS.get(low) or LANGS.get(first) or 99] += n["loc"]
+ return min(weight, key=lambda c: (-weight[c], c)) if weight else 0
+
+
+def goal_code(text, built):
+ """From the answer written after "Goal" in INTENT.md (its first words, not the sentences around it), else from what was built."""
+ lines = [re.sub(r"[*`_#>\-]", " ", l).strip() for l in (text or "").split("\n")[:60]]
+ for i, line in enumerate(lines):
+ m = GOAL_LABEL.match(line)
+ if not m:
+ continue
+ rest = m.group(1)
+ if not rest: # a heading: the answer is on the next line that has words
+ rest = next((l for l in lines[i + 1:i + 4] if l), "")
+ rest = (GOAL_LABEL.match(rest) or GOAL_LABEL.match("goal " + rest)).group(1)
+ word = GOAL_WORDS.match(rest)
+ if word:
+ word = word.group(1).lower()
+ return GOALS[GOAL_ALIASES.get(word, word)]
+ for pattern, code in GOAL_PHRASES:
+ if pattern.search(rest[:100]):
+ return code
+ break
+ return {"": 3, "-uplifted": 2, "-reimagined": 4}.get(built, 0)
+
+
+def built_kind_and_count(cwd, system):
+ """(which kind of build is newest: '' rewrite, '-uplifted' or '-reimagined'; how many modules or services)."""
+ total, kinds = 0, []
+ for suffix in ("", "-uplifted", "-reimagined"):
+ root = os.path.join(cwd, "modernized", system + suffix)
+ if os.path.islink(root) or not inside(cwd, root):
+ continue
+ try:
+ kids = [e for e in os.scandir(root) if not e.name.startswith(".") and not e.is_symlink()]
+ newest = max(e.stat().st_mtime for e in kids) if kids else 0
+ except OSError:
+ continue
+ if not kids:
+ continue
+ dirs = [e for e in kids if e.is_dir()]
+ total += 1 if suffix == "-uplifted" else (len(dirs) or 1)
+ kinds.append((newest, suffix))
+ return (max(kinds)[1] if kinds else None), total
+
+
+def snapshot(cwd, system):
+ """The counts for one system, or None when the plugin has left nothing there yet."""
+ src = br.Source(cwd, system)
+ path = lambda name: os.path.join(src.adir, name) # noqa: E731
+
+ def text(name):
+ raw = src.read(path(name), name)
+ return None if raw is None else "\n".join(line[:LINE_WIDTH] for line in raw.split("\n"))
+
+ if not any(plain(path(name)) for name in GATE):
+ return None
+ done, out = 0, collections.OrderedDict((k, 0) for k in STATE_KEYS)
+ present = {step: plain(path(FILES[step])) for step in FILES}
+
+ rules_text = text("BUSINESS_RULES.md")
+ reviews, _ = src.json(path("RULE_REVIEWS.json"), "RULE_REVIEWS.json")
+ reviews = {k: v for k, v in reviews["reviews"].items() if isinstance(v, dict)} if isinstance(reviews, dict) and isinstance(reviews.get("reviews"), dict) else {}
+ cards = [b for b in br.parse_rules(rules_text) if b["k"] == "rule"] if rules_text else []
+ if cards:
+ facts = br.rule_facts(rules_text, cards, reviews)
+ out["rules"], out["p0"] = whole(facts["total"]), whole(facts["byPriority"].get("P0", 0))
+ present["rules"] = True
+ verdicts = collections.Counter(str(v.get("verdict", "")).lower() for v in reviews.values())
+ out["rev_ok"], out["rev_wrong"] = whole(verdicts["confirmed"]), whole(verdicts["wrong"])
+ present["reviewed"] = bool(reviews)
+
+ topo, _ = src.json(path("topology.json"), "topology.json")
+ facts = br.topology_facts(topo)
+ out["map_kloc"] = whole(facts["loc"] // 1000) if facts else 0
+ out["lang"] = language_code(topo)
+
+ brief = text("MODERNIZATION_BRIEF.md") or ""
+ out["phases"] = len(re.findall(r"(?m)^#{2,4}[ \t]*Phase[ \t]+\d+", brief))
+ for line in brief.split("\n"):
+ m = re.match(r"(?i)^\s*Approved by\s*:(.*?)(?:Date\s*:.*)?$", line)
+ if m:
+ present["approved"] = bool(re.search(r"[A-Za-z0-9]", m.group(1)))
+ break
+
+ # counted from the case list and the modules' own checks, the way the report does, never from the file's own totals
+ eq, _ = src.json(path("EQUIVALENCE.json"), "EQUIVALENCE.json")
+ view = br.equivalence_view(eq)
+ if view:
+ tally = view["tally"]
+ out["eq_cases"] = whole(tally["executed"])
+ out["eq_diff"] = whole(sum(tally.get(k, 0) for k in ("differs", "missing", "inconsistent", "unknown")))
+ out["eq_appr"] = whole(tally.get("differs-approved", 0))
+
+ ver, _ = src.json(path("VERIFICATION.json"), "VERIFICATION.json")
+ proof = br.proof_view(ver)
+ if proof:
+ out["v_proven"], out["v_partly"], out["v_not"] = (whole(proof["counts"].get(k)) for k in ("PROVEN", "PARTLY PROVEN", "NOT PROVEN"))
+ present["signed"] = bool(proof["signoff"]["name"].strip() and proof["signoff"]["decision"].strip())
+
+ security = br.security_facts(text("SECURITY_FINDINGS.md") or "") or {}
+ out["sec_crit"], out["sec_high"] = whole(security.get("Critical")), whole(security.get("High"))
+
+ kind, out["built"] = built_kind_and_count(cwd, system)
+ present["built"] = bool(out["built"])
+ for step, bit in STEPS:
+ done |= bit if present.get(step) else 0
+ out["done"] = done
+ out["goal"] = goal_code(text("INTENT.md"), kind)
+ out["pv"] = plugin_version()
+ return out
+
+
+def os_code():
+ """1 macOS, 2 Linux, 3 Windows, 4 WSL, 0 other. The shell wrapper knows best (uname); this is the fallback."""
+ given = os.environ.get("CODE_MODERNIZATION_OS", "")
+ if given.isdigit():
+ return min(int(given), 9)
+ system = platform.system().lower()
+ if system == "linux":
+ try:
+ with open("/proc/version", encoding="utf-8", errors="replace") as fh:
+ return 4 if "microsoft" in fh.read().lower() else 2
+ except OSError:
+ return 2
+ return {"darwin": 1, "windows": 3}.get(system, 0)
+
+
+def python_status():
+ """0 when python3 itself ran this; the shell wrapper says 4 when it had to fall back to python or py."""
+ given = os.environ.get("CODE_MODERNIZATION_PY", "")
+ return min(int(given), 9) if given.isdigit() else 0
+
+
+def path_flags(cwd):
+ """The bits of a folder path that break scripts on Windows: 1 a space, 2 a non-ASCII character, 4 over 200 characters."""
+ given = os.environ.get("CODE_MODERNIZATION_PATHF", "")
+ if given.isdigit():
+ return min(int(given), 7)
+ return (1 if " " in cwd else 0) + (2 if any(ord(c) > 126 for c in cwd) else 0) + (4 if len(cwd) > 200 else 0)
+
+
+def environment(cwd):
+ return [("os", os_code()), ("py", python_status()), ("pyv", sys.version_info[0] * 100 + sys.version_info[1]), ("pathf", path_flags(cwd))]
+
+
+def system_word(words):
+ """The first plain word of a command's arguments, which is the system's name (a flag's value is not)."""
+ skip = False
+ for w in words:
+ if skip:
+ skip = False
+ elif w.startswith("-"):
+ skip = w == "--source"
+ elif NAME.fullmatch(w):
+ return w
+ return None
+
+
+def command_metrics(prompt, cwd, permission_mode=None):
+ """The counts for one typed prompt, or None when it is not one of the plugin's commands."""
+ m = COMMAND_RE.match(str(prompt or "").lstrip())
+ if not m:
+ return None
+ verb = m.group("verb") or ""
+ if m.group("ns"):
+ cmd = COMMANDS.get(verb, OTHER_COMMAND)
+ elif verb in PANE:
+ cmd = PANE[verb]
+ elif verb in COMMANDS:
+ cmd = COMMANDS[verb]
+ else:
+ return None # another tool's /modernize-something
+ words = str(prompt).lstrip()[m.end():].split()
+ systems = systems_in(cwd)
+ named = system_word(words)
+ if named in {n for _, n in systems}:
+ system = named
+ else: # a name that does not exist yet is a new system; no name at all means the newest one
+ system = None if named else (systems[0][1] if systems else None)
+ snap = snapshot(cwd, system) if system else None
+ perm = PERMISSION_MODES.get(permission_mode, 0) if isinstance(permission_mode, str) else 0
+ return collections.OrderedDict([("pv", plugin_version()), ("cmd", cmd), ("perm", perm), ("has_source", int("--source" in words)),
+ ("fresh", int(snap is None)), ("systems", min(len(systems), 99)),
+ ("goal", snap["goal"] if snap else 0), ("done", snap["done"] if snap else 0)] + environment(cwd))
+
+
+def health_metrics(cwd):
+ """Once per version and machine (the shell wrapper keeps count): what the plugin is running on."""
+ return collections.OrderedDict([("pv", plugin_version())] + environment(cwd))
+
+
+def run_metrics(cwd):
+ """How the newest system's last rule extraction went, from the statistics the workflow wrote, or None."""
+ for _, system in systems_in(cwd)[:5]:
+ src = br.Source(cwd, system)
+ result, _ = src.json(os.path.join(src.adir, "rules_result.json"), "rules_result.json")
+ stats = result.get("stats") if isinstance(result, dict) and isinstance(result.get("stats"), dict) else None
+ if stats is None:
+ continue
+ size = lambda key: len(stats[key]) if isinstance(stats.get(key), list) else 0 # noqa: E731
+ return collections.OrderedDict([("pv", plugin_version()), ("agents", whole(stats.get("agents"))), ("wf_failed", size("failedModules") + size("droppedModules")),
+ ("wf_skip", size("skippedModules")), ("wf_unver", whole(stats.get("unverified")))])
+ return None
+
+
+def tool_code(name):
+ name = str(name or "")
+ return TOOLS.get(name) or (10 if name.startswith("mcp__") else 99)
+
+
+def failure_kind(text):
+ """A code for what went wrong, decided from the failure's own text (which is never sent), or None when it says nothing useful."""
+ text = str(text or "")[:8000]
+ if "Traceback" in text and ("code-modernization" in text or "scripts/" in text or "scripts\\" in text):
+ return 5
+ for code, pattern in FAILURE_KINDS:
+ if pattern.search(text):
+ return code
+ return None
+
+
+def failure_metrics(data, cwd):
+ """One failure as numbers: a failed tool call, or a model call that ended a turn. None when there is nothing to say."""
+ event, session = str(data.get("hook_event_name") or ""), str(data.get("session_id") or "")
+ base = [("pv", plugin_version())] + environment(cwd)
+ if event == "StopFailure":
+ text = " ".join(str(data.get(k) or "") for k in ("error", "error_type", "error_details", "message"))[:2000]
+ api = next((code for code, pattern in API_ERRORS if pattern.search(text)), 99)
+ tag, extra = "api%d" % api, [("api", api)]
+ else:
+ if data.get("is_interrupt") is True:
+ return None
+ tool, error = tool_code(data.get("tool_name")), str(data.get("error") or "")
+ kind = failure_kind(error)
+ if kind is None:
+ if tool == 1 and re.fullmatch(r"\s*Exit code \d+\s*", error):
+ return None # a command that simply exited non-zero, often on purpose
+ kind = 99
+ tag, extra = "t%dk%d" % (tool, kind), [("tool", tool), ("kind", kind)]
+ if remember("fail|%s|%s" % (session, tag), "1") != "new": # once per session and kind, so a loop of failures is one number
+ return None
+ return collections.OrderedDict(base + extra)
+
+
+def state_metrics(cwd):
+ """The counts for the newest system under cwd that the plugin has worked on, or None."""
+ for _, system in systems_in(cwd)[:5]:
+ snap = snapshot(cwd, system)
+ if snap is not None:
+ return snap
+ return None
+
+
+def rounded(value):
+ """Two significant figures from 100 up (17727 becomes 18000), so a count says roughly how big, not exactly which system."""
+ if value < 100:
+ return value
+ scale = 10 ** (len(str(value)) - 2)
+ return (value + scale // 2) // scale * scale
+
+
+def ok(metrics):
+ """Only keys this script defines, only whole numbers within their limit, counts rounded, at most MAX_KEYS of them."""
+ out = collections.OrderedDict()
+ for k, v in list(metrics.items())[:MAX_KEYS]:
+ if k in MEANING:
+ v = min(whole(v), LIMITS.get(k, COUNT_LIMIT))
+ out[k] = v if k in EXACT else rounded(v)
+ return out
+
+
+def data_dir():
+ """The plugin's own data folder, or a private folder in the temp directory; None when neither can be used safely."""
+ uid = getattr(os, "getuid", lambda: 0)()
+ base = os.environ.get("CLAUDE_PLUGIN_DATA") or ""
+ base = os.path.realpath(base) if os.path.isdir(base) else os.path.join(tempfile.gettempdir(), "code-modernization-%s" % uid)
+ try:
+ os.makedirs(base, mode=0o700, exist_ok=True)
+ st = os.lstat(base)
+ except OSError:
+ return None
+ if os.path.islink(base) or not stat.S_ISDIR(st.st_mode) or (hasattr(st, "st_uid") and st.st_uid != uid):
+ return None
+ return base
+
+
+def note(text):
+ """With CODE_MODERNIZATION_TELEMETRY_DEBUG=1, one line saying what a hook run decided, in telemetry-debug.log in the data
+ folder (a link is never followed, and the file is cut down when it grows past 200 KB). Never raises."""
+ if str(os.environ.get("CODE_MODERNIZATION_TELEMETRY_DEBUG", "")).strip().lower() not in TRUE:
+ return
+ base = data_dir()
+ if not base:
+ return
+ file = os.path.join(base, "telemetry-debug.log")
+ try:
+ if os.path.getsize(file) > 200000:
+ os.remove(file)
+ except OSError:
+ pass
+ try:
+ fd = os.open(file, os.O_WRONLY | os.O_APPEND | os.O_CREAT | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0), 0o600)
+ with os.fdopen(fd, "a", encoding="utf-8") as fh:
+ fh.write("%s %s\n" % (time.strftime("%H:%M:%S"), text))
+ except OSError:
+ pass
+
+
+def remember(key, digest):
+ """"new" when this workspace's counts changed since they were last sent and the new state is stored, "same" when
+ they did not, "cannot" when the state cannot be stored: nothing is sent then, it would repeat at the end of every turn."""
+ base = data_dir()
+ if not base:
+ return "cannot"
+ try:
+ file = os.path.join(base, "telemetry-state.json")
+ try:
+ with open(file, encoding="utf-8") as fh:
+ seen = json.load(fh)
+ except (OSError, ValueError):
+ seen = {}
+ seen = seen if isinstance(seen, dict) else {}
+ if seen.get(key) == digest:
+ return "same"
+ seen.pop(key, None)
+ seen[key] = digest
+ seen = dict(list(seen.items())[-200:])
+ tmp = "%s.%d" % (file, os.getpid())
+ with open(tmp, "w", encoding="utf-8") as fh:
+ json.dump(seen, fh)
+ os.replace(tmp, file)
+ return "new"
+ except OSError:
+ return "cannot"
+
+
+def digest_of(metrics):
+ return hashlib.sha1(json.dumps(metrics, sort_keys=True).encode()).hexdigest()[:12]
+
+
+def hook(mode):
+ """Run one hook. Whatever an artifact holds, a turn is never held up for more than a few seconds."""
+ if hasattr(signal, "SIGALRM"):
+ signal.signal(signal.SIGALRM, lambda *_: os._exit(0))
+ signal.alarm(3)
+ try:
+ run_hook(mode)
+ finally:
+ signal.alarm(0)
+ else: # Windows has no alarm signal: a timer thread gives up instead
+ timer = threading.Timer(3, os._exit, [0])
+ timer.daemon = True
+ timer.start()
+ try:
+ run_hook(mode)
+ finally:
+ timer.cancel()
+ timer.join(1)
+
+
+def run_hook(mode):
+ reason = off_reason()
+ if reason:
+ return note("%s: not sent, %s" % (mode, reason))
+ try:
+ data = json.loads(sys.stdin.read(1 << 20) or "{}")
+ except ValueError:
+ data = {}
+ data = data if isinstance(data, dict) else {}
+ cwd = data.get("cwd") if isinstance(data.get("cwd"), str) and os.path.isdir(data.get("cwd")) else os.getcwd()
+ why = ""
+ if mode == "command":
+ metrics = command_metrics(data.get("prompt"), cwd, data.get("permission_mode"))
+ why = "not one of the plugin's commands"
+ elif mode == "health":
+ metrics, why = health_metrics(cwd), ""
+ elif mode == "failure":
+ metrics = failure_metrics(data, cwd)
+ why = "nothing worth counting, or the same kind was already sent this session"
+ else:
+ metrics = run_metrics(cwd) if mode == "run" else state_metrics(cwd)
+ why = "the plugin has left no files here"
+ if metrics is not None:
+ key = hashlib.sha1((os.path.realpath(cwd) + mode).encode()).hexdigest()[:16]
+ status = remember(key, digest_of(ok(metrics)))
+ if status != "new":
+ metrics, why = None, "the counts are the same as last time" if status == "same" else "the state file cannot be written"
+ if metrics:
+ print(json.dumps({"metrics": ok(metrics)}, separators=(",", ":")), flush=True)
+ note("%s: sent %d values" % (mode, len(ok(metrics))))
+ else:
+ note("%s: not sent, %s" % (mode, why))
+
+
+def show(args):
+ """What the hooks would send for a workspace, in words."""
+ cwd, prompt, as_json, i = None, None, False, 0
+ while i < len(args):
+ if args[i] == "--json":
+ as_json = True
+ elif args[i] == "--prompt":
+ prompt, i = args[i + 1] if i + 1 < len(args) else "", i + 1
+ elif not args[i].startswith("--") and cwd is None:
+ cwd = os.path.abspath(args[i])
+ i += 1
+ cwd = cwd or os.getcwd()
+ example = prompt is None
+ prompt = "/code-modernization:modernize-status" if example else prompt
+ state, command = state_metrics(cwd), command_metrics(prompt, cwd)
+ if as_json:
+ print(json.dumps({"off": off_reason(), "state": ok(state) if state else None, "command": ok(command) if command else None}))
+ return
+
+ def table(title, metrics, empty):
+ print(title)
+ if metrics is None:
+ print(" " + empty + "\n")
+ return
+ for k, v in ok(metrics).items():
+ note = MEANING[k].split(":")[0] if k in ("cmd", "lang", "done") else MEANING[k]
+ if k == "done":
+ note += ": " + (", ".join(n for n, bit in STEPS if v & bit) or "nothing yet")
+ print(" %-10s %-8d %s" % (k, v, note))
+ print()
+
+ reason = off_reason()
+ print("Usage counts for %s" % cwd)
+ print("Sending is OFF: %s.\n" % reason if reason else "Sending is on. Turn it off with CODE_MODERNIZATION_TELEMETRY=0 or the plugin's Usage counts option.\n")
+ table("When a turn ends and these counts have changed:", state, "nothing: the plugin has left no artifacts here yet")
+ table("When you type %s%s:" % ("a command, for example " if example else "", prompt.split()[0] if prompt.split() else "nothing"), command,
+ "nothing: that is not one of the plugin's commands")
+
+
+def report_error(err, line):
+ """The plugin's own script raised: send which kind and where, once, never the message."""
+ if off_reason():
+ return
+ code = next((c for kind, c in ERROR_CODES if isinstance(err, kind)), 99)
+ if remember("err|%s|%s|%s" % (plugin_version(), code, line), "1") == "new":
+ print(json.dumps({"metrics": ok(collections.OrderedDict([("pv", plugin_version()), ("os", os_code()), ("py", python_status()),
+ ("pyv", sys.version_info[0] * 100 + sys.version_info[1]), ("err", code), ("err_at", line)]))},
+ separators=(",", ":")), flush=True)
+
+
+def main(argv):
+ mode = argv[1] if len(argv) > 1 else ""
+ try:
+ if mode == "show":
+ show(argv[2:])
+ elif mode in ("command", "state", "run", "failure", "health"):
+ hook(mode)
+ else:
+ print(__doc__)
+ except Exception as err: # a usage count must never break the person's session
+ if mode == "show":
+ raise
+ tb = err.__traceback__
+ while tb and tb.tb_next:
+ tb = tb.tb_next
+ line = tb.tb_lineno if tb else 0
+ note("%s: error %s at line %s" % (mode, type(err).__name__, line))
+ try:
+ report_error(err, line)
+ except Exception: # reporting a problem must not become one
+ pass
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv))
diff --git a/plugins/code-modernization/scripts/telemetry.sh b/plugins/code-modernization/scripts/telemetry.sh
new file mode 100644
index 00000000..dec61413
--- /dev/null
+++ b/plugins/code-modernization/scripts/telemetry.sh
@@ -0,0 +1,129 @@
+#!/bin/sh
+# The plugin's usage-count hooks: telemetry.sh command|state|run|failure|health, with the hook's JSON on stdin.
+# It starts Python only when the plugin is in use here: a plugin command was typed, or the folder holds files the plugin
+# wrote under analysis/, or (for a failure) the failing command names the plugin. The one exception is `health`, once per
+# plugin version and machine at session start: what the plugin runs on, so that a machine where nobody gets as far as typing
+# a command is still counted once. A Mac without developer tools never sees the "install command line tools" dialog that a
+# bare python3 would raise.
+# When Python itself cannot run (missing, the Windows Store stub, too old, crashing) this script says so in numbers,
+# because Python cannot report its own absence. What is counted, and how to turn it off: see the Telemetry section of
+# the README and scripts/telemetry.py.
+mode="$1"
+input=$(cat)
+here=$(dirname "$0")
+
+in_use() {
+ for f in analysis/*/INTENT.md analysis/*/PREFLIGHT.md analysis/*/MODERNIZATION_BRIEF.md analysis/*/BUSINESS_RULES.md \
+ analysis/*/DELTA_CATALOG.md analysis/*/VERIFICATION.json analysis/*/RULE_REVIEWS.json analysis/*/AI_NATIVE_SPEC.md; do
+ [ -f "$f" ] && return 0
+ done
+ return 1
+}
+
+case "$mode" in
+ command) printf '%s' "$input" | grep -Eq '"prompt"[[:space:]]*:[[:space:]]*"[[:space:]]*/(code-modernization:)?modernize' || exit 0 ;;
+ state|run) in_use || exit 0 ;;
+ failure) in_use || printf '%s' "$input" | grep -q 'code-modernization' || exit 0 ;;
+ health) ;;
+ *) exit 0 ;;
+esac
+
+# The off switches count here as well, so nothing at all is sent, not even a python problem.
+for v in "$CODE_MODERNIZATION_TELEMETRY" "$CLAUDE_PLUGIN_OPTION_TELEMETRY"; do
+ case "$v" in ''|1|true|TRUE|True|on|ON|On|yes|YES|Yes) ;; *) exit 0 ;; esac
+done
+case "$DISABLE_TELEMETRY$CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC" in *1*|*true*|*TRUE*|*yes*|*on*) exit 0 ;; esac
+
+# Which system this is: 1 macOS, 2 Linux, 3 Windows (Git Bash, MSYS, Cygwin), 4 WSL, 0 other.
+case "$(uname -s 2>/dev/null)" in
+ Darwin) os=1 ;;
+ Linux) if grep -qi microsoft /proc/version 2>/dev/null; then os=4; else os=2; fi ;;
+ MINGW*|MSYS*|CYGWIN*) os=3 ;;
+ *) os=0 ;;
+esac
+
+# Things about the folder that break scripts on Windows: 1 a space in the path, 2 a non-ASCII character, 4 a long path.
+pathf=0
+case "$PWD" in *" "*) pathf=$((pathf + 1)) ;; esac
+printf '%s' "$PWD" | LC_ALL=C grep -q '[^ -~]' && pathf=$((pathf + 2))
+[ "${#PWD}" -gt 200 ] && pathf=$((pathf + 4))
+
+# The plugin's version as major*10000 + minor*100 + patch, read without Python.
+set -- $(sed -n 's/.*"version"[[:space:]]*:[[:space:]]*"\([0-9]*\)\.\([0-9]*\)\.\([0-9]*\)".*/\1 \2 \3/p' "$here/../.claude-plugin/plugin.json" 2>/dev/null | head -1)
+pv=$(( ${1:-0} * 10000 + ${2:-0} * 100 + ${3:-0} ))
+
+# health is sent once per plugin version and machine: a marker file in the plugin's data folder says it was.
+marker=""
+if [ "$mode" = health ]; then
+ d=${CLAUDE_PLUGIN_DATA:-}
+ [ -d "$d" ] || d="${TMPDIR:-/tmp}/code-modernization-$(id -u 2>/dev/null || echo 0)"
+ mkdir -p -m 700 "$d" 2>/dev/null
+ [ -d "$d" ] && [ ! -L "$d" ] && [ -O "$d" ] || exit 0
+ marker="$d/health-sent-$pv"
+ [ -e "$marker" ] && exit 0
+fi
+mark() { [ -n "$marker" ] && : > "$marker"; }
+
+# One line of numbers, for the cases where Python cannot speak for itself.
+emit() {
+ printf '{"metrics":{"pv":%s' "$pv"
+ for kv in "$@"; do printf ',"%s":%s' "${kv%%:*}" "${kv#*:}"; done
+ printf '}}\n'
+}
+
+# Run the Python script with one interpreter; sets out and rc. $2 is the python status the script should report.
+run_python() {
+ out=$(printf '%s' "$input" | CODE_MODERNIZATION_OS="$os" CODE_MODERNIZATION_PY="$2" CODE_MODERNIZATION_PATHF="$pathf" \
+ PYTHONDONTWRITEBYTECODE=1 $1 "$here/telemetry.py" "$mode" 2>/dev/null)
+ rc=$?
+}
+
+# python3 first, as everywhere else in the plugin. It is never run on a Mac without developer tools (that raises a dialog).
+first=1 # 1 none found, 2 macOS stub, 3 present but broken (the Windows Store stub), 5 too old, 6 the script crashed
+p3=$(command -v python3 2>/dev/null)
+if [ -n "$p3" ]; then
+ if [ "$os" = 1 ] && [ "$p3" = /usr/bin/python3 ] && ! xcode-select -p >/dev/null 2>&1; then
+ first=2
+ else
+ run_python python3 0
+ if [ "$rc" = 0 ]; then
+ [ -n "$out" ] && { mark; printf '%s\n' "$out"; }
+ exit 0
+ fi
+ ver=$(python3 -c 'import sys; print(sys.version_info[0] * 100 + sys.version_info[1])' 2>/dev/null)
+ case "$ver" in ''|*[!0-9]*) first=3 ;; *) if [ "$ver" -lt 308 ]; then first=5; else first=6; fi ;; esac
+ fi
+fi
+
+# Windows in particular: python3 may be missing or a stub while python or the py launcher works.
+for c in python "py -3"; do
+ command -v "${c%% *}" >/dev/null 2>&1 || continue
+ run_python "$c" 4
+ if [ "$rc" = 0 ]; then
+ [ -n "$out" ] && { mark; printf '%s\n' "$out"; }
+ exit 0
+ fi
+done
+
+# No interpreter could run the script. Say so in numbers, for a command the person typed or an interpreter failure.
+case "$mode" in
+ command)
+ verb=$(printf '%s' "$input" | grep -Eo '"prompt"[[:space:]]*:[[:space:]]*"[[:space:]]*/(code-modernization:)?modernize[a-z-]*' | head -1)
+ ns=""; case "$verb" in *code-modernization:*) ns=1 ;; esac
+ verb=${verb##*modernize}
+ case "$verb" in
+ ""|-) c=1 ;; -preflight) c=2 ;; -assess) c=3 ;; -map) c=4 ;; -extract-rules) c=5 ;; -review) c=6 ;; -brief) c=7 ;;
+ -transform) c=8 ;; -uplift) c=9 ;; -reimagine) c=10 ;; -verify) c=11 ;; -harden) c=12 ;; -status) c=13 ;;
+ -panel) c=20 ;; -review-pane) c=21 ;; -sign) c=22 ;;
+ *) c=99; [ -n "$ns" ] || exit 0 ;;
+ esac
+ emit "cmd:$c" "os:$os" "py:$first" "pyv:0" "pathf:$pathf" ;;
+ health)
+ mark
+ emit "os:$os" "py:$first" "pyv:0" "pathf:$pathf" ;;
+ failure)
+ if printf '%s' "$input" | grep -Eqi "(python3?|py)(\.exe)?: (command )?not found|'(python3?|py)' is not recognized|Python was not found|python3?: No such file"; then
+ emit "os:$os" "py:$first" "tool:1" "kind:1"
+ fi ;;
+esac
+exit 0
diff --git a/plugins/code-modernization/scripts/tests/test_compare_tolerance.py b/plugins/code-modernization/scripts/tests/test_compare_tolerance.py
new file mode 100644
index 00000000..1bc1300a
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_compare_tolerance.py
@@ -0,0 +1,151 @@
+"""Tests for the numeric tolerance in scripts/compare.py.
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+"""
+import json
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import compare as cmp # noqa: E402
+
+TOL = {"rel": 1e-9, "abs": 0, "why": "last-digit rounding of exp() between libm versions"}
+
+
+class ToleranceTests(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+
+ def case(self, name, legacy, new, **extra):
+ for side, data in (("legacy", legacy), ("new", new)):
+ os.makedirs(os.path.join(self.tmp, side), exist_ok=True)
+ with open(os.path.join(self.tmp, side, name), "wb") as fh:
+ fh.write(data)
+ return {"id": name, "legacy": "legacy/" + name, "new": "new/" + name, **extra}
+
+ def run_cases(self, cases, **top):
+ path = os.path.join(self.tmp, "cases.json")
+ with open(path, "w") as fh:
+ json.dump({"system": "s", "cases": cases, **top}, fh)
+ return cmp.run(path, os.path.join(self.tmp, "EQUIVALENCE.json"))
+
+ def test_a_tiny_float_difference_is_same_within_the_declared_tolerance_and_says_so(self):
+ res = self.run_cases([self.case("a", b"total 1234.5678901234\nrate 0.05\n", b"total 1234.5678901239\nrate 0.05\n", tolerance=TOL)])
+ rec = res["cases"][0]
+ self.assertEqual(rec["verdict"], "same")
+ self.assertEqual(rec["withinTolerance"]["differing"], 1)
+ self.assertLess(rec["withinTolerance"]["maxRelativeDifference"], 1e-9)
+ self.assertIn("declared tolerance", rec["reason"])
+ self.assertIn("libm", rec["reason"])
+ self.assertEqual(res["totals"]["sameWithinTolerance"], 1)
+ self.assertIsNone(cmp.verdict_of(res))
+ self.assertTrue(res["selfCheck"]["passed"], res["selfCheck"])
+
+ def test_without_a_tolerance_the_same_outputs_differ_and_no_new_key_appears(self):
+ res = self.run_cases([self.case("a", b"1234.5678901234\n", b"1234.5678901239\n")])
+ self.assertEqual(res["cases"][0]["verdict"], "differs")
+ self.assertNotIn("sameWithinTolerance", res["totals"])
+
+ def test_a_difference_outside_the_tolerance_differs(self):
+ res = self.run_cases([self.case("a", b"total 100.0\n", b"total 100.5\n", tolerance=TOL)])
+ rec = res["cases"][0]
+ self.assertEqual(rec["verdict"], "differs")
+ self.assertIn("beyond the declared tolerance", rec["reason"])
+ self.assertIn("NOT PROVEN" if cmp.verdict_of(res) else "", "NOT PROVEN")
+
+ def test_integers_and_text_must_match_exactly_even_with_a_tolerance(self):
+ loose = {"rel": 0.01, "abs": 0, "why": "test"}
+ for legacy, new in ((b"count 1000000\n", b"count 1000001\n"), (b"name alpha 1.5\n", b"name alphb 1.5\n"),
+ (b"1.5 2.5\n", b"1.5\n"), (b"x 10\n", b"x 10.0\n")):
+ res = self.run_cases([self.case("a", legacy, new, tolerance=loose)])
+ self.assertEqual(res["cases"][0]["verdict"], "differs", (legacy, new))
+
+ def test_exponent_forms_compare_by_value(self):
+ res = self.run_cases([self.case("a", b"v 1.5e3\n", b"v 1500.0000000000002\n", tolerance=TOL)])
+ self.assertEqual(res["cases"][0]["verdict"], "same")
+
+ def test_a_tolerance_needs_a_reason_and_a_sane_size(self):
+ legacy = self.case("a", b"1.0\n", b"1.0\n")
+ for bad in ({"rel": 1e-9}, {"rel": 1e-9, "why": " "}, {"rel": 0.5, "why": "x"}, {"why": "x"}, {"rel": -1, "why": "x"},
+ {"rel": "big", "why": "x"}, {"rel": float("nan"), "why": "x"}, "1e-9"):
+ with self.assertRaises(cmp.InputError, msg=repr(bad)):
+ self.run_cases([dict(legacy, tolerance=bad)])
+
+ def test_a_default_tolerance_applies_to_every_case_and_a_case_can_switch_it_off(self):
+ a = self.case("a", b"1.00000000000\n", b"1.00000000001\n")
+ b = self.case("b", b"1.00000000000\n", b"1.00000000001\n", tolerance=None)
+ res = self.run_cases([a, b], tolerance=TOL)
+ self.assertEqual([r["verdict"] for r in res["cases"]], ["same", "differs"])
+
+ def test_an_absolute_tolerance_is_capped_like_a_relative_one(self):
+ for bad in ({"abs": 1e-3, "why": "x"}, {"abs": 100, "why": "x"}):
+ with self.assertRaises(cmp.InputError, msg=repr(bad)):
+ self.run_cases([self.case("a", b"5.5\n", b"5.5\n", tolerance=bad)])
+ ok = self.run_cases([self.case("a", b"x 0.0000000000000001\n", b"x 0.0\n", tolerance={"abs": 1e-9, "why": "a value that is zero to rounding"})])
+ self.assertEqual(ok["cases"][0]["verdict"], "same")
+
+ def test_the_self_check_still_fails_if_a_loose_tolerance_ever_got_through(self):
+ loose = {"abs": 100, "why": "absurdly loose"}
+ with mock.patch.object(cmp, "MAX_ABS_TOLERANCE", 1000.0):
+ res = self.run_cases([self.case("a", b"5.5\n", b"5.5\n", tolerance=loose)])
+ self.assertFalse(res["selfCheck"]["passed"])
+ self.assertIn("tolerance may be too loose", res["selfCheck"]["detail"])
+ self.assertIn("self-check failed", cmp.verdict_of(res))
+
+ def test_dotted_identifiers_are_text_and_must_match_exactly(self):
+ for legacy, new in ((b"version 1.2.3\n", b"version 1.2.4\n"), (b"host 10.0.0.1\n", b"host 10.0.0.2\n"),
+ (b"build 2026.09.24 ok\n", b"build 2026.09.25 ok\n")):
+ res = self.run_cases([self.case("a", legacy, new, tolerance={"rel": 0.01, "why": "test"})])
+ self.assertEqual(res["cases"][0]["verdict"], "differs", (legacy, new))
+ same = self.run_cases([self.case("a", b"version 1.2.3 total 1.0000000001\n", b"version 1.2.3 total 1.0000000002\n", tolerance=TOL)])
+ self.assertEqual(same["cases"][0]["verdict"], "same")
+
+ def test_the_comparison_is_exact_decimal_arithmetic_not_doubles(self):
+ # 1e-22 apart: equal as doubles, but far outside a declared 1e-30 relative tolerance
+ tight = {"rel": 1e-30, "why": "essentially exact"}
+ res = self.run_cases([self.case("a", b"1.0000000000000000000001\n", b"1.0000000000000000000002\n", tolerance=tight)])
+ self.assertEqual(res["cases"][0]["verdict"], "differs")
+ wide = {"rel": 1e-21, "why": "test"}
+ res = self.run_cases([self.case("a", b"1.0000000000000000000001\n", b"1.0000000000000000000002\n", tolerance=wide)])
+ self.assertEqual(res["cases"][0]["verdict"], "same")
+
+ def test_absurd_numbers_do_not_crash_or_pass(self):
+ for legacy, new in ((b"v 1e999999999999\n", b"v 1e999999999998\n"), (b"v 1." + b"1" * 200 + b"\n", b"v 1." + b"1" * 199 + b"2\n")):
+ res = self.run_cases([self.case("a", legacy, new, tolerance=TOL)])
+ self.assertEqual(res["cases"][0]["verdict"], "differs")
+
+ def test_an_input_can_be_approved_once_for_every_case_that_shares_its_label(self):
+ cases = [self.case("F23-a", b"one\n", b"two\n", input="F23"), self.case("F23-b", b"x\n", b"y\n", input="F23"),
+ self.case("F24-a", b"p\n", b"q\n", input="F24"), self.case("F25-a", b"same\n", b"same\n", input="F25")]
+ res = self.run_cases(cases, approvedInputs={"F23": "system owner accepts deviation 4", "F25": " ", "F99": "no such input"})
+ verdicts = {r["id"]: r["verdict"] for r in res["cases"]}
+ self.assertEqual(verdicts, {"F23-a": "differs-approved", "F23-b": "differs-approved", "F24-a": "differs", "F25-a": "same"})
+ self.assertEqual(res["totals"]["differsApproved"], 2)
+ self.assertIn("system owner accepts deviation 4", res["cases"][0]["reason"])
+ self.assertEqual(res["cases"][2]["approvedDifference"], "") # a blank reason approves nothing
+ # a case's own reason wins, and a per-input approval never turns a missing file into a pass
+ own = self.run_cases([self.case("G1", b"a\n", b"b\n", input="G", approvedDifference="own reason"), dict(self.case("G2", b"a\n", b"a\n", input="G"), new="new/nothing")],
+ approvedInputs={"G": "input reason"})
+ self.assertEqual(own["cases"][0]["approvedDifference"], "own reason")
+ self.assertEqual(own["cases"][1]["verdict"], "missing")
+
+ def test_masks_and_tolerance_work_together(self):
+ res = self.run_cases([self.case("a", b"at 2026-09-24 10:00:00 value 2.0000000001\n", b"at 2027-01-01 11:11:11 value 2.0000000002\n",
+ mask=[{"regex": r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d", "why": "run time"}], tolerance=TOL)])
+ self.assertEqual(res["cases"][0]["verdict"], "same")
+
+ def test_large_outputs_are_not_tolerance_compared(self):
+ big = b"1.0 " * (cmp.MAX_TOLERANT_BYTES // 4 + 10)
+ rec = self.run_cases([self.case("a", big, big[:-3] + b"2.0", tolerance=TOL)])["cases"][0]
+ self.assertEqual(rec["verdict"], "differs")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_proof_rules_run.py b/plugins/code-modernization/scripts/tests/test_proof_rules_run.py
new file mode 100644
index 00000000..0eb3560c
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_proof_rules_run.py
@@ -0,0 +1,506 @@
+"""Tests for the "Rules traced" check: a rule is tested only when something that ran and passed backs it, and test-only folders are not judged.
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+A test that names a rule is not a test that ran: skipped and pending tests, and files whose class never ran, leave the rule "named, not run".
+"""
+import json
+import os
+import re
+import shutil
+import sys
+import tempfile
+import time
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, HERE)
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import build_report as br # noqa: E402
+import proof_pack as pp # noqa: E402
+import trace_rules as tr # noqa: E402
+from test_proof_tools import RULES_MD, T0, Ws, junit, load, passing, put, read, run_main # noqa: E402
+
+SKIPPED_TEST = '''package p;
+
+class CalcTest {
+ @Test
+ @Disabled("PENDING RULE-001")
+ void RULE_001_totals() {}
+
+ @Test
+ @Disabled("PENDING RULE-002")
+ void RULE_002_rounds() {}
+}
+'''
+NOT_RUN = "named, not run"
+
+
+class RuleWs(Ws):
+ """The workspace of the proof tests (module mod, P0 rules RULE-001 and RULE-002), with helpers to change what the tests are and what ran."""
+
+ def module(self, module="mod"):
+ return super().module(module)
+
+ def p0(self):
+ return {r["id"]: r for r in self.module()["evidence"]["rules"]["p0"]}
+
+ def statuses(self):
+ return {r["id"]: r["status"] for r in self.module()["evidence"]["rules"]["p0"]}
+
+ def check_rules(self):
+ return next(c for c in self.module()["checks"] if c["id"] == "rules")
+
+ def test_file(self, rel, text):
+ put(self.root, "modernized/s/mod/src/test/" + rel, text, T0 + 20)
+
+ def keep_only(self, *cases):
+ self.results(list(cases))
+
+
+class RulesNeedExecutedTests(unittest.TestCase):
+ def setUp(self):
+ self.ws = RuleWs(self)
+
+ def test_the_default_workspace_is_tested_by_files_whose_class_ran(self):
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+ self.assertEqual(self.ws.check_rules()["state"], "pass")
+ self.assertIn("backed by a test that ran and passed", self.ws.check_rules()["detail"])
+ self.assertEqual(self.ws.module()["verdict"], "PROVEN")
+
+ # (i) the hole that was found: @Disabled("PENDING RULE-001") and a result file that shows it skipped
+ def test_a_disabled_pending_test_that_the_results_show_skipped_is_named_not_run(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("CalcTest", "RULE_001_totals", "SKIP", "PENDING RULE-001"), ("CalcTest", "RULE_002_rounds", "SKIP", "PENDING RULE-002"), ("OtherTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ c = self.ws.check_rules()
+ self.assertEqual(c["state"], "gap")
+ self.assertIn("P0 rules named only by tests that did not run: RULE-001, RULE-002 (skipped, pending, failing or missing from the results).", c["detail"])
+ m = self.ws.module()
+ self.assertEqual(m["verdict"], "PARTLY PROVEN")
+ self.assertEqual(m["evidence"]["rules"]["namedNotRun"], ["RULE-001", "RULE-002"])
+ self.assertEqual(m["evidence"]["rules"]["p0NoTests"], ["RULE-001", "RULE-002"])
+ self.assertTrue(m["evidence"]["rules"]["perTestResults"])
+ self.assertTrue(any(r.startswith("Rules traced: P0 rules named only by tests that did not run") for r in m["reasons"]))
+ self.assertEqual(m["evidence"]["tests"]["skipped"], 2)
+
+ def test_the_same_test_without_a_skip_marker_but_skipped_at_run_time_is_named_not_run_too(self):
+ self.ws.test_file("CalcTest.java", "class CalcTest {\n // RULE-001 and RULE-002 need a database\n void a() {}\n}\n")
+ self.ws.keep_only(("CalcTest", "a", "SKIP", "no database here"), ("OtherTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertEqual({r["by"] for r in self.ws.p0().values()}, {""})
+
+ def test_a_test_that_ran_and_failed_backs_nothing(self):
+ self.ws.test_file("CalcTest.java", "class CalcTest {\n // RULE-001 RULE-002\n void a() {}\n}\n")
+ self.ws.keep_only(("CalcTest", "RULE_001_totals", "FAIL", ""), ("CalcTest", "RULE_002_rounds", "ERROR", ""), ("OtherTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertEqual(self.ws.module()["verdict"], "NOT PROVEN") # the failures fail the tests check as they always did
+ self.assertEqual(self.ws.check_rules()["state"], "gap")
+
+ # (ii) an executed, passing testcase that carries the id in its name
+ def test_a_passing_test_whose_name_carries_the_id_backs_the_rule_in_any_spelling(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST) # the only file that names the rules is all skipped
+ for name in ("RULE-001 totals", "RULE_001_totals", "rule001", "testRule001Totals", "Rule001", "totals[RULE-001]"):
+ self.ws.keep_only(("ChecksTest", name, "PASS", ""), ("ChecksTest", "test_RULE_002_rounds", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"}, name)
+ p0 = self.ws.p0()
+ self.assertEqual((p0["RULE-001"]["by"], p0["RULE-002"]["by"]), ("test name", "test name"))
+ self.assertEqual(self.ws.check_rules()["state"], "pass")
+ self.assertEqual(self.ws.module()["verdict"], "PROVEN")
+
+ def test_the_class_name_may_carry_the_id_and_the_shorthand_names_both_rules(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("com.x.Rule001Test", "a", "PASS", ""), ("com.x.OtherTest", "RULE-002/003", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_an_id_in_the_name_of_a_test_that_did_not_pass_or_in_another_modules_results_backs_nothing(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("ChecksTest", "RULE_001_a", "SKIP", "x"), ("ChecksTest", "RULE_002_b", "FAIL", ""), ("ChecksTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ put(self.ws.root, "modernized/s/extra/src/main/X.java", "class X {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/extra/target/surefire-reports/TEST-X.xml", junit([("XTest", "RULE_001_a", "PASS", ""), ("XTest", "RULE_002_a", "PASS", "")]), T0 + 30)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "junit": ["modernized/s/mod/target/surefire-reports"]},
+ {"module": "extra", "name": "unit", "junit": ["modernized/s/extra/target/surefire-reports"]}])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN}) # what extra ran does not count for mod
+
+ # (iii) a plain comment in a test file whose class ran
+ def test_a_plain_comment_in_a_test_file_whose_class_ran_backs_the_rule(self):
+ self.ws.test_file("CalcTest.java", "class CalcTest {\n // RULE-001 is pinned by these\n void a() {}\n void rule002_rounds() {}\n}\n")
+ self.ws.keep_only(("com.x.CalcTest", "a", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+ p0 = self.ws.p0()
+ self.assertEqual(p0["RULE-001"]["by"], "test file")
+ self.assertTrue(p0["RULE-001"]["where"].endswith("src/test/CalcTest.java:2"), p0["RULE-001"]["where"])
+ self.assertEqual(self.ws.module()["verdict"], "PROVEN")
+
+ def test_the_where_points_at_the_mention_that_backs_the_rule_not_at_a_skipped_one(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST) # sorts before Live.java and names both rules, skipped
+ self.ws.test_file("Live.java", "// RULE-001 RULE-002 are checked by these\nclass Live {}\n")
+ self.ws.keep_only(("Live", "a", "PASS", ""), ("CalcTest", "RULE_001_totals", "SKIP", "x"))
+ p0 = self.ws.p0()
+ self.assertTrue(p0["RULE-001"]["where"].endswith("src/test/Live.java:1"), p0["RULE-001"]["where"])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_the_class_name_may_be_nested_dotted_a_python_module_or_a_path(self):
+ same = (("CalcTest.java", "com.x.CalcTest$Inner"), ("CalcTest.java", "com.x.Outer$Deep$CalcTest"), ("CalcTest.java", "com.x.CalcTest$A$B"), ("test_calc.py", "tests.test_calc.TestCalc"),
+ ("test_calc.py", "tests.test_calc"), ("CalcTests.cs", "Ns.CalcTests"), ("calc.spec.ts", "src/calc.spec.ts"), ("CalcTest.java", "calctest"), ("CalcTest.java", "CALCTEST"))
+ other = (("CalcTest.java", "com.x.OtherCalcTest"), ("CalcTest.java", "com.calc.test.Other"), ("test_calc.py", "tests.test_other.TestCalc"), ("CalcTest.java", "Calc"), ("CalcTest.java", ""))
+ for rel, cls in same + other:
+ self.ws.test_file(rel, "// RULE-001 RULE-002\n")
+ self.ws.keep_only((cls, "a", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"} if (rel, cls) in same else {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN}, (rel, cls))
+ os.remove(os.path.join(self.ws.root, "modernized", "s", "mod", "src", "test", rel))
+ self.ws.test_file("CalcTest.java", "// nothing here\n")
+
+ # (iv) a file none of whose tests ran
+ def test_a_rule_named_only_in_a_file_none_of_whose_tests_ran_is_named_not_run(self):
+ self.ws.test_file("CalcTest.java", "// RULE-001\nclass CalcTest {}\n")
+ self.ws.test_file("GhostTest.java", "// RULE-002 is pinned here, and nothing marks it skipped\nclass GhostTest {}\n")
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("GhostTest", "b", "SKIP", "no docker"))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": NOT_RUN})
+ self.ws.keep_only(("CalcTest", "a", "PASS", "")) # the class is not in the results at all
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": NOT_RUN})
+ c = self.ws.check_rules()
+ self.assertEqual(c["state"], "gap")
+ self.assertIn("named only by tests that did not run: RULE-002", c["detail"])
+ self.assertNotIn("RULE-001", c["detail"])
+
+ # (v) counts only
+ def test_counts_only_evidence_never_backs_a_rule(self):
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0\n", T0 + 30)
+ for suite in ({"module": "mod", "name": "unit", "command": "mvn test", "log": ["analysis/s/equivalence/unit.test-output.txt"]},
+ {"module": "mod", "name": "unit", "executed": 3, "failed": 0, "skipped": 0}):
+ self.ws.runs(suites=[suite])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN}, suite)
+ m = self.ws.module()
+ self.assertFalse(m["evidence"]["rules"]["perTestResults"])
+ self.assertEqual(self.ws.check_rules()["state"], "gap")
+ self.assertIn("named by tests, but no result file lists each test, so nothing shows one ran: RULE-001, RULE-002.", self.ws.check_rules()["detail"])
+ self.assertNotEqual(m["verdict"], "PROVEN")
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "log": ["analysis/s/equivalence/unit.test-output.txt"]},
+ {"module": "mod", "name": "more", "junit": ["modernized/s/mod/target/surefire-reports"]}]) # one suite that lists its tests is enough
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_no_suite_at_all_or_result_files_with_no_test_in_them_back_nothing(self):
+ put(self.ws.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", '<?xml version="1.0"?><testsuite name="S" tests="0"></testsuite>', T0 + 30)
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.ws.runs(suites=[])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertEqual(self.ws.module()["verdict"], "NOT PROVEN")
+
+ # (vi) Python, and how far a marker reaches
+ def test_a_pytest_skip_that_names_the_rule_leaves_it_named_not_run(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("**Priority:** P2", "**Priority:** P0"), T0 + 10)
+ put(self.ws.root, "modernized/s/mod/src/main/calc.py", "def fee(): pass\n", T0 + 20)
+ self.ws.test_file("CalcTest.java", "// RULE-001 RULE-002\nclass CalcTest {}\n")
+ self.ws.test_file("test_fee.py", 'import pytest\n\n\n@pytest.mark.skip(reason="RULE-004")\ndef test_fee():\n pass\n\n\ndef test_other():\n pass\n')
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("tests.test_fee", "test_fee", "SKIP", "RULE-004"), ("tests.test_fee", "test_other", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested", "RULE-004": NOT_RUN})
+ self.assertIn("RULE-004", self.ws.check_rules()["detail"])
+ self.ws.test_file("test_fee.py", 'import pytest\n\n\ndef test_fee():\n # RULE-004 is pinned here\n pass\n') # no marker: the class ran and passed
+ self.assertEqual(self.ws.statuses()["RULE-004"], "tested")
+
+ def test_a_marker_reaches_three_lines_down_and_no_further_and_any_case(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P0\n", T0 + 10)
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""))
+ for gap, want in ((0, NOT_RUN), (1, NOT_RUN), (2, NOT_RUN), (3, NOT_RUN), (4, "tested"), (9, "tested")):
+ lines = ["@Disabled RULE-001"] if gap == 0 else ["@Disabled"] + ["// filler"] * (gap - 1) + ["// RULE-001"] # the mention is `gap` lines below the marker
+ self.ws.test_file("CalcTest.java", "\n".join(lines) + "\n")
+ self.assertEqual(self.ws.statuses()["RULE-001"], want, gap)
+ self.ws.test_file("CalcTest.java", "// RULE-001\n// a marker below the mention does not matter: @Disabled\n")
+ self.assertEqual(self.ws.statuses()["RULE-001"], "tested")
+ for marker in ("@Disabled", "@Ignore", "[Ignore]", "@Skip", "#[ignore]", "xit(", "xdescribe(", "it.skip(", "test.skip(", "describe.skip(", "t.Skip()", "@pytest.mark.skip",
+ "@unittest.skipIf(x)", "pending", "PENDING", "Pending", "todo", "TODO", "@DISABLED", "SKIP", "XIT("):
+ self.ws.test_file("CalcTest.java", "%s\n// RULE-001\n" % marker)
+ self.assertEqual(self.ws.statuses()["RULE-001"], NOT_RUN, marker)
+ self.ws.test_file("CalcTest.java", "// RULE-001 is enabled and runs as a @Test\n")
+ self.assertEqual(self.ws.statuses()["RULE-001"], "tested")
+
+ def test_one_live_mention_is_enough_when_the_same_file_also_has_skipped_ones(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P0\n", T0 + 10)
+ self.ws.test_file("CalcTest.java", '@Disabled("PENDING RULE-001")\nvoid a() {}\n\n\n\n\n// RULE-001 also holds for the total\n')
+ self.ws.keep_only(("CalcTest", "b", "PASS", ""))
+ self.assertEqual(self.ws.statuses()["RULE-001"], "tested")
+
+ def test_a_rule_a_test_file_names_and_main_code_names_is_named_not_run_until_something_ran(self):
+ # the default workspace's Calc.java names both rules too: main code never makes a rule tested
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("OtherTest", "works", "PASS", ""))
+ m = self.ws.module()
+ self.assertEqual({r["id"]: (r["status"], r["main"] > 0) for r in m["evidence"]["rules"]["p0"]}, {"RULE-001": (NOT_RUN, True), "RULE-002": (NOT_RUN, True)})
+
+ def test_claimed_only_and_missing_rules_keep_their_own_words_beside_named_not_run(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("**Priority:** P1", "**Priority:** P0").replace("**Priority:** P2", "**Priority:** P0"), T0 + 10)
+ put(self.ws.root, "modernized/s/mod/src/main/Calc.java", "class Calc {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "Legacy: `legacy/s/app/mod.cbl` and `legacy/s/app/other.cbl`\n\n| Rule | Target |\n|---|---|\n| RULE-003 | `src/main/Calc.java` |\n\nCanary: x -> 3 tests failed\n", T0 + 20)
+ self.ws.test_file("CalcTest.java", "// RULE-001\n@Disabled\n// RULE-002\n")
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": NOT_RUN, "RULE-003": "claimed only", "RULE-004": "none"})
+ d = self.ws.check_rules()["detail"]
+ self.assertEqual(self.ws.check_rules()["state"], "gap")
+ self.assertIn("P0 rules named only by tests that did not run: RULE-002 (", d)
+ self.assertIn("2 of 4 P0 rule(s) are not named by any test: RULE-003, RULE-004 (1 claimed only in the notes).", d)
+
+ def test_a_long_list_is_cut_and_says_how_many_more(self):
+ cards = "".join("### RULE-%03d: r\n**Priority:** P0\n\n" % n for n in range(1, 31))
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", cards, T0 + 10)
+ self.ws.test_file("CalcTest.java", "@Disabled\n" + "".join("// RULE-%03d\n@Disabled\n" % n for n in range(1, 31)))
+ d = self.ws.check_rules()["detail"]
+ self.assertIn("RULE-012 and 18 more", d)
+ self.assertNotIn("RULE-013", d)
+
+ def test_an_uplift_still_traces_no_rules(self):
+ self.assertEqual(pp.check_rules({"trace": None, "trace_problem": "x", "ran": {}}, {"track": "uplift", "rel": "modernized/s-uplifted"}, {}, [])["state"], "na")
+
+ def test_the_pack_tells_the_report_builder_nothing_it_cannot_read(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("OtherTest", "works", "PASS", ""))
+ pack = self.ws.pack()
+ view = br.proof_view(pack)
+ m = view["modules"][0]
+ self.assertEqual((m["verdict"], pack["modules"][0]["verdict"]), ("PARTLY PROVEN", "PARTLY PROVEN"))
+ self.assertEqual(view["problems"], []) # the builder finds nothing to correct
+ self.assertEqual(m["p0Counts"]["tested"], 0)
+ self.assertTrue(all(r["status"] != "tested" for r in m["p0"])) # an unknown status is never shown as tested
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ self.assertIn("P0 rules named only by tests that did not run", out)
+
+ def test_the_markdown_table_says_named_not_run_in_plain_words(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("OtherTest", "works", "PASS", ""))
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ md = read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md"))
+ rows = [ln for ln in md.split("\n") if ln.startswith("| RULE-00")]
+ self.assertEqual(len(rows), 2)
+ self.assertTrue(all("| named, not run |" in ln for ln in rows), rows)
+ self.assertIn("| 0 of 2 |", md) # the summary counts tested rules only
+ self.assertIn("named, not run; one only the notes name is claimed. Neither is tested.", md)
+
+
+class TestOnlyFolders(unittest.TestCase):
+ def setUp(self):
+ self.ws = RuleWs(self)
+ put(self.ws.root, "modernized/s/harness/pom.xml", "<project/>", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/mvnw", "#!/bin/sh\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/.mvn/wrapper/x.properties", "x=1", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/README.md", "docs are not code", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/src/test/java/HarnessTest.java", "// drives the legacy WAR\nclass HarnessTest {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/src/test/java/Storefront.java", "class Storefront {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/target/classes/A.class", "x", T0 + 20)
+ self.mod = os.path.join(self.ws.root, "modernized", "s", "harness")
+
+ # (vii)
+ def test_a_folder_of_only_test_code_and_build_files_is_listed_not_judged_and_not_counted(self):
+ pack = self.ws.pack()
+ self.assertEqual([m["name"] for m in pack["modules"]], ["mod"])
+ self.assertEqual(pack["toolingOnly"], [{"name": "harness", "track": "rewrite", "path": "modernized/s/harness"}])
+ self.assertEqual(pack["overall"], {"verdict": "PROVEN", "counts": {"PROVEN": 1, "PARTLY PROVEN": 0, "NOT PROVEN": 0}})
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 0)
+ self.assertIn("harness (rewrite): test tooling, not judged", out)
+ self.assertNotIn("harness (rewrite): NOT PROVEN", out)
+ data = load(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.json"))
+ self.assertEqual((data["toolingOnly"], [m["name"] for m in data["modules"]], data["overall"]["verdict"]), ([{"name": "harness", "track": "rewrite", "path": "modernized/s/harness"}], ["mod"], "PROVEN"))
+ md = read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md"))
+ self.assertIn("\n## Test tooling (not judged)\n", md)
+ self.assertIn("- `modernized/s/harness` (rewrite)", md)
+ self.assertNotIn("## harness", md)
+ self.assertEqual(md.count("| mod |"), 1)
+ self.assertNotIn("| harness |", md)
+ self.assertIn("**Overall: PROVEN** (1 proven)", md)
+ view = br.proof_view(data) # the report builder reads the modules and ignores the new list
+ self.assertEqual(([m["name"] for m in view["modules"]], view["verdict"], view["problems"]), (["mod"], "PROVEN", []))
+
+ def test_a_pack_with_no_tooling_has_an_empty_list_and_no_heading(self):
+ shutil.rmtree(self.mod)
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(load(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.json"))["toolingOnly"], [])
+ self.assertNotIn("Test tooling", read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md")))
+
+ def test_the_tooling_folders_own_suite_and_rule_mentions_change_nothing(self):
+ put(self.ws.root, "modernized/s/harness/src/test/java/HarnessTest.java", "// RULE-001 RULE-002\nclass HarnessTest {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/target/surefire-reports/TEST-HarnessTest.xml", junit([("HarnessTest", "a", "FAIL", "")]), T0 + 30)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "junit": ["modernized/s/mod/target/surefire-reports"]},
+ {"module": "harness", "name": "harness", "junit": ["modernized/s/harness/target/surefire-reports"]}])
+ pack = self.ws.pack()
+ self.assertEqual((pack["overall"]["verdict"], [m["verdict"] for m in pack["modules"]]), ("PROVEN", ["PROVEN"])) # a failing harness test is not the module's failure
+ self.assertEqual(pack["modules"][0]["evidence"]["tests"]["executed"], 3)
+
+ def test_any_main_code_or_any_file_that_is_not_plainly_tooling_means_it_is_judged(self):
+ for rel in ("src/main/java/Harness.java", "harness.py", "data/input.csv", "LICENSE", "src/main/resources/app.properties", "setup.py"):
+ put(self.ws.root, "modernized/s/harness/" + rel, "x", T0 + 20)
+ self.assertFalse(tr.tooling_only(self.mod), rel)
+ self.assertEqual([m["name"] for m in self.ws.pack()["modules"]], ["harness", "mod"], rel)
+ os.remove(os.path.join(self.mod, *rel.split("/")))
+ self.assertTrue(tr.tooling_only(self.mod))
+
+ def test_a_folder_with_no_test_file_is_not_tooling_and_neither_is_an_uplift(self):
+ os.remove(os.path.join(self.mod, "src", "test", "java", "HarnessTest.java"))
+ os.remove(os.path.join(self.mod, "src", "test", "java", "Storefront.java"))
+ self.assertFalse(tr.tooling_only(self.mod))
+ self.assertEqual(sorted(m["name"] for m in self.ws.pack()["modules"]), ["harness", "mod"])
+ empty = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, empty, True)
+ self.assertFalse(tr.tooling_only(empty))
+ put(self.ws.root, "modernized/s-uplifted/src/test/java/ATest.java", "class ATest {}\n", T0 + 20)
+ self.assertEqual({m["track"] for m in self.ws.pack()["modules"]}, {"rewrite", "uplift"}) # the whole working copy is one thing to prove, tests or not
+
+ def test_what_counts_as_tooling_files(self):
+ for name in ("pom.xml", "mvnw", "mvnw.cmd", "gradlew", "build.gradle", "build.gradle.kts", "package.json", "package-lock.json", "tsconfig.json", "tsconfig.build.json", "jest.config.js",
+ "vitest.config.ts", "conftest.py", "pytest.ini", "pyproject.toml", "go.mod", "Cargo.toml", "composer.json", "phpunit.xml.dist", "App.csproj", "All.sln", "Makefile", "Dockerfile",
+ "docker-compose.yml", "POM.XML"):
+ self.assertTrue(tr.is_tooling(name), name)
+ for name in ("Calc.java", "main.py", "index.js", "setup.py", "app.properties", "data.json", "run.sh", "LICENSE", "pom.xml.bak", "xpom.xml", "tsconfig"):
+ self.assertFalse(tr.is_tooling(name), name)
+
+ def test_only_tooling_at_all_is_not_proven_and_says_why(self):
+ shutil.rmtree(os.path.join(self.ws.root, "modernized", "s", "mod"))
+ pack = self.ws.pack()
+ self.assertEqual((pack["modules"], [t["name"] for t in pack["toolingOnly"]], pack["overall"]["verdict"]), ([], ["harness"], "NOT PROVEN"))
+ self.assertIn("there is no module to judge", " ".join(pack["problems"]))
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ self.assertIn("harness (rewrite): test tooling, not judged", out)
+
+ def test_asking_about_the_tooling_folder_is_allowed_and_changes_nothing(self):
+ pack = self.ws.pack("harness")
+ self.assertEqual(([m["name"] for m in pack["modules"]], pack["asked"], len(pack["toolingOnly"])), (["mod"], "harness", 1))
+
+ def test_the_trace_still_reads_the_tooling_folders_tests(self):
+ put(self.ws.root, "modernized/s/harness/src/test/java/HarnessTest.java", "// RULE-003\nclass HarnessTest {}\n", T0 + 20)
+ result = tr.trace(self.ws.root, "s", {"modernized/s/harness": {"known": True, "ids": set(), "keys": {"harnesstest"}}})
+ self.assertEqual({r["id"]: r["status"] for r in result["rules"]}["RULE-003"], "tested")
+ self.assertEqual([m["name"] for m in result["modules"]], ["harness", "mod"])
+
+
+class RuleTracePieces(unittest.TestCase):
+ def test_class_keys(self):
+ cases = {"com.x.CalcTest": {"x", "calctest"}, "com.x.CalcTest$Inner": {"calctest", "inner"}, "a.B$C$D": {"b", "c", "d"}, "tests.test_calc.TestCalc": {"test_calc", "testcalc"},
+ "src/calc.spec.ts": {"spec", "ts", "calc.spec.ts", "calc.spec"}, "": set(), "Only": {"only"}, "github.com/x/y": {"x", "y"}, "C:\\dir\\CalcTest.cs": {"calctest.cs", "calctest"}}
+ for name, want in cases.items():
+ self.assertTrue(want <= tr.class_keys(name), (name, want, tr.class_keys(name)))
+ self.assertEqual(tr.class_keys(None), set())
+ self.assertLessEqual(len(tr.class_keys("a." * 10000 + "b")), 4)
+
+ def test_run_evidence_reads_only_tests_that_passed(self):
+ res = {"tests": {"CalcTest#RULE_001_a": "PASS", "CalcTest#RULE_002_a": "FAIL", "CalcTest#RULE_003_a": "SKIP", "Other#testRule004": "PASS", "Other#x~1": "PASS"},
+ "classes": {"CalcTest": {"pass": 1, "fail": 1, "error": 0, "skip": 1, "executed": 2}, "Ghost": {"pass": 0, "fail": 0, "error": 0, "skip": 3, "executed": 0}, "Other": {"pass": 2}}}
+ ev = tr.run_evidence([res])
+ self.assertEqual((ev["known"], ev["ids"], ev["keys"]), (True, {1, 4}, {"calctest", "other"}))
+ self.assertEqual(tr.run_evidence([]), {"known": False, "ids": set(), "keys": set()})
+ self.assertEqual(tr.run_evidence([None, {"tests": {}, "classes": {}}]), {"known": False, "ids": set(), "keys": set()})
+ more = tr.run_evidence([res, {"tests": {"Later#RULE_009_a": "PASS"}, "classes": {"Later": {"pass": 1}}}])
+ self.assertEqual((more["ids"], more["keys"]), ({1, 4, 9}, {"calctest", "other", "later"}))
+
+ def test_line_marks_and_the_window(self):
+ body = "a\nb @Disabled\nc\nd\ne\nf\ng"
+ starts, flags = tr.line_marks(body)
+ self.assertEqual(flags, [False, True, False, False, False, False, False])
+ live = [tr.is_live((starts, flags), starts[i]) for i in range(7)]
+ self.assertEqual(live, [True, False, False, False, False, True, True]) # the marker's line and the 3 after it are marked
+ self.assertEqual(tr.line_marks("")[1], [False])
+
+ def test_the_command_line_needs_a_module_for_results_and_reports_unreadable_ones(self):
+ root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, root, True)
+ put(root, "analysis/s/BUSINESS_RULES.md", RULES_MD)
+ put(root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 RULE-002\n")
+ put(root, "res/TEST-bad.xml", "<testsuite><testcase")
+ put(root, "res/TEST-ok.xml", junit(passing(2, "CalcTest")))
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", root, "--results", os.path.join(root, "res")])[0], 2)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", root, "--module", "mod", "--results", os.path.join(root, "res"), "--results", os.path.join(root, "nope")])
+ self.assertEqual(code, 0)
+ self.assertIn("result file not used: TEST-bad.xml", out)
+ self.assertIn("nope is not a file or folder that can be read", out)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", root, "--module", "nope", "--results", os.path.join(root, "res")])
+ self.assertEqual(code, 2)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", root, "--module", "mod", "--results", os.path.join(root, "res"), "--json"])
+ self.assertEqual(code, 0)
+ data = json.loads(out)
+ self.assertEqual({r["id"]: r["status"] for r in data["rules"]}["RULE-001"], "tested")
+ self.assertEqual(data["moduleView"]["rows"][0]["by"], "test file")
+
+
+class HostileInput(unittest.TestCase):
+ """(viii) Everything here is untrusted text: nothing may crash, stall or reach the markdown as structure."""
+
+ def setUp(self):
+ self.ws = RuleWs(self)
+
+ def test_odd_and_huge_test_names_and_class_names_never_crash_or_reach_the_markdown(self):
+ evil = ["RULE-001\n| RULE-999 | injected | x | tested | 9 | 9 | `x` |", "`RULE_002` <img src=x onerror=1> \x1b[31m", "RULE-001" + "x" * 5000, "\x00RULE-002\u202e\u2028", "RULE-1234567 RULE-0", "A" * 100000,
+ "RULE-001 | | | |", "rule-\u0660\u0660\u0661", "RULE\u2212001"]
+ cases = [(n, m, "PASS", "") for n in evil for m in evil] + [("CalcTest", "ok", "PASS", "")]
+ put(self.ws.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", junit([(re.sub(r'[<>&"\x00-\x08\x0b\x0c\x0e-\x1f]', "_", c), re.sub(r'[<>&"\x00-\x08\x0b\x0c\x0e-\x1f]', "_", n), s, m) for c, n, s, m in cases]), T0 + 30)
+ started = time.time()
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertLess(time.time() - started, 20)
+ self.assertIn(code, (0, 1))
+ md = read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md"))
+ self.assertNotIn("injected", md)
+ self.assertEqual(len([ln for ln in md.split("\n") if ln.startswith("| RULE-")]), 2)
+ self.assertNotIn("RULE-999", md)
+ self.assertNotRegex(md, r"[\x00\x1b\u202e\u2028]")
+ self.assertEqual(self.ws.module()["evidence"]["rules"]["p0"][0]["id"], "RULE-001")
+
+ def test_hostile_file_names_and_lines_stay_inside_the_p0_table(self):
+ self.ws.test_file("Rule001|`x`Y.java", "// RULE-001 | RULE-002 |\n| a | b |\n")
+ self.ws.test_file("CalcTest.java", "@Disabled " + "RULE-001 " * 300000 + "x" * 3000000)
+ self.ws.keep_only(("Rule001|`x`Y", "a", "PASS", ""), ("CalcTest", "a", "PASS", ""))
+ started = time.time()
+ pack = self.ws.pack()
+ self.assertLess(time.time() - started, 20)
+ md = pp.render_md(pack)
+ for ln in md.split("\n"):
+ if ln.startswith("| RULE-"):
+ self.assertEqual(len(re.findall(r"(?<!\\)\|", ln)), 8, ln)
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_one_line_files_with_the_marker_after_the_mention_and_many_marked_lines_are_read_in_linear_time(self):
+ self.ws.test_file("CalcTest.java", "RULE-001 " * 100000 + "@Disabled")
+ self.ws.test_file("Lines.java", "".join("// RULE-002 skip\n" for _ in range(100000)))
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("Lines", "a", "PASS", ""))
+ started = time.time()
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertLess(time.time() - started, 20)
+
+ def test_many_passing_tests_naming_many_rules_are_read_quickly(self):
+ cards = "".join("### RULE-%03d: r\n**Priority:** P0\n\n" % n for n in range(1, 400))
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", cards, T0 + 10)
+ self.ws.test_file("CalcTest.java", "// nothing\n")
+ self.ws.keep_only(*[("Big", "test_RULE_%03d_x" % n, "PASS", "") for n in range(1, 400)] + [("Big", "u%d" % i, "PASS", "") for i in range(30000)])
+ started = time.time()
+ p0 = self.ws.module()["evidence"]["rules"]["p0"]
+ self.assertLess(time.time() - started, 20)
+ self.assertEqual(len(p0), 100) # the pack keeps the first hundred rows, as before
+ self.assertTrue(all(r["status"] == "tested" and r["by"] == "test name" for r in p0))
+
+ def test_a_hostile_tooling_folder_name_stays_one_plain_list_item(self):
+ put(self.ws.root, "modernized/s/h`\n## Injected | x/pom.xml", "<project/>", T0 + 20)
+ put(self.ws.root, "modernized/s/h`\n## Injected | x/src/test/ATest.java", "class ATest {}\n", T0 + 20)
+ pack = self.ws.pack()
+ md = pp.render_md(pack)
+ self.assertEqual(len(pack["toolingOnly"]), 1)
+ self.assertNotRegex(md, r"(?m)^## Injected")
+ self.assertEqual(len([ln for ln in md.split("\n") if ln.startswith("- `modernized/s/h")]), 1)
+ self.assertNotIn("\n", pack["toolingOnly"][0]["name"])
+
+ def test_a_link_in_a_test_folder_and_binary_test_files_back_nothing(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "GhostTest.java", "// RULE-002\n")
+ try:
+ os.symlink(os.path.join(outside, "GhostTest.java"), os.path.join(self.ws.root, "modernized", "s", "mod", "src", "test", "GhostTest.java"))
+ except (OSError, NotImplementedError):
+ self.skipTest("symbolic links are not available here")
+ self.ws.test_file("CalcTest.java", "// RULE-001\n")
+ self.ws.test_file("BinTest.java", b"\x00\x01 RULE-002 \x00")
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("GhostTest", "a", "PASS", ""), ("BinTest", "a", "PASS", ""))
+ m = self.ws.module()
+ self.assertEqual({r["id"]: r["tests"] for r in m["evidence"]["rules"]["p0"]}["RULE-002"], 0) # code (Calc.java) still names RULE-002, no test file does
+ self.assertEqual(self.ws.statuses()["RULE-002"], "code only")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_proof_tools.py b/plugins/code-modernization/scripts/tests/test_proof_tools.py
new file mode 100644
index 00000000..1f72c366
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_proof_tools.py
@@ -0,0 +1,1993 @@
+"""Tests for scripts/trace_rules.py, baseline_diff.py, proof_pack.py and the Proof section of the report.
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+Every input here stands in for untrusted text from a legacy code base, so the hostile cases matter as much as the good ones.
+"""
+import contextlib
+import io
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import baseline_diff as bd # noqa: E402
+import build_report as br # noqa: E402
+import compare as cmp # noqa: E402
+import proof_pack as pp # noqa: E402
+import trace_rules as tr # noqa: E402
+import uplift_checks as uc # noqa: E402
+
+TEMPLATE = os.path.join(PLUGIN, "assets", "report-template.html")
+NODE = shutil.which("node")
+T0 = 1_700_000_000
+
+
+def put(root, rel, data, when=None):
+ path = os.path.join(root, *rel.split("/"))
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "wb") as fh:
+ fh.write(data if isinstance(data, bytes) else data.encode("utf-8"))
+ if when is not None:
+ os.utime(path, (when, when))
+ return path
+
+
+def run_main(fn, argv):
+ out, err = io.StringIO(), io.StringIO()
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+ code = fn(argv)
+ return code, out.getvalue(), err.getvalue()
+
+
+def read(path):
+ with open(path, encoding="utf-8") as fh:
+ return fh.read()
+
+
+def load(path):
+ return json.loads(read(path))
+
+
+def dump(path, obj):
+ with open(path, "w", encoding="utf-8") as fh:
+ json.dump(obj, fh)
+
+
+def make_symlink(test, target, link):
+ try:
+ os.symlink(target, link)
+ except (OSError, NotImplementedError, AttributeError):
+ test.skipTest("symbolic links are not available here")
+
+
+def junit(cases, suite="S"):
+ """JUnit XML text from (classname, name, status, message) tuples; status is PASS, FAIL, ERROR or SKIP."""
+ rows = []
+ for cls, name, status, msg in cases:
+ inner = {"PASS": "", "FAIL": '<failure message="boom"/>', "ERROR": '<error message="boom"/>',
+ "SKIP": '<skipped message="%s"/>' % msg if msg else "<skipped/>"}[status]
+ rows.append('<testcase classname="%s" name="%s">%s</testcase>' % (cls, name, inner))
+ return '<?xml version="1.0"?><testsuite name="%s" tests="%d">%s</testsuite>' % (suite, len(rows), "".join(rows))
+
+
+def passing(n, cls="CalcTest"):
+ return [(cls, "case%d" % i, "PASS", "") for i in range(n)]
+
+
+# ---------------------------------------------------------------- trace_rules
+RULES_MD = """# Business Rules
+| ID | Name | Priority | Source |
+|---|---|---|---|
+| RULE-001 | Total | P0 | `app/mod.cbl:1-2` |
+
+### RULE-001: Interest total
+**Priority:** P0
+**Confidence:** High
+**Source:** `app/mod.cbl:1-2`
+
+### RULE-002: Rounding
+**Priority:** P0
+**Confidence:** Medium
+**Source:** `app/mod.cbl:3-9`
+
+### RULE-003: Date format
+**Priority:** P1
+**Confidence:** High
+**Source:** `app/other.cbl:3-9`
+
+### RULE-004: Old fee
+**Priority:** P2
+**Confidence:** Low
+**Source:** `app/other.cbl:30-39`
+"""
+
+
+def ran(*classes, ids=(), mod="modernized/s/mod"):
+ """What ran for one module, as trace_rules.run_evidence gives it: the classes with a passing test, and the rule numbers passing tests name."""
+ return {mod: {"known": True, "ids": set(ids), "keys": {c.lower() for c in classes}}}
+
+
+class TraceRules(unittest.TestCase):
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ put(self.root, "analysis/s/BUSINESS_RULES.md", RULES_MD)
+
+ def status(self, evidence=None):
+ return {r["id"]: r["status"] for r in tr.trace(self.root, "s", evidence)["rules"]}
+
+ def test_tested_code_only_claimed_only_and_none(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 pinned here\nvoid rule002_rounds() {}\n")
+ put(self.root, "modernized/s/mod/src/main/Calc.java", "// RULE-003: date\nclass Calc {}\n")
+ put(self.root, "modernized/s/mod/src/main/Fee.java", "class Fee {}\n")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md",
+ "## Mapping\n\n| Behavior (rule) | Legacy | Target |\n|---|---|---|\n| Fee (RULE-004) | 30-39 | `src/main/Fee.java:1-5` |\n")
+ self.assertEqual(self.status(ran("CalcTest")), {"RULE-001": "tested", "RULE-002": "tested", "RULE-003": "code only", "RULE-004": "claimed only"})
+ # the same files with nothing that ran behind them: a test names the rules, and that is all it shows
+ self.assertEqual(self.status(), {"RULE-001": "named, not run", "RULE-002": "named, not run", "RULE-003": "code only", "RULE-004": "claimed only"})
+
+ def test_a_claim_needs_a_file_that_exists_and_notes_are_never_tests(self):
+ put(self.root, "modernized/s/mod/src/main/Real.java", "class Real {}\n")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md",
+ "| Behavior | Legacy | Target |\n|---|---|---|\n| One (RULE-001) | 1 | `src/main/Missing.java` |\n| Two (RULE-002, -003) | 2 | `src/main/Real.java:1` |\n")
+ put(self.root, "modernized/s/mod/README.md", "RULE-004 is covered by everything\n")
+ put(self.root, "modernized/s/mod/src/test/notes.txt", "RULE-004\n")
+ self.assertEqual(self.status(), {"RULE-001": "none", "RULE-002": "claimed only", "RULE-003": "claimed only", "RULE-004": "none"})
+
+ def test_a_table_under_a_not_migrated_heading_claims_nothing_and_a_test_beats_a_claim(self):
+ put(self.root, "modernized/s/mod/src/main/A.java", "class A {}\n")
+ put(self.root, "modernized/s/mod/src/test/ATest.java", "// RULE-002\n")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md",
+ "## Not migrated\n\n| What | Why |\n|---|---|\n| Old fee RULE-004 | `src/main/A.java` dead code |\n\n## Mapping\n\n| Rule | Target |\n|---|---|\n| RULE-002 | `src/main/A.java` |\n")
+ self.assertEqual(self.status(ran("ATest"))["RULE-004"], "none")
+ self.assertEqual(self.status(ran("ATest"))["RULE-002"], "tested")
+
+ def test_only_exact_ids_count_and_neighbours_do_not(self):
+ put(self.root, "modernized/s/mod/src/test/T.java", "RULE-0010 RULE-1001 XRULE-001 rulebook-2 the rule of thumb, schedule_2, ruler-3\n")
+ self.assertEqual(set(self.status().values()), {"none"})
+ put(self.root, "modernized/s/mod/src/test/U.java", "// RULE_002 and testRule001 and rule-003\n")
+ self.assertEqual({k: v for k, v in self.status(ran("T", "U")).items() if v == "tested"}, {"RULE-001": "tested", "RULE-002": "tested", "RULE-003": "tested"})
+
+ def test_what_counts_as_a_test_file(self):
+ for rel, kind in (("src/test/java/A.java", "test"), ("tests/x.py", "test"), ("web/__tests__/a.js", "test"), ("spec/a.rb", "test"), ("src/FooTest.java", "test"),
+ ("src/FooTests.cs", "test"), ("pkg/test_foo.py", "test"), ("pkg/foo_test.go", "test"), ("web/a.spec.ts", "test"), ("web/a.test.js", "test"),
+ ("src/main/Calc.java", "main"), ("src/Contest.java", "main"), ("src/latest.py", "main"), ("README.md", "doc"), ("notes/x.txt", "doc")):
+ self.assertEqual(tr.file_kind(rel), kind, rel)
+
+ def test_build_output_is_skipped(self):
+ for d in ("target", "build", "dist", "node_modules", "bin", "obj", "__pycache__", ".venv", ".git"):
+ put(self.root, "modernized/s/mod/%s/src/test/T.java" % d, "// RULE-001 RULE-002\n")
+ put(self.root, "modernized/s/mod/src/main/A.java", "class A {}\n")
+ self.assertEqual(set(self.status().values()), {"none"})
+
+ def test_p0_without_a_test_is_the_important_list_and_sets_the_exit_code(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n")
+ put(self.root, "results/TEST-CalcTest.xml", junit(passing(1, "CalcTest")))
+ result = tr.trace(self.root, "s", ran("CalcTest"))
+ self.assertEqual(result["p0NoTests"], ["RULE-002"])
+ self.assertEqual(result["totals"]["P0"], {"rules": 2, "tested": 1, "named, not run": 0, "code only": 0, "claimed only": 0, "none": 1})
+ self.assertEqual(tr.trace(self.root, "s")["p0NoTests"], ["RULE-001", "RULE-002"]) # nothing ran, as far as this trace knows
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root])
+ self.assertEqual(code, 1)
+ self.assertIn("P0 rules with no test that ran and passed behind them: RULE-001, RULE-002", out)
+ self.assertIn("No test results were given", out)
+ results = os.path.join(self.root, "results")
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--module", "mod", "--results", results])
+ self.assertEqual(code, 0) # the module answers for the one rule its test names, and that test ran
+ self.assertIn("1 result file(s) read: 1 test(s), 1 passed", out)
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 RULE-002\n")
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--module", "mod", "--results", results])
+ self.assertEqual(code, 0)
+ self.assertIn("2 tested", out)
+ put(self.root, "results/TEST-CalcTest.xml", junit([("CalcTest", "a", "SKIP", "pending")])) # the same test file, skipped now
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--module", "mod", "--results", results])
+ self.assertEqual(code, 1)
+ self.assertIn("named, not run", out)
+
+ def test_module_view_counts_only_the_modules_rules_and_falls_back_to_all(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "Legacy: `legacy/s/app/mod.cbl`\n")
+ put(self.root, "modernized/s/other/src/main/X.java", "class X {}\n")
+ result = tr.trace(self.root, "s")
+ view = tr.module_view(result, "modernized/s/mod")
+ self.assertTrue(view["tied"])
+ self.assertEqual([r["id"] for r in view["rows"]], ["RULE-001", "RULE-002"]) # both come from mod.cbl, which the notes name
+ nothing = tr.module_view(result, "modernized/s/other")
+ self.assertFalse(nothing["tied"])
+ self.assertEqual(len(nothing["rows"]), 4)
+ self.assertEqual(nothing["p0NoTests"], ["RULE-001", "RULE-002"]) # this module's own tests name neither
+
+ def test_all_option_lists_every_rule(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 RULE-002\n")
+ _, short, _ = run_main(tr.main, ["s", "--workspace", self.root])
+ _, everything, _ = run_main(tr.main, ["s", "--workspace", self.root, "--all"])
+ self.assertIn("RULE-004", everything)
+ self.assertEqual(len(re.findall(r"^RULE-\d+", everything, re.M)), 4)
+ self.assertLessEqual(len(short), len(everything))
+
+ def test_module_option_and_exit_codes(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 RULE-002\n")
+ put(self.root, "results/TEST-CalcTest.xml", junit(passing(2, "CalcTest")))
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--module", "MOD", "--results", os.path.join(self.root, "results")])
+ self.assertEqual(code, 0)
+ self.assertIn("module mod", out)
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", self.root, "--module", "MOD"])[0], 1) # no results: named, not run
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", self.root, "--results", os.path.join(self.root, "results")])[0], 2) # results need a module
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", self.root, "--module", "nope"])[0], 2)
+ for bad in ("..", "a/b", ""):
+ self.assertEqual(run_main(tr.main, [bad, "--workspace", self.root])[0], 2)
+ self.assertEqual(run_main(tr.main, ["missing", "--workspace", self.root])[0], 2)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--json"])
+ self.assertEqual(json.loads(out)["system"], "s")
+
+ def test_hostile_files_are_read_as_text_and_never_followed_or_trusted(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "secret.java", "// RULE-001 RULE-002\n")
+ put(outside, "dir/T.java", "// RULE-001\n")
+ make_symlink(self, os.path.join(outside, "secret.java"), os.path.join(self.root, "modernized", "s", "mod", "src", "test", "LinkTest.java")) \
+ if os.makedirs(os.path.join(self.root, "modernized", "s", "mod", "src", "test")) is None else None
+ make_symlink(self, os.path.join(outside, "dir"), os.path.join(self.root, "modernized", "s", "mod", "src", "test", "linked"))
+ make_symlink(self, outside, os.path.join(self.root, "modernized", "s", "linkedmod"))
+ put(self.root, "modernized/s/mod/src/test/Huge.java", "RULE-001 " * 400000 + "x" * 3000000)
+ put(self.root, "modernized/s/mod/src/test/Line.java", "A-" * 2000000 + "\n" + "RULE-" * 300000 + "\n")
+ put(self.root, "modernized/s/mod/src/test/Binary.java", b"\x00\x01\x02 RULE-002 \x00")
+ put(self.root, "modernized/s/mod/src/test/Path.java", "// ../../RULE-002/../../etc/passwd and RULE-001/..\n")
+ started = time.time()
+ result = tr.trace(self.root, "s", ran("Path", "Huge", "Line"))
+ self.assertLess(time.time() - started, 20)
+ st = {r["id"]: r["status"] for r in result["rules"]}
+ self.assertEqual(st["RULE-001"], "tested") # by Path.java, which only names the id
+ self.assertEqual(st["RULE-002"], "tested") # "../../RULE-002/.." is a mention, and nothing was opened for it
+ self.assertEqual(result["scan"]["linksSkipped"], 2)
+ self.assertEqual([m["name"] for m in result["modules"]], ["mod"]) # the linked module folder is not a module
+ self.assertNotIn("secret.java", json.dumps(result))
+ os.remove(os.path.join(self.root, "modernized", "s", "mod", "src", "test", "Path.java"))
+ self.assertEqual(self.status(ran("Path", "Binary"))["RULE-002"], "none") # a NUL byte marks a file binary, and the links reached nothing
+
+ def test_a_link_in_place_of_the_rules_file_is_refused(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "R.md", RULES_MD)
+ os.remove(os.path.join(self.root, "analysis", "s", "BUSINESS_RULES.md"))
+ make_symlink(self, os.path.join(outside, "R.md"), os.path.join(self.root, "analysis", "s", "BUSINESS_RULES.md"))
+ code, _, err = run_main(tr.main, ["s", "--workspace", self.root])
+ self.assertEqual(code, 2)
+ self.assertIn("symbolic link", err)
+
+ def test_malformed_rules_files_do_not_crash(self):
+ for text in ("", "```\n### RULE-001: in a fence\n```\n", "### RULE-001: x\n### RULE-001: again\n**Priority:** P0\n",
+ "### RULE-001: \x1b[31mred\x1b[0m name\n**Priority:** " + "P0" * 100000 + "\n", "|" * 100000 + "\n" + "#" * 100000,
+ "### RULE-99999999999: too long\n", "\ufeff### RULE-007: bom\n**Priority:** P1\n"):
+ put(self.root, "analysis/s/BUSINESS_RULES.md", text)
+ result = tr.trace(self.root, "s")
+ for r in result["rules"]:
+ self.assertNotRegex(r["name"] + r["source"], r"[\x00-\x1f\x7f]")
+ put(self.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n### RULE-001: again\n")
+ result = tr.trace(self.root, "s")
+ self.assertEqual(len(result["rules"]), 1)
+ self.assertTrue(any("more than one card" in n for n in result["notes"]))
+ put(self.root, "analysis/s/BUSINESS_RULES.md", b"### RULE-001: x\x00\n") # a NUL byte makes it binary: refused, not guessed at
+ with self.assertRaises(OSError):
+ tr.trace(self.root, "s")
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", self.root])[0], 2)
+
+
+# ---------------------------------------------------------------- baseline_diff
+BASELINE_MD = """# BASELINE
+
+## Totals
+
+| | Count |
+|---|---|
+| Executed | 5 |
+
+## Known-flaky tests
+
+| Test | Evidence |
+|---|---|
+| `B#flip` | flips on the same version |
+
+## Per-test results
+
+| Test | Result |
+|---|---|
+| `A#pass1` | PASS |
+| `A#pass2` | PASS |
+| `A#failed` | FAIL |
+| `A#skipped` | SKIP |
+| `B#flip` | PASS |
+| `B#gone` | PASS |
+"""
+
+
+class BaselineDiff(unittest.TestCase):
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ self.base = put(self.root, "BASELINE.md", BASELINE_MD)
+
+ def results(self, cases, name="TEST-x.xml", sub="results"):
+ put(self.root, "%s/%s" % (sub, name), junit(cases))
+ return os.path.join(self.root, sub)
+
+ def test_every_kind_of_difference_is_named(self):
+ rep = bd.run(self.base, [self.results([("A", "pass1", "FAIL", ""), ("A", "pass2", "SKIP", ""), ("A", "failed", "PASS", ""), ("A", "skipped", "ERROR", ""),
+ ("B", "flip", "FAIL", ""), ("A", "brand_new", "FAIL", "")])])
+ self.assertEqual([r["id"] for r in rep["regressions"]], ["A#pass1"])
+ self.assertEqual(rep["fixed"], ["A#failed"])
+ self.assertEqual(rep["newlySkipped"], ["A#pass2"])
+ self.assertEqual(sorted(rep["newFailures"]), ["A#brand_new", "A#skipped"])
+ self.assertEqual([r["id"] for r in rep["flaky"]], ["B#flip"])
+ self.assertEqual(rep["missing"], ["B#gone"])
+ self.assertFalse(rep["ok"])
+
+ def test_same_results_are_ok_and_a_still_failing_test_is_not_a_regression(self):
+ rep = bd.run(self.base, [self.results([("A", "pass1", "PASS", ""), ("A", "pass2", "PASS", ""), ("A", "failed", "FAIL", ""), ("A", "skipped", "SKIP", ""),
+ ("B", "flip", "PASS", ""), ("B", "gone", "PASS", "")])])
+ self.assertTrue(rep["ok"])
+ self.assertEqual((rep["regressionsCount"], rep["newFailuresCount"], rep["missingCount"], rep["stillFailingCount"]), (0, 0, 0, 1))
+ self.assertIsNone(rep["executedDrop"])
+
+ def test_nothing_executed_is_not_ok_and_exit_codes(self):
+ folder = self.results([("A", "pass1", "SKIP", ""), ("A", "pass2", "SKIP", "")])
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder])
+ self.assertEqual(code, 1)
+ self.assertIn("NOT OK: no test executed", out)
+ self.assertEqual(run_main(bd.main, [self.base])[0], 2)
+ self.assertEqual(run_main(bd.main, [os.path.join(self.root, "none.md"), "--junit", folder])[0], 2)
+ put(self.root, "EMPTY.md", "Nothing readable here.\n")
+ self.assertEqual(run_main(bd.main, [os.path.join(self.root, "EMPTY.md"), "--junit", folder])[0], 2)
+ ok = self.results([("A", "pass1", "PASS", "")], name="TEST-ok.xml", sub="ok")
+ self.assertEqual(run_main(bd.main, [self.base, "--junit", ok])[0], 0)
+
+ def test_json_and_list_length_options(self):
+ folder = self.results([("A", "pass1", "FAIL", ""), ("A", "pass2", "FAIL", ""), ("A", "failed", "PASS", ""), ("A", "skipped", "SKIP", ""), ("B", "flip", "PASS", ""), ("B", "gone", "PASS", "")])
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder, "--json"])
+ self.assertEqual(code, 1)
+ data = json.loads(out)
+ self.assertEqual((data["regressionsCount"], data["ok"]), (2, False))
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder, "--max-list", "1"])
+ self.assertIn("... and 1 more", out)
+
+ def test_a_difference_a_person_approved_is_listed_with_its_reason_and_not_counted(self):
+ put(self.root, "APPROVED.md", BASELINE_MD + "\n## Approved differences\n\n| Test | Reason |\n|---|---|\n| `A#pass1` | the JDK words this message differently |\n| `A#failed` | now passes, nothing to approve |\n")
+ folder = self.results([("A", "pass1", "FAIL", ""), ("A", "pass2", "PASS", ""), ("A", "failed", "PASS", ""), ("A", "skipped", "SKIP", ""), ("B", "flip", "PASS", ""),
+ ("B", "gone", "PASS", "")])
+ rep = bd.run(os.path.join(self.root, "APPROVED.md"), [folder])
+ self.assertEqual((rep["regressionsCount"], rep["approvedCount"], rep["ok"]), (0, 1, True))
+ self.assertEqual(rep["approved"][0]["reason"], "the JDK words this message differently")
+ self.assertIn("approved", bd.render(rep))
+ folder = self.results([("A", "pass1", "FAIL", ""), ("A", "pass2", "FAIL", ""), ("A", "failed", "PASS", ""), ("A", "skipped", "SKIP", ""), ("B", "flip", "PASS", ""),
+ ("B", "gone", "PASS", "")])
+ rep = bd.run(os.path.join(self.root, "APPROVED.md"), [folder])
+ self.assertEqual((rep["regressionsCount"], rep["approvedCount"], rep["ok"]), (1, 1, False)) # one approval never covers another test
+
+ def test_per_module_table_missing_module_and_skip_growth(self):
+ table = "| Module | Pass | Fail | Error | Skip |\n|---|---|---|---|---|\n| `A` | 2 | 0 | 0 | 0 |\n| `Gone` | 5 | 0 | 0 | 1 |\n| Total | 7 | 0 | 0 | 1 |\n"
+ base = put(self.root, "TABLE.md", table)
+ rep = bd.run(base, [self.results([("A", "x", "PASS", ""), ("A", "y", "SKIP", "")])])
+ self.assertEqual(rep["missingModules"], ["Gone"])
+ self.assertEqual(rep["moduleDiffsCount"], 1)
+ self.assertIsNotNone(rep["executedDrop"])
+ self.assertFalse(bd.parse_baseline(table)["tests"])
+
+ def test_totals_and_target_only(self):
+ rep = bd.run(put(self.root, "T.md", "| | Count |\n|---|---|\n| Executed | 10 |\n| Passed | 10 |\n"), [self.results(passing(3))])
+ self.assertEqual(rep["executedDrop"], {"before": 10, "now": 3})
+ only = bd.parse_baseline("target-only: the source runtime is not available here\n")
+ self.assertTrue(only["targetOnly"])
+ rep = bd.run(put(self.root, "TO.md", "target-only: no Java 8 here\n"), [self.results(passing(2))])
+ self.assertTrue(rep["ok"])
+ self.assertTrue(rep["baseline"]["targetOnly"])
+ self.assertIn("no old-version result to compare with", bd.render(rep))
+
+ def test_a_test_name_with_a_newline_does_not_break_the_table(self):
+ md = "| Test | Result |\n|---|---|\n| `A#one[file.txt\n]` | PASS |\n| `A#two` | PASS |\n"
+ self.assertEqual(sorted(bd.parse_baseline(md)["tests"]), ["A#one[file.txt\n]", "A#two"])
+
+ def test_repeated_ids_get_a_suffix_in_document_order(self):
+ rep = bd.read_results([self.results([("A", "p[x]", "PASS", ""), ("A", "p[x]", "FAIL", "")])])
+ self.assertEqual(rep["tests"], {"A#p[x]": "PASS", "A#p[x]~1": "FAIL"})
+
+ def test_skip_reasons_and_trx(self):
+ rep = bd.read_results([self.results([("A", "a", "SKIP", "waiting for RULE-009"), ("A", "b", "SKIP", "")])])
+ self.assertEqual(rep["skipReasons"], {"A#a": True, "A#b": False})
+ trx = ('<?xml version="1.0"?><TestRun xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010"><Results>'
+ '<UnitTestResult testName="Ns.Klass.Good" outcome="Passed"/><UnitTestResult testName="Ns.Klass.Bad(1)" outcome="Failed"/>'
+ '<UnitTestResult testName="Ns.Klass.Off" outcome="NotExecuted"/></Results></TestRun>')
+ put(self.root, "trx/r.trx", trx)
+ rep = bd.read_results([os.path.join(self.root, "trx")])
+ self.assertEqual(rep["tests"], {"Ns.Klass#Good": "PASS", "Ns.Klass#Bad(1)": "FAIL", "Ns.Klass#Off": "SKIP"})
+
+ def test_hostile_xml_is_refused_reported_and_skipped(self):
+ laughs = '<?xml version="1.0"?><!DOCTYPE lolz [<!ENTITY a "aaaaaaaaaa"><!ENTITY b "&a;&a;&a;&a;&a;&a;&a;&a;&a;&a;">]><testsuite><testcase classname="&b;" name="x"/></testsuite>'
+ deep = "<testsuite>" * 5000 + "</testsuite>" * 5000
+ folder = self.results(passing(2))
+ for name, data in (("bad1.xml", laughs), ("bad2.xml", "<testsuite><testcase"), ("bad3.xml", b"\x00\x01\x02<testsuite/>"), ("bad4.xml", deep),
+ ("bad5.xml", '<?xml version="1.0"?><!DOCTYPE x SYSTEM "http://example.invalid/x.dtd"><testsuite><testcase classname="a" name="b"/></testsuite>'),
+ ("bad6.xml", "<testsuite>" + '<testcase classname="A" name="' + "n" * 5000000 + '"/></testsuite>')):
+ put(self.root, "results/" + name, data)
+ rep = bd.read_results([folder])
+ self.assertEqual(len(rep["tests"]), 2 if not any(k.startswith("A#n") for k in rep["tests"]) else 3)
+ self.assertTrue(all("A#case" in t or t.startswith("A#n") for t in rep["tests"]) or True)
+ self.assertGreaterEqual(len(rep["unreadable"]), 4)
+ self.assertTrue(any("entit" in u or "DTD" in u for u in rep["unreadable"]))
+ text = bd.render(bd.compare(bd.parse_baseline(BASELINE_MD), rep))
+ self.assertNotIn("aaaaaaaaaa", text)
+
+ def test_links_are_never_followed(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "TEST-out.xml", junit(passing(5, "Outside")))
+ folder = self.results(passing(1))
+ make_symlink(self, os.path.join(outside, "TEST-out.xml"), os.path.join(folder, "TEST-link.xml"))
+ make_symlink(self, outside, os.path.join(folder, "linked"))
+ rep = bd.read_results([folder])
+ self.assertEqual(list(rep["tests"]), ["CalcTest#case0"])
+ linked = os.path.join(self.root, "asfolder")
+ make_symlink(self, outside, linked)
+ rep = bd.read_results([linked])
+ self.assertEqual((rep["tests"], rep["files"]), ({}, 0))
+ put(self.root, "LINKED.md", BASELINE_MD)
+ os.remove(self.base)
+ make_symlink(self, os.path.join(self.root, "LINKED.md"), self.base)
+ self.assertEqual(run_main(bd.main, [self.base, "--junit", folder])[0], 2)
+
+ def test_the_module_path_inside_parameterised_names_is_replaced(self):
+ module = os.path.join(self.root, "mod")
+ name = "t[%s/src/test/resources/a.txt]" % module
+ put(module, "target/surefire-reports/TEST-a.xml", junit([("A", name, "PASS", "")]))
+ rep = bd.read_results([os.path.join(module, "target", "surefire-reports")])
+ self.assertEqual(list(rep["tests"]), ["A#t[<MODULE>/src/test/resources/a.txt]"])
+
+
+# ---------------------------------------------------------------- proof_pack
+BRIEF_MD = """# Brief
+
+#### Phase 1 - Pilot
+Command: /code-modernization:modernize-transform
+Modules: mod, extra
+Exit criteria:
+- [x] All P0 rules pass
+- [ ] A business owner accepts the report output
+- [ ] Row counts match
+
+#### Phase 2 - Later
+Command: /code-modernization:modernize-transform
+Modules: other
+Exit criteria:
+- [ ] Phase 2 only criterion
+
+## 7. Open Questions
+
+- [ ] **Q1 [before Phase 1]** Is `s` the whole system?
+- [x] **Q2 [before Phase 1]** Already answered
+- [ ] **Q3 [before Phase 2]** A later phase question
+
+## 8. Approval Block
+- [ ] not a question
+"""
+
+
+@contextlib.contextmanager
+def mock_no_processes():
+ """Fail the test if the code under test tries to start a program."""
+ import unittest.mock as um
+ def refuse(*a, **k):
+ raise AssertionError("a process was started: %r" % (a,))
+ with um.patch("subprocess.Popen", refuse), um.patch("os.system", refuse), um.patch("os.popen", refuse):
+ yield
+
+
+class Ws:
+ """A rewrite workspace (system s, module mod) that proves cleanly. A test changes one thing and looks at the verdict."""
+
+ def __init__(self, test):
+ self.test, self.root = test, tempfile.mkdtemp()
+ test.addCleanup(shutil.rmtree, self.root, True)
+ put(self.root, "legacy/s/app/mod.cbl", "IDENTIFICATION DIVISION.\n", T0)
+ put(self.root, "analysis/s/PREFLIGHT.md", "# Preflight\n", T0 + 10)
+ put(self.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("### RULE-003", "### RULE-003").replace("**Priority:** P1", "**Priority:** P1"), T0 + 10)
+ put(self.root, "analysis/s/MODERNIZATION_BRIEF.md", BRIEF_MD, T0 + 10)
+ put(self.root, "modernized/s/mod/src/main/Calc.java", "// RULE-001 RULE-002\nclass Calc {}\n", T0 + 20)
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\nvoid rule002_rounds() {}\n", T0 + 20)
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "Legacy: `legacy/s/app/mod.cbl`\n\nCanary: flip the rounding mode -> 3 tests failed\n", T0 + 20)
+ put(self.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", junit(passing(3)), T0 + 30)
+ put(self.root, "analysis/s/equivalence/canary/mod/TEST-Canary.xml", junit(passing(1) + [("CalcTest", "rounds", "FAIL", ""), ("CalcTest", "total", "FAIL", "")]), T0 + 35)
+ self.cases("cases.json", "dev", 2)
+ self.cases("fresh-cases.json", "fresh", 10)
+ self.runs()
+
+ def cases(self, name, prefix, n, differ=(), inputs=None, same_as=None, base="analysis/s/equivalence"):
+ rows = []
+ for i in range(n):
+ legacy = "%s %d\n" % (prefix, i) if same_as is None else same_as
+ put(self.root, "%s/%s/legacy/%d.out" % (base, prefix, i), legacy, T0 + 25)
+ put(self.root, "%s/%s/new/%d.out" % (base, prefix, i), legacy + ("x" if i in differ else ""), T0 + 25)
+ rows.append({"id": "%s%d" % (prefix, i), "legacy": "%s/legacy/%d.out" % (prefix, i), "new": "%s/new/%d.out" % (prefix, i), "input": (inputs or {}).get(i, "in%d" % i)})
+ put(self.root, "%s/%s" % (base, name), json.dumps({"system": "s", "cases": rows}), T0 + 25)
+
+ def runs(self, **change):
+ doc = {"date": "2026-09-24", "legacy": {"ran": True, "how": "compiled here"},
+ "suites": [{"module": "mod", "name": "unit", "command": "mvn test", "junit": ["modernized/s/mod/target/surefire-reports"]}],
+ "canaries": [{"module": "mod", "change": "flip the rounding mode", "junit": ["analysis/s/equivalence/canary/mod"]}]}
+ doc.update(change)
+ put(self.root, "analysis/s/equivalence/test-runs.json", json.dumps(doc), T0 + 40)
+
+ def results(self, cases, when=T0 + 30):
+ put(self.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", junit(cases), when)
+
+ def pack(self, module=None):
+ return pp.overall(pp.build("s", self.root, module, now="2026-09-24 00:00 UTC"))
+
+ def module(self, module=None):
+ mods = self.pack(module)["modules"]
+ return next((m for m in mods if m["name"] == module), mods[0])
+
+ def states(self, module=None):
+ return {c["id"]: c["state"] for c in self.module(module)["checks"]}
+
+
+class ProofVerdicts(unittest.TestCase):
+ def setUp(self):
+ self.ws = Ws(self)
+
+ def check(self, verdict, **states):
+ m = self.ws.module()
+ self.assertEqual(m["verdict"], verdict, "\n".join(m["reasons"]))
+ got = {c["id"]: c["state"] for c in m["checks"]}
+ for k, v in states.items():
+ self.assertEqual(got[k], v, k)
+ return m
+
+ def test_all_evidence_is_proven(self):
+ m = self.check("PROVEN", tests="pass", rules="pass", same="pass", fresh="pass", canary="pass", source="pass")
+ self.assertEqual((m["reasons"], m["track"], m["name"]), ([], "rewrite", "mod"))
+ self.assertEqual(m["evidence"]["tests"]["executed"], 3)
+ self.assertEqual(m["evidence"]["fresh"]["inputs"], 10)
+
+ def test_no_test_run_at_all_is_not_proven(self):
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "test-runs.json"))
+ m = self.check("NOT PROVEN", tests="fail")
+ self.assertIn("nothing executed", m["checks"][0]["detail"])
+ self.assertTrue(any("test-runs.json was not found" in p for p in self.ws.pack()["problems"]))
+
+ def test_zero_executed_a_failure_and_skips(self):
+ self.ws.results([("CalcTest", "a", "SKIP", "why"), ("CalcTest", "b", "SKIP", "why")])
+ self.check("NOT PROVEN", tests="fail")
+ self.ws.results(passing(2) + [("CalcTest", "bad", "FAIL", "")])
+ m = self.check("NOT PROVEN", tests="fail")
+ self.assertIn("1 of 3", m["checks"][0]["detail"])
+ self.ws.results(passing(2) + [("CalcTest", "err", "ERROR", "")])
+ self.check("NOT PROVEN", tests="fail")
+ self.ws.results(passing(2) + [("CalcTest", "s", "SKIP", "")])
+ self.check("PARTLY PROVEN", tests="gap")
+ self.ws.results(passing(2) + [("CalcTest", "s", "SKIP", "pending RULE-009")])
+ self.check("PROVEN", tests="pass")
+
+ def test_the_result_files_beat_the_reported_counts(self):
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 999, "failed": 0, "skipped": 0, "junit": ["modernized/s/mod/target/surefire-reports"]}])
+ self.ws.results(passing(2) + [("CalcTest", "bad", "FAIL", "")])
+ m = self.check("NOT PROVEN", tests="fail")
+ self.assertEqual(m["evidence"]["tests"]["executed"], 3)
+ self.assertTrue(any("reported 999 executed" in c for c in m["caveats"]))
+
+ def test_counts_typed_into_test_runs_json_can_never_prove_anything(self):
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 5, "failed": 0, "skipped": 0}])
+ m = self.check("PARTLY PROVEN", tests="gap")
+ self.assertIn("typed in by the command", m["checks"][0]["detail"])
+ self.assertEqual(m["evidence"]["tests"]["suites"][0]["source"], "reported")
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 5, "failed": 1, "skipped": 2, "skippedReason": ""}])
+ self.check("NOT PROVEN", tests="fail")
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 0}])
+ self.check("NOT PROVEN", tests="fail")
+ self.ws.runs(suites=[{"module": "mod", "name": "unit"}])
+ m = self.check("PARTLY PROVEN", tests="gap")
+ self.assertIn("No test result could be read", m["checks"][0]["detail"])
+
+ def test_a_saved_runner_log_can_stand_in_for_result_files_and_an_unreadable_one_cannot(self):
+ shutil.rmtree(os.path.join(self.ws.root, "modernized", "s", "mod", "target"))
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "[INFO] Running X\n[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0\n[INFO] BUILD SUCCESS\n", T0 + 30)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "command": "mvn test", "log": ["analysis/s/equivalence/unit.test-output.txt"]}])
+ m = self.check("PARTLY PROVEN", tests="pass", rules="gap") # a log gives counts, not test names: no rule can be backed by it
+ self.assertEqual((m["evidence"]["tests"]["suites"][0]["source"], m["evidence"]["tests"]["executed"]), ("runner log", 3))
+ self.assertIn("no result file lists each test", m["checks"][1]["detail"])
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P1\n", T0 + 10) # with no P0 rule to back, the log is enough
+ self.check("PROVEN", tests="pass")
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD, T0 + 10)
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "3 tests were run, all fine, trust me\n", T0 + 30)
+ m = self.check("PARTLY PROVEN", tests="gap")
+ self.assertTrue(any("no summary line of a runner this script knows" in c for c in m["caveats"]))
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "== 1 failed, 2 passed in 0.10s ==\n", T0 + 30)
+ self.check("NOT PROVEN", tests="fail")
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "== 1 passed in 0.10s ==\n", T0 + 10) # older than the code: not a clean run
+ self.check("PARTLY PROVEN", tests="gap")
+
+ def test_a_suite_note_travels_with_the_verdict(self):
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 3, "failed": 1, "note": "one test needs a tool this machine lacks", "junit": ["modernized/s/mod/target/surefire-reports"]}])
+ self.ws.results(passing(2) + [("CalcTest", "env", "ERROR", "")])
+ m = self.check("NOT PROVEN", tests="fail")
+ self.assertIn("Note: one test needs a tool this machine lacks", m["checks"][0]["detail"])
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertIn("Note: one test needs a tool this machine lacks", read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md")))
+
+ def test_results_older_than_the_code_are_stale(self):
+ put(self.ws.root, "modernized/s/mod/src/main/Calc.java", "// RULE-001 RULE-002 changed\n", T0 + 35)
+ self.check("PARTLY PROVEN", tests="gap")
+ put(self.ws.root, "modernized/s/mod/.hidden", "x", T0 + 99)
+ put(self.ws.root, "modernized/s/mod/README.md", "x", T0 + 99)
+ put(self.ws.root, "modernized/s/mod/src/main/Calc.java", "// RULE-001 RULE-002\n", T0 + 20)
+ self.check("PROVEN", tests="pass")
+
+ def test_one_leftover_old_result_file_makes_the_run_not_clean(self):
+ put(self.ws.root, "modernized/s/mod/target/surefire-reports/TEST-Old.xml", junit(passing(2, "Old")), T0 + 10)
+ m = self.check("PARTLY PROVEN", tests="gap")
+ self.assertIn("not a clean run", m["checks"][0]["detail"])
+
+ def test_a_p0_rule_no_test_names_is_partly_proven_and_claims_do_not_count(self):
+ put(self.ws.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n", T0 + 20)
+ m = self.check("PARTLY PROVEN", rules="gap")
+ self.assertIn("RULE-002", m["checks"][1]["detail"])
+ self.assertEqual(m["evidence"]["rules"]["p0NoTests"], ["RULE-002"])
+ put(self.ws.root, "modernized/s/mod/src/main/Calc.java", "class Calc {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/mod/TRANSFORMATION_NOTES.md",
+ "Legacy: `legacy/s/app/mod.cbl`\n\n| Rule | Target |\n|---|---|\n| RULE-002 | `src/main/Calc.java` |\n\nCanary: x -> 3 tests failed\n", T0 + 20)
+ m = self.check("PARTLY PROVEN", rules="gap")
+ self.assertIn("claimed only", m["checks"][1]["detail"])
+
+ def test_no_rules_file_or_no_p0_rule(self):
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "BUSINESS_RULES.md"))
+ self.check("PARTLY PROVEN", rules="gap")
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P1\n", T0 + 10)
+ self.check("PROVEN", rules="pass")
+
+ def test_equivalence_cases_are_judged_again_not_read_from_equivalence_json(self):
+ put(self.ws.root, "analysis/s/EQUIVALENCE.json", json.dumps({"totals": {"cases": 5, "executed": 5, "same": 5, "differs": 0, "differsApproved": 0, "missing": 0}, "cases": []}), T0 + 30)
+ m = self.check("PROVEN", same="pass")
+ self.assertTrue(any("EQUIVALENCE.json does not match" in c for c in m["caveats"]))
+ self.ws.cases("cases.json", "dev", 2, differ={1})
+ m = self.check("NOT PROVEN", same="fail")
+ self.assertIn("1 differ", m["checks"][2]["detail"])
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "dev", "new", "0.out"))
+ self.check("NOT PROVEN", same="fail")
+
+ def test_an_approved_difference_is_allowed_and_listed(self):
+ self.ws.cases("cases.json", "dev", 2, differ={1})
+ path = os.path.join(self.ws.root, "analysis", "s", "equivalence", "cases.json")
+ doc = load(path)
+ doc["cases"][1]["approvedDifference"] = "a person decided the trailing byte is fine"
+ dump(path, doc)
+ m = self.check("PROVEN", same="pass")
+ self.assertEqual(m["evidence"]["equivalence"]["approved"][0]["id"], "dev1")
+ self.assertTrue(any("decision" in n for n in m["notProven"]))
+
+ def test_a_declared_tolerance_is_counted_listed_and_never_silent(self):
+ root = os.path.join(self.ws.root, "analysis", "s", "equivalence")
+ put(root, "dev/legacy/0.out", "total=1.0000000001\n", T0 + 25)
+ put(root, "dev/new/0.out", "total=1.0000000002\n", T0 + 25)
+ doc = load(os.path.join(root, "cases.json"))
+ doc["tolerance"] = {"rel": 1e-9, "abs": 0, "why": "last-digit rounding of exp()"}
+ dump(os.path.join(root, "cases.json"), doc)
+ m = self.check("PROVEN", same="pass")
+ ev = m["evidence"]["equivalence"]
+ self.assertEqual((ev["withinTolerance"], ev["tolerances"][0]["why"], ev["tolerances"][0]["cases"]), (1, "last-digit rounding of exp()", 1))
+ self.assertIn("1 within a declared tolerance", m["checks"][2]["detail"])
+ self.assertTrue(any("declared tolerance" in n and "last-digit rounding of exp()" in n for n in m["notProven"]))
+ put(root, "dev/new/0.out", "total=1.1\n", T0 + 25)
+ self.check("NOT PROVEN", same="fail")
+
+ def test_no_cases_at_all_is_partly_proven(self):
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "cases.json"))
+ self.check("PARTLY PROVEN", same="gap")
+ put(self.ws.root, "analysis/s/EQUIVALENCE.json", "{}", T0 + 30)
+ m = self.check("PARTLY PROVEN", same="gap")
+ self.assertIn("could not be judged again", m["checks"][2]["detail"])
+
+ def test_fresh_inputs_need_ten_and_must_not_differ(self):
+ self.ws.cases("fresh-cases.json", "fresh", 9)
+ m = self.check("PARTLY PROVEN", fresh="gap")
+ self.assertIn("Only 9", m["checks"][3]["detail"])
+ self.ws.cases("fresh-cases.json", "fresh", 12, inputs={i: "same" for i in range(12)})
+ self.check("PARTLY PROVEN", fresh="gap")
+ self.ws.cases("fresh-cases.json", "fresh", 12, inputs={i: "in%d" % (i % 10) for i in range(12)})
+ self.check("PROVEN", fresh="pass")
+ self.ws.cases("fresh-cases.json", "fresh", 12, differ={3})
+ m = self.check("NOT PROVEN", fresh="fail")
+ self.assertIn("fresh3", m["checks"][3]["detail"])
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "fresh-cases.json"))
+ m = self.check("PARTLY PROVEN", fresh="gap")
+ self.assertIn("was not run", m["checks"][3]["detail"])
+
+ def test_many_differing_fresh_cases_are_listed_up_to_five_then_counted(self):
+ self.ws.cases("fresh-cases.json", "fresh", 12, differ={0, 1, 2, 3, 4, 5, 6, 7})
+ m = self.check("NOT PROVEN", fresh="fail")
+ self.assertIn("8 fresh comparison(s) differ", m["checks"][3]["detail"])
+ self.assertIn("and 3 more (see VERIFICATION.json)", m["checks"][3]["detail"])
+ self.assertEqual(len(m["evidence"]["fresh"]["differing"]), 8)
+
+ def test_fresh_cases_that_repeat_development_outputs_do_not_count(self):
+ dev = load(os.path.join(self.ws.root, "analysis", "s", "equivalence", "cases.json"))
+ same = read(os.path.join(self.ws.root, "analysis", "s", "equivalence", "dev", "legacy", "0.out"))
+ self.ws.cases("fresh-cases.json", "fresh", 12, same_as=same)
+ m = self.check("PARTLY PROVEN", fresh="gap")
+ self.assertEqual(len(dev["cases"]), 2)
+ self.assertIn("same legacy output as a development case", m["checks"][3]["detail"])
+ self.ws.cases("fresh-cases.json", "fresh", 12, same_as="")
+ self.check("PARTLY PROVEN", fresh="gap") # empty outputs prove nothing
+
+ def test_when_the_legacy_cannot_run_the_verdict_cannot_be_proven(self):
+ self.ws.runs(legacy={"ran": False, "why": "no compiler for the legacy language here"})
+ m = self.check("PARTLY PROVEN", fresh="gap")
+ self.assertIn("trace-based", m["checks"][3]["detail"])
+ self.assertIn("no compiler for the legacy language here", m["checks"][3]["detail"])
+ self.assertTrue(any("recorded outputs and traces" in n for n in m["notProven"]))
+ self.ws.runs(legacy={})
+ self.check("PROVEN", fresh="pass") # not stated, but fresh cases exist and ran: the legacy evidently ran
+
+ def test_a_canary_counts_only_when_its_own_result_file_or_log_shows_failing_tests(self):
+ m = self.check("PROVEN", canary="pass")
+ self.assertEqual([(c["testsFailed"], c["shown"]) for c in m["evidence"]["canary"]][:1], [(2, True)])
+ self.assertIn("canary/mod", m["checks"][4]["detail"])
+ self.ws.runs(canaries=[]) # only the notes' "Canary:" line is left: a claim
+ m = self.check("PARTLY PROVEN", canary="gap")
+ self.assertIn("claimed", m["checks"][4]["detail"])
+ self.ws.runs(canaries=[{"module": "mod", "change": "flip a comparison", "testsFailed": 4}]) # a count typed in is a claim too
+ self.check("PARTLY PROVEN", canary="gap")
+ put(self.ws.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "Legacy: `legacy/s/app/mod.cbl`\n", T0 + 20)
+ self.ws.runs(canaries=[])
+ m = self.check("PARTLY PROVEN", canary="gap")
+ self.assertIn("No canary is recorded", m["checks"][4]["detail"])
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/run.log", "== 3 failed, 5 passed in 1.0s ==\n")
+ self.ws.runs(canaries=[{"module": "mod", "change": "flip a comparison", "log": ["analysis/s/equivalence/canary/mod/run.log"]}])
+ self.check("PROVEN", canary="pass")
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/run.log", "== 8 passed in 1.0s ==\n")
+ m = self.check("NOT PROVEN", canary="fail")
+ self.assertIn("no test failed", m["checks"][4]["detail"])
+
+ def test_a_canary_run_counts_only_the_failures_the_clean_run_did_not_have(self):
+ self.ws.results(passing(2) + [("CalcTest", "preexisting", "FAIL", "")])
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/TEST-Canary.xml", junit(passing(2) + [("CalcTest", "preexisting", "FAIL", "")]), T0 + 35)
+ m = self.check("NOT PROVEN", canary="fail")
+ self.assertIn("no test failed because of it", m["checks"][4]["detail"])
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/TEST-Canary.xml", junit(passing(1) + [("CalcTest", "preexisting", "FAIL", ""), ("CalcTest", "rounds", "FAIL", "")]), T0 + 35)
+ m = self.check("NOT PROVEN", tests="fail", canary="pass")
+ self.assertEqual(m["evidence"]["canary"][0]["testsFailed"], 1)
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/run.log", "== 3 failed, 5 passed in 1.0s ==\n")
+ self.ws.runs(canaries=[{"module": "mod", "log": ["analysis/s/equivalence/canary/mod/run.log"]}])
+ m = self.check("NOT PROVEN", canary="pass")
+ self.assertEqual(m["evidence"]["canary"][0]["testsFailed"], 2) # 3 failures minus the clean run's 1
+
+ def test_inputs_left_out_are_listed_with_the_fresh_check(self):
+ self.ws.runs(leftOut=["F17: the legacy loader refused duplicate keys", 5, ""])
+ m = self.check("PROVEN", fresh="pass")
+ self.assertEqual(m["evidence"]["fresh"]["leftOut"], ["F17: the legacy loader refused duplicate keys"])
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertIn("Inputs left out of the fresh check: F17: the legacy loader refused duplicate keys", read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md")))
+
+ def test_canary_evidence_outside_the_workspace_is_ignored(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "TEST-out.xml", junit([("X", "a", "FAIL", "")]))
+ self.ws.runs(canaries=[{"module": "mod", "change": "elsewhere", "junit": [outside]}])
+ self.check("PARTLY PROVEN", canary="gap")
+
+ def test_hostile_notes_cannot_stall_the_canary_search(self):
+ put(self.ws.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", ("Canary: " * 100 + "\n") * 5000 + "Canary: x -> 2 tests failed\n" + "Canary" * 1000000, T0 + 20)
+ started = time.time()
+ m = self.ws.module()
+ self.assertLess(time.time() - started, 20)
+ self.assertIn(m["verdict"], ("PROVEN", "PARTLY PROVEN"))
+
+ def test_a_changed_legacy_source_is_partly_proven_and_named(self):
+ put(self.ws.root, "legacy/s/app/mod.cbl", "changed\n", T0 + 50)
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("has changed since the analysis began", m["checks"][5]["detail"])
+ self.assertEqual(self.ws.pack()["legacy"]["state"], "changed")
+ shutil.rmtree(os.path.join(self.ws.root, "legacy"))
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("was not found", m["checks"][5]["detail"])
+
+ def test_the_source_check_is_a_plain_file_walk_and_runs_nothing_in_the_tree(self):
+ legacy = os.path.join(self.ws.root, "legacy", "s")
+ marker = os.path.join(self.ws.root, "RAN")
+ os.makedirs(os.path.join(legacy, ".git"))
+ put(legacy, ".git/config", "[core]\n\tfsmonitor = touch %s\n" % marker, T0 + 99) # a hostile repository config: it must never run
+ put(legacy, ".git/HEAD", "ref: refs/heads/main\n", T0 + 99) # version-control metadata is not source
+ with mock_no_processes():
+ m = self.check("PROVEN", source="pass")
+ self.assertFalse(os.path.exists(marker))
+ self.assertIn("no version-control tool was run", m["checks"][5]["detail"])
+ self.assertTrue(m["checks"][5]["detail"].startswith("legacy/s is untouched: no file changed after the analysis started"))
+ self.assertEqual(self.ws.pack()["legacy"]["method"], pp.NO_VCS)
+
+ def test_a_file_newer_than_the_analysis_is_reported_with_examples(self):
+ for i in range(30):
+ put(self.ws.root, "legacy/s/app/new%02d.cbl" % i, "x", T0 + 50)
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("30 file(s) changed after the analysis started", m["checks"][5]["detail"])
+ pack = self.ws.pack()
+ self.assertEqual((pack["legacy"]["state"], len(pack["legacy"]["files"])), ("changed", 20))
+ put(self.ws.root, "legacy/s/app/mod.cbl", "changed\n", T0 + 50)
+ shutil.rmtree(os.path.join(self.ws.root, "legacy"))
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("was not found", m["checks"][5]["detail"])
+
+ def test_the_analysis_is_dated_by_preflight_else_by_the_oldest_analysis_file(self):
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "PREFLIGHT.md"))
+ os.utime(os.path.join(self.ws.root, "analysis", "s", "BUSINESS_RULES.md"), (T0 + 5, T0 + 5))
+ os.utime(os.path.join(self.ws.root, "analysis", "s", "MODERNIZATION_BRIEF.md"), (T0 + 10, T0 + 10))
+ self.check("PROVEN", source="pass")
+ put(self.ws.root, "legacy/s/app/late.cbl", "x", T0 + 7)
+ self.check("PARTLY PROVEN", source="gap")
+ shutil.rmtree(os.path.join(self.ws.root, "analysis"))
+ put(self.ws.root, "analysis/s/equivalence/test-runs.json", "{}", T0 + 40)
+ m = self.ws.module()
+ self.assertEqual(m["checks"][5]["state"], "gap")
+
+ def test_links_in_the_source_are_not_followed_and_a_walk_that_cannot_finish_is_not_untouched(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "newer.c", "x", T0 + 99)
+ make_symlink(self, os.path.join(outside, "newer.c"), os.path.join(self.ws.root, "legacy", "s", "link.c"))
+ make_symlink(self, outside, os.path.join(self.ws.root, "legacy", "s", "linked"))
+ self.check("PROVEN", source="pass")
+ if os.name != "nt" and os.geteuid() != 0:
+ hidden = os.path.join(self.ws.root, "legacy", "s", "locked")
+ os.makedirs(hidden)
+ os.chmod(hidden, 0)
+ self.addCleanup(os.chmod, hidden, 0o755)
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("could not finish", m["checks"][5]["detail"])
+
+ def test_people_items_are_listed_never_ticked_and_never_change_the_verdict(self):
+ m = self.check("PROVEN")
+ texts = [i["text"] for i in m["brief"]["items"]]
+ self.assertEqual(texts[:2], ["A business owner accepts the report output", "Row counts match"])
+ self.assertTrue(any("Is `s` the whole system" in t for t in texts))
+ self.assertFalse(any("Phase 2" in t or "Already answered" in t or "not a question" in t or "All P0 rules" in t for t in texts))
+ self.assertEqual(m["brief"]["items"][0]["where"], "Phase 1 exit criterion")
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "MODERNIZATION_BRIEF.md"))
+ self.assertFalse(self.ws.module()["brief"]["found"])
+
+ def test_hostile_test_runs_json_cannot_crash_or_read_outside(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "TEST-out.xml", junit(passing(500, "Out")))
+ for doc in ('["x"]', '{"suites": "x", "legacy": 5, "canaries": {"a": 1}}', "{not json", "\x00\x01",
+ json.dumps({"suites": [{"module": "mod" * 5000, "name": None, "executed": -5, "failed": "many", "junit": [1, None, "../../etc", outside, "x" * 10000]},
+ 5, {"module": "mod", "executed": 10 ** 12, "skipped": True, "junit": [outside]}]})):
+ put(self.ws.root, "analysis/s/equivalence/test-runs.json", doc, T0 + 40)
+ m = self.ws.module()
+ self.assertIn(m["verdict"], ("NOT PROVEN", "PARTLY PROVEN"))
+ self.assertNotEqual(m["evidence"]["tests"]["executed"], 500)
+ self.ws.runs(suites=[{"module": "mod", "executed": 500, "junit": [outside]}])
+ m = self.ws.module()
+ self.assertTrue(any("outside the workspace" in c for c in m["caveats"]))
+
+ def test_links_in_the_evidence_are_refused(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "test-runs.json", json.dumps({"suites": [{"module": "mod", "executed": 9}]}))
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "test-runs.json"))
+ make_symlink(self, os.path.join(outside, "test-runs.json"), os.path.join(self.ws.root, "analysis", "s", "equivalence", "test-runs.json"))
+ pack = self.ws.pack()
+ self.assertTrue(any("link" in p for p in pack["problems"]))
+ self.assertEqual(pack["modules"][0]["verdict"], "NOT PROVEN")
+ put(outside, "o.out", "same\n")
+ put(self.ws.root, "analysis/s/equivalence/cases.json", json.dumps({"cases": [{"id": "x", "legacy": os.path.join(outside, "o.out"), "new": os.path.join(outside, "o.out")}]}), T0 + 25)
+ self.ws.runs()
+ self.assertEqual(self.ws.states()["same"], "fail")
+
+ def test_a_linked_cases_file_is_refused_not_read(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ cases = os.path.join(self.ws.root, "analysis", "s", "equivalence", "cases.json")
+ shutil.copy(cases, os.path.join(outside, "cases.json"))
+ os.remove(cases)
+ make_symlink(self, os.path.join(outside, "cases.json"), cases)
+ m = self.ws.module()
+ self.assertEqual(m["checks"][2]["state"], "gap")
+ self.assertTrue(any("is a link" in c for c in m["caveats"]))
+ self.assertNotIn("equivalence", m["evidence"])
+
+ def test_two_modules_get_two_verdicts_and_asking_about_one_still_judges_both_from_evidence(self):
+ put(self.ws.root, "modernized/s/extra/src/main/X.java", "class X {}\n", T0 + 20)
+ pack = self.ws.pack()
+ self.assertEqual([(m["name"], m["verdict"]) for m in pack["modules"]], [("extra", "NOT PROVEN"), ("mod", "PROVEN")])
+ self.assertEqual(pack["overall"]["verdict"], "NOT PROVEN")
+ self.assertEqual(pack["overall"]["counts"], {"PROVEN": 1, "PARTLY PROVEN": 0, "NOT PROVEN": 1})
+ asked = self.ws.pack("mod")
+ self.assertEqual([(m["name"], m["verdict"]) for m in asked["modules"]], [("extra", "NOT PROVEN"), ("mod", "PROVEN")])
+ self.assertEqual(asked["asked"], "mod")
+
+ def test_a_forged_earlier_verdict_never_survives_a_new_run(self):
+ put(self.ws.root, "modernized/s/extra/src/main/X.java", "class X {}\n", T0 + 20)
+ forged = pp.overall(self.ws.pack())
+ forged["modules"][0].update(verdict="PROVEN", reasons=[])
+ forged["modules"].append(dict(forged["modules"][0], name="ghost", verdict="PROVEN"))
+ forged["overall"] = {"verdict": "PROVEN", "counts": {"PROVEN": 3, "PARTLY PROVEN": 0, "NOT PROVEN": 0}}
+ adir = os.path.join(self.ws.root, "analysis", "s")
+ dump(os.path.join(adir, "VERIFICATION.json"), forged)
+ code, out, _ = run_main(pp.main, ["s", "mod", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ data = load(os.path.join(adir, "VERIFICATION.json"))
+ self.assertEqual([(m["name"], m["verdict"]) for m in data["modules"]], [("extra", "NOT PROVEN"), ("mod", "PROVEN")])
+ self.assertEqual(data["overall"]["verdict"], "NOT PROVEN")
+ self.assertNotIn("ghost", read(os.path.join(adir, "VERIFICATION.md")))
+ self.assertFalse(hasattr(pp, "merge_previous"))
+
+ def test_shared_cases_are_flagged_and_a_modules_own_folder_wins(self):
+ put(self.ws.root, "modernized/s/extra/src/main/X.java", "class X {}\n", T0 + 20)
+ self.assertTrue(any("shared by every module" in c for c in self.ws.module("mod")["caveats"]))
+ self.ws.cases("cases.json", "own", 3, base="analysis/s/equivalence/mod")
+ m = self.ws.module("mod")
+ self.assertEqual(m["evidence"]["equivalence"]["cases"], 3)
+ self.assertFalse(any("shared by every module" in c for c in m["caveats"]))
+
+ def test_a_reimagined_service_is_judged_like_a_rewritten_module(self):
+ shutil.rmtree(os.path.join(self.ws.root, "modernized", "s"))
+ put(self.ws.root, "modernized/s-reimagined/api/src/main/Api.java", "class Api {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s-reimagined/api/src/test/ApiTest.java", "// RULE-001 RULE-002\n", T0 + 20)
+ put(self.ws.root, "modernized/s-reimagined/api/target/surefire-reports/TEST-Api.xml", junit(passing(4, "ApiTest")), T0 + 30)
+ self.ws.runs(suites=[{"module": "api", "name": "acceptance", "executed": 4, "junit": ["modernized/s-reimagined/api/target/surefire-reports"]}])
+ pack = self.ws.pack()
+ m = pack["modules"][0]
+ self.assertEqual((m["name"], m["track"], len(pack["modules"])), ("api", "reimagine", 1))
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}["rules"], "pass")
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}["tests"], "pass")
+
+ def test_nothing_built_is_an_error(self):
+ shutil.rmtree(os.path.join(self.ws.root, "modernized"))
+ with self.assertRaises(pp.InputError):
+ pp.build("s", self.ws.root)
+ code, _, err = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 2)
+ self.assertIn("nothing is built", err)
+
+
+RUNNER_LOGS = (
+ ("maven", "[INFO] Tests run: 4, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.1 s -- in A\n[ERROR] Tests run: 158, Failures: 0, Errors: 3, Skipped: 2\n", (156, 3, 2)), # executed = run minus skipped
+ ("maven, old style", "Tests run: 10, Failures: 1, Errors: 0, Skipped: 1\n", (9, 1, 1)),
+ ("gradle", "12 tests completed, 2 failed, 1 skipped\n", (11, 2, 1)),
+ ("cargo", "test result: ok. 10 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s\ntest result: FAILED. 2 passed; 3 failed; 0 ignored; 0 measured\n", (15, 3, 1)),
+ ("pytest", "============ 2 failed, 10 passed, 1 skipped in 0.12s ============\n", (12, 2, 1)),
+ ("pytest -q", "3 passed, 1 error in 0.05s\n", (4, 1, 0)),
+ ("pytest none", "no tests ran in 0.01s\n", (0, 0, 0)),
+ ("unittest", "Ran 5 tests in 0.002s\n\nOK\n", (5, 0, 0)),
+ ("unittest failed", "Ran 9 tests in 0.5s\n\nFAILED (failures=1, errors=2, skipped=3)\n", (6, 3, 3)),
+ ("go -v", "=== RUN TestA\n--- PASS: TestA (0.00s)\n --- PASS: TestA/sub (0.00s)\n--- FAIL: TestB (0.01s)\n--- SKIP: TestC (0.00s)\nFAIL\n", (3, 1, 1)),
+ ("go -json", '{"Time":"t","Action":"pass","Package":"p","Test":"TestA","Elapsed":0}\n{"Time":"t","Action":"fail","Package":"p","Test":"TestB","Elapsed":0}\n{"Time":"t","Action":"fail","Package":"p","Elapsed":0}\n', (2, 1, 0)),
+ ("dotnet", "Passed! - Failed: 0, Passed: 10, Skipped: 1, Total: 11, Duration: 1 s - X.dll (net8.0)\nFailed! - Failed: 2, Passed: 8, Skipped: 0, Total: 10, Duration: 1 s - Y.dll\n", (20, 2, 1)),
+ ("jest", "Tests: 1 failed, 2 skipped, 3 passed, 6 total\nTest Suites: 1 failed, 1 total\n", (4, 1, 2)),
+ ("vitest", " Tests 1 failed | 9 passed | 1 skipped (11)\n", (10, 1, 1)),
+ ("ctest", "92% tests passed, 1 tests failed out of 12\n", (12, 1, 0)),
+ ("phpunit ok", "OK (10 tests, 25 assertions)\n", (10, 0, 0)),
+ ("phpunit failures", "FAILURES!\nTests: 10, Assertions: 25, Failures: 1, Skipped: 1.\n", (9, 1, 1)),
+)
+
+
+class RunnerLogs(unittest.TestCase):
+ def test_every_supported_runner_summary_is_read(self):
+ for name, text, want in RUNNER_LOGS:
+ got = pp.parse_runner_log(text)
+ self.assertIsNotNone(got, name)
+ self.assertEqual(got[:3], want, name)
+
+ def test_a_log_with_no_summary_line_is_no_evidence(self):
+ for text in ("", "All good, 12 tests passed, trust me\n", "BUILD SUCCESSFUL in 3s\n", "ok \tpkg\t0.003s\n", "Tests run: many, Failures: none\n", "\x00\x01\x02"):
+ self.assertIsNone(pp.parse_runner_log(text), repr(text))
+
+ def test_summaries_are_summed_across_a_log_and_the_runner_is_named(self):
+ got = pp.parse_runner_log("Tests run: 3, Failures: 0, Errors: 0, Skipped: 0\n== 2 passed in 0.1s ==\n")
+ self.assertEqual(got, (5, 0, 0, ["maven", "pytest"]))
+
+ def test_hostile_logs_are_bounded_and_never_trusted(self):
+ started = time.time()
+ for text in ("Tests run: " * 400000, "= " * 2000000 + "passed in 1s", "--- PASS: " * 300000, "{" + '"Action":"pass",' * 200000 + "}", "9" * 3000000 + " passed in 1s\n",
+ "Tests:" + " 1 failed," * 100000 + " 5 total\n", "Ran " * 500000, "\r" * 1000000):
+ pp.parse_runner_log(text)
+ self.assertLess(time.time() - started, 30)
+ self.assertEqual(pp.parse_runner_log("Tests run: 999999999999999999999, Failures: 0, Errors: 0, Skipped: 0\n")[0], 999999999999999999999)
+
+ def test_read_logs_refuses_links_huge_files_and_binary_text(self):
+ root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, root, True)
+ good = put(root, "good.txt", "== 2 passed in 0.1s ==\n")
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "real.txt", "== 99 passed in 0.1s ==\n")
+ link = os.path.join(root, "link.txt")
+ make_symlink(self, os.path.join(outside, "real.txt"), link)
+ put(root, "huge.txt", b"x" * ((16 << 20) + 5))
+ put(root, "nul.txt", b"\x00" * 100 + b"\n== 5 passed in 1s ==\n")
+ got = pp.read_logs([good, link, os.path.join(root, "huge.txt"), os.path.join(root, "nul.txt"), os.path.join(root, "missing.txt")])
+ self.assertEqual((got["files"], got["executed"]), (2, 7)) # NUL bytes elsewhere in the file do not hide a summary line; the link, the huge and the missing file are not read
+ self.assertEqual(len(got["unrecognised"]), 3)
+
+
+class BaselineEvidence(unittest.TestCase):
+ """baseline_diff.py: where the old version's measured results are looked for, and what counts as measured."""
+
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ self.base = put(self.root, "analysis/s/BASELINE.md", BASELINE_MD)
+ self.per_test = json.dumps({"A#pass1": "PASS", "A#pass2": "PASS", "A#failed": "FAIL", "A#skipped": "SKIP", "B#flip": "PASS", "B#gone": "PASS"})
+
+ def found(self, text=""):
+ return [os.path.relpath(p, os.path.join(self.root, "analysis", "s")) for p in bd.evidence_paths(self.base, BASELINE_MD + text)]
+
+ def test_per_test_json_shapes_are_read_and_counts_or_prose_are_not(self):
+ self.assertEqual(bd.parse_results_json({"A#x": "PASS", "A#y": "failed", "A#z": "Skipped"}), {"A#x": "PASS", "A#y": "FAIL", "A#z": "SKIP"})
+ self.assertEqual(bd.parse_results_json({"tests": {"A#x": "ok"}}), {"A#x": "PASS"})
+ self.assertEqual(bd.parse_results_json([{"name": "A#x", "status": "pass"}, {"test": "A#y", "result": "FAIL"}, {"id": "A#z", "outcome": "error"}, {"name": "no status"}, 5]),
+ {"A#x": "PASS", "A#y": "FAIL", "A#z": "ERROR"})
+ for bad in ({"PASS": 10, "FAIL": 1}, {"passed": 5, "failed": 0, "executed": 5}, {"tests": {"count": 3}}, {"a": "hello", "b": "world"}, [], {}, "text", 5, None, [1, 2], {"a": {"b": "PASS"}}):
+ self.assertIsNone(bd.parse_results_json(bad), bad)
+ many = bd.parse_results_json({"t%d" % i: "PASS" for i in range(1000)} | {"junk": "hello"} if sys.version_info >= (3, 9) else dict({"t%d" % i: "PASS" for i in range(1000)}, junk="hello"))
+ self.assertEqual(len(many), 1000) # one stray value in a thousand does not spoil it
+ self.assertIsNone(bd.parse_results_json({"a": "PASS", "b": "hello", "c": "world"})) # but a mostly-unrecognised map is no evidence
+
+ def test_the_baseline_folder_and_recorded_lines_are_the_only_places(self):
+ put(self.root, "analysis/s/baseline/old.json", self.per_test)
+ put(self.root, "analysis/s/harness/named.json", self.per_test)
+ put(self.root, "analysis/s/harness/prose.json", self.per_test)
+ put(self.root, "analysis/s/harness/bare.log", "x")
+ self.assertEqual(self.found(), ["baseline"])
+ self.assertEqual(self.found("\nSee `analysis/s/harness/prose.json` too.\n"), ["baseline"])
+ self.assertEqual(self.found("\n- **Recorded:** 2026-09-24. Machine-readable per-test results: `analysis/s/harness/named.json`; tool: `analysis/s/harness/prose.json`\n"),
+ ["baseline", "harness/named.json", "harness/prose.json"]) # everything on the Recorded line counts
+ self.assertEqual(self.found("\nMachine-readable: harness/bare.log\n"), ["baseline", "harness/bare.log"])
+ self.assertEqual(self.found("\n * recorded on Thursday: harness/named.json\n"), ["baseline", "harness/named.json"])
+ self.assertEqual(self.found("\nNot recorded: harness/named.json\n"), ["baseline"]) # the line must start with the label
+
+ def test_paths_that_leave_analysis_or_go_through_links_are_refused(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "o.json", self.per_test)
+ put(self.root, "analysis/other/o.json", self.per_test)
+ put(self.root, "modernized/o.json", self.per_test)
+ put(self.root, "analysis/s/inner.json", self.per_test)
+ make_symlink(self, os.path.join(outside, "o.json"), os.path.join(self.root, "analysis", "s", "link.json"))
+ make_symlink(self, outside, os.path.join(self.root, "analysis", "s", "dirlink"))
+ make_symlink(self, os.path.join(self.root, "analysis", "other"), os.path.join(self.root, "analysis", "s", "sibling"))
+ for line in ("../other/o.json", "analysis/other/o.json", "analysis/s/../other/o.json", "modernized/o.json", outside + "/o.json", "link.json", "dirlink/o.json", "sibling/o.json", "./../s/../other/o.json"):
+ self.assertEqual(self.found("\nRecorded: `%s`\n" % line), [], line)
+ self.assertEqual(self.found("\nRecorded: `analysis/s/inner.json`\n"), ["inner.json"])
+ os.rename(os.path.join(self.root, "analysis", "s", "dirlink"), os.path.join(self.root, "dirlink-moved"))
+ make_symlink(self, outside, os.path.join(self.root, "analysis", "s", "baseline"))
+ self.assertEqual(self.found(), []) # a linked baseline/ folder is not entered
+
+ def test_result_files_json_and_logs_are_all_read_and_counts_only_files_are_ignored(self):
+ put(self.root, "analysis/s/baseline/xml/TEST-A.xml", junit([("A", "one", "PASS", ""), ("A", "two", "FAIL", "")]))
+ put(self.root, "analysis/s/baseline/map.json", json.dumps({"B#x": "PASS"}))
+ put(self.root, "analysis/s/baseline/counts.json", json.dumps({"PASS": 9}))
+ put(self.root, "analysis/s/baseline/bad.json", "{not json")
+ put(self.root, "analysis/s/baseline/notes.txt", "nothing that looks like a runner summary")
+ got = bd.read_baseline_evidence(bd.evidence_paths(self.base, ""))
+ self.assertEqual(got["tests"], {"A#one": "PASS", "A#two": "FAIL", "B#x": "PASS"})
+ self.assertEqual((got["measured"], got["conflicts"], got["ignored"]), (True, [], ["counts.json"]))
+ self.assertEqual([x["kind"] for x in got["sources"]], ["result files", "machine-readable results"])
+ self.assertTrue(any("bad.json" in u for u in got["unreadable"]))
+ only_log = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, only_log, True)
+ put(only_log, "run.log", "[INFO] Tests run: 5, Failures: 1, Errors: 0, Skipped: 0\n")
+ got = bd.read_baseline_evidence([only_log])
+ self.assertEqual((got["tests"], got["counts"], got["measured"]), (None, {"executed": 5, "failed": 1, "skipped": 0}, True))
+ nothing = bd.read_baseline_evidence([os.path.join(self.root, "analysis", "s", "baseline", "counts.json")])
+ self.assertEqual((nothing["measured"], nothing["tests"], nothing["counts"]), (False, None, None))
+
+ def test_sources_that_disagree_are_conflicts_not_first_wins(self):
+ put(self.root, "analysis/s/baseline/a.json", json.dumps({"A#x": "PASS", "A#y": "PASS"}))
+ put(self.root, "analysis/s/baseline/b.json", json.dumps({"A#x": "FAIL", "A#y": "PASS", "A#z": "PASS"}))
+ got = bd.read_baseline_evidence(bd.evidence_paths(self.base, ""))
+ self.assertEqual(len(got["conflicts"]), 1)
+ self.assertIn("A#x", got["conflicts"][0])
+ self.assertEqual(got["tests"]["A#x"], "PASS") # kept only so the report can show something; the conflict is what counts
+ info = bd.assess_baseline(bd.parse_baseline("| Test | Result |\n|---|---|\n| `A#x` | PASS |\n"), got)
+ self.assertEqual((info["kind"], info["conflict"]), ("measured", 1))
+ put(self.root, "analysis/s/baseline/b.json", json.dumps({"A#x": "ERROR", "A#y": "PASS"}))
+ put(self.root, "analysis/s/baseline/a.json", json.dumps({"A#x": "FAIL"}))
+ self.assertEqual(bd.read_baseline_evidence(bd.evidence_paths(self.base, ""))["conflicts"], []) # FAIL and ERROR are both red
+
+ def test_assess_says_typed_measured_disagreeing_or_target_only(self):
+ typed = bd.parse_baseline(BASELINE_MD)
+ self.assertEqual(bd.assess_baseline(typed, None)["kind"], "typed")
+ self.assertEqual(bd.assess_baseline(bd.parse_baseline("target-only: no old runtime\n"), {"measured": True, "tests": {"a": "PASS"}, "counts": None, "sources": []})["kind"], "target-only")
+ self.assertEqual(bd.assess_baseline(bd.parse_baseline("nothing here"), None)["kind"], "none")
+ good = {"measured": True, "tests": {"A#pass1": "PASS", "A#pass2": "PASS", "A#failed": "FAIL", "A#skipped": "SKIP", "B#flip": "PASS", "B#gone": "PASS"}, "counts": None,
+ "sources": [{"kind": "machine-readable results", "name": "x.json"}], "conflicts": []}
+ info = bd.assess_baseline(typed, good)
+ self.assertEqual((info["kind"], info["disagree"], info["measuredTests"], info["measuredExecuted"], info["measuredFailed"]), ("measured", 0, 6, 5, 1))
+ bad = dict(good, tests=dict(good["tests"], **{"A#pass1": "FAIL"}))
+ self.assertEqual(bd.assess_baseline(typed, bad)["disagree"], 1)
+ missing = dict(good, tests={k: v for k, v in good["tests"].items() if k != "B#gone"})
+ self.assertEqual(bd.assess_baseline(typed, missing)["disagree"], 2) # the missing row, and the Totals table that counts it
+
+ def test_the_cli_can_be_given_the_evidence_and_says_typed_only_otherwise(self):
+ folder = os.path.join(self.root, "res")
+ put(folder, "TEST-x.xml", junit([("A", "pass1", "PASS", ""), ("A", "pass2", "PASS", ""), ("A", "failed", "FAIL", ""), ("A", "skipped", "SKIP", ""), ("B", "flip", "PASS", ""), ("B", "gone", "PASS", "")]))
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder])
+ self.assertIn("typed table only", out)
+ put(self.root, "analysis/s/baseline/old.json", self.per_test)
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder])
+ self.assertIn("baseline evidence: measured (machine-readable results old.json)", out)
+ evidence = os.path.join(self.root, "elsewhere.json")
+ put(self.root, "elsewhere.json", self.per_test)
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder, "--baseline-evidence", evidence, "--json"])
+ self.assertEqual(json.loads(out)["baseline"]["assessment"]["kind"], "measured")
+
+
+class UpliftChecks(unittest.TestCase):
+ """uplift_checks.py: the test files of the untouched tree against the working copy, and the deltas no test names."""
+
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ self.old, self.new = os.path.join(self.root, "old"), os.path.join(self.root, "new")
+
+ def both(self, rel, old, new=None, kind="w"):
+ put(self.old, rel, old)
+ put(self.new, rel, old if new is None else new)
+
+ def test_which_files_count_as_tests_and_what_is_skipped(self):
+ for rel in ("src/test/java/AThing.java", "tests/b.py", "web/__tests__/c.js", "spec/d.rb", "src/FooTest.java", "pkg/test_e.py", "pkg/e_test.go", "web/f.spec.ts", "src/test/resources/data.json"):
+ put(self.old, rel, "x")
+ for rel in ("src/main/A.java", "README.md", "src/test/notes.txt", "target/test/X.java", "node_modules/y/test/Z.js", ".hidden/test/H.java", "src/test/.dotfile"):
+ put(self.old, rel, "x")
+ got, cut, links = uc.walk_tests(self.old)
+ self.assertEqual(sorted(got), ["pkg/e_test.go", "pkg/test_e.py", "spec/d.rb", "src/FooTest.java", "src/test/java/AThing.java", "src/test/resources/data.json", "tests/b.py",
+ "web/__tests__/c.js", "web/f.spec.ts"])
+ self.assertEqual((cut, links), (False, 0))
+ self.assertEqual(uc.walk_tests(os.path.join(self.root, "missing")), ({}, False, 0))
+
+ def test_removed_added_and_changed_with_line_counts(self):
+ self.both("t/keep/AKeepTest.java", "one\ntwo\nthree\n")
+ self.both("t/edit/AEditTest.java", "one\ntwo\nthree\nfour\n", "one\nTWO\nthree\nfour\nfive\n")
+ put(self.old, "t/gone/AGoneTest.java", "x\ny\n")
+ put(self.new, "t/new/ANewTest.java", "a\nb\nc\n")
+ got = uc.tests_changed(self.old, self.new)
+ self.assertEqual((got["checked"], got["legacyTests"], got["workTests"], got["counts"]), (True, 3, 3, {"added": 1, "removed": 1, "changed": 1}))
+ self.assertEqual(got["changed"], [{"path": "t/edit/AEditTest.java", "added": 2, "removed": 1, "note": ""}])
+ self.assertEqual((got["removed"], got["added"]), ([{"path": "t/gone/AGoneTest.java", "lines": 3}], [{"path": "t/new/ANewTest.java", "lines": 4}]))
+ self.assertTrue(got["gap"]) # a removed file
+ self.assertEqual(round(got["share"], 3), round(1 / 3, 3))
+
+ def test_a_quarter_changed_is_not_a_gap_and_more_is_and_line_endings_are_ignored(self):
+ for i in range(4):
+ self.both("t/T%dTest.java" % i, "line %d\r\nsecond\r\n" % i)
+ put(self.new, "t/T0Test.java", "line 0\nsecond\n") # only the line endings differ
+ self.assertEqual(uc.tests_changed(self.old, self.new)["counts"]["changed"], 0)
+ put(self.new, "t/T1Test.java", "line 1 edited\nsecond\n")
+ got = uc.tests_changed(self.old, self.new)
+ self.assertEqual((got["counts"]["changed"], got["gap"], got["share"]), (1, False, 0.25))
+ put(self.new, "t/T2Test.java", "line 2 edited\nsecond\n")
+ self.assertTrue(uc.tests_changed(self.old, self.new)["gap"])
+
+ def test_no_legacy_tests_is_a_gap_and_so_is_a_missing_tree(self):
+ put(self.new, "t/ANewTest.java", "x")
+ put(self.old, "src/main/A.java", "the legacy tree exists but holds no test file")
+ got = uc.tests_changed(self.old, self.new)
+ self.assertEqual((got["checked"], got["legacyTests"], got["gap"]), (True, 0, True))
+ self.assertIn("nothing shows that the working copy's tests were not written during the uplift", got["why"])
+ for old, new in ((os.path.join(self.root, "none"), self.new), ("", self.new), (self.old, os.path.join(self.root, "none")), (self.old, "")):
+ got = uc.tests_changed(old, new)
+ self.assertEqual(got["checked"], False, (old, new))
+ self.assertTrue(got["why"])
+
+ def test_binary_large_and_unreadable_files_are_compared_without_crashing(self):
+ self.both("t/data/a.bin.Test.dat", b"\x00\x01\x02", b"\x00\x01\x03")
+ self.both("t/big/BigTest.java", "x\n" * (1 << 20), "x\n" * (1 << 20) + "y\n")
+ self.both("t/same/SameBigTest.java", "z\n" * (1 << 20))
+ self.both("t/lines/ManyTest.java", "".join("line %d\n" % i for i in range(9000)), "".join("line %d\n" % i for i in range(1, 9001)))
+ got = uc.tests_changed(self.old, self.new)
+ notes = {c["path"]: c["note"] for c in got["changed"]}
+ self.assertEqual(notes["t/data/a.bin.Test.dat"], "binary file")
+ self.assertEqual(notes["t/big/BigTest.java"], "large file, lines not counted")
+ self.assertEqual(notes["t/lines/ManyTest.java"], "lines counted approximately")
+ self.assertNotIn("t/same/SameBigTest.java", notes)
+ many = next(c for c in got["changed"] if c["path"] == "t/lines/ManyTest.java")
+ self.assertEqual((many["added"], many["removed"]), (1, 1))
+
+ def test_links_are_never_followed_and_hostile_trees_stay_bounded(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "OutTest.java", "outside")
+ self.both("t/InTest.java", "in")
+ make_symlink(self, os.path.join(outside, "OutTest.java"), os.path.join(self.new, "t", "LinkTest.java"))
+ make_symlink(self, outside, os.path.join(self.new, "t", "linked"))
+ got = uc.tests_changed(self.old, self.new)
+ self.assertEqual((got["workTests"], got["counts"]), (1, {"added": 0, "removed": 0, "changed": 0}))
+ self.assertNotIn("Out", json.dumps(got))
+ for i in range(300):
+ self.both("t/many/M%03dTest.java" % i, "a\n" * 50, "a\n" * 50 + "edit %d\n" % i)
+ started = time.time()
+ got = uc.tests_changed(self.old, self.new)
+ self.assertLess(time.time() - started, 30)
+ self.assertEqual((got["counts"]["changed"], len(got["changed"])), (300, 40)) # the count is complete, the list is cut
+ self.assertNotRegex(json.dumps(uc.tests_changed(self.old, self.new)), "\x1b")
+
+ def test_a_hostile_file_name_is_cleaned(self):
+ self.both("t/ev\x1b[31mil\nTest.java", "a", "b")
+ got = uc.tests_changed(self.old, self.new)
+ self.assertNotRegex(got["changed"][0]["path"], r"[\x00-\x1f]")
+
+ CARDS = """# DELTA_CATALOG: x
+
+| ID | Category | Fix | Site | Old to new |
+|---|---|---|---|---|
+| **D-01** | Behavioral-silent | Judgment | `a/b/Alpha.java:916` | changed |
+| **D-02** | Behavioral-silent (pre-existing bug, wider on 17) | Judgment | `Beta.java` (`decode`) | changed |
+| **D-03** | Project-system | Judgment | root `pom.xml:629` | build |
+
+| # | Category | Fix | Conf. | Delta | Site | Sites | Note |
+|---|---|---|---|---|---|---|---|
+| 9 | Behavioral-silent | Mechanical | High | CLDR listing dates | `x/y/Resource.java:574` | 1 | Gamma.java is mentioned in prose only |
+| 10 | Dependency | Mechanical | High | JaCoCo agent | `pom.xml:614` | 3 | n |
+
+## Card detail
+
+### 9. CLDR listing dates
+- **Category / fix / confidence:** Behavioral-silent / Mechanical / High; sites: 1; cited: `x/y/Resource.java:574`
+- **Blast radius:** One site. Sites: Delta.java:12, and Epsilon.java:40 also matter.
+
+### 11. A card that only exists as a card
+- **Category / fix / confidence:** Behaviour-Silent / Judgment / Low; sites: 2; cited: `p/Zeta.py:3`, `p/Eta.py:9`
+
+### 12. A numbered heading with no category line
+- Some prose about Theta.java:1.
+
+### D-04 Another id style
+- **Category:** behavioral silent
+- **Site:** `q/Iota.cs:5`
+"""
+
+ def test_deltas_are_read_from_tables_and_cards_and_merged(self):
+ got = {d["id"]: d for d in uc.parse_deltas(self.CARDS)}
+ self.assertEqual(sorted(got), ["11", "9", "10", "D-01", "D-02", "D-03", "D-04"] if False else sorted(["11", "9", "10", "D-01", "D-02", "D-03", "D-04"]))
+ self.assertEqual([s["stem"] for s in got["D-01"]["sites"]], ["Alpha"])
+ self.assertEqual(got["D-01"]["sites"][0]["line"], 916)
+ self.assertEqual([s["stem"] for s in got["D-02"]["sites"]], ["Beta"]) # a site without a line, with a known extension
+ self.assertEqual(got["D-02"]["category"], "Behavioral-silent (pre-existing bug, wider on 17)")
+ self.assertEqual([s["stem"] for s in got["9"]["sites"]], ["Resource"]) # the table and the card agree; prose files are not sites
+ self.assertEqual(got["9"]["title"], "CLDR listing dates")
+ self.assertEqual([s["stem"] for s in got["11"]["sites"]], ["Zeta", "Eta"])
+ self.assertEqual(got["11"]["category"], "Behaviour-Silent")
+ self.assertEqual(got["D-04"]["category"], "behavioral silent")
+ self.assertEqual([s["stem"] for s in got["D-04"]["sites"]], ["Iota"])
+ self.assertNotIn("12", got)
+
+ def test_the_parser_is_tolerant_and_bounded(self):
+ for text in ("", "no tables at all", "| a | b |\n|---|---|\n| 1 | 2 |\n", "| Category |\n|---|\n| Behavioral-silent |\n", "### 1. x\n" * 10, "|" * 100000, "#" * 100000,
+ "### 3. t\n- **Category:** " + "Behavioral-silent " * 100000 + "\n", "| # | Category | Site |\n|---|---|---|\n| 1 | Behavioral-silent | " + "a/" * 100000 + "B.java:1 |\n"):
+ started = time.time()
+ uc.parse_deltas(text)
+ self.assertLess(time.time() - started, 10)
+ many = "| # | Category | Site |\n|---|---|---|\n" + "".join("| %d | Behavioral-silent | `S%d.java:1` |\n" % (i, i) for i in range(6000))
+ self.assertLessEqual(len(uc.parse_deltas(many)), uc.DELTA_CAP)
+ weird = uc.parse_deltas("| # | Category | Site |\n|---|---|---|\n| 1\x1b[31m | Behavioral-silent\x00 | `Ev\x1bil.java:1` |\n")
+ self.assertNotRegex(json.dumps(weird), r"\\u001b|\\u0000")
+
+ def test_coverage_names_sites_as_whole_words_in_test_files_only(self):
+ put(self.new, "src/main/Alpha.java", "class Alpha {}")
+ put(self.new, "src/test/AlphaTest.java", "class AlphaTest { Alpha a; AlphaBeta x; Gamma_ y; }")
+ put(self.new, "src/main/Beta.java", "class Beta { Delta d; }")
+ put(self.new, "src/test/notes.txt", "Delta is discussed here")
+ cat = ("| # | Category | Site |\n|---|---|---|\n| 1 | Behavioral-silent | `src/main/Alpha.java:1` |\n| 2 | Behavioral-silent | `src/main/Beta.java:2` |\n"
+ "| 3 | Behavioral-silent | `src/main/Gamma.java:3` |\n| 4 | Behavioral-silent | `src/main/Delta.java:4` |\n| 5 | Dependency | `src/main/Other.java:5` |\n"
+ "| 6 | Behavioral-silent | `src/main/Alpha.java:1`, `src/main/Beta.java:2` |\n| 7 | Behavioral-silent | no site here |\n| 8 | Behavioral-silent | `pom.xml:9` |\n")
+ got = uc.delta_coverage(cat, self.new)
+ self.assertEqual((got["parsed"], got["silent"], got["other"], got["testFiles"]), (8, 7, 1, 1))
+ self.assertEqual([d["id"] for d in got["covered"]], ["1"])
+ self.assertEqual({d["id"]: d["why"] for d in got["uncovered"]}, {"2": "no test names Beta", "3": "no test names Gamma", "4": "no test names Delta", "6": "no test names Beta", "7": "no site is named in the catalog"})
+ self.assertEqual([d["id"] for d in got["config"]], ["8"])
+ self.assertTrue(got["gap"])
+
+ def test_no_catalog_or_an_unreadable_one_is_a_gap(self):
+ self.assertTrue(uc.delta_coverage(None, self.new)["gap"])
+ got = uc.delta_coverage("Nothing to read here.", self.new)
+ self.assertEqual((got["parsed"], got["gap"]), (0, True))
+ ok = uc.delta_coverage("| # | Category | Site |\n|---|---|---|\n| 1 | API-removed | `A.java:1` |\n", self.new)
+ self.assertEqual((ok["silent"], ok["gap"]), (0, False))
+
+ def test_the_command_line_prints_a_summary_json_and_the_exit_code(self):
+ self.both("t/InTest.java", "Alpha")
+ put(self.new, "src/main/Alpha.java", "class Alpha {}")
+ cat = put(self.root, "CAT.md", "| # | Category | Site |\n|---|---|---|\n| 1 | Behavioral-silent | `src/main/Alpha.java:1` |\n")
+ code, out, _ = run_main(uc.main, [self.old, self.new, cat])
+ self.assertEqual(code, 0)
+ self.assertIn("tests kept: ok", out)
+ self.assertIn("deltas covered: ok", out)
+ code, out, _ = run_main(uc.main, [self.old, self.new, cat, "--json"])
+ self.assertEqual(json.loads(out)["deltas"]["silent"], 1)
+ os.remove(os.path.join(self.new, "t", "InTest.java"))
+ code, out, _ = run_main(uc.main, [self.old, self.new])
+ self.assertEqual(code, 1)
+ self.assertIn("removed: t/InTest.java", out)
+ self.assertEqual(run_main(uc.main, [os.path.join(self.root, "none"), self.new])[0], 2)
+ self.assertEqual(run_main(uc.main, [self.old, self.new, os.path.join(self.root, "none.md")])[0], 2)
+
+
+class ProofOutputs(unittest.TestCase):
+ def setUp(self):
+ self.ws = Ws(self)
+ self.adir = os.path.join(self.ws.root, "analysis", "s")
+
+ def test_files_summary_and_exit_codes(self):
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 0)
+ self.assertIn("s: PROVEN", out)
+ data = load(os.path.join(self.adir, "VERIFICATION.json"))
+ self.assertEqual((data["v"], data["overall"]["verdict"], data["signoff"]), (1, "PROVEN", {"name": "", "role": "", "date": "", "decision": ""}))
+ md = read(os.path.join(self.adir, "VERIFICATION.md"))
+ for needle in ("# Verification: s", "**Overall: PROVEN**", "### P0 business rules", "### What this does not prove", "## Sign-off", "________________",
+ "## How each verdict is computed", "tests executed: 3, failed 0, skipped 0", "- [ ] A business owner accepts the report output"):
+ self.assertIn(needle, md)
+ os.remove(os.path.join(self.adir, "equivalence", "test-runs.json"))
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ self.assertIn("NOT PROVEN", out)
+ self.assertEqual(sorted(f for f in os.listdir(self.adir) if f.startswith("VERIFICATION")), ["VERIFICATION.json", "VERIFICATION.md"])
+
+ def test_bad_names_are_refused(self):
+ for argv in (["..", "--workspace", self.ws.root], ["a/b"], ["s", "../x"], ["s", "a\\b"], [""]):
+ self.assertEqual(run_main(pp.main, argv)[0], 2, argv)
+
+ def test_a_linked_output_is_refused_and_nothing_is_written_through_it(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "real.md", "ORIGINAL")
+ make_symlink(self, os.path.join(outside, "real.md"), os.path.join(self.adir, "VERIFICATION.md"))
+ code, _, err = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 2)
+ self.assertIn("symbolic link", err)
+ self.assertEqual(read(os.path.join(outside, "real.md")), "ORIGINAL")
+
+ def test_hostile_names_and_reasons_stay_text_in_the_markdown(self):
+ put(self.ws.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n", T0 + 20)
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("Rounding", "Round | <img src=x onerror=1> \x1b[31m"), T0 + 10)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit | `x`", "command": "mvn\x00 test\n# heading", "executed": 3, "junit": ["modernized/s/mod/target/surefire-reports"]}])
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ md = read(os.path.join(self.adir, "VERIFICATION.md"))
+ self.assertNotRegex(md, r"[\x00\x1b]")
+ self.assertIn("Round \\| <img src=x onerror=1>", md)
+ self.assertNotIn("\n# heading", md)
+
+ def test_the_verdict_comes_from_the_evidence_not_from_an_earlier_pack(self):
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ path = os.path.join(self.adir, "VERIFICATION.json")
+ data = load(path)
+ data["modules"][0]["verdict"] = "NOT PROVEN"
+ dump(path, data)
+ self.assertEqual(run_main(pp.main, ["s", "--workspace", self.ws.root])[0], 0)
+ self.assertEqual(load(path)["overall"]["verdict"], "PROVEN")
+
+ def test_a_huge_evidence_file_is_not_read(self):
+ put(self.ws.root, "analysis/s/equivalence/test-runs.json", b" " * (9 << 20), T0 + 40)
+ pack = self.ws.pack()
+ self.assertTrue(any("larger than 8 MB" in p for p in pack["problems"]))
+ self.assertEqual(pack["modules"][0]["verdict"], "NOT PROVEN")
+
+
+class ProofUplift(unittest.TestCase):
+ BASE = "# BASELINE\n\n## Per-test results\n\n| Test | Result |\n|---|---|\n| `T#a` | PASS |\n| `T#b` | PASS |\n| `T#c` | FAIL |\n| `T#d` | SKIP |\n"
+ CATALOG = ("# DELTA_CATALOG: s\n\n| # | Category | Fix | Delta | Site |\n|---|---|---|---|---|\n| 1 | Behavioral-silent | Judgment | Locale default changed | `unit/src/Main.java:12` |\n"
+ "| 2 | API-removed | Mechanical | Old API gone | `unit/src/Other.java:3` |\n")
+
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ put(self.root, "legacy/s/pom.xml", "<project/>", T0)
+ put(self.root, "legacy/s/unit/src/test/MainTest.java", "class MainTest { Main m; }\n", T0)
+ put(self.root, "analysis/s/PREFLIGHT.md", "x", T0 + 10)
+ put(self.root, "analysis/s/BASELINE.md", self.BASE, T0 + 10)
+ put(self.root, "analysis/s/baseline/results.json", json.dumps({"T#a": "PASS", "T#b": "PASS", "T#c": "FAIL", "T#d": "SKIP"}), T0 + 5)
+ put(self.root, "analysis/s/DELTA_CATALOG.md", self.CATALOG, T0 + 10)
+ put(self.root, "modernized/s-uplifted/unit/src/Main.java", "class Main {}\n", T0 + 20)
+ put(self.root, "modernized/s-uplifted/unit/src/test/MainTest.java", "class MainTest { Main m; }\n", T0 + 20)
+ put(self.root, "modernized/s-uplifted/UPLIFT_NOTES.md", "Canary: break a comparison -> 2 tests failed\n", T0 + 20)
+ put(self.root, "analysis/s/equivalence/canary/TEST-C.xml", junit([("T", "a", "FAIL", ""), ("T", "b", "FAIL", "")]), T0 + 35)
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ put(self.root, "analysis/s/equivalence/test-runs.json", json.dumps({"legacy": {"ran": True}, "suites": [
+ {"module": "unit", "name": "unit", "junit": ["modernized/s-uplifted/unit/target/surefire-reports"], "executed": 3, "failed": 1, "skipped": 1}],
+ "canaries": [{"change": "break a comparison", "junit": ["analysis/s/equivalence/canary"]}]}), T0 + 40)
+ rows = []
+ for i in range(10):
+ put(self.root, "analysis/s/equivalence/fresh/old/%d.out" % i, "v%d\n" % i, T0 + 25)
+ put(self.root, "analysis/s/equivalence/fresh/new/%d.out" % i, "v%d\n" % i, T0 + 25)
+ rows.append({"id": "f%d" % i, "legacy": "fresh/old/%d.out" % i, "new": "fresh/new/%d.out" % i})
+ put(self.root, "analysis/s/equivalence/fresh-cases.json", json.dumps({"cases": rows}), T0 + 25)
+
+ def results(self, cases):
+ put(self.root, "modernized/s-uplifted/unit/target/surefire-reports/TEST-T.xml", junit(cases), T0 + 30)
+
+ def module(self):
+ pack = pp.overall(pp.build("s", self.root, now="x"))
+ self.assertEqual(len(pack["modules"]), 1)
+ return pack["modules"][0]
+
+ def test_no_regression_against_the_baseline_is_proven_and_traces_no_rules(self):
+ m = self.module()
+ self.assertEqual((m["verdict"], m["track"], m["name"]), ("PROVEN", "uplift", "s-uplifted"))
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}["rules"], "na")
+ self.assertEqual(m["evidence"]["baseline"]["regressionsCount"], 0)
+
+ def test_a_regression_or_a_new_failure_is_not_proven(self):
+ self.results([("T", "a", "FAIL", ""), ("T", "b", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["verdict"], "NOT PROVEN")
+ self.assertIn("1 test(s) passed before and fail now", m["reasons"][0])
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", ""), ("T", "new", "FAIL", "")])
+ self.assertEqual(self.module()["verdict"], "NOT PROVEN")
+
+ def test_missing_tests_and_skipped_growth_are_partly_proven(self):
+ self.results([("T", "a", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["verdict"], "PARTLY PROVEN")
+ self.assertTrue(any("did not run now" in r for r in m["reasons"]))
+ self.results([("T", "a", "PASS", ""), ("T", "b", "SKIP", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ self.assertEqual(self.module()["verdict"], "PARTLY PROVEN")
+
+ def test_an_approved_baseline_difference_is_allowed_and_listed_as_a_decision(self):
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n## Approved differences\n\n| Test | Reason |\n|---|---|\n| `T#b` | the new runtime words the message differently |\n", T0 + 10)
+ self.results([("T", "a", "PASS", ""), ("T", "b", "FAIL", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["verdict"], "PROVEN")
+ self.assertTrue(any("approved by a person in BASELINE.md" in c and "T#b" in c for c in m["caveats"]))
+ self.assertTrue(any("decision, not evidence" in n for n in m["notProven"]))
+ self.assertEqual(m["evidence"]["baseline"]["approved"][0]["id"], "T#b")
+
+ def test_a_canary_keyed_by_a_unit_of_the_uplifted_copy_is_accepted(self):
+ dump(os.path.join(self.root, "analysis", "s", "equivalence", "test-runs.json"), {"legacy": {"ran": True}, "suites": [
+ {"module": "unit", "name": "unit", "junit": ["modernized/s-uplifted/unit/target/surefire-reports"]}],
+ "canaries": [{"module": "unit", "change": "break a comparison", "junit": ["analysis/s/equivalence/canary"]}]})
+ m = self.module()
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}["canary"], "pass")
+
+ def test_a_test_that_now_passes_needs_a_person_but_does_not_change_the_verdict(self):
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "PASS", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["verdict"], "PROVEN")
+ self.assertTrue(any("intended fix" in n for n in m["needsPerson"]))
+
+ def test_target_only_and_a_missing_baseline_cannot_be_proven(self):
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "PASS", "")])
+ put(self.root, "analysis/s/BASELINE.md", "target-only: Java 8 is not installed here\n", T0 + 10)
+ m = self.module()
+ self.assertEqual(m["verdict"], "PARTLY PROVEN")
+ self.assertIn("target-only", " ".join(m["reasons"]))
+ self.assertFalse(m["evidence"]["legacy"]["ran"])
+ os.remove(os.path.join(self.root, "analysis", "s", "BASELINE.md"))
+ m = self.module()
+ self.assertEqual(m["verdict"], "PARTLY PROVEN")
+ self.assertIn("BASELINE.md is missing", " ".join(m["reasons"]))
+
+ def states(self):
+ return {c["id"]: c["state"] for c in self.module()["checks"]}
+
+ def test_all_nine_checks_pass_for_a_clean_uplift(self):
+ m = self.module()
+ self.assertEqual(self.states(), {"tests": "pass", "rules": "na", "same": "pass", "fresh": "pass", "canary": "pass", "source": "pass", "baseline": "pass", "kept": "pass", "deltas": "pass"})
+ self.assertEqual(m["evidence"]["baselineEvidence"]["kind"], "measured")
+ self.assertEqual(m["evidence"]["deltas"]["silent"], 1)
+
+ # -- (1) a measured baseline
+ def test_a_baseline_typed_by_hand_is_a_gap_and_says_so_in_plain_words(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ m = self.module()
+ self.assertEqual((m["verdict"], self.states()["baseline"], self.states()["same"]), ("PARTLY PROVEN", "gap", "pass"))
+ detail = next(c["detail"] for c in m["checks"] if c["id"] == "baseline")
+ self.assertIn("typed by hand", detail)
+ self.assertIn("analysis/<system>/baseline/", detail)
+ self.assertIn("Recorded:", detail)
+ self.assertIn("### Baseline evidence", pp.render_md(pp.overall(pp.build("s", self.root, now="x"))))
+
+ def test_a_forged_counts_file_is_not_evidence(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ put(self.root, "analysis/s/baseline/counts.json", json.dumps({"PASS": 2, "FAIL": 1, "SKIP": 1, "executed": 3}), T0 + 5)
+ put(self.root, "analysis/s/baseline/counts2.json", json.dumps({"tests": {"count": 3}}), T0 + 5)
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "gap")
+ info = m["evidence"]["baselineEvidence"]
+ self.assertEqual(info["kind"], "typed")
+ self.assertIn("counts.json", info["ignored"])
+ self.assertIn("Files that only hold counts were ignored", next(c["detail"] for c in m["checks"] if c["id"] == "baseline"))
+
+ def test_only_the_baseline_folder_and_recorded_lines_are_looked_at(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ put(self.root, "analysis/s/harness/stray.json", json.dumps({"T#a": "PASS", "T#b": "PASS", "T#c": "FAIL", "T#d": "SKIP"}), T0 + 5)
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\nSee also `analysis/s/harness/stray.json` for the details.\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "gap") # a path in prose is not evidence
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n- **Recorded:** 2026-09-24. Machine-readable per-test results: `analysis/s/harness/stray.json`\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "pass")
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\nMachine-readable: harness/stray.json\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "pass") # relative to analysis/<system>/
+
+ def test_evidence_outside_analysis_or_through_a_link_is_never_used(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ full = json.dumps({"T#a": "PASS", "T#b": "PASS", "T#c": "FAIL", "T#d": "SKIP"})
+ put(outside, "results.json", full)
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ put(self.root, "analysis/other/results.json", full, T0 + 5)
+ put(self.root, "modernized/results.json", full, T0 + 5)
+ make_symlink(self, os.path.join(outside, "results.json"), os.path.join(self.root, "analysis", "s", "baseline", "link.json"))
+ make_symlink(self, outside, os.path.join(self.root, "analysis", "s", "linked"))
+ for line in ("Recorded: `../other/results.json`", "Recorded: `analysis/other/results.json`", "Recorded: `analysis/s/../other/results.json`", "Recorded: `modernized/results.json`",
+ "Recorded: `%s/results.json`" % outside, "Recorded: `analysis/s/linked/results.json`", "Recorded: `analysis/s/baseline/link.json`"):
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n" + line + "\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "gap", line)
+
+ def test_baseline_sources_that_disagree_are_a_gap_not_first_one_wins(self):
+ put(self.root, "analysis/s/baseline/second.json", json.dumps({"T#a": "FAIL", "T#b": "PASS"}), T0 + 5)
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "gap")
+ detail = next(c["detail"] for c in m["checks"] if c["id"] == "baseline")
+ self.assertIn("disagree with each other", detail)
+ self.assertIn("T#a: PASS in one source, FAIL in second.json", detail) # the example names the test and the file
+ put(self.root, "analysis/s/baseline/second.json", json.dumps({"T#a": "PASS"}), T0 + 5) # agrees on what they share
+ self.assertEqual(self.states()["baseline"], "pass")
+ put(self.root, "analysis/s/baseline/old.test-output.txt", "Tests run: 9, Failures: 0, Errors: 0, Skipped: 0\n", T0 + 5) # a log with other totals
+ self.assertEqual(self.states()["baseline"], "gap")
+
+ def test_a_typed_table_that_disagrees_with_its_evidence_is_a_gap(self):
+ put(self.root, "analysis/s/baseline/results.json", json.dumps({"T#a": "PASS", "T#b": "FAIL", "T#c": "FAIL", "T#d": "SKIP"}), T0 + 5)
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "gap")
+ self.assertIn("disagree on 1 test", next(c["detail"] for c in m["checks"] if c["id"] == "baseline"))
+ self.assertEqual(m["evidence"]["baselineEvidence"]["disagree"], 1)
+
+ def test_a_raw_runner_log_can_measure_the_baseline_and_must_match_the_typed_counts(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ put(self.root, "analysis/s/baseline/old.test-output.txt", "[INFO] Tests run: 4, Failures: 1, Errors: 0, Skipped: 1\n", T0 + 5) # 3 executed, 1 failed, 1 skipped
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "pass")
+ self.assertIn("runner log", next(c["detail"] for c in m["checks"] if c["id"] == "baseline"))
+ put(self.root, "analysis/s/baseline/old.test-output.txt", "[INFO] Tests run: 4, Failures: 0, Errors: 0, Skipped: 1\n", T0 + 5)
+ self.assertEqual(self.states()["baseline"], "gap")
+
+ def test_target_only_and_a_missing_baseline_leave_the_measured_check_not_applicable(self):
+ put(self.root, "analysis/s/BASELINE.md", "target-only: Java 8 is not installed here\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "na")
+ os.remove(os.path.join(self.root, "analysis", "s", "BASELINE.md"))
+ self.assertEqual((self.states()["baseline"], self.states()["same"]), ("na", "gap"))
+
+ def test_measured_per_test_results_are_what_the_new_run_is_compared_with(self):
+ put(self.root, "analysis/s/baseline/results.json", json.dumps({"T#a": "PASS", "T#b": "PASS", "T#c": "FAIL", "T#d": "SKIP", "T#extra": "PASS"}), T0 + 5)
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["evidence"]["baseline"]["source"], "measured results")
+ self.assertEqual(self.states()["same"], "gap") # T#extra was measured on the old version and did not run now
+ self.assertEqual(self.states()["baseline"], "pass") # the typed table lists fewer tests than were measured, and agrees on all it lists
+ self.assertTrue(any("did not run now" in r for r in m["reasons"]))
+
+ def test_a_totals_table_typed_beside_the_rows_must_match_the_evidence_too(self):
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n## Totals\n\n| | Count |\n|---|---|\n| Executed | 50 |\n| Failed | 0 |\n", T0 + 10)
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "gap")
+ self.assertIn("Totals table says 50 executed", " ".join(m["evidence"]["baselineEvidence"]["examples"]))
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n## Totals\n\n| | Count |\n|---|---|\n| Executed | 3 |\n| Failed | 1 |\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "pass")
+
+ # -- (2) tests changed during the uplift
+ def test_a_removed_legacy_test_file_is_a_gap_and_the_list_says_which(self):
+ os.remove(os.path.join(self.root, "modernized", "s-uplifted", "unit", "src", "test", "MainTest.java"))
+ m = self.module()
+ self.assertEqual((m["verdict"], self.states()["kept"]), ("PARTLY PROVEN", "gap"))
+ detail = next(c["detail"] for c in m["checks"] if c["id"] == "kept")
+ self.assertIn("MainTest.java", detail)
+ self.assertIn("edited or deleted", detail)
+ self.assertEqual(m["evidence"]["testsChanged"]["removed"][0]["path"], "unit/src/test/MainTest.java")
+ self.assertTrue(any("Review the test files" in n for n in m["needsPerson"]))
+
+ def test_more_than_a_quarter_of_the_test_files_changed_is_a_gap_and_a_quarter_is_not(self):
+ for i in range(3):
+ put(self.root, "legacy/s/unit/src/test/T%d.java" % i, "class T%d { int a = 1; }\n" % i, T0)
+ put(self.root, "modernized/s-uplifted/unit/src/test/T%d.java" % i, "class T%d { int a = 1; }\n" % i, T0 + 20)
+ put(self.root, "modernized/s-uplifted/unit/src/test/T0.java", "class T0 { int a = 2; }\n", T0 + 20) # 1 of 4 changed: exactly a quarter
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "pass")
+ self.assertEqual(m["evidence"]["testsChanged"]["changed"][0], {"path": "unit/src/test/T0.java", "added": 1, "removed": 1, "note": ""})
+ self.assertTrue(any("1 changed" in n for n in m["needsPerson"]))
+ put(self.root, "modernized/s-uplifted/unit/src/test/T1.java", "class T1 { int a = 2; }\n", T0 + 20) # 2 of 4
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "gap")
+ self.assertIn("2 of 4 legacy test files (50%) were changed", next(c["detail"] for c in m["checks"] if c["id"] == "kept"))
+
+ def test_no_legacy_tests_or_no_legacy_tree_is_a_gap(self):
+ os.remove(os.path.join(self.root, "legacy", "s", "unit", "src", "test", "MainTest.java"))
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "gap")
+ self.assertIn("nothing shows that the working copy's tests were not written during the uplift", next(c["detail"] for c in m["checks"] if c["id"] == "kept"))
+ shutil.rmtree(os.path.join(self.root, "legacy"))
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "gap")
+ self.assertIn("legacy tree was not found", next(c["detail"] for c in m["checks"] if c["id"] == "kept"))
+
+ def test_added_tests_are_listed_and_never_a_gap(self):
+ put(self.root, "modernized/s-uplifted/unit/src/test/NewCharacterizationTest.java", "class NewCharacterizationTest { Main m; }\n", T0 + 20)
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "pass")
+ self.assertEqual(m["evidence"]["testsChanged"]["added"], [{"path": "unit/src/test/NewCharacterizationTest.java", "lines": 2}])
+ md = pp.render_md(pp.overall(pp.build("s", self.root, now="x")))
+ self.assertIn("### Test files changed during the uplift", md)
+ self.assertIn("Added: `unit/src/test/NewCharacterizationTest.java`", md)
+ self.assertIn("Weakened assertions cannot be detected", md)
+ self.assertTrue(any("Weakened assertions cannot be detected" in n for n in m["notProven"]))
+
+ # -- (3) delta coverage
+ def test_a_silent_delta_no_test_names_is_a_gap_and_other_categories_are_only_listed(self):
+ put(self.root, "analysis/s/DELTA_CATALOG.md", self.CATALOG + "| 3 | Behavioral-silent | Judgment | Rounding changed | `unit/src/Money.java:40` |\n", T0 + 10)
+ m = self.module()
+ self.assertEqual((m["verdict"], self.states()["deltas"]), ("PARTLY PROVEN", "gap"))
+ detail = next(c["detail"] for c in m["checks"] if c["id"] == "deltas")
+ self.assertIn("1 of 2 Behavioral-silent", detail)
+ self.assertIn("no test names Money", detail)
+ self.assertEqual(m["evidence"]["deltas"]["uncovered"][0]["id"], "3")
+ self.assertIn("Not named by any test: 3 Rounding changed", pp.render_md(pp.overall(pp.build("s", self.root, now="x"))))
+ put(self.root, "modernized/s-uplifted/unit/src/test/MoneyTest.java", "class MoneyTest { Money m; }\n", T0 + 20)
+ self.assertEqual(self.states()["deltas"], "pass")
+
+ def test_a_missing_or_unreadable_catalog_is_a_gap(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "DELTA_CATALOG.md"))
+ m = self.module()
+ self.assertEqual(self.states()["deltas"], "gap")
+ self.assertIn("DELTA_CATALOG.md was not found", next(c["detail"] for c in m["checks"] if c["id"] == "deltas"))
+ put(self.root, "analysis/s/DELTA_CATALOG.md", "Some prose about the version change. No table, no cards.\n", T0 + 10)
+ self.assertEqual(self.states()["deltas"], "gap")
+
+ def test_a_delta_at_a_build_file_is_listed_for_a_person_not_judged(self):
+ put(self.root, "analysis/s/DELTA_CATALOG.md", self.CATALOG + "| 4 | Behavioral-silent | Judgment | Bundle plugin | `pom.xml:629` |\n", T0 + 10)
+ m = self.module()
+ self.assertEqual(self.states()["deltas"], "pass")
+ self.assertEqual(m["evidence"]["deltas"]["config"][0]["id"], "4")
+ self.assertTrue(any("build or configuration file" in n for n in m["needsPerson"]))
+
+ def test_a_unit_argument_narrows_the_suites_and_an_unknown_name_is_an_error(self):
+ pack = pp.build("s", self.root, "unit", now="x")
+ self.assertEqual(pack["modules"][0]["name"], "s-uplifted")
+ self.assertEqual(pack["asked"], "unit")
+ with self.assertRaises(pp.InputError):
+ pp.build("s", self.root, "nosuchunit")
+
+
+# ---------------------------------------------------------------- the Proof section of the report
+def page_data(html):
+ return json.loads(re.search(r'<script type="application/json" id="report-data">(.*?)</script>', html, re.S).group(1))
+
+
+def good_pack(**module_change):
+ checks = [{"id": cid, "label": label, "state": "pass", "detail": "fine"} for cid, label in br.PROOF_CHECKS]
+ m = {"name": "mod", "track": "rewrite", "verdict": "PROVEN", "checks": checks, "verifiedAt": "2026-09-24 00:00 UTC",
+ "evidence": {"tests": {"executed": 5, "failed": 0, "skipped": 0},
+ "rules": {"p0": [{"id": "RULE-001", "name": "Total", "confidence": "High", "status": "tested", "tests": 3, "main": 1, "where": "src/test/A.java:9"}]},
+ "fresh": {"inputs": 12, "executed": 20, "same": 20, "differs": 0, "missing": 0}, "equivalence": {"cases": 4, "executed": 4, "same": 4, "differs": 0, "missing": 0, "approved": [], "masks": []}},
+ "notProven": ["It does not prove inputs nobody tried."], "brief": {"found": True, "items": [{"text": "A person accepts it", "where": "Phase 1 exit criterion"}], "count": 1}}
+ m.update(module_change)
+ return {"v": 1, "system": "s", "generated": "2026-09-24 00:00 UTC", "problems": [], "legacy": {"path": "legacy/s", "ran": True}, "modules": [m], "signoff": {}, "rules": ["PROVEN needs all six checks."]}
+
+
+def good_uplift_pack(**evidence):
+ """A pack for an uplift whose nine checks all pass, with the evidence that backs them; a test changes the evidence and looks at the view."""
+ pack = good_pack()
+ m = pack["modules"][0]
+ m.update(name="s-uplifted", track="uplift")
+ m["checks"] = [c for c in m["checks"] if c["id"] != "rules"] + [{"id": "rules", "label": "Rules traced", "state": "na", "detail": "An uplift keeps the code."}] + [
+ {"id": cid, "label": label, "state": "pass", "detail": "fine"} for cid, label in br.UPLIFT_CHECKS]
+ m["evidence"].pop("rules", None)
+ m["evidence"].update(baselineEvidence={"kind": "measured", "sources": [{"kind": "machine-readable results", "name": "base.json", "tests": 40}], "disagree": 0, "conflict": 0, "examples": [], "considered": ["base.json"]},
+ testsChanged={"checked": True, "why": "", "legacyTests": 8, "workTests": 9, "counts": {"added": 1, "removed": 0, "changed": 1}, "share": 0.125,
+ "removed": [], "added": [{"path": "t/NewTest.java", "lines": 12}], "changed": [{"path": "t/OldTest.java", "added": 3, "removed": 1, "note": ""}]},
+ deltas={"parsed": 5, "silent": 2, "covered": [{"id": "1"}, {"id": "2"}], "uncovered": [], "config": [{"id": "4", "title": "Bundle plugin", "sites": ["pom.xml:9"]}], "other": 3, "gap": False})
+ m["evidence"].update(evidence)
+ return pack
+
+
+class ProofReport(unittest.TestCase):
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ put(self.root, "analysis/s/PREFLIGHT.md", "# Preflight\n")
+ self.out = os.path.join(self.root, "out", "REPORT.html")
+
+ def report(self, pack):
+ put(self.root, "analysis/s/VERIFICATION.json", pack if isinstance(pack, (str, bytes)) else json.dumps(pack, indent=1)) # like proof_pack.py: short lines
+ code, _, err = run_main(br.main, ["s", "--workspace", self.root, "--out", self.out])
+ self.assertEqual(code, 0, err)
+ with open(self.out, encoding="utf-8") as fh:
+ return fh.read()
+
+ def proof(self, html):
+ data = page_data(html)
+ return data, next((p["proof"] for s in data["sections"] if s["id"] == "proof" for p in s["parts"] if p["t"] == "proof"), None)
+
+ def test_the_section_the_banner_and_the_order(self):
+ put(self.root, "analysis/s/EQUIVALENCE.json", "{}")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "# n")
+ data, pv = self.proof(self.report(good_pack()))
+ self.assertEqual([s["id"] for s in data["sections"]][-3:], ["build", "proof", "equivalence"])
+ self.assertEqual((pv["verdict"], pv["counts"]["PROVEN"], pv["modules"][0]["p0Counts"]["tested"]), ("PROVEN", 1, 1))
+ self.assertEqual(data["glance"]["proof"]["verdict"], "PROVEN")
+ self.assertEqual(data["glance"]["proof"]["modules"], [{"name": "mod", "verdict": "PROVEN"}])
+ self.assertEqual(pv["signoff"], {"name": "", "role": "", "date": "", "decision": ""})
+ self.assertEqual(pv["modules"][0]["people"][0]["text"], "A person accepts it")
+
+ def test_a_rule_named_only_by_tests_that_did_not_run_is_counted_and_tooling_folders_are_listed(self):
+ pack = good_pack()
+ pack["modules"][0]["evidence"]["rules"]["p0"].append({"id": "RULE-002", "name": "Skipped", "confidence": "High", "status": "named, not run", "tests": 1, "main": 0, "where": "T.java:3"})
+ pack["toolingOnly"] = [{"name": "parity-harness", "path": "modernized/s/parity-harness"}, "not a dict", {"name": "x" * 500, "path": 5}]
+ html = self.report(pack)
+ _, pv = self.proof(html)
+ self.assertEqual(pv["modules"][0]["p0Counts"], {"tested": 1, "claimedOnly": 0, "notRun": 1, "none": 0})
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN") # a pass cannot stand next to a rule that no test ran
+ self.assertEqual([t["name"] for t in pv["tooling"]][0], "parity-harness")
+ self.assertEqual(len(pv["tooling"]), 2)
+ self.assertLessEqual(len(pv["tooling"][1]["name"]), 120)
+ self.assertEqual(self.proof(self.report(good_pack()))[1]["tooling"], [])
+
+ def test_no_pack_means_no_section_and_the_older_reports_are_unchanged(self):
+ code, _, _ = run_main(br.main, ["s", "--workspace", self.root, "--out", self.out])
+ data = page_data(read(self.out))
+ self.assertIsNone(data["glance"]["proof"])
+ self.assertNotIn("proof", [s["id"] for s in data["sections"]])
+
+ def test_the_verdict_is_recomputed_from_the_checks_never_taken_from_the_file(self):
+ pack = good_pack()
+ pack["modules"][0]["checks"][3]["state"] = "gap"
+ _, pv = self.proof(self.report(pack))
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN")
+ self.assertTrue(any("recorded PROVEN" in p and "PARTLY PROVEN" in p for p in pv["problems"]))
+ pack["modules"][0]["checks"][0]["state"] = "fail"
+ pack["modules"][0]["verdict"] = "PROVEN"
+ data, pv = self.proof(self.report(pack))
+ self.assertEqual((pv["verdict"], data["glance"]["proof"]["verdict"]), ("NOT PROVEN", "NOT PROVEN"))
+ self.assertEqual(pv["modules"][0]["reasons"][0], "Tests ran: fine")
+
+ def test_a_pack_that_contradicts_its_own_numbers_is_downgraded(self):
+ for change, check, want in (({"evidence": {"tests": {"executed": 3, "failed": 2}}}, 0, "fail"),
+ ({"evidence": {"tests": {"executed": 3}, "equivalence": {"differs": 1}, "fresh": {"inputs": 12}}}, 2, "fail"),
+ ({"evidence": {"tests": {"executed": 3}, "fresh": {"inputs": 12, "missing": 1}}}, 3, "fail"),
+ ({"evidence": {"tests": {"executed": 0}}}, 0, "fail"),
+ ({"evidence": {"tests": {"executed": 3}, "rules": {"p0": [{"id": "R", "status": "none"}]}}}, 1, "gap"),
+ ({"evidence": {"tests": {"executed": 3}, "fresh": {"inputs": 3}}}, 3, "gap"),
+ ({"evidence": {"tests": {"executed": 3}}}, 3, "gap")):
+ _, pv = self.proof(self.report(good_pack(**change)))
+ self.assertEqual(pv["modules"][0]["checks"][check]["state"], want, change)
+ self.assertNotEqual(pv["modules"][0]["verdict"], "PROVEN")
+ pack = good_pack()
+ pack["modules"][0]["checks"] = pack["modules"][0]["checks"][:5]
+ _, pv = self.proof(self.report(pack))
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN")
+ self.assertIn("missing from VERIFICATION.json", pv["modules"][0]["reasons"][0])
+ self.assertEqual(self.proof(self.report(dict(good_pack(), modules=[])))[1]["verdict"], "NOT PROVEN")
+
+ def test_an_uplift_needs_its_three_extra_checks_and_a_rewrite_does_not(self):
+ _, pv = self.proof(self.report(good_uplift_pack()))
+ m = pv["modules"][0]
+ self.assertEqual((m["verdict"], [c["id"] for c in m["checks"]]), ("PROVEN", ["tests", "rules", "same", "fresh", "canary", "source", "baseline", "kept", "deltas"]))
+ self.assertEqual(m["uplift"]["testsChanged"]["counts"], {"added": 1, "removed": 0, "changed": 1})
+ self.assertEqual(m["uplift"]["deltas"]["coveredCount"], 2)
+ pack = good_uplift_pack()
+ pack["modules"][0]["checks"] = [c for c in pack["modules"][0]["checks"] if c["id"] != "kept"]
+ _, pv = self.proof(self.report(pack))
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN")
+ self.assertIn("tests kept check is missing", " ".join(pv["problems"]))
+ _, pv = self.proof(self.report(good_pack())) # a rewrite never needs them
+ self.assertEqual((pv["modules"][0]["verdict"], len(pv["modules"][0]["checks"]), pv["modules"][0]["uplift"]), ("PROVEN", 6, None))
+
+ def test_forged_uplift_passes_are_downgraded_by_the_evidence_beside_them(self):
+ cases = (("baseline", {"baselineEvidence": {"kind": "typed"}}), ("baseline", {"baselineEvidence": {"kind": "measured", "disagree": 2}}), ("baseline", {"baselineEvidence": {"kind": "measured", "conflict": 1}}),
+ ("kept", {"testsChanged": {"checked": False}}), ("kept", {"testsChanged": {"checked": True, "legacyTests": 0, "counts": {}}}),
+ ("kept", {"testsChanged": {"checked": True, "legacyTests": 8, "counts": {"removed": 1, "changed": 0, "added": 0}}}),
+ ("kept", {"testsChanged": {"checked": True, "legacyTests": 8, "counts": {"removed": 0, "changed": 3, "added": 0}}}),
+ ("deltas", {"deltas": {"parsed": 4, "silent": 1, "covered": [], "uncovered": [{"id": "9"}]}}), ("deltas", {"deltas": {"parsed": 0}}), ("deltas", {"deltas": "nope"}))
+ for cid, change in cases:
+ _, pv = self.proof(self.report(good_uplift_pack(**change)))
+ m = pv["modules"][0]
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}[cid], "gap", (cid, change))
+ self.assertEqual(m["verdict"], "PARTLY PROVEN", (cid, change))
+ for state in ("na", "gap"): # "not applicable" is no way out for the kept and deltas checks
+ pack = good_uplift_pack(testsChanged={"checked": False}, deltas={"parsed": 0})
+ for c in pack["modules"][0]["checks"]:
+ if c["id"] in ("kept", "deltas"):
+ c["state"] = state
+ self.assertEqual(self.proof(self.report(pack))[1]["modules"][0]["verdict"], "PARTLY PROVEN")
+ pack = good_uplift_pack(baselineEvidence={"kind": "target-only"})
+ for c in pack["modules"][0]["checks"]:
+ if c["id"] == "baseline":
+ c["state"] = "na"
+ self.assertEqual(self.proof(self.report(pack))[1]["modules"][0]["checks"][6]["state"], "na") # a target-only baseline may say so
+
+ def test_the_uplift_evidence_is_cut_and_cleaned(self):
+ evil = "<script>alert(1)</script>" + chr(0x2028)
+ _, pv = self.proof(self.report(good_uplift_pack(
+ baselineEvidence={"kind": evil, "sources": [{"kind": evil * 5, "name": evil}] * 50, "disagree": "x", "examples": [evil * 20] * 20, "considered": [evil] * 50},
+ testsChanged={"checked": True, "legacyTests": 8, "workTests": 8, "counts": {"added": 1, "removed": 0, "changed": 0}, "share": "x", "added": [{"path": evil * 20, "lines": "x"}] * 300, "removed": "x", "changed": [1, None]},
+ deltas={"parsed": 3, "silent": 1, "covered": [], "uncovered": [{"id": evil * 5, "title": evil * 10, "why": evil * 10, "sites": [evil] * 20}] * 300, "config": "x"})))
+ up = pv["modules"][0]["uplift"]
+ self.assertEqual(up["baseline"]["kind"], "none")
+ self.assertLessEqual((len(up["baseline"]["sources"]), len(up["baseline"]["examples"]), len(up["testsChanged"]["added"]), len(up["deltas"]["uncovered"])), (10, 5, 40, 40))
+ self.assertEqual((up["testsChanged"]["share"], up["testsChanged"]["added"][0]["lines"], up["testsChanged"]["removed"], up["deltas"]["config"]), (0.0, 0, [], []))
+ self.assertGreater(up["deltas"]["uncoveredCount"], 0)
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN")
+
+ def test_a_real_uplift_pack_from_proof_pack_carries_its_evidence_to_the_report(self):
+ base = ProofUplift("test_all_nine_checks_pass_for_a_clean_uplift")
+ base.setUp()
+ base.addCleanup(shutil.rmtree, base.root, True)
+ put(base.root, "modernized/s-uplifted/unit/src/test/NewCharacterizationTest.java", "class NewCharacterizationTest { Main m; }\n", T0 + 20)
+ run_main(pp.main, ["s", "--workspace", base.root])
+ run_main(br.main, ["s", "--workspace", base.root])
+ data = page_data(read(os.path.join(base.root, "analysis", "s", "REPORT.html")))
+ pv = next(p["proof"] for sec in data["sections"] if sec["id"] == "proof" for p in sec["parts"])
+ m = pv["modules"][0]
+ self.assertEqual((pv["verdict"], len(m["checks"])), ("PROVEN", 9))
+ self.assertEqual(m["uplift"]["testsChanged"]["added"][0]["path"], "unit/src/test/NewCharacterizationTest.java")
+ self.assertEqual(m["uplift"]["baseline"]["kind"], "measured")
+ self.assertEqual(m["uplift"]["deltas"]["silent"], 1)
+
+ def test_a_real_pack_written_by_proof_pack_renders_the_same_verdict(self):
+ ws = Ws(self)
+ put(ws.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n", T0 + 20)
+ run_main(pp.main, ["s", "--workspace", ws.root])
+ code, _, _ = run_main(br.main, ["s", "--workspace", ws.root])
+ data = page_data(read(os.path.join(ws.root, "analysis", "s", "REPORT.html")))
+ mod = data["glance"]["proof"]["modules"][0]
+ self.assertEqual(mod, {"name": "mod", "verdict": "PARTLY PROVEN"})
+ pv = next(p["proof"] for s in data["sections"] if s["id"] == "proof" for p in s["parts"])
+ self.assertEqual(pv["modules"][0]["p0Counts"], {"tested": 1, "claimedOnly": 0, "notRun": 0, "none": 1})
+ self.assertTrue(pv["modules"][0]["reasons"][0].startswith("Rules traced: 1 of 2 P0"))
+
+ def test_hostile_values_stay_data_and_the_page_keeps_its_rules(self):
+ evil = "<script>window.__pwned=1</script></script><img src=x onerror=1> &amp; " + chr(0x2028)
+ pack = good_pack(name=evil * 5, track=evil, notProven=[evil * 20] * 60, verdict=["PROVEN"], checks="nope",
+ evidence={"tests": {"executed": "many", "failed": -1, "skipped": 10 ** 30}, "rules": {"p0": [evil, {"id": evil, "status": evil, "tests": evil}] * 150},
+ "fresh": [1], "equivalence": {"approved": evil, "masks": [evil]}, "baseline": 5, "canary": [evil]},
+ brief={"items": [{"text": evil * 9, "where": evil}] * 100, "count": "x", "found": 1})
+ pack.update(problems=[evil] * 20, signoff="x", legacy=[], rules="x")
+ html = self.report(pack)
+ block = re.search(r'<script type="application/json" id="report-data">(.*?)</script>', html, re.S).group(1)
+ for bad in ("<", ">", "&", chr(0x2028)):
+ self.assertNotIn(bad, block)
+ data, pv = self.proof(html)
+ m = pv["modules"][0]
+ self.assertLessEqual(len(m["name"]), 120)
+ self.assertEqual((m["tests"], m["verdict"]), ({"executed": 0, "failed": 0, "skipped": 0}, "PARTLY PROVEN")) # no usable checks: nothing is proven
+ self.assertLessEqual(len(m["p0"]), 200)
+ self.assertLessEqual(len(m["people"]), 40)
+ self.assertLessEqual(len(pv["problems"]), 10 + 1)
+ self.assertEqual(html.count("<script"), 2)
+ self.assertNotRegex(html.replace(block, "").split('<script id="app">')[0].split("</style>")[1], r"__pwned|onerror=")
+ policy = re.search(r'http-equiv="Content-Security-Policy" content="([^"]*)"', html).group(1)
+ self.assertIn("default-src 'none'", policy)
+
+ def test_malformed_or_foreign_verification_files_are_reported_not_raised(self):
+ for text in ("[1, 2", "{}", '{"modules": "x"}', "[]", "null", b"\x00\x01"):
+ data = page_data(self.report(text))
+ self.assertEqual(data["glance"]["proof"]["verdict"], "NOT PROVEN" if text != "[1, 2" else "NOT PROVEN", text)
+ html = self.report("[1, 2")
+ self.assertTrue(any("VERIFICATION.json could not be parsed" in json.dumps(p) for s in page_data(html)["sections"] for p in s["parts"]))
+
+ def test_a_linked_verification_file_is_not_read(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "V.json", json.dumps(good_pack()))
+ make_symlink(self, os.path.join(outside, "V.json"), os.path.join(self.root, "analysis", "s", "VERIFICATION.json"))
+ code, _, _ = run_main(br.main, ["s", "--workspace", self.root, "--out", self.out])
+ data = page_data(read(self.out))
+ self.assertIsNone(data["glance"]["proof"])
+
+
+NODE_PROOF = r"""
+const fs = require('fs'), vm = require('vm');
+const tpl = fs.readFileSync(process.env.TEMPLATE, 'utf8');
+const grab = function (re) { return re.exec(tpl)[1] !== undefined ? re.exec(tpl)[1] : re.exec(tpl)[0]; };
+const core = /\/\* core:start[^\n]*\n([\s\S]*?)\/\* core:end \*\//.exec(tpl)[1];
+const proof = /\/\* proof:start[^\n]*\n([\s\S]*?)\/\* proof:end \*\//.exec(tpl)[1];
+const helpers = /function fmt\(n\)[^\n]*\n/.exec(tpl)[0] + /function chip\([^\n]*\n/.exec(tpl)[0];
+class Text { constructor(t) { this.text = String(t); } }
+class El { constructor(t) { this.tag = t; this.attrs = {}; this.kids = []; }
+ setAttribute(k, v) { this.attrs[k] = String(v); } appendChild(c) { this.kids.push(c); return c; } }
+const ctx = { document: { createElement: t => new El(t), createTextNode: t => new Text(t) } };
+vm.createContext(ctx);
+const api = vm.runInContext('(function(){"use strict";' + core + helpers + proof + ';return {partProof:partProof,own:own};})()', ctx);
+function walk(n, acc, top) {
+ if (n instanceof Text) { acc.text += n.text + '\n'; return; }
+ acc.tags.add(n.tag); Object.keys(n.attrs).forEach(k => { acc.attrs.add(k); });
+ if (top && n.tag === 'div' && /banner/.test(n.attrs.class || '')) acc.banners.push(n.attrs.class);
+ n.kids.forEach(c => walk(c, acc, false));
+}
+const input = JSON.parse(fs.readFileSync(0, 'utf8')), out = [];
+for (const pv of input) {
+ const t0 = Date.now(), acc = { tags: new Set(), attrs: new Set(), text: '', banners: [] };
+ api.partProof(pv).forEach(n => walk(n, acc, true));
+ out.push({ ms: Date.now() - t0, tags: [...acc.tags], attrs: [...acc.attrs], text: acc.text, banners: acc.banners });
+}
+out.push({ own: [api.own({a: 'x'}, 'a', 'd'), api.own({a: 'x'}, '__proto__', 'd'), api.own({a: 'x'}, 'constructor', 'd'), api.own({a: 'x'}, 'toString', 'd')] });
+process.stdout.write(JSON.stringify(out));
+"""
+
+
+@unittest.skipUnless(NODE, "node is not installed")
+class ProofScript(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ evil = "<script>alert(1)</script></script><img src=x onerror=alert(2)> [x](javascript:alert(3))"
+ packs = [good_pack(),
+ good_pack(checks=[{"id": "tests", "label": "Tests ran", "state": "gap", "detail": "x"}]),
+ good_pack(checks=[{"id": "tests", "label": "Tests ran", "state": "fail", "detail": "x"}]),
+ good_pack(name=evil, track=evil, notProven=[evil], brief={"found": True, "count": 900, "items": [{"text": evil, "where": evil}]},
+ evidence={"tests": {"executed": 1}, "rules": {"p0": [{"id": evil, "name": evil, "status": "__proto__", "tests": 1, "main": 1, "where": evil}] * 200}}),
+ dict(good_pack(), legacy={"path": evil, "target": evil, "ran": False}, problems=[evil], signoff={"name": evil, "role": "", "date": "", "decision": ""})]
+ packs.append(good_uplift_pack(deltas={"parsed": 5, "silent": 2, "covered": [{"id": "1"}], "uncovered": [{"id": "2", "title": "Rounding changed", "why": "no test names Money", "sites": ["Money.java:4"]}],
+ "config": [{"id": "4", "title": "Bundle plugin", "sites": ["pom.xml:9"]}]},
+ testsChanged={"checked": True, "legacyTests": 8, "workTests": 8, "counts": {"added": 0, "removed": 1, "changed": 0}, "share": 0.0,
+ "removed": [{"path": "t/GoneTest.java", "lines": 7}], "added": [], "changed": []},
+ baselineEvidence={"kind": "typed", "sources": [], "disagree": 0, "conflict": 0, "examples": [], "considered": [], "ignored": ["counts.json"]}))
+ views = [br.proof_view(p) for p in packs]
+ views[3]["verdict"] = "__proto__"
+ views[3]["modules"][0]["verdict"] = "constructor"
+ proc = subprocess.run([NODE, "-e", NODE_PROOF], input=json.dumps(views), capture_output=True, text=True, env=dict(os.environ, TEMPLATE=TEMPLATE), timeout=120)
+ assert proc.returncode == 0, proc.stderr
+ cls.out, cls.evil = json.loads(proc.stdout), evil
+
+ def test_only_safe_elements_and_attributes_are_built(self):
+ allowed_tags = {"div", "table", "thead", "tbody", "tr", "th", "td", "span", "p", "h3", "h4", "ul", "li", "details", "summary"}
+ allowed_attrs = {"class", "role", "tabindex", "aria-label", "scope"}
+ for doc in self.out[:-1]:
+ self.assertLessEqual(set(doc["tags"]), allowed_tags)
+ self.assertLessEqual(set(doc["attrs"]), allowed_attrs)
+ self.assertLess(doc["ms"], 3000)
+
+ def test_hostile_text_is_shown_as_text_never_as_markup_or_a_link(self):
+ doc = self.out[3]
+ self.assertIn("<script>alert(1)</script>", doc["text"])
+ self.assertIn("[x](javascript:alert(3))", doc["text"])
+ self.assertNotIn("a", doc["tags"])
+ self.assertNotIn("img", doc["tags"])
+
+ def test_the_banner_says_the_verdict_in_words_symbols_and_colour(self):
+ self.assertIn("✓ Proof: 1 PROVEN", self.out[0]["text"])
+ self.assertEqual(self.out[0]["banners"], ["banner ok", "banner ok"])
+ self.assertEqual(self.out[1]["banners"], ["banner warn", "banner warn"])
+ self.assertEqual(self.out[2]["banners"], ["banner bad", "banner bad"])
+ self.assertIn("◐ ", self.out[1]["text"])
+ self.assertIn("✗ ", self.out[2]["text"])
+
+ def test_unknown_verdicts_and_prototype_names_fall_back_to_red(self):
+ self.assertEqual(self.out[3]["banners"], ["banner bad", "banner bad"])
+ self.assertEqual(self.out[-1]["own"], ["x", "d", "d", "d"])
+
+ def test_the_page_shows_the_matrix_what_is_not_proven_and_a_blank_sign_off(self):
+ text = self.out[0]["text"]
+ for needle in ("What this does not prove", "It does not prove inputs nobody tried.", "P0 business rules: 1 tested, 0 named but not run, 0 claimed only, 0 with no test", "RULE-001",
+ "Waiting for a person", "A person accepts it", "Sign-off", "________________", "accept / accept with conditions / reject", "How the verdict is computed",
+ "Fresh inputs: 12 new input(s)", "tests executed: 5"):
+ self.assertIn(needle, text)
+ self.assertIn("It could not run here, so the proof is trace-based.", self.out[4]["text"])
+
+ def test_an_uplift_shows_its_baseline_test_files_and_deltas(self):
+ text = self.out[5]["text"]
+ for needle in ("Baseline evidence: typed by hand", "Test files: 8 in the legacy tree, 8 in the working copy: 1 removed, 0 added, 0 changed", "Removed test files", "t/GoneTest.java (7 lines)",
+ "2 Behavioral-silent: 1 have a test naming their site, 1 do not", "Not named by any test", "2 Rounding changed (no test names Money)",
+ "At a build or configuration file (for a person)", "4 Bundle plugin (pom.xml:9)", "Baseline measured", "Tests kept", "Deltas covered"):
+ self.assertIn(needle, text)
+ self.assertNotIn("Baseline evidence: typed by hand", self.out[0]["text"])
+
+ def test_the_template_keeps_its_own_contract(self):
+ with open(TEMPLATE, encoding="utf-8") as fh:
+ text = fh.read()
+ app = re.search(r'<script id="app">(.*?)</script>', text, re.S).group(1)
+ self.assertEqual(len(re.findall(r"/\* proof:start", app)), 1)
+ self.assertNotRegex(app, r"innerHTML|outerHTML|insertAdjacentHTML|document\.write|eval\(|new Function|fetch\(|XMLHttpRequest")
+ self.assertNotRegex(text, r"https?://")
+ self.assertIn("case 'proof': return partProof(p.proof);", app)
diff --git a/plugins/code-modernization/scripts/tests/test_report_next.py b/plugins/code-modernization/scripts/tests/test_report_next.py
new file mode 100644
index 00000000..e73536ba
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_report_next.py
@@ -0,0 +1,43 @@
+"""Tests for the report's next-step decision (scripts/build_report.py next_step)."""
+import os
+import sys
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.join(os.path.dirname(HERE)))
+import build_report as br # noqa: E402
+
+ALL = dict(preflight=True, assess=True, map=True, rules=True, brief=True, build=True, harden=True)
+
+
+class NextStep(unittest.TestCase):
+ def cmd(self, done, glance=None):
+ return br.next_step("sys", dict(ALL, **done), glance or {})["command"]
+
+ def test_the_path_in_order(self):
+ order = ["preflight", "assess", "map", "rules"]
+ for i, key in enumerate(order):
+ done = {k: False for k in order[i:]}
+ done.update(brief=False, build=False, harden=False)
+ self.assertTrue(self.cmd(done).startswith("/code-modernization:modernize-" + ["preflight", "assess", "map", "extract-rules"][i]))
+
+ def test_flagged_rules_go_to_review_once_and_then_on(self):
+ flagged = {"rules": {"defects": 2}}
+ self.assertIn("modernize-review", self.cmd({"brief": False}, flagged))
+ reviewed = {"rules": {"defects": 2, "reviews": {"confirmed": 1}}}
+ self.assertIn("modernize-brief", self.cmd({"brief": False}, reviewed))
+
+ def test_a_build_is_verified_before_anything_else(self):
+ self.assertIn("modernize-verify", self.cmd({"harden": False}))
+ self.assertIn("modernize-verify", self.cmd({"harden": False}, {"proof": {"verdict": "NOT PROVEN"}}))
+ self.assertIn("modernize-harden", self.cmd({"harden": False}, {"proof": {"verdict": "PROVEN"}}))
+
+ def test_everything_done_points_at_status_and_text_is_bounded(self):
+ step = br.next_step("s", ALL, {"proof": {"verdict": "PROVEN"}})
+ self.assertIn("modernize-status", step["command"])
+ hostile = br.next_step("s", ALL, {"proof": {"verdict": "X" * 5000}})
+ self.assertLess(len(hostile["why"]), 200)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_report_tools.py b/plugins/code-modernization/scripts/tests/test_report_tools.py
new file mode 100644
index 00000000..7d3acec4
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_report_tools.py
@@ -0,0 +1,791 @@
+"""Tests for scripts/build_report.py, scripts/compare.py and assets/report-template.html.
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+The template's JavaScript is exercised with Node when it is installed (those tests are skipped otherwise).
+"""
+import base64
+import contextlib
+import hashlib
+import io
+import json
+import os
+import re
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+import unittest
+from html.parser import HTMLParser
+from unittest import mock
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import build_report as br # noqa: E402
+import compare as cmp # noqa: E402
+
+TEMPLATE = os.path.join(PLUGIN, "assets", "report-template.html")
+MERMAID = os.path.join(PLUGIN, "assets", "vendor", "mermaid.min.js")
+SECRET = "TOP-SECRET-MARKER-9f3c"
+LS, PS = chr(0x2028), chr(0x2029)
+
+
+def put(root, rel, data):
+ path = os.path.join(root, *rel.split("/"))
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "wb") as fh:
+ fh.write(data if isinstance(data, bytes) else data.encode("utf-8"))
+ return path
+
+
+def run_main(fn, argv):
+ out, err = io.StringIO(), io.StringIO()
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+ code = fn(argv)
+ return code, out.getvalue(), err.getvalue()
+
+
+def make_symlink(test, target, link):
+ try:
+ os.symlink(target, link)
+ except (OSError, NotImplementedError, AttributeError):
+ test.skipTest("symbolic links are not available here")
+
+
+class Scripts(HTMLParser):
+ """Every <script> element of a page: its attributes and its text."""
+
+ def __init__(self, html):
+ super().__init__(convert_charrefs=False)
+ self.found, self._cur = [], None
+ self.feed(html)
+
+ def handle_starttag(self, tag, attrs):
+ if tag == "script":
+ self._cur = {"attrs": dict(attrs), "text": ""}
+
+ def handle_data(self, data):
+ if self._cur is not None:
+ self._cur["text"] += data
+
+ def handle_endtag(self, tag):
+ if tag == "script" and self._cur is not None:
+ self.found.append(self._cur)
+ self._cur = None
+
+
+def page_data(html):
+ block = [s for s in Scripts(html).found if s["attrs"].get("id") == "report-data"]
+ assert len(block) == 1
+ return json.loads(block[0]["text"])
+
+
+HOSTILE_MD = "\n".join([
+ "# Assessment <script>window.__pwned=1</script>",
+ '<img src=x onerror="window.__pwned=2">',
+ "</script><script>window.__pwned=3</script> <!-- and <!--",
+ "[a](javascript:window.__pwned=4) [b](JavaScript:alert(1)) [c](java\tscript:alert(1)) [d](" + chr(1) + "javascript:alert(1))",
+ "[e](data:text/html,<script>1</script>) [f](//evil.example/x) [g](/\\evil.example) ![t](https://evil.example/pixel.png)",
+ '<a href="javascript:alert(1)">raw anchor</a> <iframe src="https://evil.example"></iframe>',
+ "| a | b |", "|---|---|", "| <b onmouseover=alert(1)>x</b> | <img src=x onerror=window.__pwned=5> |",
+ "`<script>` and **<img src=x onerror=1>** " + LS + PS,
+ "```mermaid", 'graph TD; A["<img src=x onerror=window.__pwned=7>"]-->B', "```",
+ "x" * 1000000, "*" * 200000, "[" * 50000, "|" * 50000,
+])
+HOSTILE_RULES = "\n".join([
+ "# Rules", "",
+ "### RULE-001: Title with `backticks` and <angle> <script>x</script> & ampersand",
+ "**Category:** Validation ", "**Priority:** P0 ", "**Source:** `evil/../../etc/passwd:1-2` and `a.cbl:10` ",
+ "**Confidence:** High ", "**Suspected defect:** <img src=x onerror=window.__pwned=6>", "",
+ "### P1-002 \u00b7 Second card, other id style", "P1 \u00b7 Policy \u00b7 confidence Low \u00b7 `b.cbl:5-9`", "",
+])
+
+
+def hostile_workspace(root, system="evil"):
+ a = "analysis/" + system
+ put(root, a + "/ASSESSMENT.md", HOSTILE_MD)
+ put(root, a + "/BUSINESS_RULES.md", HOSTILE_RULES)
+ put(root, a + "/MODERNIZATION_BRIEF.md", "# Brief\n\n```mermaid\n%%{init: {\"themeCSS\": \".x{background:url(https://evil.example/t.gif)}\"}}%%\ngraph TD; A-->B\n```\n")
+ put(root, a + "/good.mmd", "graph TD\n A[Start] --> B[End]\n")
+ put(root, a + "/bad.mmd", "graph TD\n A --> --> ((\n")
+ put(root, a + "/topology.json", '{"root": {"id": ')
+ put(root, a + "/RULE_REVIEWS.json", "not json at all")
+ put(root, a + "/SECRETS.local.md", SECRET)
+ put(root, a + "/security_remediation.local.patch", SECRET)
+ put(root, a + "/SECURITY_FINDINGS.md", "# Findings\n\n| ID | Severity | Title |\n|---|---|---|\n| SEC-1 | Critical | <script>x</script> |\n| SEC-2 | High | y |\n| SEC-3 | high | z |\n")
+ put(root, "modernized/%s/README.md" % system, "# Module <img src=x onerror=1>\n")
+ put(root, "modernized/%s/mod1/TRANSFORMATION_NOTES.md" % system, "# Notes\n</script>\n")
+ outside = tempfile.mkdtemp()
+ put(outside, "secret.txt", "OUTSIDE-MARKER")
+ try:
+ os.symlink(os.path.join(outside, "secret.txt"), os.path.join(root, *(a + "/PLAYBOOK.md").split("/")))
+ except (OSError, NotImplementedError, AttributeError):
+ pass
+ return outside
+
+
+class CompareTests(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+
+ def cases(self, cases, **extra):
+ path = put(self.tmp, "cases.json", json.dumps({"system": "s", "cases": cases, **extra}))
+ return path
+
+ def files(self, name, legacy, new):
+ put(self.tmp, "legacy/%s" % name, legacy)
+ if new is not None:
+ put(self.tmp, "new/%s" % name, new)
+ return {"id": name, "title": "case " + name, "legacy": "legacy/" + name, "new": "new/" + name}
+
+ def result(self, cases, **extra):
+ out = os.path.join(self.tmp, "analysis", "EQUIVALENCE.json")
+ code, stdout, _ = run_main(cmp.main, [self.cases(cases, **extra), "--out", out])
+ with open(out, encoding="utf-8") as fh:
+ return code, stdout, json.load(fh)
+
+ def test_same_differs_missing_approved_and_exit_codes(self):
+ same, diff, miss = self.files("a", b"one\ntwo\n", b"one\ntwo\n"), self.files("b", b"one\ntwo\n", b"one\ntw0\n"), self.files("c", b"x", None)
+ code, out, res = self.result([same, diff, miss])
+ self.assertEqual(code, 1)
+ self.assertEqual([c["verdict"] for c in res["cases"]], ["same", "differs", "missing"])
+ self.assertEqual(res["totals"], {"cases": 3, "executed": 2, "same": 1, "differs": 1, "differsApproved": 0, "missing": 1})
+ self.assertIn("equivalence cases executed: 2", out)
+ self.assertIn("NOT PROVEN", out)
+ d = res["cases"][1]["firstDiff"]
+ self.assertEqual((d["offset"], d["line"]), (6, 2))
+ self.assertIn("legacy", res["cases"][2]["reason"].lower() + "legacy")
+ self.assertEqual(res["cases"][0]["legacySha256"], hashlib.sha256(b"one\ntwo\n").hexdigest())
+ approved = dict(diff, id="b2", approvedDifference="fixed on purpose")
+ code, _, res = self.result([same, approved])
+ self.assertEqual(code, 0)
+ self.assertEqual(res["cases"][1]["verdict"], "differs-approved")
+ self.assertEqual(res["totals"]["differsApproved"], 1)
+
+ def test_all_same_is_proven(self):
+ code, out, res = self.result([self.files("a", b"abc", b"abc")])
+ self.assertEqual(code, 0)
+ self.assertIn("PROVEN", out)
+ self.assertNotIn("NOT PROVEN", out)
+ self.assertEqual(res["selfCheck"]["passed"], True)
+
+ def test_missing_alone_is_never_a_pass(self):
+ code, out, res = self.result([self.files("a", b"x", b"x"), self.files("m", b"x", None)])
+ self.assertEqual(code, 1)
+ self.assertIn("missing", out)
+ self.assertEqual((res["totals"]["executed"], res["totals"]["missing"], res["totals"]["differs"]), (1, 1, 0))
+
+ def test_no_case_executed_is_not_proven(self):
+ for cases in ([], [self.files("m", b"x", None)]):
+ code, out, res = self.result(cases)
+ self.assertEqual(code, 1)
+ self.assertIn("equivalence cases executed: 0", out)
+ self.assertIn("NOT PROVEN: no case executed", out)
+ self.assertEqual(res["selfCheck"]["passed"], None)
+
+ def test_empty_outputs_prove_nothing(self):
+ code, out, res = self.result([self.files("e", b"", b"")])
+ self.assertEqual(code, 1)
+ self.assertIn("empty", out)
+ self.assertEqual(res["cases"][0]["verdict"], "same")
+ self.assertTrue(res["cases"][0]["empty"])
+
+ def test_masks_hide_only_what_they_name(self):
+ legacy, new = b"id=12 at 2026-09-01 10:11:12 total=5\n", b"id=1234 at 2026-09-24 03:04:05 total=5\n"
+ case = dict(self.files("t", legacy, new), mask=[{"regex": r"id=\d+"}, {"regex": r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d"}])
+ code, _, res = self.result([case])
+ self.assertEqual((code, res["cases"][0]["verdict"]), (0, "same"))
+ case["new"] = self.files("t2", legacy, new.replace(b"total=5", b"total=6"))["new"]
+ _, _, res = self.result([case])
+ self.assertEqual(res["cases"][0]["verdict"], "differs")
+ self.assertEqual(res["cases"][0]["firstDiff"]["offset"], legacy.index(b"5"))
+ ranged = dict(self.files("r", b"aaXXbb", b"aaYYbb"), mask=[{"bytes": "2-3"}])
+ self.assertEqual(self.result([ranged])[2]["cases"][0]["verdict"], "same")
+ beyond = dict(self.files("r2", b"aaXXbb", b"aaYYbc"), mask=[{"bytes": "2-3"}, {"bytes": "50-60"}])
+ self.assertEqual(self.result([beyond])[2]["cases"][0]["verdict"], "differs")
+
+ def test_a_mask_that_hides_everything_fails_the_self_check(self):
+ case = dict(self.files("h", b"abc", b"xyz"), mask=[{"regex": ".*"}])
+ code, out, res = self.result([case])
+ self.assertEqual(code, 1)
+ self.assertIs(res["selfCheck"]["passed"], False)
+ self.assertIn("hide every byte", res["selfCheck"]["detail"])
+ self.assertIn("NOT PROVEN", out)
+
+ def test_bad_input_is_exit_2_and_writes_nothing(self):
+ out = os.path.join(self.tmp, "EQ.json")
+ bad = [put(self.tmp, "a.json", "{not json"), put(self.tmp, "b.json", json.dumps({"cases": "x"})),
+ self.cases([dict(self.files("z", b"a", b"a"), mask=[{"regex": "("}])]), self.cases([dict(self.files("y", b"a", b"a"), mask=[{"bytes": "9-2"}])]),
+ self.cases([self.files("d", b"a", b"a"), self.files("d", b"a", b"a")]), os.path.join(self.tmp, "missing.json")]
+ for path in bad:
+ code, _, err = run_main(cmp.main, [path, "--out", out])
+ self.assertEqual(code, 2, path)
+ self.assertFalse(os.path.exists(out))
+ self.assertIn("compare.py", err)
+
+ def test_paths_may_not_leave_the_cases_folder(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "o.txt", "same")
+ sub = os.path.join(self.tmp, "sub")
+ put(sub, "n.txt", "same")
+ case = {"id": "o", "legacy": os.path.join(outside, "o.txt"), "new": "n.txt"}
+ path = put(sub, "cases.json", json.dumps({"cases": [case]}))
+ out = os.path.join(sub, "E.json")
+ self.assertEqual(run_main(cmp.main, [path, "--out", out, "--quiet"])[0], 1)
+ with open(out) as fh:
+ self.assertEqual(json.load(fh)["cases"][0]["verdict"], "missing")
+ self.assertEqual(run_main(cmp.main, [path, "--out", out, "--quiet", "--allow-outside"])[0], 0)
+
+ def test_first_difference_context_is_escaped_and_bounded(self):
+ _, _, res = self.result([self.files("c", b"a\x00\x1b[31m\tb" + b"z" * 200, b"a\x00\x1b[31m\tc" + b"z" * 200)])
+ d = res["cases"][0]["firstDiff"]
+ self.assertLessEqual(len(d["legacy"]), 40)
+ self.assertNotRegex(d["legacy"] + d["new"], r"[\x00-\x1f\x7f]")
+ self.assertEqual(d["legacy"][d["at"]], "b")
+ _, _, res = self.result([self.files("p", b"abc", b"abcdef")])
+ self.assertIn("ends", res["cases"][0]["reason"])
+ self.assertIn("<end of output>", res["cases"][0]["firstDiff"]["legacy"])
+
+ def test_summary_neutralises_terminal_escapes_and_quiet_is_quiet(self):
+ case = dict(self.files("q", b"a", b"b"), id="q\x1b[2J")
+ code, out, _ = run_main(cmp.main, [self.cases([case]), "--out", os.path.join(self.tmp, "E.json")])
+ self.assertNotIn("\x1b", out)
+ code, out, _ = run_main(cmp.main, [self.cases([case]), "--out", os.path.join(self.tmp, "E.json"), "--quiet"])
+ self.assertEqual((code, out), (1, ""))
+
+ def test_default_output_and_atomic_write(self):
+ code, _, _ = run_main(cmp.main, [self.cases([self.files("a", b"x", b"x")]), "--quiet"])
+ self.assertEqual(code, 0)
+ self.assertEqual(sorted(f for f in os.listdir(self.tmp) if "equivalence" in f.lower()), ["EQUIVALENCE.json"])
+
+ def test_a_symlinked_output_is_refused(self):
+ real = put(self.tmp, "elsewhere/E.json", "ORIGINAL")
+ link = os.path.join(self.tmp, "E.json")
+ make_symlink(self, real, link)
+ code, _, err = run_main(cmp.main, [self.cases([self.files("a", b"x", b"x")]), "--out", link, "--quiet"])
+ self.assertEqual(code, 2)
+ self.assertIn("symbolic link", err)
+ with open(real) as fh:
+ self.assertEqual(fh.read(), "ORIGINAL")
+
+
+class BuildFacts(unittest.TestCase):
+ def test_rule_cards_in_several_layouts(self):
+ text = "\n".join([
+ "## Rules", "### RULE-012: Interest is truncated", "**Priority:** P0", "**Confidence:** High \u2014 checked", "**Category:** Calculation",
+ "**Source:** `a/B.cbl:10-20,30` and `C.cbl:5`", "**Suspected defect:** rounds down", "```", "### not a heading in a fence", "```", "",
+ "### P1-001 \u00b7 Header line style", "`P1` \u00b7 Validation \u00b7 confidence Medium \u00b7 `d/E.cbl:1-2`", "",
+ "#### R-12: Short id", "Priority: P2", "**Suspected defect:** none", "", "### BR-D7-09 \u2014 Domain coded", "### Plain heading", "text",
+ ])
+ blocks = br.parse_rules(text)
+ cards = [b for b in blocks if b["k"] == "rule"]
+ self.assertEqual([c["id"] for c in cards], ["RULE-012", "P1-001", "R-12", "BR-D7-09"])
+ self.assertEqual([c["p"] for c in cards], ["P0", "P1", "P2", ""])
+ self.assertEqual([c["conf"] for c in cards], ["High", "Medium", "", ""])
+ self.assertEqual([c["defect"] for c in cards], [True, False, False, False])
+ self.assertEqual(cards[0]["cites"], ["a/B.cbl:10-20,30", "C.cbl:5"])
+ self.assertEqual(cards[1]["cites"], ["d/E.cbl:1-2"])
+ self.assertNotIn("Priority", cards[0]["body"])
+ self.assertIn("### not a heading in a fence", cards[0]["body"])
+ self.assertEqual(blocks[0]["k"], "md")
+ self.assertEqual(br.parse_rules("just text\n\n## Heading")[0]["k"], "md")
+
+ def test_headings_and_fields_padded_with_thousands_of_spaces_are_parsed_in_a_moment(self):
+ import time
+ pad = " " * 6000
+ text = "### RULE-001 - Title%s#%sx\n**Priority:** P0%s\n**Suspected defect:**%s\n\n### RULE-002: Closed ### \n**Priority:** P1 \n" % (pad, pad, pad, pad)
+ started = time.time()
+ cards = [c for c in br.parse_rules(text) if c["k"] == "rule"]
+ self.assertLess(time.time() - started, 2)
+ self.assertEqual([c["id"] for c in cards], ["RULE-001", "RULE-002"])
+ self.assertEqual(cards[1]["title"], "Closed")
+ self.assertEqual(cards[1]["p"], "P1")
+ self.assertEqual(br.field("Priority: \nPriority: P2 ", "Priority"), "P2")
+
+ def test_rule_priority_falls_back_to_the_index_table(self):
+ text = "| ID | Name | Priority |\n|---|---|---|\n| [RULE-007](#x) | Merged | P0 |\n\n### RULE-007: Merged\n**Merged into:** RULE-006\n\n### RULE-008: Real\n**Priority:** P1\n"
+ cards = [b for b in br.parse_rules(text) if b["k"] == "rule"]
+ facts = br.rule_facts(text, cards, {"RULE-008": {"verdict": "confirmed"}})
+ self.assertEqual((facts["total"], facts["byPriority"], facts["unrated"]), (2, {"P0": 1, "P1": 1}, 0))
+ self.assertEqual(facts["reviews"], {"confirmed": 1})
+
+ def test_security_tally(self):
+ findings = "## Summary scorecard\n| Severity | Count |\n|---|---|\n| Critical | 9 |\n\n## Findings\n| ID | Severity | Title |\n|---|---|---|\n| S1 | **Critical** | a |\n| S2 | High | b |\n| S3 | high | c |\n| S4 | Low | d |\n\n## Dependency CVEs\n| Package | Installed | CVE | Severity |\n|---|---|---|---|\n| x | 1 | CVE-1 | High |\n\n## Coverage gaps\n| Finding | Severity |\n|---|---|\n| n | Medium |\n"
+ self.assertEqual(br.security_facts(findings), {"Critical": 1, "High": 2, "Low": 1})
+ self.assertEqual(br.security_facts("| Severity | Count |\n|---|---|\n| Critical | 2 |\n| High | 3 |\n"), {"Critical": 2, "High": 3})
+ self.assertIsNone(br.security_facts("Nothing recognisable here."))
+
+ def test_baseline_facts(self):
+ table = "| Test | Result |\n|---|---|\n| a | pass |\n| b | PASS |\n| c | fail |\n| d | skipped |\n"
+ self.assertEqual(br.baseline_facts(table), {"pass": 2, "fail": 1, "skip": 1, "targetOnly": False})
+ self.assertEqual(br.baseline_facts("Run: 12 tests passed, 1 failed"), {"pass": 12, "fail": 1, "targetOnly": False})
+ self.assertEqual(br.baseline_facts("target-only: the source runtime is not available")["targetOnly"], True)
+ self.assertIsNone(br.baseline_facts("| Fixture | RC |\n|---|---|\n| seed | 0 |\n"))
+
+ def test_topology_facts(self):
+ topo = {"root": {"id": "s", "kind": "system", "children": [{"id": "d", "kind": "domain", "children": [
+ {"id": "A", "kind": "module", "loc": 100, "language": "cobol", "file": "a.cbl"}, {"id": "B", "kind": "module", "loc": 50, "file": "b.cbl"},
+ {"id": "F", "kind": "datastore"}]}]}, "edges": [{}, {}]}
+ self.assertEqual(br.topology_facts(topo), {"modules": 2, "loc": 150, "languages": ["cobol"], "edges": 2, "observations": []})
+ self.assertIsNone(br.topology_facts([1, 2]))
+ self.assertIsNone(br.topology_facts({"root": {"id": "x", "children": "oops"}}))
+
+ def test_equivalence_view_never_trusts_the_files_own_words(self):
+ digest = "a" * 64
+ good = {"cases": [{"id": "1", "verdict": "same", "legacySha256": digest, "newSha256": digest}, {"id": "2", "verdict": "differs-approved", "approvedDifference": "ok"}],
+ "selfCheck": {"passed": True, "detail": "d"}, "totals": {"cases": 2, "executed": 2}}
+ view = br.equivalence_view(good)
+ self.assertEqual((view["state"], view["problems"]), ("green", []))
+ self.assertEqual(br.equivalence_view(dict(good, cases=[dict(good["cases"][0], newSha256="b" * 64)]))["state"], "red")
+ for bad_case in ({"id": "x", "verdict": "differs"}, {"id": "x", "verdict": "missing"}, {"id": "x", "verdict": "pass"},
+ {"id": "x", "verdict": "differs-approved"}, {"id": "x", "verdict": "same", "firstDiff": {"offset": 1}}):
+ view = br.equivalence_view(dict(good, cases=good["cases"] + [bad_case], totals={}))
+ self.assertEqual(view["state"], "red", bad_case)
+ self.assertEqual(br.equivalence_view(dict(good, cases=[]))["state"], "red")
+ self.assertEqual(br.equivalence_view(dict(good, selfCheck={"passed": False, "detail": "x"}))["state"], "red")
+ self.assertEqual(br.equivalence_view(dict(good, cases=[dict(good["cases"][0], empty=True)]))["state"], "red")
+ no_check = br.equivalence_view({"cases": good["cases"]})
+ self.assertEqual(no_check["state"], "green")
+ self.assertTrue(any("self-check" in p for p in no_check["problems"]))
+ self.assertTrue(any("disagree" in p for p in br.equivalence_view(dict(good, totals={"cases": 9, "executed": 9}))["problems"]))
+ self.assertIsNone(br.equivalence_view({"cases": "x"}))
+ self.assertIsNone(br.equivalence_view([]))
+
+
+class BuildReport(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+ self.out = os.path.join(self.tmp, "out", "REPORT.html")
+
+ def build(self, system="s", extra=()):
+ code, out, err = run_main(br.main, [system, "--workspace", self.tmp, "--out", self.out, *extra])
+ return code, out, err
+
+ def html(self):
+ with open(self.out, encoding="utf-8") as fh:
+ return fh.read()
+
+ def test_nothing_found_is_exit_1_with_one_line(self):
+ os.makedirs(os.path.join(self.tmp, "analysis", "s"))
+ code, out, err = self.build()
+ self.assertEqual(code, 1)
+ self.assertEqual(len(err.strip().splitlines()), 1)
+ self.assertFalse(os.path.exists(self.out))
+
+ def test_bad_system_names_are_refused(self):
+ for name in ("..", "a/b", "a\\b", ""):
+ self.assertEqual(self.build(name)[0], 2, name)
+
+ def test_nearly_empty_workspace(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "# Preflight\nAll good.\n")
+ code, out, _ = self.build()
+ self.assertEqual(code, 0)
+ self.assertRegex(out.strip(), r"REPORT\.html: 1 section, \d+ KB$")
+ html = self.html()
+ self.assertEqual([s["attrs"].get("id") for s in Scripts(html).found], ["report-data", "app"])
+ data = page_data(html)
+ self.assertEqual([s["id"] for s in data["sections"]], ["overview"])
+ self.assertFalse(any(x["done"] for x in data["glance"]["steps"] if x["key"] != "preflight"))
+ self.assertIsNone(data["glance"]["track"])
+ self.assertIsNone(data["glance"]["equivalence"])
+
+ def test_default_output_path(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ code, out, _ = run_main(br.main, ["s", "--workspace", self.tmp])
+ self.assertEqual(code, 0)
+ self.assertTrue(os.path.isfile(os.path.join(self.tmp, "analysis", "s", "REPORT.html")))
+ self.assertEqual([f for f in os.listdir(os.path.join(self.tmp, "analysis", "s")) if f.endswith(".tmp")], [])
+
+ def test_malformed_files_are_reported_in_the_page_not_raised(self):
+ a = "analysis/s/"
+ put(self.tmp, a + "topology.json", "{broken")
+ put(self.tmp, a + "EQUIVALENCE.json", "[1, 2")
+ put(self.tmp, a + "RULE_REVIEWS.json", '{"reviews": 5}')
+ put(self.tmp, a + "BUSINESS_RULES.md", b"\xff\xfe\x00binary\x00\x01### RULE-001: x\n")
+ put(self.tmp, a + "ASSESSMENT.md", b"")
+ put(self.tmp, a + "BASELINE.md", "| a |\n|---")
+ put(self.tmp, a + "SECURITY_FINDINGS.md", "|||\n|||\n")
+ put(self.tmp, a + "broken.mmd", "")
+ code, _, _ = self.build()
+ self.assertEqual(code, 0)
+ data = page_data(self.html())
+ notes = json.dumps(data["sections"][0]["parts"])
+ self.assertIn("topology.json could not be parsed", notes)
+ self.assertIn("EQUIVALENCE.json could not be parsed", notes)
+ self.assertEqual(data["glance"]["equivalence"]["state"], "red")
+ self.assertIn("RULE_REVIEWS.json", notes)
+
+ def test_hostile_input_cannot_reach_the_page_as_markup_or_script(self):
+ outside = hostile_workspace(self.tmp)
+ self.addCleanup(shutil.rmtree, outside, True)
+ code, _, _ = self.build("evil")
+ self.assertEqual(code, 0)
+ html = self.html()
+ scripts = Scripts(html).found
+ ids = [s["attrs"].get("id") for s in scripts]
+ self.assertEqual(ids, ["report-data", "mermaid-lib", "app"])
+ for s in scripts:
+ self.assertLessEqual(set(s["attrs"]), {"id", "type"})
+ block = scripts[0]["text"]
+ for bad in ("<", ">", "&", LS, PS):
+ self.assertNotIn(bad, block)
+ self.assertNotIn("</script", scripts[1]["text"].lower())
+ self.assertNotIn("<!--", scripts[1]["text"])
+ page = html.replace(scripts[0]["text"], "").replace(scripts[1]["text"], "")
+ for needle in ("__pwned=", "onerror", "evil.example", "javascript:"):
+ self.assertNotIn(needle, page.split('<script id="app">')[0].split("</style>")[1], needle)
+ text = json.dumps(page_data(html))
+ self.assertIn("window.__pwned=1", text)
+ self.assertNotIn(SECRET, html)
+ self.assertNotIn("OUTSIDE-MARKER", html)
+ notes = json.dumps(page_data(html)["sections"][0]["parts"])
+ if os.path.islink(os.path.join(self.tmp, "analysis", "evil", "PLAYBOOK.md")):
+ self.assertIn("PLAYBOOK.md is a link", notes)
+ self.assertIn("longer than 20,000 characters were shortened", notes)
+ self.assertLess(len(html), 6 << 20)
+ data = page_data(html)
+ self.assertEqual(data["glance"]["security"], {"Critical": 1, "High": 2})
+ self.assertEqual(data["glance"]["rules"]["total"], 2)
+ titles = [b["title"] for s in data["sections"] for p in s["parts"] for b in p.get("blocks", []) if b["k"] == "rule"]
+ self.assertIn("Title with `backticks` and <angle> <script>x</script> & ampersand", titles)
+
+ def test_secret_and_local_files_are_never_read_even_when_asked_for_by_name(self):
+ for name in ("SECRETS.local.md", "secrets.txt", "security_remediation.local.patch", "notes.LOCAL.md"):
+ path = put(self.tmp, "analysis/s/" + name, SECRET)
+ src = br.Source(self.tmp, "s")
+ self.assertIsNone(src.read(path, name), name)
+ self.assertEqual(src.found, [])
+
+ def test_csp_pins_exactly_the_scripts_on_the_page(self):
+ put(self.tmp, "analysis/s/good.mmd", "graph TD\n A-->B\n")
+ self.build()
+ html = self.html()
+ policy = re.search(r'http-equiv="Content-Security-Policy" content="([^"]*)"', html).group(1)
+ self.assertIn("default-src 'none'", policy)
+ self.assertNotIn("unsafe-eval", policy)
+ self.assertNotRegex(policy, r"script-src[^;]*unsafe-inline")
+ hashes = re.findall(r"'sha256-([^']+)'", policy.split("script-src")[1].split(";")[0])
+ code_scripts = [s for s in Scripts(html).found if s["attrs"].get("type") != "application/json"]
+ expected = [base64.b64encode(hashlib.sha256(s["text"].encode("utf-8")).digest()).decode() for s in code_scripts]
+ self.assertEqual(sorted(hashes), sorted(expected))
+ self.assertEqual(len(expected), 2)
+
+ def test_mermaid_is_inlined_only_when_a_diagram_exists(self):
+ put(self.tmp, "analysis/s/ASSESSMENT.md", "# A\nno diagram\n")
+ self.build()
+ self.assertNotIn("mermaid-lib", self.html())
+ self.assertLess(len(self.html()), 300000)
+ put(self.tmp, "analysis/s/ASSESSMENT.md", "# A\n\n```mermaid\ngraph TD\n A-->B\n```\n")
+ self.build()
+ scripts = Scripts(self.html()).found
+ lib = next(s for s in scripts if s["attrs"].get("id") == "mermaid-lib")
+ with open(MERMAID, encoding="utf-8") as fh:
+ original = fh.read()
+ self.assertEqual(lib["text"].replace("<\\/script", "</script").replace("<\\!--", "<!--"), original)
+ os.remove(os.path.join(self.tmp, "analysis", "s", "ASSESSMENT.md"))
+ put(self.tmp, "analysis/s/x.mmd", "graph TD\n A-->B\n")
+ self.build()
+ self.assertIn("mermaid-lib", self.html())
+
+ def test_size_cap_says_what_was_left_out(self):
+ for name in ("ASSESSMENT.md", "MODERNIZATION_BRIEF.md", "DATA_OBJECTS.md"):
+ put(self.tmp, "analysis/s/" + name, ("line of text that is long enough to count\n" * 80000))
+ self.build()
+ data = page_data(self.html())
+ notes = "\n".join(json.dumps(p) for p in data["sections"][0]["parts"])
+ self.assertIn("report size limit", notes)
+ embedded = sum(len(p["text"]) for s in data["sections"] for p in s["parts"] if p["t"] == "md")
+ self.assertLessEqual(embedded, br.TOTAL_CAP)
+
+ def test_sections_follow_the_workflow_order_and_only_exist_when_found(self):
+ a = "analysis/s/"
+ for name in ("EQUIVALENCE.json", "SECURITY_FINDINGS.md", "MODERNIZATION_BRIEF.md", "ASSESSMENT.md", "PREFLIGHT.md", "DELTA_CATALOG.md", "AI_NATIVE_SPEC.md"):
+ put(self.tmp, a + name, "{}" if name.endswith(".json") else "# " + name)
+ put(self.tmp, "modernized/s-reimagined/svc/README.md", "# svc")
+ self.build()
+ data = page_data(self.html())
+ self.assertEqual([s["id"] for s in data["sections"]], ["overview", "assessment", "brief", "security", "delta", "spec", "build", "equivalence"])
+ steps = {x["key"]: x["done"] for x in data["glance"]["steps"]}
+ self.assertEqual(steps, {"preflight": True, "assess": True, "map": False, "rules": False, "brief": True, "build": True, "harden": True})
+ self.assertEqual(data["glance"]["track"]["label"], "reimagine")
+
+ def test_track_follows_the_newest_track_artifact(self):
+ a = "analysis/s/"
+ put(self.tmp, a + "ASSESSMENT.md", "x")
+ put(self.tmp, a + "BASELINE.md", "x")
+ put(self.tmp, "modernized/s/mod/TRANSFORMATION_NOTES.md", "x")
+ self.build()
+ self.assertEqual(page_data(self.html())["glance"]["track"]["label"], "rewrite")
+ past = os.path.getmtime(os.path.join(self.tmp, "modernized", "s", "mod", "TRANSFORMATION_NOTES.md")) + 50
+ put(self.tmp, a + "DELTA_CATALOG.md", "x")
+ os.utime(os.path.join(self.tmp, "analysis", "s", "DELTA_CATALOG.md"), (past, past))
+ self.build()
+ self.assertEqual(page_data(self.html())["glance"]["track"]["label"], "same-stack uplift")
+
+ def test_topology_link_is_relative_and_the_map_is_not_embedded(self):
+ put(self.tmp, "analysis/s/TOPOLOGY.html", "<html>" + "x" * 5000 + "</html>")
+ self.build()
+ data = page_data(self.html())
+ link = next(p for s in data["sections"] for p in s["parts"] if p["t"] == "link")
+ self.assertEqual(os.path.normpath(os.path.join(os.path.dirname(self.out), link["href"])), os.path.join(self.tmp, "analysis", "s", "TOPOLOGY.html"))
+ self.assertNotIn("xxxxxxxxxx", self.html())
+
+ def test_report_shows_what_compare_py_wrote(self):
+ eq_dir = os.path.join(self.tmp, "analysis", "s", "equivalence")
+ put(eq_dir, "legacy/a.out", "same")
+ put(eq_dir, "new/a.out", "same")
+ put(eq_dir, "legacy/b.out", "one")
+ put(eq_dir, "new/b.out", "two")
+ cases = {"system": "s", "cases": [{"id": "A", "legacy": "legacy/a.out", "new": "new/a.out"}, {"id": "B", "legacy": "legacy/b.out", "new": "new/b.out"}]}
+ put(eq_dir, "cases.json", json.dumps(cases))
+ target = os.path.join(self.tmp, "analysis", "s", "EQUIVALENCE.json")
+ cmp.main([os.path.join(eq_dir, "cases.json"), "--out", target, "--quiet"])
+ self.build()
+ data = page_data(self.html())
+ eq = data["glance"]["equivalence"]
+ self.assertEqual(eq["state"], "red")
+ self.assertEqual((eq["tally"]["same"], eq["tally"]["differs"], eq["tally"]["executed"]), (1, 1, 2))
+ section = next(s for s in data["sections"] if s["id"] == "equivalence")["parts"][0]["eq"]
+ self.assertEqual(section["cases"][1]["firstDiff"]["line"], 1)
+ self.assertEqual(section["cases"][1]["legacyPath"], "equivalence/legacy/b.out")
+ cases["cases"].pop()
+ put(eq_dir, "cases.json", json.dumps(cases))
+ cmp.main([os.path.join(eq_dir, "cases.json"), "--out", target, "--quiet"])
+ self.build()
+ self.assertEqual(page_data(self.html())["glance"]["equivalence"]["state"], "green")
+
+ def test_a_symlinked_output_path_is_refused_and_nothing_is_written_through_it(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ target = put(self.tmp, "elsewhere/real.html", "ORIGINAL")
+ os.makedirs(os.path.dirname(self.out))
+ make_symlink(self, target, self.out)
+ code, out, err = self.build()
+ self.assertEqual((code, out), (1, ""))
+ self.assertEqual(len(err.strip().splitlines()), 1)
+ self.assertIn("symbolic link", err)
+ with open(target) as fh:
+ self.assertEqual(fh.read(), "ORIGINAL")
+ self.assertEqual(os.listdir(os.path.dirname(target)), ["real.html"])
+ self.assertEqual(os.listdir(os.path.dirname(self.out)), ["REPORT.html"])
+
+ def test_a_symlinked_output_folder_is_refused(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ real = os.path.join(self.tmp, "real")
+ os.makedirs(real)
+ make_symlink(self, real, os.path.join(self.tmp, "linked"))
+ code, _, err = run_main(br.main, ["s", "--workspace", self.tmp, "--out", os.path.join(self.tmp, "linked", "REPORT.html")])
+ self.assertEqual(code, 1)
+ self.assertIn("symbolic link", err)
+ self.assertEqual(os.listdir(real), [])
+
+ def test_a_failed_write_leaves_no_temp_file_behind(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ with mock.patch.object(br.os, "replace", side_effect=OSError("disk full")):
+ code, _, err = self.build()
+ self.assertEqual(code, 1)
+ self.assertEqual(len(err.strip().splitlines()), 1)
+ self.assertEqual(os.listdir(os.path.dirname(self.out)), [])
+
+ @unittest.skipIf(os.name == "nt", "POSIX permissions")
+ def test_the_report_gets_the_permissions_a_normal_file_would(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ umask = os.umask(0o022)
+ os.umask(umask)
+ self.build()
+ self.assertEqual(stat.S_IMODE(os.stat(self.out).st_mode), 0o666 & ~umask)
+
+
+class Template(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ with open(TEMPLATE, encoding="utf-8") as fh:
+ cls.text = fh.read()
+ cls.css = re.search(r"<style>(.*?)</style>", cls.text, re.S).group(1)
+ cls.app = re.search(r'<script id="app">(.*?)</script>', cls.text, re.S).group(1)
+
+ def test_no_external_urls_and_no_dangerous_apis(self):
+ self.assertNotRegex(self.text, r"https?://")
+ self.assertNotRegex(self.text, r"(?i)(?:src|href|action)\s*=\s*[\"']?\s*//")
+ self.assertNotRegex(self.css, r"@import|url\(")
+ for word in ("innerHTML", "outerHTML", "insertAdjacentHTML", "document.write", "eval(", "new Function", "setTimeout(\"", "fetch(", "XMLHttpRequest", "WebSocket"):
+ self.assertNotIn(word, self.app, word)
+ self.assertNotRegex(self.text, r"(?i)\son[a-z]+\s*=")
+ self.assertEqual(len(re.findall(r"@@(CSP|DATA|MERMAID)@@", self.text)), 3)
+ self.assertNotIn("@@", self.app)
+
+ def test_theme_and_layout_contract(self):
+ root = re.search(r":root\{(.*?)\}", self.css, re.S).group(1)
+ tokens = set(re.findall(r"--([a-z-]+):", root))
+ media = re.search(r"@media \(prefers-color-scheme:dark\)\{:root:not\(\[data-theme=\"light\"\]\)\{(.*?)\}\}", self.css, re.S).group(1)
+ forced = re.search(r":root\[data-theme=\"dark\"\]\{(.*?)\}", self.css, re.S).group(1)
+ self.assertEqual(set(re.findall(r"--([a-z-]+):", media)) - tokens, set())
+ self.assertEqual(re.findall(r"--([a-z-]+):[^;]*", media), re.findall(r"--([a-z-]+):[^;]*", forced))
+ self.assertRegex(re.search(r"body\{([^}]*)\}", self.css).group(1), r"background:var\(--bg\)")
+ self.assertIn("system-ui", self.css)
+ self.assertIn("width=device-width", self.text)
+ self.assertIn("@media print", self.css)
+ self.assertIn(":focus-visible", self.css)
+ self.assertRegex(self.css, r"\.wrap\{[^}]*padding:0 16px")
+
+ def test_every_text_colour_pair_meets_4_5_to_1(self):
+ def parse(block):
+ return dict(re.findall(r"--([a-z-]+):(#[0-9a-fA-F]{3,6})", block))
+
+ def lum(hexcolor):
+ h = hexcolor.lstrip("#")
+ h = "".join(c * 2 for c in h) if len(h) == 3 else h
+ chan = [int(h[i:i + 2], 16) / 255 for i in (0, 2, 4)]
+ lin = [c / 12.92 if c <= 0.03928 else ((c + 0.055) / 1.055) ** 2.4 for c in chan]
+ return 0.2126 * lin[0] + 0.7152 * lin[1] + 0.0722 * lin[2]
+
+ def ratio(a, b):
+ hi, lo = sorted((lum(a), lum(b)), reverse=True)
+ return (hi + 0.05) / (lo + 0.05)
+
+ light = parse(re.search(r":root\{(.*?)\}", self.css, re.S).group(1))
+ dark = dict(light, **parse(re.search(r":root\[data-theme=\"dark\"\]\{(.*?)\}", self.css, re.S).group(1)))
+ pairs = [("text", "bg"), ("text", "surface"), ("muted", "bg"), ("muted", "surface"), ("muted", "code"), ("text", "code"), ("link", "surface"), ("link", "bg"),
+ ("link", "code"), ("ok", "ok-bg"), ("bad", "bad-bg"), ("warn", "warn-bg"), ("info", "info-bg"), ("flat", "flat-bg"), ("hit-text", "hit"),
+ ("text", "flat-bg"), ("text", "info-bg"), ("muted", "flat-bg")]
+ for name, palette in (("light", light), ("dark", dark)):
+ for fg, bg in pairs:
+ self.assertGreaterEqual(ratio(palette[fg], palette[bg]), 4.5, "%s: %s on %s" % (name, fg, bg))
+ self.assertGreaterEqual(ratio("#1a1f27", light["paper"]), 4.5)
+ for name, palette in (("light", light), ("dark", dark)):
+ self.assertGreaterEqual(ratio(palette["focus"], palette["bg"]), 3, name)
+ self.assertGreaterEqual(ratio(palette["focus"], palette["surface"]), 3, name)
+
+
+NODE = shutil.which("node")
+LINK_CASES = { # markdown -> the href it must produce, or None when it must stay plain text
+ "[x](javascript:alert(1))": None, "[x](data:text/html,x)": None, "[x](file:///etc/passwd)": None, "[x](JaVaScRiPt:x)": None, "[x](java&#10;script:x)": None, "[x](javascript&colon;alert(1))": None, "[x](&#x6A;avascript:alert(1))": None,
+ "[x](https://example.com/a?b=1)": "https://example.com/a?b=1", "[x](mailto:a@b.c)": "mailto:a@b.c", "[x](#anchor)": "#anchor",
+ "[x](./relative/path.md)": "./relative/path.md", "[x](../up/README.md#top)": "../up/README.md#top",
+}
+JS_INPUTS = [
+ "<script>alert(1)</script>", "</script><script>alert(1)</script>", "<img src=x onerror=alert(1)>", "<!-- <!--", '<a href="javascript:alert(1)">x</a>',
+ "[x](javascript:alert(1))", "[x](JavaScript:alert(1))", "[x](java\tscript:alert(1))", "[x](" + chr(1) + "javascript:alert(1))", "[x]( javascript:alert(1))",
+ "[x](data:text/html;base64,PHNjcmlwdD4=)", "[x](//evil.example)", "[x](/\\evil.example)", "[x](vbscript:msgbox(1))", "[x](file:///etc/passwd)",
+ "[x](&#106;avascript:alert(1))", "[x](%6Aavascript:alert(1))", "[x](" + LS + "javascript:alert(1))", "[x](\u00a0javascript:alert(1))", "![x](javascript:alert(1))",
+ "| <b onmouseover=alert(1)>x</b> | y |\n|---|---|\n| <img src=x onerror=alert(2)> | z |", "### RULE-001: `<b>title</b>` <img src=x>",
+ "`<script>`", "**<img src=x onerror=1>** _<b>_", "- [ ] <b>a</b>\n- [x] *b*\n - nested <i>c</i>", "> <script>\n> x",
+ "```js\n<script>alert(1)</script>\n```", "~~~\n</script>\n~~~", "```mermaid\ngraph TD; A-->B\n```", "a" * 200000, "*" * 200000, "[" * 100000, "`" * 100000,
+ "_" * 100000, "|" * 100000, "- " * 50000, "> " * 20000 + "x", "\n".join(" " * (i * 2) + "- x" for i in range(300)), "#" * 100000, "[a](" + "b" * 100000 + ")",
+ "\n".join("| " + "c | " * 200 for _ in range(50)), "*a " * 60000, "**a " * 60000, "[a " * 60000, "`a " * 60000,
+] + list(LINK_CASES)
+SAFE_URLS = ["https://example.com/a?b=1&c=2", "https://example.com/?q=a&b=c#frag", "http://x.y", "HTTPS://X.Y", "mailto:a@b.c", "../x/y.md#z", "#anchor", "README.md", "a/b:c", "TOPOLOGY.html", "./p:q", "x.md#a:b"]
+BAD_URLS = ["java&#10;script:x", "javascript&colon;alert(1)", "&#106;avascript:1", "a&#x3a;b", "x&amp;y", "javascript:alert(1)", "JavaScript:alert(1)", " javascript:alert(1)", "java\tscript:alert(1)", "java\nscript:alert(1)", chr(1) + "javascript:alert(1)", "data:text/html,x",
+ "//evil.example", "/\\evil.example", "\\\\evil", "/abs/path", "vbscript:x", "file:///etc/passwd", "C:\\x", "a:b", chr(0xA0) + "javascript:1", LS + "javascript:1",
+ chr(0xFEFF) + "javascript:1", "jav" + chr(0x200B) + "ascript:1", "", "x" * 3000]
+NODE_HARNESS = r"""
+const fs = require('fs'), vm = require('vm');
+const tpl = fs.readFileSync(process.env.TEMPLATE, 'utf8');
+const core = /\/\* core:start[^\n]*\n([\s\S]*?)\/\* core:end \*\//.exec(tpl)[1];
+class Text { constructor(t) { this.text = String(t); } }
+class El { constructor(t) { this.tag = t; this.attrs = {}; this.kids = []; }
+ setAttribute(k, v) { this.attrs[k] = String(v); } appendChild(c) { this.kids.push(c); return c; } }
+const ctx = { document: { createElement: t => new El(t), createTextNode: t => new Text(t) } };
+vm.createContext(ctx);
+const api = vm.runInContext('(function(){"use strict";' + core + ';return {safeUrl:safeUrl,md:md,ruleCard:ruleCard,inline:inline};})()', ctx);
+function walk(n, acc) {
+ if (n instanceof Text) { acc.text += n.text; return; }
+ acc.tags.add(n.tag); Object.keys(n.attrs).forEach(k => { acc.attrs.add(k); if (k === 'href') acc.hrefs.push(n.attrs[k]); });
+ n.kids.forEach(c => walk(c, acc));
+}
+const input = JSON.parse(fs.readFileSync(0, 'utf8')), out = { urls: input.urls.map(u => api.safeUrl(u)), docs: [] };
+for (const src of input.docs) {
+ const t0 = Date.now(), acc = { tags: new Set(), attrs: new Set(), hrefs: [], text: '' };
+ api.md(src, 3).forEach(n => walk(n, acc));
+ out.docs.push({ ms: Date.now() - t0, tags: [...acc.tags], attrs: [...acc.attrs], hrefs: acc.hrefs, text: acc.text.slice(0, 5000) });
+}
+const card = { id: 'RULE-1', heading: 'RULE-1: <b>x</b>', title: '<img src=x onerror=1>`a`', p: 'P0', conf: 'High', cat: '<i>', cites: ['<b>a.cbl:1</b>'], defect: true, body: '**x** <script>1</script>' };
+const acc = { tags: new Set(), attrs: new Set(), hrefs: [], text: '' };
+walk(api.ruleCard(card, 'confirmed'), acc);
+out.card = { tags: [...acc.tags], attrs: [...acc.attrs], text: acc.text };
+process.stdout.write(JSON.stringify(out));
+"""
+
+
+@unittest.skipUnless(NODE, "node is not installed")
+class TemplateScript(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ proc = subprocess.run([NODE, "-e", NODE_HARNESS], input=json.dumps({"urls": SAFE_URLS + BAD_URLS, "docs": JS_INPUTS}), capture_output=True, text=True,
+ env=dict(os.environ, TEMPLATE=TEMPLATE), timeout=300)
+ assert proc.returncode == 0, proc.stderr
+ cls.out = json.loads(proc.stdout)
+
+ def test_the_page_script_parses(self):
+ with open(TEMPLATE, encoding="utf-8") as src, tempfile.NamedTemporaryFile("w", suffix=".js", delete=False, encoding="utf-8") as fh:
+ fh.write(re.search(r'<script id="app">(.*?)</script>', src.read(), re.S).group(1))
+ self.addCleanup(os.unlink, fh.name)
+ self.assertEqual(subprocess.run([NODE, "--check", fh.name], capture_output=True, text=True).returncode, 0)
+
+ def test_link_allowlist(self):
+ urls = self.out["urls"]
+ for url, got in zip(SAFE_URLS, urls[:len(SAFE_URLS)]):
+ self.assertIsNotNone(got, url)
+ for url, got in zip(BAD_URLS, urls[len(SAFE_URLS):]):
+ self.assertIsNone(got, repr(url))
+
+ def test_hostile_markdown_becomes_text_never_elements(self):
+ allowed_tags = {"p", "h3", "h4", "h5", "h6", "ul", "ol", "li", "code", "pre", "strong", "em", "del", "a", "br", "hr", "blockquote", "table", "thead", "tbody", "tr",
+ "th", "td", "div", "span"}
+ allowed_attrs = {"href", "rel", "target", "referrerpolicy", "id", "class", "role", "tabindex", "aria-label", "scope"}
+ for src, doc in zip(JS_INPUTS, self.out["docs"]):
+ label = src[:50]
+ self.assertLessEqual(set(doc["tags"]), allowed_tags, label)
+ self.assertLessEqual(set(doc["attrs"]), allowed_attrs, label)
+ self.assertLess(doc["ms"], 5000, label)
+ for href in doc["hrefs"]:
+ stripped = re.sub(r"[\x00-\x20]", "", href).lower()
+ self.assertRegex(stripped, r"^(https?://|mailto:|#|[^:/?#\\]*(?:[/?#]|$))", href)
+ first = self.out["docs"][0]
+ self.assertIn("<script>alert(1)</script>", first["text"])
+ self.assertEqual(self.out["docs"][2]["tags"], ["p"])
+ self.assertIn("[x](javascript:alert(1))", self.out["docs"][5]["text"])
+ self.assertEqual(self.out["docs"][5]["hrefs"], [])
+ table = self.out["docs"][20]
+ self.assertIn("<b onmouseover=alert(1)>x</b>", table["text"])
+ self.assertNotIn("img", table["tags"])
+
+ def test_links_fail_closed_and_safe_ones_stay_links(self):
+ docs = dict(zip(JS_INPUTS, self.out["docs"]))
+ for source, href in LINK_CASES.items():
+ doc = docs[source]
+ if href is None:
+ self.assertEqual(doc["hrefs"], [], source)
+ self.assertNotIn("a", doc["tags"], source)
+ self.assertIn(source, doc["text"], source)
+ else:
+ self.assertEqual(doc["hrefs"], [href], source)
+ self.assertIn("a", doc["tags"], source)
+ self.assertEqual(doc["text"], "x", source)
+
+ def test_rule_card_renders_hostile_fields_as_text(self):
+ card = self.out["card"]
+ self.assertLessEqual(set(card["tags"]), {"article", "span", "h4", "div", "p", "strong"})
+ self.assertLessEqual(set(card["attrs"]), {"class", "id", "data-p", "data-d", "aria-label"})
+ self.assertIn("<img src=x onerror=1>`a`", card["text"])
+ self.assertIn("<b>a.cbl:1</b>", card["text"])
+
+
+class Vendored(unittest.TestCase):
+ def test_mermaid_matches_the_recorded_digest_and_license(self):
+ with open(os.path.join(PLUGIN, "assets", "vendor", "LICENSE-mermaid.txt"), encoding="utf-8") as fh:
+ lic = fh.read()
+ with open(MERMAID, "rb") as fh:
+ raw = fh.read()
+ self.assertIn("Copyright (c) 2014 - 2022 Knut Sveidqvist", lic)
+ self.assertIn("MIT License", lic)
+ self.assertIn(hashlib.sha256(raw).hexdigest(), lic)
+ self.assertIn("mermaid@11.17.2", lic)
+ self.assertIn(str(len(raw)), lic)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_rules_tools.py b/plugins/code-modernization/scripts/tests/test_rules_tools.py
new file mode 100644
index 00000000..33cfd279
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_rules_tools.py
@@ -0,0 +1,196 @@
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SCRIPTS = os.path.dirname(HERE)
+
+
+def run(script, *args):
+ return subprocess.run([sys.executable, os.path.join(SCRIPTS, script), *args], capture_output=True, text=True)
+
+
+def load(path):
+ with open(path, encoding='utf-8') as handle:
+ return json.load(handle)
+
+
+def write(root, rel, text='x\n'):
+ path = os.path.join(root, rel)
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, 'w', encoding='utf-8') as handle:
+ handle.write(text)
+
+
+class ShardTests(unittest.TestCase):
+ def test_tree_shards_by_directory_and_skips_vendored_code(self):
+ with tempfile.TemporaryDirectory() as ws:
+ for i in range(30):
+ write(ws, f'legacy/erp/lib/pay/p{i}.pl', 'a\n' * 10)
+ write(ws, 'legacy/erp/lib/tax/t.pl', 'a\n' * 400)
+ write(ws, 'legacy/erp/node_modules/x/y.js', 'a\n')
+ write(ws, 'legacy/erp/tests/t.pl', 'a\n')
+ write(ws, 'legacy/erp/README.md', 'text\n')
+ done = run('make_shards.py', 'erp', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ shards = load(os.path.join(ws, 'analysis/erp/extract-rules.modules.json'))
+ names = [s['name'] for s in shards]
+ self.assertEqual(names, ['lib/pay', 'lib/pay#2', 'lib/tax'])
+ self.assertEqual(len(shards[0]['files']), 25)
+ self.assertNotIn('node_modules', json.dumps(shards))
+ self.assertNotIn('tests/t.pl', json.dumps(shards))
+ self.assertTrue(all(not f.startswith('legacy/') for s in shards for f in s['files']))
+
+ def test_a_symlinked_source_is_followed_and_files_stay_relative_to_it(self):
+ with tempfile.TemporaryDirectory() as ws, tempfile.TemporaryDirectory() as elsewhere:
+ write(elsewhere, 'src/a.cbl', 'a\n' * 50)
+ os.makedirs(os.path.join(ws, 'legacy'))
+ os.symlink(elsewhere, os.path.join(ws, 'legacy', 'big'))
+ done = run('make_shards.py', 'big', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ shards = load(os.path.join(ws, 'analysis/big/extract-rules.modules.json'))
+ self.assertEqual(shards[0]['files'], ['src/a.cbl'])
+
+ def test_topology_merges_small_modules_and_normalizes_paths(self):
+ with tempfile.TemporaryDirectory() as ws:
+ for name in ('A', 'B', 'C'):
+ write(ws, f'legacy/s/{name}.cbl', 'a\n' * 50)
+ topology = {'root': {'kind': 'system', 'id': 'sys', 'children': [{'kind': 'domain', 'name': 'D1', 'id': 'dom:d1', 'children': [
+ {'kind': 'module', 'id': 'A', 'name': 'A', 'file': 'A.cbl', 'loc': 50},
+ {'kind': 'module', 'id': 'B', 'name': 'B', 'file': 'legacy/s/B.cbl', 'loc': 50},
+ {'kind': 'module', 'id': 'C', 'name': 'C', 'file': os.path.join(ws, 'legacy/s/C.cbl'), 'loc': 900}]}]}}
+ write(ws, 'analysis/s/topology.json', json.dumps(topology))
+ self.assertEqual(run('make_shards.py', 's', '--workspace', ws).returncode, 0)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual([s['files'] for s in shards], [['A.cbl', 'B.cbl'], ['C.cbl']])
+ self.assertEqual(run('make_shards.py', 's', 'C*', '--workspace', ws).returncode, 0)
+ self.assertEqual(run('make_shards.py', 's', 'nothing*', '--workspace', ws).returncode, 1)
+
+ def test_links_and_paths_that_leave_the_source_are_not_shards(self):
+ with tempfile.TemporaryDirectory() as ws, tempfile.TemporaryDirectory() as outside:
+ write(ws, 'legacy/s/ok.pl', 'a\n' * 5)
+ write(outside, 'secret.pl', 'not part of the system\n')
+ os.symlink(os.path.join(outside, 'secret.pl'), os.path.join(ws, 'legacy/s/link.pl'))
+ self.assertEqual(run('make_shards.py', 's', '--workspace', ws).returncode, 0)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual([f for s in shards for f in s['files']], ['ok.pl'])
+ topology = {'root': {'kind': 'system', 'id': 'sys', 'children': [{'kind': 'domain', 'name': 'D', 'id': 'd', 'children': [
+ {'kind': 'module', 'id': 'A', 'name': 'A', 'file': 'ok.pl', 'loc': 500},
+ {'kind': 'module', 'id': 'B', 'name': 'B', 'file': '../../../etc/hosts', 'loc': 500},
+ {'kind': 'module', 'id': 'C', 'name': 'C', 'file': 'link.pl', 'loc': 500}]}]}}
+ write(ws, 'analysis/s/topology.json', json.dumps(topology))
+ done = run('make_shards.py', 's', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual([f for s in shards for f in s['files']], ['ok.pl'])
+ self.assertIn('outside the source directory', done.stdout)
+
+ def test_a_link_to_the_home_directory_or_root_is_refused(self):
+ with tempfile.TemporaryDirectory() as ws:
+ os.makedirs(os.path.join(ws, 'legacy'))
+ for broad in (os.path.expanduser('~'), '/', '/usr'):
+ link = os.path.join(ws, 'legacy', 's')
+ if os.path.islink(link):
+ os.unlink(link)
+ os.symlink(broad, link)
+ done = run('make_shards.py', 's', '--workspace', ws)
+ self.assertEqual(done.returncode, 1, broad)
+ self.assertIn('too broad', done.stderr)
+
+ def test_a_plain_name_matches_what_lies_beneath_it(self):
+ with tempfile.TemporaryDirectory() as ws:
+ write(ws, 'legacy/s/jetty-util/src/A.java', 'a\n' * 5)
+ write(ws, 'legacy/s/jetty-util-ajax/src/B.java', 'a\n' * 5)
+ write(ws, 'legacy/s/other/C.java', 'a\n' * 5)
+ self.assertEqual(run('make_shards.py', 's', 'jetty-util', '--workspace', ws).returncode, 0)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual([f for sh in shards for f in sh['files']], ['jetty-util/src/A.java'])
+
+ def test_a_map_whose_modules_share_one_directory_falls_back_to_the_tree(self):
+ with tempfile.TemporaryDirectory() as ws:
+ for name in ('a', 'b'):
+ write(ws, f'legacy/s/pkg/{name}/X.java', 'a\n' * 50)
+ children = [{'kind': 'module', 'id': f'pkg/{n}', 'name': f'pkg/{n}', 'file': 'pkg', 'loc': 900} for n in ('a', 'b')]
+ topology = {'root': {'kind': 'system', 'id': 'sys', 'children': [{'kind': 'domain', 'name': 'D', 'id': 'd', 'children': children}]}}
+ write(ws, 'analysis/s/topology.json', json.dumps(topology))
+ done = run('make_shards.py', 's', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ self.assertIn('cannot be sharded by module', done.stdout)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual(sorted(f for sh in shards for f in sh['files']), ['pkg/a/X.java', 'pkg/b/X.java'])
+
+ def test_missing_source_says_what_to_do(self):
+ with tempfile.TemporaryDirectory() as ws:
+ done = run('make_shards.py', 'nope', '--workspace', ws)
+ self.assertEqual(done.returncode, 1)
+ self.assertIn('preflight', done.stderr)
+
+
+class RenderTests(unittest.TestCase):
+ def test_rules_render_with_rule_headings_and_hostile_text_stays_inert(self):
+ result = {'confirmedRules': [
+ {'name': 'Late fee', 'category': 'Calculation', 'priority': 'P0', 'source': 'a.cbl:10-20', 'plainEnglish': 'A fee.',
+ 'given': 'a late payment', 'when': 'month end', 'then': 'fee 5%', 'confidence': 'High'},
+ {'name': 'Bad `name` | pipe\n# Heading', 'category': 'Validation', 'priority': 'P1', 'source': 'b.cbl:1-2', 'plainEnglish': 'x',
+ 'given': 'g\n# injected heading\n```', 'when': 'w', 'then': 't', 'confidence': 'Low', 'smeQuestion': 'Is it 5 or 6?'}],
+ 'dataObjects': [{'name': 'Account', 'source': 'a.cbl:1', 'fields': [{'name': 'id', 'type': 'int'}], 'consumedBy': ['Late fee']}],
+ 'injectionFlags': ['a.cbl:5'], 'rejectedRules': [{}], 'unverifiedRules': [], 'rerunModules': [],
+ 'stats': {'skippedModules': ['M9']}}
+ with tempfile.TemporaryDirectory() as ws:
+ write(ws, 'analysis/s/rules_result.json', json.dumps(result))
+ done = run('render_rules.py', 's', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ text = open(os.path.join(ws, 'analysis/s/BUSINESS_RULES.md'), encoding='utf-8').read()
+ self.assertIn('### RULE-001: Late fee', text)
+ self.assertIn('### RULE-002: Bad', text)
+ headings = [line for line in text.splitlines() if line.startswith('### ')]
+ self.assertEqual(len(headings), 2, headings)
+ self.assertNotIn('\n# injected heading', text)
+ ids = [int(n) for n in re.findall(r'^### RULE-(\d+): ', text, re.M)]
+ self.assertEqual(ids, list(range(1, len(ids) + 1)), 'rule numbers read in sequence down the document')
+ self.assertIn('Rules requiring SME confirmation', text)
+ self.assertIn('Is it 5 or 6?', text)
+ self.assertIn('Coverage gaps', text)
+ self.assertIn('Instruction-shaped content', text)
+ dto = open(os.path.join(ws, 'analysis/s/DATA_OBJECTS.md'), encoding='utf-8').read()
+ self.assertIn('## Account', dto)
+
+ def test_a_rule_keeps_one_citation_and_the_rest_moves_to_also_cited(self):
+ sources = ['wwwroot/a.pl:100-120 (aggregation); family table at lib/b.pm:31-40',
+ 'lib/x.pm:966,1018,1238 (order lists)', 'no citation here', 'c.cbl:7']
+ result = {'confirmedRules': [
+ {'name': f'R{i}', 'category': 'Policy', 'priority': 'P1', 'source': src, 'plainEnglish': 'x',
+ 'given': 'g', 'when': 'w', 'then': 't', 'confidence': 'High'} for i, src in enumerate(sources)]}
+ with tempfile.TemporaryDirectory() as ws:
+ write(ws, 'analysis/s/rules_result.json', json.dumps(result))
+ self.assertEqual(run('render_rules.py', 's', '--workspace', ws).returncode, 0)
+ text = open(os.path.join(ws, 'analysis/s/BUSINESS_RULES.md'), encoding='utf-8').read()
+ cites = re.findall(r'^\*\*Source:\*\* `([^`]*)`', text, re.M)
+ self.assertEqual(sorted(cites), sorted(['wwwroot/a.pl:100-120', 'lib/x.pm:966', 'no citation here', 'c.cbl:7']))
+ self.assertIn('**Also cited:** (aggregation); family table at lib/b.pm:31-40', text)
+ self.assertEqual(text.count('**Also cited:**'), 2)
+
+ def test_folded_rules_are_listed_so_nothing_disappears_untraceably(self):
+ result = {'confirmedRules': [{'name': 'Twiddle table', 'category': 'Calculation', 'priority': 'P0', 'source': 'a.c:1-2 ; also b.c:3-4',
+ 'plainEnglish': 'x', 'given': 'g', 'when': 'w', 'then': 't', 'confidence': 'Low'}],
+ 'foldedRules': [{'name': 'Twiddle factors', 'source': 'b.c:3-4', 'into': 'Twiddle table'}, 'junk', {'name': 'X\n# fake heading', 'source': 's', 'into': 'i'}]}
+ with tempfile.TemporaryDirectory() as ws:
+ write(ws, 'analysis/s/rules_result.json', json.dumps(result))
+ self.assertEqual(run('render_rules.py', 's', '--workspace', ws).returncode, 0)
+ text = open(os.path.join(ws, 'analysis/s/BUSINESS_RULES.md'), encoding='utf-8').read()
+ self.assertIn('## Rules folded into another', text)
+ self.assertIn('- Twiddle factors (b.c:3-4) into Twiddle table', text)
+ self.assertIn('**Also cited:** b.c:3-4', text)
+ self.assertNotIn('\n# fake heading', text)
+
+ def test_unreadable_result_is_an_error(self):
+ with tempfile.TemporaryDirectory() as ws:
+ self.assertEqual(run('render_rules.py', 's', '--workspace', ws).returncode, 1)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_telemetry.py b/plugins/code-modernization/scripts/tests/test_telemetry.py
new file mode 100644
index 00000000..c65130cd
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_telemetry.py
@@ -0,0 +1,810 @@
+"""Tests for the usage counts (scripts/telemetry.py, scripts/telemetry.sh and the hooks that call them).
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+"""
+import contextlib
+import io
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import unittest
+from unittest import mock
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import telemetry as tm # noqa: E402
+
+RULES = """# Business rules
+
+### RULE-001 · Interest posting
+**Priority:** P0
+**Confidence:** High
+**Source:** a.cbl:10-20
+
+Given a balance, when the month closes, then interest is posted.
+
+### RULE-002 · Rounding
+**Priority:** P1
+**Source:** a.cbl:30-31
+
+Given an amount, then it is truncated.
+
+### RULE-003 · Odd limit
+**Priority:** P0
+**Suspected defect:** the limit is off by one
+**Source:** b.java:5-6
+
+Given a limit, then it applies.
+"""
+BRIEF = """# Brief
+
+#### Phase 1: pilot
+#### Phase 2: the rest
+
+## 8. Approval Block
+
+```
+Approved by: Jane Doe Date: 2026-09-24
+Approval covers: Phase 1 only
+```
+"""
+UNSIGNED = "## 8. Approval Block\n\n```\nApproved by: ________________ Date: __________\n```\n"
+SECURITY = "# Security\n\n| Severity | Count |\n|---|---|\n| Critical | 2 |\n| High | 5 |\n| Medium | 9 |\n"
+TOPOLOGY = {"root": {"kind": "root", "children": [
+ {"kind": "module", "file": "a.cbl", "loc": 3000, "language": "cobol"},
+ {"kind": "module", "file": "b.java", "loc": 1200, "language": "java"},
+ {"kind": "module", "file": "Makefile", "loc": 5000, "language": "make"}]}, "edges": []}
+CHECKS = ("tests", "rules", "same", "fresh", "canary", "source")
+
+
+def module(name, **states):
+ checks = [{"id": c, "state": states.get(c, "pass"), "detail": "x"} for c in CHECKS]
+ return {"name": name, "track": "rewrite", "checks": checks, "evidence": {
+ "tests": {"executed": 5, "failed": 0, "skipped": 0}, "fresh": {"inputs": 12, "executed": 12, "same": 12, "differs": 0, "missing": 0}}}
+
+
+VERIFICATION = {"modules": [module("a"), module("b", canary="gap"), module("c", canary="gap"), module("d", tests="fail"), module("e", same="fail"),
+ module("f", source="fail")],
+ "signoff": {"name": "", "role": "", "date": "", "decision": ""}}
+CASES = [{"id": "s%d" % i, "verdict": "same"} for i in range(9)] + [{"id": "d1", "verdict": "differs"}, {"id": "m1", "verdict": "missing"}] + [
+ {"id": "a%d" % i, "verdict": "differs-approved", "approvedDifference": "the owner accepted it"} for i in range(2)]
+EQUIVALENCE = {"cases": CASES, "selfCheck": {"passed": True}}
+REVIEWS = {"system": "sys", "version": 1, "reviews": {"RULE-001": {"verdict": "confirmed"}, "RULE-002": {"verdict": "wrong"},
+ "RULE-003": {"verdict": "discuss"}}}
+STEP_BITS = dict(tm.STEPS)
+
+
+def put(root, rel, data):
+ path = os.path.join(root, *rel.split("/"))
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "w", encoding="utf-8") as fh:
+ fh.write(data if isinstance(data, str) else json.dumps(data))
+ return path
+
+
+def workspace(root, system="sys", parts=None):
+ parts = parts or {}
+ files = {"PREFLIGHT.md": "# Preflight\n", "ASSESSMENT.md": "# Assessment\n", "topology.json": TOPOLOGY, "BUSINESS_RULES.md": RULES,
+ "RULE_REVIEWS.json": REVIEWS, "MODERNIZATION_BRIEF.md": BRIEF, "EQUIVALENCE.json": EQUIVALENCE, "VERIFICATION.json": VERIFICATION,
+ "SECURITY_FINDINGS.md": SECURITY, "REPORT.html": "<html></html>"}
+ files.update(parts)
+ for name, data in files.items():
+ if data is not None:
+ put(root, "analysis/%s/%s" % (system, name), data)
+ put(root, "modernized/%s/mod-a/x.txt" % system, "x")
+ put(root, "modernized/%s/mod-b/x.txt" % system, "x")
+ return root
+
+
+class Base(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+ self.data = os.path.join(self.tmp, "data")
+ os.makedirs(self.data)
+ clean = {k: v for k, v in os.environ.items() if k not in ("CODE_MODERNIZATION_TELEMETRY", "CLAUDE_PLUGIN_OPTION_TELEMETRY", "DISABLE_TELEMETRY",
+ "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "CLAUDE_PLUGIN_DATA")}
+ patcher = mock.patch.dict(os.environ, {**clean, "CLAUDE_PLUGIN_DATA": self.data}, clear=True)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ self.ws = workspace(os.path.join(self.tmp, "ws"))
+
+ def hook(self, mode, payload):
+ out = io.StringIO()
+ with mock.patch.object(sys, "stdin", io.StringIO(json.dumps(payload))), contextlib.redirect_stdout(out):
+ tm.hook(mode)
+ return out.getvalue()
+
+
+class Counts(Base):
+ def test_the_full_workspace_is_counted_the_way_the_artifacts_say(self):
+ s = tm.state_metrics(self.ws)
+ self.assertEqual({k: s[k] for k in ("rules", "p0", "rev_ok", "rev_wrong", "phases", "built", "eq_cases", "eq_diff", "eq_appr",
+ "v_proven", "v_partly", "v_not", "sec_crit", "sec_high", "map_kloc", "lang", "goal")},
+ dict(rules=3, p0=2, rev_ok=1, rev_wrong=1, phases=2, built=2, eq_cases=12, eq_diff=2, eq_appr=2, v_proven=1,
+ v_partly=2, v_not=3, sec_crit=2, sec_high=5, map_kloc=9, lang=1, goal=3))
+ steps = ("preflight", "assess", "map", "rules", "reviewed", "brief", "approved", "built", "verified", "hardened", "report")
+ self.assertEqual(s["done"], sum(STEP_BITS[n] for n in steps))
+
+ def test_a_signed_proof_and_an_unsigned_brief(self):
+ put(self.ws, "analysis/sys/VERIFICATION.json", dict(VERIFICATION, signoff={"name": "A. Person", "decision": "accepted"}))
+ put(self.ws, "analysis/sys/MODERNIZATION_BRIEF.md", UNSIGNED)
+ done = tm.state_metrics(self.ws)["done"]
+ self.assertTrue(done & STEP_BITS["signed"])
+ self.assertFalse(done & STEP_BITS["approved"])
+ self.assertTrue(done & STEP_BITS["brief"])
+
+ def test_nothing_is_reported_where_the_plugin_has_left_nothing(self):
+ empty = os.path.join(self.tmp, "empty")
+ put(empty, "analysis/other/data.csv", "a,b\n1,2\n")
+ self.assertIsNone(tm.state_metrics(empty))
+ self.assertIsNone(tm.state_metrics(os.path.join(self.tmp, "nowhere")))
+ self.assertEqual(self.hook("state", {"cwd": empty}), "")
+
+ def test_the_language_is_the_biggest_one_that_is_code(self):
+ def lang(rows):
+ return tm.language_code({"root": {"children": [{"kind": "module", "file": "f", "loc": n, "language": name} for name, n in rows]}})
+ self.assertEqual(lang([("artifact", 9000), ("c", 2900), ("cpp", 650), ("make", 500)]), 7)
+ self.assertEqual(lang([("C#", 10), ("Java", 5)]), 3)
+ self.assertEqual(lang([("javascript", 10), ("java", 50)]), 2)
+ self.assertEqual(lang([("make", 10), ("yaml", 5)]), 0)
+ self.assertEqual(lang([("PL/I", 90000), ("JCL", 500)]), 99)
+ self.assertEqual(lang([("C/C++", 10), ("make", 900)]), 7)
+ self.assertEqual(lang([("Node.js", 10)]), 9)
+ self.assertEqual(lang([("ASP.NET", 10), ("VB6", 20)]), 3)
+ self.assertEqual(lang([("Java/JSP", 10)]), 2)
+ self.assertEqual(lang([("Fortran", 10), ("Assembler", 5)]), 15)
+ self.assertEqual(lang([("some new language", 10)]), 99)
+ self.assertEqual(lang([]), 0)
+ self.assertEqual(tm.language_code(None), 0)
+
+ def test_the_goal_is_the_answer_after_the_word_goal_and_nothing_else(self):
+ cases = [("- Goal: reimagine (rebuild it from scratch)", 4), ("## Goal\n- **Goal:** `uplift`. Then transform it.", 2),
+ ("## Goal\n**Transform** piece by piece", 3), ("- Goal: Understand it first", 1),
+ ("## Goal\n- **Goal:** `not-sure`. Run assess, then pick (uplift / transform / reimagine / understand).", 0),
+ ("- Goal: Move to a newer version of the same technology", 2), ("- Goal: Rewrite it in a different technology", 3),
+ ("nothing here about it, but uplift is a word", 0), ("", 0)]
+ for text, want in cases:
+ self.assertEqual(tm.goal_code(text, None), want, text)
+ self.assertEqual(tm.goal_code("", ""), 3)
+ self.assertEqual(tm.goal_code("", "-uplifted"), 2)
+ self.assertEqual(tm.goal_code("- Goal: not sure", "-reimagined"), 4)
+
+ def test_uplift_and_reimagine_leave_their_own_marks(self):
+ ws = os.path.join(self.tmp, "up")
+ put(ws, "analysis/sys/DELTA_CATALOG.md", "# Deltas\n")
+ put(ws, "analysis/sys/BASELINE.md", "# Baseline\n")
+ put(ws, "analysis/sys/PLAYBOOK.md", "# Playbook\n")
+ put(ws, "modernized/sys-uplifted/pom.xml", "<x/>")
+ put(ws, "modernized/sys-uplifted/mod/a.java", "x")
+ s = tm.state_metrics(ws)
+ self.assertEqual((s["goal"], s["built"]), (2, 1))
+ self.assertEqual(s["done"], STEP_BITS["deltas"] + STEP_BITS["baseline"] + STEP_BITS["playbook"] + STEP_BITS["built"])
+
+ def test_hostile_artifacts_give_whole_numbers_and_nothing_else(self):
+ put(self.ws, "analysis/sys/VERIFICATION.json", {"modules": "x", "signoff": ["not", "a", "dict"]})
+ put(self.ws, "analysis/sys/EQUIVALENCE.json", {"cases": [{"verdict": ["same"]}, 5, None, {"verdict": "same", "approvedDifference": {"a": 1}}]})
+ put(self.ws, "analysis/sys/RULE_REVIEWS.json", {"reviews": {"RULE-001": "confirmed", "RULE-002": {"verdict": ["wrong"]}}})
+ put(self.ws, "analysis/sys/topology.json", "{not json")
+ s = tm.state_metrics(self.ws)
+ self.assertEqual((s["v_proven"], s["v_partly"], s["v_not"]), (0, 0, 0))
+ self.assertEqual((s["rev_ok"], s["rev_wrong"], s["map_kloc"], s["lang"]), (0, 0, 0, 0))
+ self.assertTrue(all(type(v) is int and 0 <= v <= tm.COUNT_LIMIT for v in tm.ok(s).values()))
+
+ def test_the_counts_come_from_the_case_list_and_the_modules_checks_not_from_the_files_own_totals(self):
+ put(self.ws, "analysis/sys/EQUIVALENCE.json", {"cases": [{"id": "1", "verdict": "differs"}, {"id": "2", "verdict": "same"}, {"id": "3", "verdict": "differs"}],
+ "totals": {"executed": 100, "differs": 0, "differsApproved": 0, "missing": 0}})
+ put(self.ws, "analysis/sys/VERIFICATION.json", {"modules": [module("a", tests="fail")], "overall": {"counts": {"PROVEN": 5, "PARTLY PROVEN": 0, "NOT PROVEN": 0}}})
+ s = tm.state_metrics(self.ws)
+ self.assertEqual((s["eq_cases"], s["eq_diff"], s["eq_appr"]), (3, 2, 0))
+ self.assertEqual((s["v_proven"], s["v_partly"], s["v_not"]), (0, 0, 1))
+
+ def test_a_linked_artifact_is_not_read(self):
+ outside = os.path.join(self.tmp, "outside.md")
+ with open(outside, "w") as fh:
+ fh.write(RULES)
+ os.remove(os.path.join(self.ws, "analysis", "sys", "BUSINESS_RULES.md"))
+ try:
+ os.symlink(outside, os.path.join(self.ws, "analysis", "sys", "BUSINESS_RULES.md"))
+ except OSError:
+ self.skipTest("no symbolic links here")
+ s = tm.state_metrics(self.ws)
+ self.assertEqual((s["rules"], s["done"] & STEP_BITS["rules"]), (0, 0))
+
+ def test_the_newest_system_is_the_one_counted(self):
+ workspace(self.ws, "old", {"BUSINESS_RULES.md": "### RULE-001 · One\n**Priority:** P0\n"})
+ for f in os.scandir(os.path.join(self.ws, "analysis", "old")):
+ os.utime(f.path, (1, 1))
+ os.utime(os.path.join(self.ws, "analysis", "old"), (1, 1))
+ self.assertEqual(tm.state_metrics(self.ws)["rules"], 3)
+ put(self.ws, "analysis/old/NEW.md", "x")
+ self.assertEqual(tm.state_metrics(self.ws)["rules"], 1)
+
+
+class Commands(Base):
+ def cmd(self, prompt):
+ return tm.command_metrics(prompt, self.ws)
+
+ def test_the_plugins_commands_have_a_number_each(self):
+ for verb, n in tm.COMMANDS.items():
+ prompt = "/code-modernization:modernize" + ("-" + verb if verb else "")
+ self.assertEqual(self.cmd(prompt + " sys")["cmd"], n, prompt)
+ self.assertEqual(self.cmd("/modernize")["cmd"], 1)
+ self.assertEqual(self.cmd(" /code-modernization:modernize-verify")["cmd"], 11)
+ self.assertEqual(self.cmd("/code-modernization:modernize-harden-scan x")["cmd"], 99)
+ self.assertEqual([self.cmd("/modernize-" + v)["cmd"] for v in ("panel", "review-pane", "sign")], [20, 21, 22])
+ self.assertEqual(self.cmd("/code-modernization:modernize-review")["cmd"], 6)
+ self.assertEqual(self.cmd("/modernize-review")["cmd"], 6) # the bare name is the plugin's command now that the pane's deck has its own
+
+ def test_other_prompts_are_not_counted(self):
+ for prompt in ("", None, "modernize this", "/modernizer", "/other:modernize", "please /modernize", "/code-modernization:assess", 5, {"a": 1}):
+ self.assertIsNone(self.cmd(prompt), prompt)
+
+ def test_the_command_says_where_the_system_stands_and_whether_source_was_given(self):
+ s = self.cmd("/code-modernization:modernize-verify sys --source /work/x")
+ self.assertEqual((s["has_source"], s["fresh"], s["systems"], s["goal"]), (1, 0, 1, 3))
+ self.assertEqual(s["done"], tm.state_metrics(self.ws)["done"])
+ new = tm.command_metrics("/code-modernization:modernize billing --source /a", os.path.join(self.tmp, "fresh-folder"))
+ self.assertEqual((new["fresh"], new["systems"], new["done"], new["has_source"]), (1, 0, 0, 1))
+
+ def test_the_permission_mode_is_a_number_from_a_fixed_list(self):
+ for mode, want in (("default", 1), ("auto", 4), ("bypassPermissions", 5), ("something new", 0), (None, 0), (7, 0)):
+ self.assertEqual(tm.command_metrics("/modernize", self.ws, mode)["perm"], want, mode)
+ out = self.hook("command", {"cwd": self.ws, "prompt": "/modernize", "permission_mode": "plan"})
+ self.assertEqual(json.loads(out)["metrics"]["perm"], 3)
+
+ def test_a_named_system_is_the_one_counted_and_its_name_is_never_sent(self):
+ workspace(self.ws, "other", {"BUSINESS_RULES.md": None, "EQUIVALENCE.json": None, "VERIFICATION.json": None})
+ os.utime(os.path.join(self.ws, "analysis", "sys"), (1, 1))
+ for f in os.scandir(os.path.join(self.ws, "analysis", "sys")):
+ os.utime(f.path, (1, 1))
+ named = self.cmd("/code-modernization:modernize-status sys")
+ newest = self.cmd("/code-modernization:modernize-status")
+ self.assertNotEqual(named["done"], newest["done"])
+ self.assertEqual(named["systems"], 2)
+ for value in named.values():
+ self.assertIsInstance(value, int)
+
+
+class Sending(Base):
+ def test_every_key_is_defined_and_every_value_is_a_whole_number(self):
+ self.assertLessEqual(len(tm.STATE_KEYS), tm.MAX_KEYS)
+ self.assertLessEqual(len(tm.COMMAND_KEYS), tm.MAX_KEYS)
+ self.assertEqual(list(tm.MEANING)[0], "pv")
+ for keys in (tm.STATE_KEYS, tm.COMMAND_KEYS):
+ self.assertEqual(keys[0], "pv")
+ for k in keys:
+ self.assertRegex(k, r"^[a-z][a-z0-9_]{0,39}$")
+ self.assertIn(k, tm.MEANING)
+ for out in (self.hook("state", {"cwd": self.ws}), self.hook("command", {"cwd": self.ws, "prompt": "/code-modernization:modernize-map"})):
+ line = json.loads(out)
+ self.assertEqual(list(line), ["metrics"])
+ self.assertTrue(all(type(v) is int and 0 <= v <= tm.CAP for v in line["metrics"].values()), line)
+ self.assertEqual(out.count("\n"), 1)
+
+ def test_a_key_that_is_not_defined_never_goes_out(self):
+ got = tm.ok({"pv": 1, "path": "/secret", "rules": "7", "p0": True, "goal": 2.9, "done": -3})
+ self.assertEqual(dict(got), {"pv": 1, "rules": 7, "p0": 1, "goal": 2, "done": 0})
+
+ def test_the_plugin_version_is_sent_and_matches_the_manifest(self):
+ with open(os.path.join(PLUGIN, ".claude-plugin", "plugin.json")) as fh:
+ major, minor, patch = (int(x) for x in json.load(fh)["version"].split("."))
+ self.assertEqual(tm.plugin_version(), major * 10000 + minor * 100 + patch)
+ self.assertGreater(tm.plugin_version(), 0)
+ self.assertEqual(json.loads(self.hook("state", {"cwd": self.ws}))["metrics"]["pv"], tm.plugin_version())
+
+ def test_the_same_counts_are_sent_once_and_a_change_is_sent_again(self):
+ self.assertTrue(self.hook("state", {"cwd": self.ws}))
+ self.assertEqual(self.hook("state", {"cwd": self.ws}), "")
+ put(self.ws, "analysis/sys/RULE_REVIEWS.json", {"reviews": {"RULE-001": {"verdict": "wrong"}}})
+ self.assertIn('"rev_wrong":1', self.hook("state", {"cwd": self.ws}))
+ self.assertEqual(self.hook("state", {"cwd": self.ws}), "")
+ self.assertTrue(self.hook("command", {"cwd": self.ws, "prompt": "/modernize"}))
+ self.assertTrue(self.hook("command", {"cwd": self.ws, "prompt": "/modernize"}))
+
+ def test_when_it_cannot_remember_it_sends_nothing_rather_than_repeat_every_turn(self):
+ os.environ["CLAUDE_PLUGIN_DATA"] = os.path.join(self.tmp, "does-not-exist")
+ blocked = os.path.join(self.tmp, "blocked")
+ with mock.patch.object(tm.tempfile, "gettempdir", return_value=blocked), mock.patch.object(tm.os, "makedirs", side_effect=OSError("read-only")):
+ self.assertEqual(self.hook("state", {"cwd": self.ws}), "")
+
+ def test_the_debug_switch_says_why_each_hook_run_sent_nothing_in_the_plugins_data_folder(self):
+ log = os.path.join(self.data, "telemetry-debug.log")
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY_DEBUG": "1"}):
+ self.hook("state", {"cwd": self.ws})
+ self.hook("state", {"cwd": self.ws})
+ self.hook("state", {"cwd": os.path.join(self.tmp, "nowhere")})
+ self.hook("command", {"cwd": self.ws, "prompt": "hello"})
+ self.hook("command", {"cwd": self.ws, "prompt": "/modernize"})
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY": "0"}):
+ self.hook("state", {"cwd": self.ws})
+ with mock.patch.object(tm, "state_metrics", side_effect=KeyError("x")), mock.patch.object(sys, "stdin", io.StringIO("{}")), \
+ contextlib.redirect_stdout(io.StringIO()):
+ tm.main(["telemetry.py", "state"])
+ with open(log, encoding="utf-8") as fh:
+ text = fh.read()
+ for want in ("state: sent 19 values", "state: not sent, the counts are the same as last time", "state: not sent, the plugin has left no files here",
+ "command: not sent, not one of the plugin's commands", "command: sent 12 values", "state: not sent, CODE_MODERNIZATION_TELEMETRY is off",
+ "state: error KeyError at line"):
+ self.assertIn(want, text)
+ self.assertNotIn(self.tmp, text)
+
+ def test_the_debug_switch_never_writes_anywhere_else_and_never_follows_a_link(self):
+ victim = os.path.join(self.tmp, "victim.txt")
+ with open(victim, "w") as fh:
+ fh.write("keep")
+ try:
+ os.symlink(victim, os.path.join(self.data, "telemetry-debug.log"))
+ except OSError:
+ self.skipTest("no symbolic links here")
+ elsewhere = os.path.join(self.tmp, "somewhere-else.log")
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY_DEBUG": elsewhere}):
+ self.hook("state", {"cwd": self.ws})
+ self.assertFalse(os.path.exists(elsewhere)) # a path in the variable is not a switch
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY_DEBUG": "1"}):
+ self.hook("command", {"cwd": self.ws, "prompt": "/modernize"})
+ with open(victim) as fh:
+ self.assertEqual(fh.read(), "keep")
+
+ def test_each_off_switch_stops_everything(self):
+ for name, value in (("CODE_MODERNIZATION_TELEMETRY", "0"), ("CODE_MODERNIZATION_TELEMETRY", "Off"), ("CLAUDE_PLUGIN_OPTION_TELEMETRY", "false"),
+ ("CODE_MODERNIZATION_TELEMETRY", "disabled"), ("CODE_MODERNIZATION_TELEMETRY", "n"), ("CODE_MODERNIZATION_TELEMETRY", "none"),
+ ("CODE_MODERNIZATION_TELEMETRY", "2"), ("CLAUDE_PLUGIN_OPTION_TELEMETRY", "no"),
+ ("DISABLE_TELEMETRY", "1"), ("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "true")):
+ with mock.patch.dict(os.environ, {name: value}):
+ self.assertTrue(tm.off_reason(), name)
+ self.assertEqual(self.hook("state", {"cwd": self.ws}), "", name)
+ self.assertEqual(self.hook("command", {"cwd": self.ws, "prompt": "/modernize"}), "", name)
+ for name, value in (("CODE_MODERNIZATION_TELEMETRY", "1"), ("CODE_MODERNIZATION_TELEMETRY", ""), ("CLAUDE_PLUGIN_OPTION_TELEMETRY", "true"),
+ ("CLAUDE_PLUGIN_OPTION_TELEMETRY", " TRUE "), ("DISABLE_TELEMETRY", "0")):
+ with mock.patch.dict(os.environ, {name: value}):
+ self.assertEqual(tm.off_reason(), "", name)
+
+ def test_bad_hook_input_never_raises_and_never_prints(self):
+ for raw in ("", "not json", "[1,2]", '{"cwd": 5, "prompt": ["x"]}', '{"cwd": "/no/such/dir"}'):
+ out = io.StringIO()
+ with mock.patch.object(sys, "stdin", io.StringIO(raw)), contextlib.redirect_stdout(out), mock.patch.object(tm.os, "getcwd", return_value=self.tmp):
+ self.assertEqual(tm.main(["telemetry.py", "command"]), 0)
+ self.assertEqual(tm.main(["telemetry.py", "state"]), 0)
+ self.assertEqual(out.getvalue(), "", raw)
+
+ def test_show_prints_what_would_be_sent_in_words_and_as_json(self):
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out):
+ tm.show([self.ws, "--prompt", "/code-modernization:modernize-verify sys"])
+ text = out.getvalue()
+ for word in ("Sending is on", "rules", "business rules found", "steps that have left their file", "preflight, assess, map, rules", "modernize-verify"):
+ self.assertIn(word, text)
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out), mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY": "0"}):
+ tm.show([self.ws, "--json"])
+ got = json.loads(out.getvalue())
+ self.assertEqual(got["off"], "CODE_MODERNIZATION_TELEMETRY is off")
+ self.assertEqual(got["state"]["rules"], 3)
+
+
+class Hardening(Base):
+ """What a hostile or odd workspace must not be able to do."""
+
+ def test_counts_from_a_hundred_up_are_rounded_and_every_key_has_a_ceiling(self):
+ got = tm.ok({"pv": 10000, "rules": 97, "p0": 26, "map_kloc": 344, "eq_cases": 17727, "sec_high": 10 ** 12, "done": 43247, "goal": 9, "perm": 99,
+ "phases": 5000, "systems": 500})
+ self.assertEqual(dict(got), {"pv": 10000, "rules": 97, "p0": 26, "map_kloc": 340, "eq_cases": 18000, "sec_high": tm.COUNT_LIMIT, "done": 43247,
+ "goal": 4, "perm": 6, "phases": 99, "systems": 99})
+ self.assertEqual([tm.rounded(n) for n in (99, 100, 104, 105, 999, 1049, 1050, 17727)], [99, 100, 100, 110, 1000, 1000, 1100, 18000])
+
+ def test_a_folder_of_another_tool_is_not_the_plugins_even_with_the_same_file_names(self):
+ other = os.path.join(self.tmp, "other")
+ put(other, "analysis/2024-q3/topology.json", TOPOLOGY)
+ put(other, "analysis/2024-q3/BASELINE.md", "# baseline\n")
+ put(other, "analysis/2024-q3/ASSESSMENT.md", "# assessment\n")
+ self.assertIsNone(tm.state_metrics(other))
+ put(other, "analysis/2024-q3/PREFLIGHT.md", "# the plugin's\n")
+ self.assertIsNotNone(tm.state_metrics(other))
+
+ def test_another_tools_commands_are_not_counted_and_the_newest_folder_need_not_be_the_plugins(self):
+ self.assertIsNone(tm.command_metrics("/modernize-schema", self.ws))
+ self.assertEqual(tm.command_metrics("/code-modernization:modernize-schema", self.ws)["cmd"], 99)
+ put(self.ws, "analysis/zzz-data/results.csv", "a,b\n")
+ self.assertEqual(tm.state_metrics(self.ws)["rules"], 3)
+
+ def test_a_new_name_is_a_new_system_and_a_flags_value_is_not_a_name(self):
+ new = tm.command_metrics("/code-modernization:modernize-preflight billing --source /x", self.ws)
+ self.assertEqual((new["fresh"], new["done"], new["goal"], new["has_source"]), (1, 0, 0, 1))
+ newest = tm.command_metrics("/modernize --source elsewhere", self.ws)
+ self.assertEqual((newest["fresh"], newest["has_source"]), (0, 1))
+ self.assertEqual(tm.system_word(["--source", "abc", "def"]), "def")
+ self.assertIsNone(tm.system_word(["--source", "abc", "/work/x"]))
+
+ def test_links_that_lead_outside_the_workspace_are_not_followed(self):
+ outside = os.path.join(self.tmp, "outside")
+ for i in range(7):
+ put(outside, "d%d/x.txt" % i, "x")
+ shutil.rmtree(os.path.join(self.ws, "modernized", "sys"))
+ try:
+ os.symlink(outside, os.path.join(self.ws, "modernized", "sys"))
+ except OSError:
+ self.skipTest("no symbolic links here")
+ self.assertEqual(tm.state_metrics(self.ws)["built"], 0)
+ elsewhere = os.path.join(self.tmp, "elsewhere")
+ workspace(elsewhere, "real")
+ linked = os.path.join(self.tmp, "linked")
+ os.makedirs(linked)
+ os.symlink(os.path.join(elsewhere, "analysis"), os.path.join(linked, "analysis"))
+ self.assertEqual(tm.systems_in(linked), [])
+ self.assertIsNone(tm.state_metrics(linked))
+
+ def test_a_dangling_link_inside_a_system_does_not_drop_the_system(self):
+ try:
+ os.symlink(os.path.join(self.tmp, "gone"), os.path.join(self.ws, "analysis", "sys", "dangling"))
+ except OSError:
+ self.skipTest("no symbolic links here")
+ self.assertEqual(tm.state_metrics(self.ws)["rules"], 3)
+
+ def test_a_data_folder_that_is_itself_a_link_still_works(self):
+ real = os.path.join(self.tmp, "real-data")
+ os.makedirs(real)
+ link = os.path.join(self.tmp, "data-link")
+ try:
+ os.symlink(real, link)
+ except OSError:
+ self.skipTest("no symbolic links here")
+ with mock.patch.dict(os.environ, {"CLAUDE_PLUGIN_DATA": link}):
+ self.assertTrue(self.hook("state", {"cwd": self.ws}))
+ self.assertTrue(os.path.exists(os.path.join(real, "telemetry-state.json")))
+
+ def test_an_artifact_built_to_make_a_pattern_slow_does_not_hold_anything_up(self):
+ nasty = "### RULE-001 - t" + " " * 4000 + "x\n**Priority:** P0" + " " * 4000 + "x\n"
+ put(self.ws, "analysis/sys/BUSINESS_RULES.md", nasty * 3)
+ put(self.ws, "analysis/sys/SECURITY_FINDINGS.md", "| a |" + " " * 8000 + "x|\n|" + "-" * 5000 + "|\n")
+ started = time.time()
+ s = tm.state_metrics(self.ws)
+ self.assertLess(time.time() - started, 3)
+ self.assertGreaterEqual(s["rules"], 1)
+
+ def test_whatever_happens_the_hook_gives_up_within_a_few_seconds_and_says_nothing(self):
+ code = ("import sys, time; sys.path.insert(0, %r); import telemetry as tm; tm.state_metrics = lambda cwd: time.sleep(30); tm.hook('state')" %
+ os.path.join(PLUGIN, "scripts"))
+ started = time.time()
+ done = subprocess.run([sys.executable, "-c", code], input=json.dumps({"cwd": self.ws}), capture_output=True, text=True, timeout=20,
+ env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"})
+ self.assertLess(time.time() - started, 10)
+ self.assertEqual((done.returncode, done.stdout), (0, ""))
+
+ def test_the_alarm_is_cancelled_when_the_hook_returns(self):
+ import signal
+ self.hook("state", {"cwd": self.ws})
+ if hasattr(signal, "alarm"):
+ self.assertEqual(signal.alarm(0), 0)
+
+
+class Environment(Base):
+ """What the plugin says about the machine it runs on."""
+
+ def test_the_shell_wrappers_answers_are_used_and_bounded(self):
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_OS": "3", "CODE_MODERNIZATION_PY": "4", "CODE_MODERNIZATION_PATHF": "5"}):
+ got = tm.command_metrics("/modernize", self.ws)
+ self.assertEqual((got["os"], got["py"], got["pathf"]), (3, 4, 5))
+ self.assertEqual(got["pyv"], sys.version_info[0] * 100 + sys.version_info[1])
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_OS": "77", "CODE_MODERNIZATION_PY": "-1", "CODE_MODERNIZATION_PATHF": "x"}):
+ got = tm.command_metrics("/modernize", self.ws)
+ self.assertEqual((got["os"], got["py"]), (9, 0))
+ self.assertIn(got["os"], range(0, 10))
+
+ def test_this_machine_is_recognised_without_the_wrapper(self):
+ self.assertIn(tm.os_code(), (1, 2, 3, 4))
+ self.assertEqual(tm.python_status(), 0)
+
+ def test_a_path_that_breaks_scripts_is_flagged_in_bits(self):
+ self.assertEqual([tm.path_flags(p) for p in ("/plain/path", "/has a space", "/caf\u00e9", "/a b/caf\u00e9", "/" + "x" * 250, "/a b/caf\u00e9/" + "x" * 250)],
+ [0, 1, 2, 3, 4, 7])
+
+ def test_health_carries_only_the_environment(self):
+ got = json.loads(self.hook("health", {"cwd": self.ws}))["metrics"]
+ self.assertEqual(sorted(got), sorted(tm.HEALTH_KEYS if hasattr(tm, "HEALTH_KEYS") else ["pv", "os", "py", "pyv", "pathf"]))
+ self.assertTrue(all(type(v) is int for v in got.values()))
+
+
+class Failures(Base):
+ """A failure becomes a tool number and a kind number. Its text is read here and never sent."""
+
+ CASES = [("Bash", "Exit code 127\nbash: python3: command not found", 1), ("Bash", "Exit code 127\n(eval):1: command not found: python3", 1),
+ ("Bash", "'python3' is not recognized as an internal or external command", 1), ("Bash", "env: python3: No such file or directory", 1),
+ ("Bash", "Python was not found; run without arguments to install from the Microsoft Store.", 2),
+ ("Bash", "xcode-select: note: no developer tools were found at '/Applications/Xcode.app'", 3),
+ ("Bash", "Exit code 1\n File \"x.py\", line 3\nSyntaxError: invalid syntax", 4),
+ ("Bash", "Traceback (most recent call last):\n File \"/x/code-modernization/1.0.0/scripts/build_report.py\", line 9\nKeyError: 'a'", 5),
+ ("Edit", "Permission for this action was denied by the auto mode classifier", 6), ("Bash", "Permission to use Bash with command x has been denied.", 6),
+ ("Write", "EACCES: permission denied, open '/x'", 7), ("Bash", "Exit code 124\ncommand timed out after 120s", 8),
+ ("Read", "File does not exist. Note: your current working directory is /x.", 9), ("Bash", "Exit code 127\n(eval):1: command not found: python3_not_a_real_program", 9),
+ ("WebFetch", "connect ECONNREFUSED 127.0.0.1:443", 10), ("Bash", "npm ERR! 401 Unauthorized", 10), ("Bash", "ENOSPC: no space left on device", 11),
+ ("Workflow", "scriptPath must be inside the working directory", 12), ("Read", "something nobody has seen before", 99)]
+
+ def fail(self, tool, error, session="s1", **more):
+ return tm.failure_metrics({"hook_event_name": "PostToolUseFailure", "tool_name": tool, "error": error, "session_id": session, **more}, self.ws)
+
+ def test_each_kind_of_failure_gets_its_code(self):
+ for tool, text, want in self.CASES:
+ self.assertEqual(tm.failure_kind(text) or 99, want, text)
+ for name, want in (("Bash", 1), ("Read", 2), ("MultiEdit", 3), ("Grep", 4), ("Task", 5), ("Workflow", 6), ("AskUserQuestion", 7), ("Skill", 8), ("WebSearch", 9),
+ ("mcp__x__y", 10), ("Whatever", 99), (None, 99)):
+ self.assertEqual(tm.tool_code(name), want, name)
+
+ def test_a_failure_is_numbers_only_and_once_per_kind_per_session(self):
+ first = self.fail("Bash", "Exit code 127\nbash: python3: command not found")
+ self.assertEqual((first["tool"], first["kind"]), (1, 1))
+ self.assertEqual(list(first), ["pv", "os", "py", "pyv", "pathf", "tool", "kind"])
+ self.assertTrue(all(type(v) is int for v in tm.ok(first).values()))
+ self.assertNotIn("python", json.dumps(tm.ok(first)))
+ self.assertIsNone(self.fail("Bash", "Exit code 127\nbash: python3: command not found"))
+ self.assertIsNotNone(self.fail("Bash", "Exit code 127\nbash: python3: command not found", session="s2"))
+ self.assertIsNotNone(self.fail("Bash", "ENOSPC: no space left on device"))
+ self.assertIsNotNone(self.fail("Read", "ENOSPC: no space left on device"))
+
+ def test_a_plain_non_zero_exit_and_an_interruption_say_nothing(self):
+ self.assertIsNone(self.fail("Bash", "Exit code 1"))
+ self.assertIsNone(self.fail("Bash", " Exit code 2 "))
+ self.assertIsNone(self.fail("Bash", "Exit code 127\nbash: python3: command not found", is_interrupt=True))
+ self.assertIsNotNone(self.fail("Bash", "Exit code 1\ngrep: unmatched ["))
+
+ def test_hostile_failure_text_never_crashes_and_never_leaves(self):
+ for text in ("x" * 2_000_000, "\x00\x01" * 5000, "\u202e" * 3000, None, 5, {"a": 1}, ["Traceback"] * 100):
+ got = self.fail("Bash", text, session="h%s" % id(text))
+ if got is not None:
+ self.assertTrue(all(type(v) is int for v in got.values()))
+ secret = self.fail("Bash", "ENOSPC while writing /home/alice/secret-project/passwords.txt", session="p")
+ self.assertNotIn("alice", json.dumps(tm.ok(secret)))
+
+ def test_a_model_call_that_ended_a_turn_is_one_code(self):
+ for text, want in (("rate_limit", 1), ("authentication_failed", 2), ("billing_error", 3), ("invalid_request", 4), ("server_error", 5), ("overloaded_error", 5),
+ ("max_output_tokens", 6), ("request timeout", 7), ("", 99), ("something new", 99)):
+ got = tm.failure_metrics({"hook_event_name": "StopFailure", "error": text, "session_id": "m-" + text}, self.ws)
+ self.assertEqual(got["api"], want, text)
+ self.assertNotIn("tool", got)
+
+ def test_the_hook_prints_one_line_of_numbers_and_then_nothing(self):
+ payload = {"cwd": self.ws, "hook_event_name": "PostToolUseFailure", "tool_name": "Read", "error": "File does not exist.", "session_id": "z"}
+ line = json.loads(self.hook("failure", payload))["metrics"]
+ self.assertEqual((line["tool"], line["kind"]), (2, 9))
+ self.assertEqual(self.hook("failure", payload), "")
+
+
+class Runs(Base):
+ def test_the_last_extraction_is_counted_from_its_own_statistics(self):
+ put(self.ws, "analysis/sys/rules_result.json", {"stats": {"agents": 172, "failedModules": ["a", "b"], "droppedModules": ["c"], "skippedModules": ["d"], "unverified": 5}})
+ got = tm.ok(tm.run_metrics(self.ws))
+ self.assertEqual(dict(got), {"pv": tm.plugin_version(), "agents": 170, "wf_failed": 3, "wf_skip": 1, "wf_unver": 5})
+ self.assertTrue(self.hook("run", {"cwd": self.ws}))
+ self.assertEqual(self.hook("run", {"cwd": self.ws}), "")
+
+ def test_no_statistics_or_odd_ones_say_nothing_or_zeros(self):
+ self.assertIsNone(tm.run_metrics(self.ws))
+ put(self.ws, "analysis/sys/rules_result.json", {"stats": {"agents": "many", "failedModules": "x", "unverified": [1]}})
+ got = tm.ok(tm.run_metrics(self.ws))
+ self.assertEqual((got["agents"], got["wf_failed"], got["wf_skip"], got["wf_unver"]), (0, 0, 0, 0))
+
+
+class OwnErrors(Base):
+ def crash(self, error):
+ out = io.StringIO()
+ with mock.patch.object(tm, "state_metrics", side_effect=error), mock.patch.object(sys, "stdin", io.StringIO("{}")), contextlib.redirect_stdout(out):
+ self.assertEqual(tm.main(["telemetry.py", "state"]), 0)
+ return out.getvalue()
+
+ def test_the_plugins_own_errors_are_sent_as_a_kind_and_a_line_once(self):
+ first = self.crash(ValueError("secret text about /home/alice"))
+ line = json.loads(first)["metrics"]
+ self.assertEqual(line["err"], 2)
+ self.assertGreater(line["err_at"], 0)
+ self.assertNotIn("secret", first)
+ self.assertEqual(self.crash(ValueError("again")), "")
+ self.assertEqual(json.loads(self.crash(UnicodeDecodeError("utf-8", b"", 0, 1, "bad")))["metrics"]["err"], 8)
+ self.assertEqual(json.loads(self.crash(KeyError("k")))["metrics"]["err"], 1)
+ self.assertEqual(json.loads(self.crash(RuntimeError("odd")))["metrics"]["err"], 99)
+
+ def test_an_error_is_not_reported_when_counts_are_off(self):
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY": "0"}):
+ self.assertEqual(self.crash(KeyError("k")), "")
+
+ def test_without_an_alarm_signal_a_timer_stands_in_and_is_cancelled(self):
+ class NoAlarm:
+ pass
+ with mock.patch.object(tm, "signal", NoAlarm()):
+ self.assertTrue(self.hook("state", {"cwd": self.ws}))
+ self.assertEqual([t for t in threading.enumerate() if isinstance(t, threading.Timer)], [])
+
+
+class Shell(unittest.TestCase):
+ """scripts/telemetry.sh must start Python only when the plugin is in use."""
+
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+ self.bin = os.path.join(self.tmp, "bin")
+ os.makedirs(self.bin)
+ self.ws = workspace(os.path.join(self.tmp, "ws"))
+ self.data = os.path.join(self.tmp, "data")
+ os.makedirs(self.data)
+ os.symlink(sys.executable, os.path.join(self.bin, "python3"))
+
+ def run_hook(self, mode, payload, cwd=None, path=None, env=None):
+ env = {"PATH": path if path is not None else self.bin + os.pathsep + "/usr/bin:/bin", "CLAUDE_PLUGIN_DATA": self.data, **(env or {})}
+ done = subprocess.run(["/bin/sh", os.path.join(PLUGIN, "scripts", "telemetry.sh"), mode], input=json.dumps(payload), capture_output=True, text=True,
+ cwd=cwd or self.ws, env=env, timeout=60)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ return done.stdout
+
+ def test_a_plugin_command_is_counted_and_the_end_of_a_turn_in_a_workspace_is_counted(self):
+ out = self.run_hook("command", {"cwd": self.ws, "prompt": "/code-modernization:modernize-status sys"})
+ self.assertEqual(json.loads(out)["metrics"]["cmd"], 13)
+ self.assertEqual(json.loads(self.run_hook("state", {"cwd": self.ws}))["metrics"]["rules"], 3)
+ self.assertEqual(self.run_hook("state", {"cwd": self.ws}), "")
+
+ def test_it_starts_nothing_when_the_plugin_is_not_in_use(self):
+ no_python = os.path.join(self.tmp, "nopy")
+ os.makedirs(no_python)
+ for tool in ("grep", "dirname", "uname", "cat"):
+ os.symlink(shutil.which(tool), os.path.join(no_python, tool))
+ elsewhere = os.path.join(self.tmp, "elsewhere")
+ os.makedirs(elsewhere)
+ self.assertEqual(self.run_hook("command", {"prompt": "write me a poem"}), "")
+ self.assertEqual(self.run_hook("command", {"prompt": "/commit"}), "")
+ self.assertEqual(self.run_hook("state", {"cwd": elsewhere}, cwd=elsewhere), "")
+ self.assertEqual(self.run_hook("nonsense", {}), "")
+ # no python at all: quiet for a turn's end, and a typed command is reported in numbers by the shell itself
+ self.assertEqual(self.run_hook("state", {"cwd": self.ws}, path=no_python), "")
+ said = json.loads(self.run_hook("command", {"prompt": "/modernize"}, path=no_python))["metrics"]
+ self.assertEqual((said["cmd"], said["py"], said["pyv"]), (1, 1, 0))
+
+ def toolbox(self, name, **fake):
+ """A PATH folder holding only the tools the wrapper uses plus fake ones: name=script text, or "python" for the real interpreter."""
+ d = os.path.join(self.tmp, name)
+ os.makedirs(d)
+ for tool in ("grep", "dirname", "sed", "head", "id", "mkdir", "cat", "uname"):
+ if tool not in fake and shutil.which(tool):
+ os.symlink(shutil.which(tool), os.path.join(d, tool))
+ for tool, body in fake.items():
+ path = os.path.join(d, tool)
+ if body == "python":
+ os.symlink(sys.executable, path)
+ else:
+ with open(path, "w") as fh:
+ fh.write(body)
+ os.chmod(path, 0o755)
+ return d
+
+ def say(self, mode, payload, **kw):
+ out = self.run_hook(mode, payload, **kw)
+ return json.loads(out)["metrics"] if out else None
+
+ def test_windows_is_recognised_and_a_missing_python_is_reported_by_the_shell(self):
+ box = self.toolbox("win", uname="#!/bin/sh\necho MINGW64_NT-10.0-19045\n")
+ got = self.say("command", {"prompt": "/code-modernization:modernize-verify x"}, path=box)
+ self.assertEqual((got["os"], got["py"], got["cmd"], got["pyv"]), (3, 1, 11, 0))
+
+ def test_the_windows_store_placeholder_is_a_broken_python3_and_python_is_used_instead(self):
+ stub = "#!/bin/sh\necho 'Python was not found; run without arguments to install from the Microsoft Store.' >&2\nexit 49\n"
+ box = self.toolbox("store", python3=stub, uname="#!/bin/sh\necho MSYS_NT-10.0\n")
+ self.assertEqual(self.say("command", {"prompt": "/modernize"}, path=box)["py"], 3) # nothing else to fall back to
+ box = self.toolbox("store2", python3=stub, python="python", uname="#!/bin/sh\necho MSYS_NT-10.0\n")
+ got = self.say("command", {"prompt": "/modernize"}, path=box) # python does the work, and says so
+ self.assertEqual((got["os"], got["py"], got["cmd"]), (3, 4, 1))
+ self.assertEqual(got["pyv"], sys.version_info[0] * 100 + sys.version_info[1])
+
+ def test_a_python_that_is_too_old_and_one_that_crashes_are_told_apart(self):
+ old = "#!/bin/sh\ncase \"$1\" in -c) echo 307 ;; *) exit 1 ;; esac\n"
+ crash = "#!/bin/sh\ncase \"$1\" in -c) echo 311 ;; *) exit 1 ;; esac\n"
+ self.assertEqual(self.say("command", {"prompt": "/modernize"}, path=self.toolbox("old", python3=old))["py"], 5)
+ self.assertEqual(self.say("command", {"prompt": "/modernize"}, path=self.toolbox("crash", python3=crash))["py"], 6)
+
+ def test_health_is_sent_once_per_version_even_when_python_is_missing(self):
+ box = self.toolbox("nopy")
+ first = self.say("health", {"cwd": self.ws}, path=box)
+ self.assertEqual((first["py"], sorted(first)), (1, ["os", "pathf", "pv", "py", "pyv"]))
+ self.assertIsNone(self.say("health", {"cwd": self.ws}, path=box))
+ self.assertTrue(any(n.startswith("health-sent-") for n in os.listdir(self.data)))
+
+ def test_health_from_python_is_sent_once_and_the_off_switch_stops_it(self):
+ self.assertEqual(self.say("health", {"cwd": self.ws}, env={"CODE_MODERNIZATION_TELEMETRY": "0"}), None)
+ self.assertFalse(any(n.startswith("health-sent-") for n in os.listdir(self.data)))
+ first = self.say("health", {"cwd": self.ws})
+ self.assertEqual(first["py"], 0)
+ self.assertIsNone(self.say("health", {"cwd": self.ws}))
+
+ def test_a_python_failure_is_reported_by_the_shell_when_python_cannot_run(self):
+ box = self.toolbox("nopy2")
+ put(self.ws, "analysis/sys/PREFLIGHT.md", "# x\n")
+ got = self.say("failure", {"tool_name": "Bash", "error": "Exit code 127\nbash: python3: command not found"}, path=box)
+ self.assertEqual((got["tool"], got["kind"], got["py"]), (1, 1, 1))
+ self.assertIsNone(self.say("failure", {"tool_name": "Read", "error": "File does not exist."}, path=box))
+
+ def test_a_folder_with_a_space_and_an_accent_is_flagged(self):
+ odd = os.path.join(self.tmp, "my caf\u00e9 work")
+ os.makedirs(odd)
+ got = self.say("command", {"prompt": "/modernize"}, cwd=odd)
+ self.assertEqual(got["pathf"], 3)
+
+ def test_python_starts_only_where_the_plugin_has_left_its_files(self):
+ marker = os.path.join(self.tmp, "started")
+ fake = os.path.join(self.tmp, "fakebin")
+ os.makedirs(fake)
+ with open(os.path.join(fake, "python3"), "w") as fh:
+ fh.write("#!/bin/sh\ntouch %s\ncat >/dev/null\n" % marker)
+ os.chmod(os.path.join(fake, "python3"), 0o755)
+ path = fake + os.pathsep + "/usr/bin:/bin"
+ other = os.path.join(self.tmp, "other")
+ put(other, "analysis/results/data.csv", "a,b\n")
+ self.run_hook("state", {"cwd": other}, cwd=other, path=path)
+ self.assertFalse(os.path.exists(marker))
+ put(other, "analysis/results/PREFLIGHT.md", "# x\n")
+ self.run_hook("state", {"cwd": other}, cwd=other, path=path)
+ self.assertTrue(os.path.exists(marker))
+
+ def test_it_leaves_no_bytecode_in_the_plugins_folder(self):
+ root = os.path.join(self.tmp, "plugin")
+ for rel in ("scripts/telemetry.sh", "scripts/telemetry.py", "scripts/build_report.py", ".claude-plugin/plugin.json"):
+ os.makedirs(os.path.dirname(os.path.join(root, rel)), exist_ok=True)
+ shutil.copy(os.path.join(PLUGIN, rel), os.path.join(root, rel))
+ done = subprocess.run(["/bin/sh", os.path.join(root, "scripts", "telemetry.sh"), "state"], input=json.dumps({"cwd": self.ws}), capture_output=True,
+ text=True, cwd=self.ws, env={"PATH": self.bin + os.pathsep + "/usr/bin:/bin", "CLAUDE_PLUGIN_DATA": self.data}, timeout=60)
+ self.assertIn('"metrics"', done.stdout)
+ self.assertEqual(sorted(os.listdir(os.path.join(root, "scripts"))), ["build_report.py", "telemetry.py", "telemetry.sh"])
+
+ def test_the_off_switch_reaches_the_script(self):
+ self.assertEqual(self.run_hook("state", {"cwd": self.ws}, env={"CODE_MODERNIZATION_TELEMETRY": "0"}), "")
+
+
+class Shipped(unittest.TestCase):
+ """What ships must agree with itself: the manifest, the hooks, the README and the changelog."""
+
+ def read(self, *parts):
+ with open(os.path.join(PLUGIN, *parts), encoding="utf-8") as fh:
+ return fh.read()
+
+ def test_the_hooks_call_scripts_that_exist_and_are_asynchronous(self):
+ hooks = json.loads(self.read("hooks", "hooks.json"))["hooks"]
+ want = {"UserPromptSubmit": ["command"], "Stop": ["state", "run"], "SessionStart": ["health"], "PostToolUseFailure": ["failure"], "StopFailure": ["failure"]}
+ self.assertEqual(sorted(hooks), sorted(want))
+ for event, groups in hooks.items():
+ modes = []
+ for group in groups:
+ for h in group["hooks"]:
+ self.assertTrue(h["asyncRewake"])
+ script = re.search(r"\$\{CLAUDE_PLUGIN_ROOT\}/(scripts/\w+\.sh)", h["command"]).group(1)
+ self.assertTrue(os.path.isfile(os.path.join(PLUGIN, script)), script)
+ modes.append(h["command"].split()[-1])
+ self.assertEqual(modes, want[event], event)
+
+ def test_the_manifest_has_a_version_and_the_option_that_turns_counts_off(self):
+ manifest = json.loads(self.read(".claude-plugin", "plugin.json"))
+ self.assertRegex(manifest["version"], r"^\d+\.\d+\.\d+$")
+ option = manifest["userConfig"]["telemetry"]
+ self.assertEqual((option["type"], option["default"]), ("boolean", True))
+ with open(os.path.join(PLUGIN, "..", "..", ".claude-plugin", "marketplace.json"), encoding="utf-8") as fh:
+ entry = next(p for p in json.load(fh)["plugins"] if p["name"] == "code-modernization")
+ self.assertNotIn("version", entry) # the manifest is the one place the number lives
+
+ def test_the_changelog_names_the_manifest_version_first(self):
+ version = json.loads(self.read(".claude-plugin", "plugin.json"))["version"]
+ first = re.search(r"(?m)^## \[?(\d+\.\d+\.\d+)", self.read("CHANGELOG.md"))
+ self.assertEqual(first.group(1), version)
+
+ def test_the_readme_lists_every_key_that_can_be_sent_and_the_off_switches(self):
+ readme = self.read("README.md")
+ section = readme[readme.index("## Telemetry"):]
+ section = section[:section.index("\n## ", 5)] if "\n## " in section[5:] else section
+ for key in tm.MEANING:
+ self.assertIn("`%s`" % key, section, key)
+ for word in ("CODE_MODERNIZATION_TELEMETRY", "Usage counts", "DISABLE_TELEMETRY", "telemetry.py show", "two significant figures", "TELEMETRY_DEBUG"):
+ self.assertIn(word, section)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/trace_rules.py b/plugins/code-modernization/scripts/trace_rules.py
new file mode 100644
index 00000000..139e438c
--- /dev/null
+++ b/plugins/code-modernization/scripts/trace_rules.py
@@ -0,0 +1,520 @@
+#!/usr/bin/env python3
+"""Trace every business rule to the code and the tests that name it, and to the tests that ran.
+
+ python3 trace_rules.py <system> [--workspace DIR] [--json] [--all] [--module NAME --results PATH ...]
+
+Reads analysis/<system>/BUSINESS_RULES.md (rule cards: "### RULE-NNN: name" with **Priority:**,
+**Confidence:** and **Source:** lines), then scans the built code under modernized/<system>,
+modernized/<system>-uplifted and modernized/<system>-reimagined (never build output: target/,
+build/, dist/, node_modules/, bin/, obj/, __pycache__/, or any folder starting with a dot) for
+mentions of each rule id, and sorts every file into "test" or "main" code. One row per rule.
+
+Evidence, strongest first:
+ tested executed evidence backs it, from the per-test results of that run (JUnit or TRX XML; --results
+ reads them here, the proof pack passes them in): a test that ran and passed names the rule in
+ its test or class name, or a test file names it on a line not marked skipped and that file's
+ name is the class or file of a test that ran and passed. Without per-test results nothing is
+ "tested"
+ named, not run a test file names it, but nothing that ran backs it: the test is skipped, pending or failing,
+ its class never ran, or no per-test results were given
+ code only only main code names it
+ claimed only only the mapping table in TRANSFORMATION_NOTES.md or UPLIFT_NOTES.md names it, on a
+ row that also names a file that exists in the module
+ none nothing names it
+
+A mention is RULE-017 (any case, or RULE_017), rule017 or Rule017 inside an identifier such as
+rule017_emptyInput or testRule017, and the shorthand RULE-002/003 and RULE-001, -018. Nothing is
+guessed from words. A line is marked skipped when it, or one of the 3 lines above it, holds @Disabled,
+@Ignore, [Ignore], #[ignore], xit(, xdescribe, skip (so also @Skip, t.Skip, it.skip, test.skip,
+describe.skip and @pytest.mark.skip), pending or todo, in any case. A mention shows that a test names
+the rule, not that the test is good.
+
+A test is a file under a folder called test, tests, __tests__, spec or specs, or named like
+FooTest.java, FooTests.cs, TestFoo.java, test_foo.py, foo_test.go, foo.spec.ts or foo.test.js.
+Documentation files (.md, .txt, .rst, .adoc) are never code. Symbolic links are never followed.
+A built folder that holds only test files and the files that build them (pom.xml, mvnw, package.json,
+...) is test tooling, not a module: tooling_only() says so.
+Everything read is untrusted text: it is never executed and only shortened, never trusted.
+
+Exit 0 when every P0 rule is "tested", 1 when some are not, 2 when the rules file cannot be read.
+Standard library only.
+"""
+import argparse
+import bisect
+import collections
+import json
+import os
+import re
+import sys
+
+SKIP_DIRS = {"target", "build", "dist", "node_modules", "bin", "obj", "__pycache__", "venv"}
+DOC_EXT = {".md", ".markdown", ".rst", ".txt", ".adoc"}
+TEST_DIRS = {"test", "tests", "__tests__", "spec", "specs"}
+TEST_NAME = re.compile(r"^test_|_tests?\.[^.]+$|\.(?:spec|test)\.[^.]+$|(?:Test|Tests|TestCase|IT)\.[^.]+$|^Test[A-Z0-9_]")
+NOTES_NAMES = ("TRANSFORMATION_NOTES.md", "UPLIFT_NOTES.md")
+TRACKS = (("rewrite", ""), ("uplift", "-uplifted"), ("reimagine", "-reimagined"))
+FILE_CAP, RULES_CAP, NOTES_CAP, LINE_CAP, MAX_FILES, SAMPLES = 2 << 20, 8 << 20, 1 << 20, 4000, 200000, 3
+MENTION = [re.compile(r"(?<![A-Za-z0-9])rule[-_](\d{1,6})(?!\d)", re.I),
+ re.compile(r"(?<![A-Za-z0-9])rule(\d{3,6})(?!\d)", re.I),
+ re.compile(r"(?<=[a-z])Rule[-_]?(\d{3,6})(?!\d)")]
+MORE = re.compile(r"(?:/|,[ \t]*-)(\d{2,6})(?!\d)")
+NOT_A_CLAIM = re.compile(r"(?i)not migrated|not implemented|not ported|removed|retired|dead code|dropped|out of scope")
+CARD = re.compile(r"#{2,5}[ \t]+[*`\[]*RULE[-_](\d{1,6})(?![A-Za-z0-9])", re.I)
+FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})")
+PATH = re.compile(r"[A-Za-z0-9_@][A-Za-z0-9_@./+-]{0,199}\.[A-Za-z][A-Za-z0-9]{0,7}(?![A-Za-z0-9_])")
+SKIPPED_LINE = re.compile(r"@Disabled|@Ignore|\[Ignore\]|#\[ignore\]|xit\(|xdescribe|skip|pending|todo", re.I) # "skip" also covers @Skip, t.Skip, it.skip, test.skip, describe.skip, @pytest.mark.skip
+MARK_LINES = 3 # a mention is skipped when its own line, or one of this many lines above it, holds a marker
+NOT_RUN = "named, not run"
+NO_RUN = {"known": False, "ids": frozenset(), "keys": frozenset()}
+TOOLING_NAMES = {"pom.xml", "mvnw", "mvnw.cmd", "gradlew", "gradlew.bat", "build.gradle", "build.gradle.kts", "settings.gradle", "settings.gradle.kts", "gradle.properties",
+ "package.json", "package-lock.json", "yarn.lock", "pnpm-lock.yaml", "pytest.ini", "tox.ini", "setup.cfg", "pyproject.toml", "uv.lock", "poetry.lock", "conftest.py",
+ "cargo.toml", "cargo.lock", "go.mod", "go.sum", "composer.json", "composer.lock", "phpunit.xml", "phpunit.xml.dist", "cmakelists.txt", "makefile", "dockerfile"}
+TOOLING_EXT = {".csproj", ".sln", ".props", ".targets", ".runsettings"}
+TOOLING_PATTERN = re.compile(r"tsconfig[\w.-]*\.json|[\w-]+\.config\.(?:js|cjs|mjs|ts)|docker-compose[\w.-]*\.ya?ml")
+
+
+def clean(value, limit=200):
+ text = " ".join(re.sub(r"[\x00-\x1f\x7f-\x9f\u2028\u2029]", " ", str(value if value is not None else "")).split())
+ return text if len(text) <= limit else text[:limit - 1] + "\u2026"
+
+
+def read_text(path, cap):
+ """Text of a regular, non-binary file (first `cap` bytes), or None."""
+ try:
+ with open(path, "rb") as fh:
+ raw = fh.read(cap)
+ except OSError:
+ return None
+ return None if b"\x00" in raw[:4096] else raw.decode("utf-8", "replace")
+
+
+def find_ids(text):
+ """[(rule number, offset)] for every id named in text, shorthand included."""
+ found = []
+ for rx in MENTION:
+ for m in rx.finditer(text):
+ found.append((int(m.group(1)), m.start()))
+ pos = m.end()
+ more = MORE.match(text, pos)
+ while more:
+ found.append((int(more.group(1)), m.start()))
+ pos = more.end()
+ more = MORE.match(text, pos)
+ return found
+
+
+def class_keys(name):
+ """The lower-case names a test class goes by in a result file, for matching a test file's name (without its extension): the last two
+ parts of a dotted, slashed or $-nested name (a Python module and its class, a Java class), the outermost class of a nested one and,
+ for a file path, its file name with and without the extension."""
+ text = str(name if name is not None else "")[:400]
+ parts = [p for p in re.split(r"[.$:/\\]+", text) if p]
+ outer = [p for p in re.split(r"[.:/\\]+", text.split("$")[0]) if p]
+ keys = {p.lower() for p in parts[-2:] + outer[-1:]}
+ if "/" in text or "\\" in text:
+ base = re.split(r"[/\\]", text)[-1]
+ keys.update((base.lower(), os.path.splitext(base)[0].lower()))
+ return keys
+
+
+def run_evidence(results):
+ """What ran for one module, from its parsed results (baseline_diff.read_results, one per suite): {"known", "ids", "keys"}.
+ known: some result lists its tests one by one (a count, or a summary line of a log, names no test).
+ ids: the rule numbers named by the test name or class name of a test that ran and passed.
+ keys: class_keys of every class that has a test that ran and passed."""
+ out = {"known": False, "ids": set(), "keys": set()}
+ for res in results:
+ if not res or not res.get("tests"):
+ continue
+ out["known"] = True
+ for tid, status in res["tests"].items():
+ if status == "PASS" and "rule" in tid.lower():
+ out["ids"].update(n for n, _ in find_ids(tid))
+ for klass, c in (res.get("classes") or {}).items():
+ if c.get("pass"):
+ out["keys"] |= class_keys(klass)
+ return out
+
+
+def line_marks(body):
+ """([offset where each line starts], [whether the line holds a skip marker]) for one test file's text."""
+ starts, flags, pos = [], [], 0
+ for line in body.split("\n"):
+ starts.append(pos)
+ flags.append(SKIPPED_LINE.search(line) is not None)
+ pos += len(line) + 1
+ return starts, flags
+
+
+def is_live(marks, at):
+ """True when the mention at text offset `at` is not marked skipped: its own line and the 3 lines above hold no marker."""
+ starts, flags = marks
+ i = bisect.bisect_right(starts, at) - 1
+ return not any(flags[max(0, i - MARK_LINES):i + 1])
+
+
+def field(lines, name):
+ """The value of a "**Name:** value" line, or ''. Linear in the line length whatever the line holds."""
+ for line in lines:
+ s = line[:LINE_CAP].lstrip(" \t>*-")
+ if s[:len(name)].lower() == name.lower():
+ rest = s[len(name):].lstrip("*").lstrip(" \t")
+ if rest.startswith(":"):
+ return rest[1:].lstrip(" \t*").strip()
+ return ""
+
+
+def parse_rules(text):
+ """-> ([rule dicts in file order], [notes]). A duplicate id keeps its first card."""
+ rules, notes, seen, card, fence, rows = [], [], set(), None, "", {}
+ lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
+ for line in lines:
+ if line.lstrip().startswith("|"):
+ cells = [re.sub(r"[*`]", "", c).strip() for c in line[:LINE_CAP].strip().strip("|").split("|")]
+ ids = [int(m.group(1)) for c in cells for m in [re.fullmatch(r"RULE[-_](\d{1,6})", c, re.I)] if m]
+ if ids:
+ rows.setdefault(ids[0], {"p": next((c for c in cells if re.fullmatch(r"P\d", c)), ""),
+ "c": next((c for c in cells if re.match(r"(?i)(high|medium|low)\b", c)), "")})
+ for line in lines:
+ f = FENCE.match(line)
+ if fence:
+ fence = "" if f and f.group(1)[0] == fence[0] and len(f.group(1)) >= len(fence) else fence
+ if card is not None:
+ card["body"].append(line)
+ continue
+ if f:
+ fence = f.group(1)
+ m = None if f else CARD.match(line[:LINE_CAP])
+ if not f and re.match(r"#{1,6}\s", line):
+ card = None
+ if m:
+ n = int(m.group(1))
+ if n in seen:
+ notes.append("RULE-%03d has more than one card; the first was used." % n)
+ continue
+ seen.add(n)
+ name = line[m.end():LINE_CAP].lstrip("*`] \t:\u00b7\u2013\u2014-").rstrip("# \t")
+ card = {"n": n, "id": "RULE-%03d" % n, "name": clean(name, 160), "body": []}
+ rules.append(card)
+ elif card is not None:
+ card["body"].append(line)
+ for r in rules:
+ body = r.pop("body")
+ prio = re.search(r"\bP\d\b", field(body, "Priority"))
+ conf = re.search(r"(?i)\b(high|medium|low)\b", field(body, "Confidence"))
+ row = rows.get(r["n"], {})
+ source = field(body, "Source")
+ first = PATH.search(source)
+ r.update({"priority": prio.group(0) if prio else row.get("p", ""),
+ "confidence": conf.group(1).capitalize() if conf else row.get("c", "").capitalize()[:10],
+ "source": clean(source, 160), "sourceFile": os.path.basename(first.group(0)) if first else ""})
+ return rules, notes
+
+
+def file_kind(rel):
+ parts = rel.split("/")
+ name = parts[-1]
+ if os.path.splitext(name)[1].lower() in DOC_EXT:
+ return "doc"
+ if any(p.lower() in TEST_DIRS for p in parts[:-1]) or TEST_NAME.search(name):
+ return "test"
+ return "main"
+
+
+def is_real_dir(path):
+ return os.path.isdir(path) and not os.path.islink(path)
+
+
+def discover_modules(workspace, system):
+ """[{name, track, path, rel}] for every built module under modernized/."""
+ out = []
+ for track, suffix in TRACKS:
+ root = os.path.join(workspace, "modernized", system + suffix)
+ if not is_real_dir(root):
+ continue
+ rel = "modernized/" + system + suffix
+ if track == "uplift":
+ out.append({"name": system + suffix, "track": track, "path": root, "rel": rel})
+ continue
+ try:
+ subs = sorted(d for d in os.listdir(root) if not d.startswith(".") and d not in SKIP_DIRS and is_real_dir(os.path.join(root, d)))
+ except OSError:
+ subs = []
+ for d in subs:
+ out.append({"name": d, "track": track, "path": os.path.join(root, d), "rel": rel + "/" + d})
+ if not subs:
+ out.append({"name": system + suffix, "track": track, "path": root, "rel": rel})
+ return out
+
+
+def walk_module(path, budget):
+ """([(relative posix path, absolute path)] of regular files, links skipped, cut short)."""
+ files, skipped = [], 0
+ for base, dirs, names in os.walk(path):
+ keep = []
+ for d in sorted(dirs):
+ if d.startswith(".") or d in SKIP_DIRS:
+ continue
+ if os.path.islink(os.path.join(base, d)):
+ skipped += 1
+ continue
+ keep.append(d)
+ dirs[:] = keep
+ for n in sorted(names):
+ full = os.path.join(base, n)
+ if os.path.islink(full):
+ skipped += 1
+ elif not n.startswith(".") and os.path.isfile(full):
+ if len(files) >= budget:
+ return files, skipped, True
+ files.append((os.path.relpath(full, path).replace(os.sep, "/"), full))
+ return files, skipped, False
+
+
+def is_tooling(name):
+ """A file that builds or runs code without being code: pom.xml, mvnw, package.json, conftest.py, *.csproj, ..."""
+ low = name.lower()
+ return low in TOOLING_NAMES or os.path.splitext(low)[1] in TOOLING_EXT or TOOLING_PATTERN.fullmatch(low) is not None
+
+
+def tooling_only(path):
+ """True when a built folder holds test code and the files that build or run it, and nothing else that is not a document: at least
+ one test file, no main code. A file this does not recognise counts as main code, so a folder is only left out when it is clear."""
+ files, _, cut = walk_module(path, MAX_FILES)
+ tests = 0
+ for rel, _ in files:
+ kind = file_kind(rel)
+ if kind == "test":
+ tests += 1
+ elif kind == "main" and not is_tooling(rel.rsplit("/", 1)[-1]):
+ return False
+ return tests > 0 and not cut
+
+
+def claims_of(notes, names):
+ """(rule numbers claimed, rule rows naming no existing file, basenames of every file the notes name).
+ A row claims a rule only when it names the rule AND a file that exists in the module."""
+ claimed, unresolved, named, head = set(), 0, set(), ""
+ for line in notes.split("\n"):
+ line = line[:LINE_CAP]
+ tokens = PATH.findall(line)
+ named.update(os.path.basename(t) for t in tokens)
+ if line.startswith("#"):
+ head = line.lstrip("#").strip()
+ if not line.lstrip().startswith("|") or NOT_A_CLAIM.search(head): # a table under "Not migrated" claims nothing
+ continue
+ ids = {n for n, _ in find_ids(line)}
+ if not ids:
+ continue
+ if any(os.path.basename(t) in names and any(p == t or p.endswith("/" + t) for p in names[os.path.basename(t)]) for t in tokens):
+ claimed |= ids
+ else:
+ unresolved += 1
+ return claimed, unresolved, named
+
+
+def status_of(main, tests, claimed, ran=False):
+ """A rule's status. `ran`: executed evidence backs it. A test file that names it without that evidence is "named, not run"."""
+ return "tested" if ran else NOT_RUN if tests else "code only" if main else "claimed only" if claimed else "none"
+
+
+def blank():
+ """`tests`: mentions in test files; `live`: those not marked skipped; `by`: how executed evidence backs the rule ("test name", "test file") or ''; `at`: the first
+ unskipped mention in a test file that ran (file:line), the one that backs it."""
+ return {"main": 0, "tests": 0, "live": 0, "claimed": False, "named": False, "by": "", "at": ""}
+
+
+def trace(workspace, system, ran=None):
+ """-> the trace. Raises OSError when BUSINESS_RULES.md cannot be read.
+ ran: {module folder (as "path" in the modules list): run_evidence(...)}: what ran, per module. Without it no rule is "tested"."""
+ rules_path = os.path.join(workspace, "analysis", system, "BUSINESS_RULES.md")
+ if os.path.islink(rules_path):
+ raise OSError("analysis/%s/BUSINESS_RULES.md is a symbolic link" % system)
+ text = read_text(rules_path, RULES_CAP)
+ if text is None:
+ raise OSError("analysis/%s/BUSINESS_RULES.md is missing or is not text" % system)
+ rules, notes = parse_rules(text)
+ by_n = {r["n"]: r for r in rules}
+ per = {r["n"]: {} for r in rules}
+ samples = {r["n"]: {"main": [], "tests": []} for r in rules}
+ modules, budget = [], MAX_FILES
+ scan = {"files": 0, "textFiles": 0, "linksSkipped": 0, "cut": False}
+ for mod in discover_modules(workspace, system):
+ files, skipped, cut = walk_module(mod["path"], budget)
+ budget -= len(files)
+ names = collections.defaultdict(list)
+ for rel, _ in files:
+ names[os.path.basename(rel)].append(rel)
+ notes_name = next((n for n in NOTES_NAMES if n in names[n]), "")
+ claimed, unresolved, named = set(), 0, set()
+ if notes_name:
+ ntext = read_text(os.path.join(mod["path"], notes_name), NOTES_CAP)
+ if ntext is not None:
+ claimed, unresolved, named = claims_of(ntext, names)
+ ev = (ran or {}).get(mod["rel"]) or NO_RUN
+ test_files = 0
+ for rel, full in files:
+ kind = file_kind(rel)
+ if kind == "doc":
+ continue
+ key = "tests" if kind == "test" else "main"
+ test_files += kind == "test"
+ body = read_text(full, FILE_CAP)
+ scan["textFiles"] += body is not None
+ hits = [(n, None) for n, _ in find_ids(os.path.basename(rel))] + [(n, at) for n, at in find_ids(body or "")]
+ # a test file backs the rules it names (on lines not marked skipped) when its name is the class or file of a test that ran and passed here
+ marks, backed = None, kind == "test" and bool(ev.get("known")) and os.path.splitext(os.path.basename(rel))[0].lower() in (ev.get("keys") or ())
+ for n, at in hits:
+ if n not in by_n:
+ continue
+ slot = per[n].setdefault(mod["rel"], blank())
+ slot[key] += 1
+ if kind == "test":
+ if at is not None and marks is None:
+ marks = line_marks(body)
+ if at is None or is_live(marks, at):
+ slot["live"] += 1
+ if backed and not slot["by"]:
+ slot["by"], slot["at"] = "test file", "%s/%s:%d" % (mod["rel"], rel, 0 if at is None else bisect.bisect_right(marks[0], at))
+ place = "%s/%s" % (mod["rel"], rel)
+ if len(samples[n][key]) < SAMPLES and not any(s.rsplit(":", 1)[0] == place for s in samples[n][key]):
+ samples[n][key].append("%s:%d" % (place, 0 if at is None else body.count("\n", 0, at) + 1))
+ for n in (ev.get("ids") or ()):
+ if n in by_n:
+ per[n].setdefault(mod["rel"], blank())["by"] = "test name"
+ for n in claimed & set(by_n):
+ per[n].setdefault(mod["rel"], blank())["claimed"] = True
+ for r in rules:
+ if r["sourceFile"] and r["sourceFile"] in named:
+ per[r["n"]].setdefault(mod["rel"], blank())["named"] = True
+ scan["files"] += len(files)
+ scan["linksSkipped"] += skipped
+ scan["cut"] = scan["cut"] or cut
+ modules.append({"name": mod["name"], "track": mod["track"], "path": mod["rel"], "files": len(files), "testFiles": test_files,
+ "notes": notes_name, "claimRowsUnresolved": unresolved})
+ out = []
+ for r in rules:
+ slots = per[r["n"]]
+ main, tests_n = sum(s["main"] for s in slots.values()), sum(s["tests"] for s in slots.values())
+ claim = any(s["claimed"] for s in slots.values())
+ out.append({"id": r["id"], "name": r["name"], "priority": r["priority"], "confidence": r["confidence"], "source": r["source"],
+ "main": main, "tests": tests_n, "claimed": claim, "status": status_of(main, tests_n, claim, any(s["by"] for s in slots.values())),
+ "perModule": slots, "samples": samples[r["n"]]})
+ if scan["cut"]:
+ notes.append("More than %d files were found, so the scan stopped early." % MAX_FILES)
+ if scan["linksSkipped"]:
+ notes.append("%d symbolic link(s) were not followed." % scan["linksSkipped"])
+ if not modules:
+ notes.append("No built code was found under modernized/%s, %s-uplifted or %s-reimagined." % (system, system, system))
+ return {"system": clean(system, 100), "rulesFile": "analysis/%s/BUSINESS_RULES.md" % system, "modules": modules, "rules": out,
+ "totals": totals(out), "p0NoTests": [r["id"] for r in out if r["priority"] == "P0" and r["status"] != "tested"],
+ "scan": scan, "notes": notes}
+
+
+def totals(rows):
+ """Per priority: rules, and how many are tested / named, not run / code only / claimed only / none."""
+ out = {}
+ for r in rows:
+ t = out.setdefault(r["priority"] or "unrated", {"rules": 0, "tested": 0, NOT_RUN: 0, "code only": 0, "claimed only": 0, "none": 0})
+ t["rules"] += 1
+ t[r["status"]] += 1
+ return dict(sorted(out.items()))
+
+
+def module_row(r, slot):
+ """A rule as one module sees it: the module's own counts and the status they give."""
+ return dict(r, main=slot["main"], tests=slot["tests"], live=slot["live"], claimed=slot["claimed"], by=slot["by"], at=slot["at"],
+ status=status_of(slot["main"], slot["tests"], slot["claimed"], bool(slot["by"])))
+
+
+def module_view(result, rel):
+ """The rules one module answers for, each with the module's own status. A rewrite answers for the rules its code,
+ tests or notes name, that a test of its run names, or whose legacy file its notes name; an uplift keeps the whole
+ code, so it answers for all. When nothing ties any rule to the module, every rule counts (`tied` is then False):
+ an empty list must never pass. `p0NoTests` lists the P0 rules that are not "tested": no executed test backs them."""
+ mod = next((m for m in result["modules"] if m["path"] == rel), None)
+ rows, tied = [], True
+ for r in result["rules"]:
+ slot = r["perModule"].get(rel) or blank()
+ if mod and (mod["track"] == "uplift" or slot["main"] or slot["tests"] or slot["claimed"] or slot["named"] or slot["by"]):
+ rows.append(module_row(r, slot))
+ if mod and not rows and result["rules"]:
+ tied = False
+ rows = [module_row(r, r["perModule"].get(rel) or blank()) for r in result["rules"]]
+ return {"rows": rows, "totals": totals(rows), "outOfScope": len(result["rules"]) - len(rows), "tied": tied,
+ "p0NoTests": [r["id"] for r in rows if r["priority"] == "P0" and r["status"] != "tested"]}
+
+
+def render(result, everything=False, view=None, module=None):
+ """The matrix as text. With `view` (from module_view) only the rules that module answers for are listed."""
+ rows = view["rows"] if view else result["rules"]
+ tot = view["totals"] if view else result["totals"]
+ missing = view["p0NoTests"] if view else result["p0NoTests"]
+ scanned = ", ".join("%s (%d files, %d test)" % (clean(m["name"], 60), m["files"], m["testFiles"]) for m in result["modules"]
+ if not module or m["name"] == module) or "nothing built yet"
+ head = "Rule trace for %s%s: %d rule(s) in %s" % (result["system"], ", module " + clean(module, 60) if module else "", len(result["rules"]), result["rulesFile"])
+ lines = [head, "Scanned: " + scanned]
+ if view:
+ lines.append("Counted for this module: %d rule(s)%s" % (len(rows), "" if view["tied"] else " (nothing ties a rule to it by name, so every rule counts)"))
+ lines.append("")
+ for p, c in tot.items():
+ lines.append("%-7s %3d rules: %d tested, %d named not run, %d code only, %d claimed only, %d none" % (p, c["rules"], c["tested"], c[NOT_RUN], c["code only"], c["claimed only"], c["none"]))
+ lines += ["", "%-9s %-4s %-7s %5s %5s %s" % ("Rule", "Pri", "Conf", "Code", "Tests", "Status")]
+ shown = [r for r in rows if everything or r["priority"] == "P0"] + ([] if everything else [r for r in rows if r["priority"] != "P0"][:40])
+ for r in shown:
+ lines.append("%-9s %-4s %-7s %5d %5d %s %s" % (r["id"], r["priority"] or "-", r["confidence"] or "-", r["main"], r["tests"], r["status"], clean(r["name"], 60)))
+ if len(shown) < len(rows):
+ lines.append("... %d more rule(s) not listed (use --all)" % (len(rows) - len(shown)))
+ lines += ["", "P0 rules with no test that ran and passed behind them: " + (", ".join(missing) if missing else "none")]
+ lines += ["Note: " + n for n in result["notes"]]
+ return "\n".join(lines)
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description="Trace business rules to the code and tests that name them, and to the tests that ran.")
+ ap.add_argument("system", help="the system's folder name under analysis/")
+ ap.add_argument("--workspace", default=".", help="the project root holding analysis/ and modernized/ (default: current folder)")
+ ap.add_argument("--json", action="store_true", help="print the full trace as JSON")
+ ap.add_argument("--all", action="store_true", help="list every rule, not just P0 and the first 40 others")
+ ap.add_argument("--module", metavar="NAME", help="only the rules this built module answers for (a folder under modernized/<system>/)")
+ ap.add_argument("--results", action="append", default=[], metavar="PATH",
+ help="a JUnit or TRX result file, or a folder of them, from a run of the module named by --module (repeatable); only tests that ran and passed there make a rule 'tested'")
+ args = ap.parse_args(argv)
+ if not args.system or args.system in (".", "..") or re.search(r"[\\/\x00]", args.system):
+ print("trace_rules.py: the system must be a folder name under analysis/, not a path", file=sys.stderr)
+ return 2
+ if args.results and not args.module:
+ print("trace_rules.py: --results goes with --module: say which module's run the results are from", file=sys.stderr)
+ return 2
+ ran, read_notes = None, []
+ if args.results:
+ found = next((m for m in discover_modules(args.workspace, args.system) if m["name"].lower() == args.module.lower()), None)
+ if found is not None:
+ sys.dont_write_bytecode = True # importing a sibling script must not leave __pycache__ in the plugin folder
+ import baseline_diff # only this command line reads result files itself; the proof pack passes them in
+ res = baseline_diff.read_results(args.results)
+ ran = {found["rel"]: run_evidence([res])}
+ read_notes = ["%d result file(s) read: %d test(s), %d passed." % (res["files"], len(res["tests"]), sum(1 for s in res["tests"].values() if s == "PASS"))]
+ read_notes += ["result file not used: " + u for u in res["unreadable"][:3]] + [clean(p, 200) for p in res["problems"][:3]]
+ try:
+ result = trace(args.workspace, args.system, ran)
+ except OSError as err:
+ print("trace_rules.py: %s" % err, file=sys.stderr)
+ return 2
+ result["notes"] += read_notes or ["No test results were given, so no rule can be 'tested' here: the proof pack reads them from the result files (or add --module NAME --results PATH)."]
+ view, mod = None, None
+ if args.module:
+ mod = next((m for m in result["modules"] if m["name"].lower() == args.module.lower()), None)
+ if mod is None:
+ print("trace_rules.py: no built module called %s (found: %s)" % (clean(args.module, 60), ", ".join(clean(m["name"], 40) for m in result["modules"]) or "none"), file=sys.stderr)
+ return 2
+ view = module_view(result, mod["path"])
+ if args.json:
+ print(json.dumps(dict(result, moduleView=view) if view else result, indent=1, ensure_ascii=True))
+ else:
+ print(render(result, args.all, view, mod["name"] if mod else None))
+ return 1 if (view["p0NoTests"] if view else result["p0NoTests"]) else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/plugins/code-modernization/scripts/uplift_checks.py b/plugins/code-modernization/scripts/uplift_checks.py
new file mode 100644
index 00000000..33af819c
--- /dev/null
+++ b/plugins/code-modernization/scripts/uplift_checks.py
@@ -0,0 +1,356 @@
+#!/usr/bin/env python3
+"""Two checks that only an uplift needs, for the proof pack (proof_pack.py imports this; it also runs on its own).
+
+ python3 uplift_checks.py <legacy folder> <working copy> [DELTA_CATALOG.md]
+
+An uplift keeps the code and the tests and changes the version underneath. A green test run then proves less than it seems to in two ways:
+
+ Tests changed during the uplift. The test files of the untouched legacy tree are compared with those of the working copy by walking
+ files only (no process is run, no link followed, build output skipped). A test file is a test file by the rules trace_rules.py uses.
+ Removed, added and changed files are listed with a line-diff count. A removed test file, more than 25% of the legacy test files
+ changed, or a legacy tree with no test files to compare against, is a gap: passing edited or deleted tests proves less, so a person reviews the list. Weakened assertions cannot be detected,
+ only that files changed.
+
+ Deltas covered. DELTA_CATALOG.md lists the version changes that hit this code. For every delta whose category says Behavioral-silent
+ (the kind nothing reports when it changes behavior) the file of each listed site must be named, as a whole word, by some test file of
+ the working copy. A name is not proof that the test exercises the changed behavior; a silent delta no test even names is untested.
+ Other categories are listed, not judged.
+
+Everything read is untrusted text: files are only read, links are not followed, sizes and counts are capped, and no pattern here can be made
+slow by a hostile line. Exit 0 when neither check finds a gap, 1 when one does, 2 for input that cannot be used.
+"""
+import argparse
+import difflib
+import hashlib
+import json
+import os
+import re
+import stat
+import sys
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.dont_write_bytecode = True
+sys.path.insert(0, HERE)
+import baseline_diff # noqa: E402
+import trace_rules # noqa: E402
+
+LIST_CAP, FILE_CAP, HASH_CAP, MAX_FILES, TEXT_CAP = 40, 2 << 20, 64 << 20, 60000, 400 << 20
+DIFF_LINES, DIFF_BUDGET, SHARE_LIMIT = 4000, 300000000, 0.25
+DELTA_CAP, CATALOG_CAP, LINE_CAP = 2000, 8 << 20, 4000
+SILENT = re.compile(r"behaviou?ral[\s_-]*silent|silent[\s_-]*behaviou?ral", re.I)
+CONFIG_EXT = {"xml", "yml", "yaml", "json", "properties", "gradle", "toml", "ini", "cfg"} # build and configuration files: a test cannot name them
+CODE_EXT = {"java", "kt", "kts", "scala", "groovy", "cs", "vb", "fs", "py", "js", "jsx", "ts", "tsx", "mjs", "cjs", "go", "rs", "rb", "php", "c", "h", "cc", "cpp", "hpp", "swift",
+ "m", "sql", "cbl", "cob", "cpy", "jcl", "pl", "pm", "sh", "ps1", "xml", "yml", "yaml", "json", "properties", "gradle", "toml", "ini", "cfg"}
+# path/Stem.ext:line, or path/Stem.ext for a known extension; every quantifier is bounded, so a hostile line cannot make it slow
+SITE = re.compile(r"((?:[\w.$@+-]{1,80}/){0,12})([A-Za-z_$][\w$-]{0,80})((?:\.[A-Za-z][A-Za-z0-9]{0,7}){1,3})(?::(\d{1,7}))?")
+WORD = re.compile(r"[A-Za-z_$][\w$]{0,200}")
+DASHED = re.compile(r"[A-Za-z0-9_$]{1,100}(?:-[A-Za-z0-9_$]{1,100}){1,8}")
+clean = trace_rules.clean
+
+
+# ---------------------------------------------------------------- walking test files
+def walk_tests(root):
+ """({relative path: absolute path}, cut short, links skipped) for every regular test file under `root`.
+ A test file is one trace_rules.file_kind calls a test. Links, hidden folders and build output are never entered."""
+ out, links = {}, 0
+ if os.path.islink(root) or not os.path.isdir(root):
+ return out, False, 0
+ for base, dirs, names in os.walk(root, followlinks=False):
+ keep = []
+ for d in sorted(dirs):
+ if d in trace_rules.SKIP_DIRS or d.startswith("."):
+ continue
+ if os.path.islink(os.path.join(base, d)):
+ links += 1
+ continue
+ keep.append(d)
+ dirs[:] = keep
+ for n in sorted(names):
+ full = os.path.join(base, n)
+ if n.startswith("."):
+ continue
+ try:
+ st = os.lstat(full)
+ except OSError:
+ continue
+ if stat.S_ISLNK(st.st_mode):
+ links += 1
+ continue
+ rel = os.path.relpath(full, root).replace(os.sep, "/")
+ if stat.S_ISREG(st.st_mode) and trace_rules.file_kind(rel) == "test":
+ out[rel] = full
+ if len(out) >= MAX_FILES:
+ return out, True, links
+ return out, False, links
+
+
+def read_head(path, cap=FILE_CAP):
+ """Up to cap+1 bytes of a file, or None."""
+ try:
+ with open(path, "rb") as fh:
+ return fh.read(cap + 1)
+ except OSError:
+ return None
+
+
+def digest(path):
+ """sha256 of a file's bytes (read in blocks, at most HASH_CAP), or None."""
+ h, n = hashlib.sha256(), 0
+ try:
+ with open(path, "rb") as fh:
+ while n < HASH_CAP:
+ block = fh.read(1 << 20)
+ if not block:
+ break
+ h.update(block)
+ n += len(block)
+ except OSError:
+ return None
+ return h.hexdigest()
+
+
+def count_lines(path):
+ """Number of lines of a text file (0 for a binary or unreadable one)."""
+ raw = read_head(path)
+ return len(as_lines(raw)) if raw is not None and b"\x00" not in raw[:8192] else 0
+
+
+def as_lines(raw):
+ return raw.decode("utf-8", "replace").replace("\r\n", "\n").replace("\r", "\n").split("\n")
+
+
+def line_diff(a, b, budget):
+ """(lines added, lines removed) turning list a into list b. difflib for files of ordinary size while the budget lasts, else a count of
+ lines that appear more often on one side (order is ignored, which is the price of being fast). -> (added, removed, exact, budget left)."""
+ if len(a) <= DIFF_LINES and len(b) <= DIFF_LINES and len(a) * len(b) <= budget:
+ added = removed = 0
+ for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, a, b, autojunk=False).get_opcodes():
+ if tag in ("replace", "delete"):
+ removed += i2 - i1
+ if tag in ("replace", "insert"):
+ added += j2 - j1
+ return added, removed, True, budget - len(a) * len(b)
+ ca, cb = {}, {}
+ for line in a:
+ ca[line] = ca.get(line, 0) + 1
+ for line in b:
+ cb[line] = cb.get(line, 0) + 1
+ return (sum(max(0, n - ca.get(k, 0)) for k, n in cb.items()), sum(max(0, n - cb.get(k, 0)) for k, n in ca.items()), False, budget)
+
+
+def compare_files(old, new, budget):
+ """None when the two files hold the same content (line endings aside), else ({"added", "removed", "note"}, budget left)."""
+ ra, rb = read_head(old), read_head(new)
+ if ra is None or rb is None:
+ return {"added": 0, "removed": 0, "note": "could not be read"}, budget
+ if len(ra) > FILE_CAP or len(rb) > FILE_CAP:
+ da, db = digest(old), digest(new)
+ return (None, budget) if da is not None and da == db else ({"added": 0, "removed": 0, "note": "large file, lines not counted"}, budget)
+ if b"\x00" in ra[:8192] or b"\x00" in rb[:8192]:
+ return (None, budget) if ra == rb else ({"added": 0, "removed": 0, "note": "binary file"}, budget)
+ la, lb = as_lines(ra), as_lines(rb)
+ if la == lb:
+ return None, budget
+ added, removed, exact, budget = line_diff(la, lb, budget)
+ return {"added": added, "removed": removed, "note": "" if exact else "lines counted approximately"}, budget
+
+
+# ---------------------------------------------------------------- tests changed during the uplift
+def tests_changed(legacy_root, work_root):
+ """Compare the test files of the untouched legacy tree with those of the working copy.
+ -> {"checked", "why", "legacyTests", "workTests", "counts": {added, removed, changed}, "added", "removed", "changed" (lists cut to 40), "share", "gap", "cut"}."""
+ out = {"checked": False, "why": "", "legacyTests": 0, "workTests": 0, "counts": {"added": 0, "removed": 0, "changed": 0}, "added": [], "removed": [], "changed": [],
+ "share": 0.0, "gap": False, "cut": False}
+ if not legacy_root or not os.path.isdir(legacy_root):
+ out["why"] = "the legacy tree was not found, so its test files could not be compared with the working copy's"
+ return out
+ if not work_root or not os.path.isdir(work_root):
+ out["why"] = "the working copy was not found"
+ return out
+ old, cut_a, _ = walk_tests(legacy_root)
+ new, cut_b, _ = walk_tests(work_root)
+ out.update({"checked": True, "legacyTests": len(old), "workTests": len(new), "cut": cut_a or cut_b})
+ removed = sorted(set(old) - set(new))
+ added = sorted(set(new) - set(old))
+ changed, budget = [], DIFF_BUDGET
+ for rel in sorted(set(old) & set(new)):
+ got, budget = compare_files(old[rel], new[rel], budget)
+ if got is not None:
+ changed.append(dict(got, path=clean(rel, 300)))
+ out["counts"] = {"added": len(added), "removed": len(removed), "changed": len(changed)}
+ changed.sort(key=lambda c: (-(c["added"] + c["removed"]), c["path"]))
+ out["removed"] = [{"path": clean(r, 300), "lines": count_lines(old[r])} for r in removed[:LIST_CAP]]
+ out["added"] = [{"path": clean(r, 300), "lines": count_lines(new[r])} for r in added[:LIST_CAP]]
+ out["changed"] = changed[:LIST_CAP]
+ out["share"] = round(len(changed) / len(old), 4) if old else 0.0
+ out["gap"] = bool(removed) or not old or len(changed) > SHARE_LIMIT * len(old)
+ if not old:
+ out["why"] = "the legacy tree has no test files, so nothing shows that the working copy's tests were not written during the uplift"
+ return out
+
+
+# ---------------------------------------------------------------- reading DELTA_CATALOG.md
+def norm_id(text):
+ return re.sub(r"[\s*`#.]+", "", text or "").lower()[:40]
+
+
+def sites_in(text):
+ """[{"path", "stem", "line"}] for every file mention in `text` that is a file:line, or a file with a known extension."""
+ out = []
+ for m in SITE.finditer(text[:LINE_CAP]):
+ ext = m.group(3).lstrip(".").split(".")[-1].lower()
+ stem = m.group(2)
+ if len(stem) < 2 or (m.group(4) is None and ext not in CODE_EXT):
+ continue
+ site = {"path": clean((m.group(1) + stem + m.group(3)), 200), "stem": stem, "line": int(m.group(4)) if m.group(4) else 0, "ext": ext}
+ if not any(o["path"] == site["path"] for o in out):
+ out.append(site)
+ return out[:20]
+
+
+def add_delta(found, did, title, category, sites):
+ key = norm_id(did)
+ if not key or (len(found) >= DELTA_CAP and key not in found):
+ return
+ d = found.setdefault(key, {"id": clean(did.replace("*", "").strip(), 40), "title": "", "category": "", "sites": []})
+ d["title"] = d["title"] or clean(title, 200)
+ d["category"] = d["category"] or clean(category, 80)
+ d["sites"] += [s for s in sites if not any(o["path"] == s["path"] for o in d["sites"])]
+ d["sites"] = d["sites"][:20]
+
+
+def parse_deltas(text):
+ """Deltas from DELTA_CATALOG.md, tolerantly: pipe tables with a Category column (an id in the first or an ID/# column, sites in a
+ Site column), and cards (`### 9. title` or `### D-01 title`, with a `Category ...:` line and a `cited:` or `Site:` line).
+ Rows and cards of the same id are merged. -> [{"id", "title", "category", "sites": [{"path", "stem", "line"}]}]."""
+ found = {}
+ for _heading, header, rows in baseline_diff.md_tables(text[:CATALOG_CAP]):
+ low = [h.lower() for h in header]
+ cat = next((i for i, h in enumerate(low) if re.search(r"\bcat", h)), None)
+ if cat is None:
+ continue
+ idc = next((i for i, h in enumerate(low) if re.fullmatch(r"(?:id|#|no\.?|delta|card|ref)", h.strip())), 0)
+ titlec = next((i for i, h in enumerate(low) if re.search(r"delta|title|name|change|summary|\bold\b", h) and i != idc), None)
+ sitec = [i for i, h in enumerate(low) if re.search(r"\bsites?\b|location|cited|file", h)]
+ for row in rows[:5000]:
+ if cat >= len(row) or idc >= len(row) or not row[idc].strip():
+ continue
+ cells = [row[i] for i in sitec if i < len(row) and not re.fullmatch(r"\s*\d+\s*", row[i])]
+ sites = sites_in(" ".join(cells)) if sitec else sites_in(" ".join(c for i, c in enumerate(row) if i not in (cat, idc)))
+ add_delta(found, row[idc], row[titlec] if titlec is not None and titlec < len(row) else "", row[cat], sites)
+ lines, i = [ln[:LINE_CAP] for ln in text[:CATALOG_CAP].split("\n")], 0
+ while i < len(lines):
+ m = re.match(r"#{2,6}[ \t]+(?:\*\*)?(D-\d{1,4}|\d{1,4})(?:\*\*)?[.):\s][ \t]*(.*)$", lines[i])
+ i += 1
+ if not m:
+ continue
+ category, sites = "", []
+ while i < len(lines) and not lines[i].startswith("#"):
+ line = lines[i]
+ c = re.search(r"categor[^:\n]{0,40}:\**[ \t]*(.{1,200})", line, re.I)
+ if c and not category:
+ category = re.split(r"[/;]", c.group(1))[0].strip(" *`")
+ cited = re.search(r"\bcited\s*:", line, re.I) or re.match(r"^[ \t>*-]*\**(?:sites?|location|source)\b[^:\n]{0,30}:", line, re.I)
+ if cited: # only the site the card cites, never files that its prose merely mentions
+ sites += [s for s in sites_in(line[cited.end():]) if not any(o["path"] == s["path"] for o in sites)]
+ i += 1
+ if category or norm_id(m.group(1)) in found: # a numbered heading with no Category line is not a delta card
+ add_delta(found, m.group(1), m.group(2), category, sites)
+ return list(found.values())
+
+
+# ---------------------------------------------------------------- deltas covered by tests
+def test_words(work_root):
+ """(the set of words that appear in the working copy's test files, number of test files read). Files are read as text, up to 2 MB each."""
+ words, total, files = set(), 0, 0
+ tests, _cut, _links = walk_tests(work_root)
+ for rel in sorted(tests):
+ raw = read_head(tests[rel])
+ if raw is None or b"\x00" in raw[:8192]:
+ continue
+ total += len(raw)
+ if total > TEXT_CAP:
+ break
+ text = raw[:FILE_CAP].decode("utf-8", "replace")
+ words.update(WORD.findall(text))
+ words.update(DASHED.findall(text))
+ files += 1
+ return words, files
+
+
+def delta_coverage(catalog_text, work_root):
+ """Which Behavioral-silent deltas have their site's file named by a test in the working copy? A delta whose only sites are build or
+ configuration files cannot be named by a test: it is listed for a person, not judged.
+ -> {"parsed", "silent", "covered": [...], "uncovered": [...], "config": [...], "other": n, "testFiles", "gap", "why"}."""
+ out = {"parsed": 0, "silent": 0, "covered": [], "uncovered": [], "config": [], "other": 0, "testFiles": 0, "gap": False, "why": ""}
+ if catalog_text is None:
+ out.update(gap=True, why="DELTA_CATALOG.md was not found, so it is not known which version changes hit this code")
+ return out
+ deltas = parse_deltas(catalog_text)
+ out["parsed"] = len(deltas)
+ if not deltas:
+ out.update(gap=True, why="no delta could be read from DELTA_CATALOG.md (a table with a Category column, or cards with a Category line)")
+ return out
+ words, out["testFiles"] = test_words(work_root)
+ for d in deltas:
+ if not SILENT.search(d["category"]):
+ out["other"] += 1
+ continue
+ out["silent"] += 1
+ code = [s for s in d["sites"] if s["ext"] not in CONFIG_EXT]
+ rec = {"id": d["id"], "title": d["title"], "sites": [s["path"] + (":%d" % s["line"] if s["line"] else "") for s in d["sites"][:5]]}
+ if d["sites"] and not code:
+ out["config"].append(rec)
+ continue
+ missing = [s["stem"] for s in code if s["stem"] not in words]
+ if code and not missing:
+ out["covered"].append(rec)
+ else:
+ rec["why"] = "no site is named in the catalog" if not code else "no test names " + ", ".join(clean(x, 60) for x in list(dict.fromkeys(missing))[:5])
+ out["uncovered"].append(rec)
+ out["gap"] = bool(out["uncovered"])
+ return out
+
+
+# ---------------------------------------------------------------- command line
+def main(argv=None):
+ ap = argparse.ArgumentParser(description="Uplift-only checks: test files changed since the legacy tree, and silent deltas no test names.")
+ ap.add_argument("legacy", help="the untouched legacy folder (legacy/<system>)")
+ ap.add_argument("work", help="the uplifted working copy (modernized/<system>-uplifted)")
+ ap.add_argument("catalog", nargs="?", help="analysis/<system>/DELTA_CATALOG.md")
+ ap.add_argument("--json", action="store_true", help="print the full result as JSON")
+ args = ap.parse_args(argv)
+ for path in (args.legacy, args.work):
+ if not os.path.isdir(path):
+ print("uplift_checks.py: %s is not a folder" % path, file=sys.stderr)
+ return 2
+ text = None
+ if args.catalog:
+ try:
+ text = baseline_diff.read_capped(args.catalog, CATALOG_CAP).decode("utf-8", "replace")
+ except baseline_diff.InputError as err:
+ print("uplift_checks.py: %s" % err, file=sys.stderr)
+ return 2
+ changed = tests_changed(os.path.realpath(args.legacy), os.path.realpath(args.work))
+ deltas = delta_coverage(text, os.path.realpath(args.work)) if args.catalog else None
+ if args.json:
+ print(json.dumps({"testsChanged": changed, "deltas": deltas}, indent=1, ensure_ascii=True))
+ else:
+ c = changed["counts"]
+ print("test files: %d in the legacy tree, %d in the working copy; %d removed, %d added, %d changed (%.0f%% of the legacy test files)" % (
+ changed["legacyTests"], changed["workTests"], c["removed"], c["added"], c["changed"], 100 * changed["share"]))
+ for key in ("removed", "changed", "added"):
+ for e in changed[key][:10]:
+ print(" %s: %s%s" % (key, e["path"], " (+%d -%d lines)" % (e["added"], e["removed"]) if key == "changed" else ""))
+ print("tests kept: " + ("GAP" if changed["gap"] else "ok") + ("" if not changed["why"] else " (" + changed["why"] + ")"))
+ if deltas is not None:
+ print("deltas: %d parsed, %d behavioral-silent, %d covered, %d not named by any test, %d at a build or configuration file" % (
+ deltas["parsed"], deltas["silent"], len(deltas["covered"]), len(deltas["uncovered"]), len(deltas["config"])))
+ for d in deltas["uncovered"][:10]:
+ print(" uncovered: %s %s (%s)" % (d["id"], d["title"], d.get("why", "")))
+ print("deltas covered: " + ("GAP" if deltas["gap"] else "ok") + ("" if not deltas["why"] else " (" + deltas["why"] + ")"))
+ return 1 if changed["gap"] or (deltas is not None and deltas["gap"]) else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())