summaryrefslogtreecommitdiff
path: root/plugins/code-modernization/scripts/tests
diff options
context:
space:
mode:
Diffstat (limited to 'plugins/code-modernization/scripts/tests')
-rw-r--r--plugins/code-modernization/scripts/tests/test_compare_tolerance.py151
-rw-r--r--plugins/code-modernization/scripts/tests/test_proof_rules_run.py506
-rw-r--r--plugins/code-modernization/scripts/tests/test_proof_tools.py1993
-rw-r--r--plugins/code-modernization/scripts/tests/test_report_next.py43
-rw-r--r--plugins/code-modernization/scripts/tests/test_report_tools.py791
-rw-r--r--plugins/code-modernization/scripts/tests/test_rules_tools.py196
-rw-r--r--plugins/code-modernization/scripts/tests/test_telemetry.py810
7 files changed, 4490 insertions, 0 deletions
diff --git a/plugins/code-modernization/scripts/tests/test_compare_tolerance.py b/plugins/code-modernization/scripts/tests/test_compare_tolerance.py
new file mode 100644
index 00000000..1bc1300a
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_compare_tolerance.py
@@ -0,0 +1,151 @@
+"""Tests for the numeric tolerance in scripts/compare.py.
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+"""
+import json
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import compare as cmp # noqa: E402
+
+TOL = {"rel": 1e-9, "abs": 0, "why": "last-digit rounding of exp() between libm versions"}
+
+
+class ToleranceTests(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+
+ def case(self, name, legacy, new, **extra):
+ for side, data in (("legacy", legacy), ("new", new)):
+ os.makedirs(os.path.join(self.tmp, side), exist_ok=True)
+ with open(os.path.join(self.tmp, side, name), "wb") as fh:
+ fh.write(data)
+ return {"id": name, "legacy": "legacy/" + name, "new": "new/" + name, **extra}
+
+ def run_cases(self, cases, **top):
+ path = os.path.join(self.tmp, "cases.json")
+ with open(path, "w") as fh:
+ json.dump({"system": "s", "cases": cases, **top}, fh)
+ return cmp.run(path, os.path.join(self.tmp, "EQUIVALENCE.json"))
+
+ def test_a_tiny_float_difference_is_same_within_the_declared_tolerance_and_says_so(self):
+ res = self.run_cases([self.case("a", b"total 1234.5678901234\nrate 0.05\n", b"total 1234.5678901239\nrate 0.05\n", tolerance=TOL)])
+ rec = res["cases"][0]
+ self.assertEqual(rec["verdict"], "same")
+ self.assertEqual(rec["withinTolerance"]["differing"], 1)
+ self.assertLess(rec["withinTolerance"]["maxRelativeDifference"], 1e-9)
+ self.assertIn("declared tolerance", rec["reason"])
+ self.assertIn("libm", rec["reason"])
+ self.assertEqual(res["totals"]["sameWithinTolerance"], 1)
+ self.assertIsNone(cmp.verdict_of(res))
+ self.assertTrue(res["selfCheck"]["passed"], res["selfCheck"])
+
+ def test_without_a_tolerance_the_same_outputs_differ_and_no_new_key_appears(self):
+ res = self.run_cases([self.case("a", b"1234.5678901234\n", b"1234.5678901239\n")])
+ self.assertEqual(res["cases"][0]["verdict"], "differs")
+ self.assertNotIn("sameWithinTolerance", res["totals"])
+
+ def test_a_difference_outside_the_tolerance_differs(self):
+ res = self.run_cases([self.case("a", b"total 100.0\n", b"total 100.5\n", tolerance=TOL)])
+ rec = res["cases"][0]
+ self.assertEqual(rec["verdict"], "differs")
+ self.assertIn("beyond the declared tolerance", rec["reason"])
+ self.assertIn("NOT PROVEN" if cmp.verdict_of(res) else "", "NOT PROVEN")
+
+ def test_integers_and_text_must_match_exactly_even_with_a_tolerance(self):
+ loose = {"rel": 0.01, "abs": 0, "why": "test"}
+ for legacy, new in ((b"count 1000000\n", b"count 1000001\n"), (b"name alpha 1.5\n", b"name alphb 1.5\n"),
+ (b"1.5 2.5\n", b"1.5\n"), (b"x 10\n", b"x 10.0\n")):
+ res = self.run_cases([self.case("a", legacy, new, tolerance=loose)])
+ self.assertEqual(res["cases"][0]["verdict"], "differs", (legacy, new))
+
+ def test_exponent_forms_compare_by_value(self):
+ res = self.run_cases([self.case("a", b"v 1.5e3\n", b"v 1500.0000000000002\n", tolerance=TOL)])
+ self.assertEqual(res["cases"][0]["verdict"], "same")
+
+ def test_a_tolerance_needs_a_reason_and_a_sane_size(self):
+ legacy = self.case("a", b"1.0\n", b"1.0\n")
+ for bad in ({"rel": 1e-9}, {"rel": 1e-9, "why": " "}, {"rel": 0.5, "why": "x"}, {"why": "x"}, {"rel": -1, "why": "x"},
+ {"rel": "big", "why": "x"}, {"rel": float("nan"), "why": "x"}, "1e-9"):
+ with self.assertRaises(cmp.InputError, msg=repr(bad)):
+ self.run_cases([dict(legacy, tolerance=bad)])
+
+ def test_a_default_tolerance_applies_to_every_case_and_a_case_can_switch_it_off(self):
+ a = self.case("a", b"1.00000000000\n", b"1.00000000001\n")
+ b = self.case("b", b"1.00000000000\n", b"1.00000000001\n", tolerance=None)
+ res = self.run_cases([a, b], tolerance=TOL)
+ self.assertEqual([r["verdict"] for r in res["cases"]], ["same", "differs"])
+
+ def test_an_absolute_tolerance_is_capped_like_a_relative_one(self):
+ for bad in ({"abs": 1e-3, "why": "x"}, {"abs": 100, "why": "x"}):
+ with self.assertRaises(cmp.InputError, msg=repr(bad)):
+ self.run_cases([self.case("a", b"5.5\n", b"5.5\n", tolerance=bad)])
+ ok = self.run_cases([self.case("a", b"x 0.0000000000000001\n", b"x 0.0\n", tolerance={"abs": 1e-9, "why": "a value that is zero to rounding"})])
+ self.assertEqual(ok["cases"][0]["verdict"], "same")
+
+ def test_the_self_check_still_fails_if_a_loose_tolerance_ever_got_through(self):
+ loose = {"abs": 100, "why": "absurdly loose"}
+ with mock.patch.object(cmp, "MAX_ABS_TOLERANCE", 1000.0):
+ res = self.run_cases([self.case("a", b"5.5\n", b"5.5\n", tolerance=loose)])
+ self.assertFalse(res["selfCheck"]["passed"])
+ self.assertIn("tolerance may be too loose", res["selfCheck"]["detail"])
+ self.assertIn("self-check failed", cmp.verdict_of(res))
+
+ def test_dotted_identifiers_are_text_and_must_match_exactly(self):
+ for legacy, new in ((b"version 1.2.3\n", b"version 1.2.4\n"), (b"host 10.0.0.1\n", b"host 10.0.0.2\n"),
+ (b"build 2026.09.24 ok\n", b"build 2026.09.25 ok\n")):
+ res = self.run_cases([self.case("a", legacy, new, tolerance={"rel": 0.01, "why": "test"})])
+ self.assertEqual(res["cases"][0]["verdict"], "differs", (legacy, new))
+ same = self.run_cases([self.case("a", b"version 1.2.3 total 1.0000000001\n", b"version 1.2.3 total 1.0000000002\n", tolerance=TOL)])
+ self.assertEqual(same["cases"][0]["verdict"], "same")
+
+ def test_the_comparison_is_exact_decimal_arithmetic_not_doubles(self):
+ # 1e-22 apart: equal as doubles, but far outside a declared 1e-30 relative tolerance
+ tight = {"rel": 1e-30, "why": "essentially exact"}
+ res = self.run_cases([self.case("a", b"1.0000000000000000000001\n", b"1.0000000000000000000002\n", tolerance=tight)])
+ self.assertEqual(res["cases"][0]["verdict"], "differs")
+ wide = {"rel": 1e-21, "why": "test"}
+ res = self.run_cases([self.case("a", b"1.0000000000000000000001\n", b"1.0000000000000000000002\n", tolerance=wide)])
+ self.assertEqual(res["cases"][0]["verdict"], "same")
+
+ def test_absurd_numbers_do_not_crash_or_pass(self):
+ for legacy, new in ((b"v 1e999999999999\n", b"v 1e999999999998\n"), (b"v 1." + b"1" * 200 + b"\n", b"v 1." + b"1" * 199 + b"2\n")):
+ res = self.run_cases([self.case("a", legacy, new, tolerance=TOL)])
+ self.assertEqual(res["cases"][0]["verdict"], "differs")
+
+ def test_an_input_can_be_approved_once_for_every_case_that_shares_its_label(self):
+ cases = [self.case("F23-a", b"one\n", b"two\n", input="F23"), self.case("F23-b", b"x\n", b"y\n", input="F23"),
+ self.case("F24-a", b"p\n", b"q\n", input="F24"), self.case("F25-a", b"same\n", b"same\n", input="F25")]
+ res = self.run_cases(cases, approvedInputs={"F23": "system owner accepts deviation 4", "F25": " ", "F99": "no such input"})
+ verdicts = {r["id"]: r["verdict"] for r in res["cases"]}
+ self.assertEqual(verdicts, {"F23-a": "differs-approved", "F23-b": "differs-approved", "F24-a": "differs", "F25-a": "same"})
+ self.assertEqual(res["totals"]["differsApproved"], 2)
+ self.assertIn("system owner accepts deviation 4", res["cases"][0]["reason"])
+ self.assertEqual(res["cases"][2]["approvedDifference"], "") # a blank reason approves nothing
+ # a case's own reason wins, and a per-input approval never turns a missing file into a pass
+ own = self.run_cases([self.case("G1", b"a\n", b"b\n", input="G", approvedDifference="own reason"), dict(self.case("G2", b"a\n", b"a\n", input="G"), new="new/nothing")],
+ approvedInputs={"G": "input reason"})
+ self.assertEqual(own["cases"][0]["approvedDifference"], "own reason")
+ self.assertEqual(own["cases"][1]["verdict"], "missing")
+
+ def test_masks_and_tolerance_work_together(self):
+ res = self.run_cases([self.case("a", b"at 2026-09-24 10:00:00 value 2.0000000001\n", b"at 2027-01-01 11:11:11 value 2.0000000002\n",
+ mask=[{"regex": r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d", "why": "run time"}], tolerance=TOL)])
+ self.assertEqual(res["cases"][0]["verdict"], "same")
+
+ def test_large_outputs_are_not_tolerance_compared(self):
+ big = b"1.0 " * (cmp.MAX_TOLERANT_BYTES // 4 + 10)
+ rec = self.run_cases([self.case("a", big, big[:-3] + b"2.0", tolerance=TOL)])["cases"][0]
+ self.assertEqual(rec["verdict"], "differs")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_proof_rules_run.py b/plugins/code-modernization/scripts/tests/test_proof_rules_run.py
new file mode 100644
index 00000000..0eb3560c
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_proof_rules_run.py
@@ -0,0 +1,506 @@
+"""Tests for the "Rules traced" check: a rule is tested only when something that ran and passed backs it, and test-only folders are not judged.
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+A test that names a rule is not a test that ran: skipped and pending tests, and files whose class never ran, leave the rule "named, not run".
+"""
+import json
+import os
+import re
+import shutil
+import sys
+import tempfile
+import time
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, HERE)
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import build_report as br # noqa: E402
+import proof_pack as pp # noqa: E402
+import trace_rules as tr # noqa: E402
+from test_proof_tools import RULES_MD, T0, Ws, junit, load, passing, put, read, run_main # noqa: E402
+
+SKIPPED_TEST = '''package p;
+
+class CalcTest {
+ @Test
+ @Disabled("PENDING RULE-001")
+ void RULE_001_totals() {}
+
+ @Test
+ @Disabled("PENDING RULE-002")
+ void RULE_002_rounds() {}
+}
+'''
+NOT_RUN = "named, not run"
+
+
+class RuleWs(Ws):
+ """The workspace of the proof tests (module mod, P0 rules RULE-001 and RULE-002), with helpers to change what the tests are and what ran."""
+
+ def module(self, module="mod"):
+ return super().module(module)
+
+ def p0(self):
+ return {r["id"]: r for r in self.module()["evidence"]["rules"]["p0"]}
+
+ def statuses(self):
+ return {r["id"]: r["status"] for r in self.module()["evidence"]["rules"]["p0"]}
+
+ def check_rules(self):
+ return next(c for c in self.module()["checks"] if c["id"] == "rules")
+
+ def test_file(self, rel, text):
+ put(self.root, "modernized/s/mod/src/test/" + rel, text, T0 + 20)
+
+ def keep_only(self, *cases):
+ self.results(list(cases))
+
+
+class RulesNeedExecutedTests(unittest.TestCase):
+ def setUp(self):
+ self.ws = RuleWs(self)
+
+ def test_the_default_workspace_is_tested_by_files_whose_class_ran(self):
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+ self.assertEqual(self.ws.check_rules()["state"], "pass")
+ self.assertIn("backed by a test that ran and passed", self.ws.check_rules()["detail"])
+ self.assertEqual(self.ws.module()["verdict"], "PROVEN")
+
+ # (i) the hole that was found: @Disabled("PENDING RULE-001") and a result file that shows it skipped
+ def test_a_disabled_pending_test_that_the_results_show_skipped_is_named_not_run(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("CalcTest", "RULE_001_totals", "SKIP", "PENDING RULE-001"), ("CalcTest", "RULE_002_rounds", "SKIP", "PENDING RULE-002"), ("OtherTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ c = self.ws.check_rules()
+ self.assertEqual(c["state"], "gap")
+ self.assertIn("P0 rules named only by tests that did not run: RULE-001, RULE-002 (skipped, pending, failing or missing from the results).", c["detail"])
+ m = self.ws.module()
+ self.assertEqual(m["verdict"], "PARTLY PROVEN")
+ self.assertEqual(m["evidence"]["rules"]["namedNotRun"], ["RULE-001", "RULE-002"])
+ self.assertEqual(m["evidence"]["rules"]["p0NoTests"], ["RULE-001", "RULE-002"])
+ self.assertTrue(m["evidence"]["rules"]["perTestResults"])
+ self.assertTrue(any(r.startswith("Rules traced: P0 rules named only by tests that did not run") for r in m["reasons"]))
+ self.assertEqual(m["evidence"]["tests"]["skipped"], 2)
+
+ def test_the_same_test_without_a_skip_marker_but_skipped_at_run_time_is_named_not_run_too(self):
+ self.ws.test_file("CalcTest.java", "class CalcTest {\n // RULE-001 and RULE-002 need a database\n void a() {}\n}\n")
+ self.ws.keep_only(("CalcTest", "a", "SKIP", "no database here"), ("OtherTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertEqual({r["by"] for r in self.ws.p0().values()}, {""})
+
+ def test_a_test_that_ran_and_failed_backs_nothing(self):
+ self.ws.test_file("CalcTest.java", "class CalcTest {\n // RULE-001 RULE-002\n void a() {}\n}\n")
+ self.ws.keep_only(("CalcTest", "RULE_001_totals", "FAIL", ""), ("CalcTest", "RULE_002_rounds", "ERROR", ""), ("OtherTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertEqual(self.ws.module()["verdict"], "NOT PROVEN") # the failures fail the tests check as they always did
+ self.assertEqual(self.ws.check_rules()["state"], "gap")
+
+ # (ii) an executed, passing testcase that carries the id in its name
+ def test_a_passing_test_whose_name_carries_the_id_backs_the_rule_in_any_spelling(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST) # the only file that names the rules is all skipped
+ for name in ("RULE-001 totals", "RULE_001_totals", "rule001", "testRule001Totals", "Rule001", "totals[RULE-001]"):
+ self.ws.keep_only(("ChecksTest", name, "PASS", ""), ("ChecksTest", "test_RULE_002_rounds", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"}, name)
+ p0 = self.ws.p0()
+ self.assertEqual((p0["RULE-001"]["by"], p0["RULE-002"]["by"]), ("test name", "test name"))
+ self.assertEqual(self.ws.check_rules()["state"], "pass")
+ self.assertEqual(self.ws.module()["verdict"], "PROVEN")
+
+ def test_the_class_name_may_carry_the_id_and_the_shorthand_names_both_rules(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("com.x.Rule001Test", "a", "PASS", ""), ("com.x.OtherTest", "RULE-002/003", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_an_id_in_the_name_of_a_test_that_did_not_pass_or_in_another_modules_results_backs_nothing(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("ChecksTest", "RULE_001_a", "SKIP", "x"), ("ChecksTest", "RULE_002_b", "FAIL", ""), ("ChecksTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ put(self.ws.root, "modernized/s/extra/src/main/X.java", "class X {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/extra/target/surefire-reports/TEST-X.xml", junit([("XTest", "RULE_001_a", "PASS", ""), ("XTest", "RULE_002_a", "PASS", "")]), T0 + 30)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "junit": ["modernized/s/mod/target/surefire-reports"]},
+ {"module": "extra", "name": "unit", "junit": ["modernized/s/extra/target/surefire-reports"]}])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN}) # what extra ran does not count for mod
+
+ # (iii) a plain comment in a test file whose class ran
+ def test_a_plain_comment_in_a_test_file_whose_class_ran_backs_the_rule(self):
+ self.ws.test_file("CalcTest.java", "class CalcTest {\n // RULE-001 is pinned by these\n void a() {}\n void rule002_rounds() {}\n}\n")
+ self.ws.keep_only(("com.x.CalcTest", "a", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+ p0 = self.ws.p0()
+ self.assertEqual(p0["RULE-001"]["by"], "test file")
+ self.assertTrue(p0["RULE-001"]["where"].endswith("src/test/CalcTest.java:2"), p0["RULE-001"]["where"])
+ self.assertEqual(self.ws.module()["verdict"], "PROVEN")
+
+ def test_the_where_points_at_the_mention_that_backs_the_rule_not_at_a_skipped_one(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST) # sorts before Live.java and names both rules, skipped
+ self.ws.test_file("Live.java", "// RULE-001 RULE-002 are checked by these\nclass Live {}\n")
+ self.ws.keep_only(("Live", "a", "PASS", ""), ("CalcTest", "RULE_001_totals", "SKIP", "x"))
+ p0 = self.ws.p0()
+ self.assertTrue(p0["RULE-001"]["where"].endswith("src/test/Live.java:1"), p0["RULE-001"]["where"])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_the_class_name_may_be_nested_dotted_a_python_module_or_a_path(self):
+ same = (("CalcTest.java", "com.x.CalcTest$Inner"), ("CalcTest.java", "com.x.Outer$Deep$CalcTest"), ("CalcTest.java", "com.x.CalcTest$A$B"), ("test_calc.py", "tests.test_calc.TestCalc"),
+ ("test_calc.py", "tests.test_calc"), ("CalcTests.cs", "Ns.CalcTests"), ("calc.spec.ts", "src/calc.spec.ts"), ("CalcTest.java", "calctest"), ("CalcTest.java", "CALCTEST"))
+ other = (("CalcTest.java", "com.x.OtherCalcTest"), ("CalcTest.java", "com.calc.test.Other"), ("test_calc.py", "tests.test_other.TestCalc"), ("CalcTest.java", "Calc"), ("CalcTest.java", ""))
+ for rel, cls in same + other:
+ self.ws.test_file(rel, "// RULE-001 RULE-002\n")
+ self.ws.keep_only((cls, "a", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"} if (rel, cls) in same else {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN}, (rel, cls))
+ os.remove(os.path.join(self.ws.root, "modernized", "s", "mod", "src", "test", rel))
+ self.ws.test_file("CalcTest.java", "// nothing here\n")
+
+ # (iv) a file none of whose tests ran
+ def test_a_rule_named_only_in_a_file_none_of_whose_tests_ran_is_named_not_run(self):
+ self.ws.test_file("CalcTest.java", "// RULE-001\nclass CalcTest {}\n")
+ self.ws.test_file("GhostTest.java", "// RULE-002 is pinned here, and nothing marks it skipped\nclass GhostTest {}\n")
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("GhostTest", "b", "SKIP", "no docker"))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": NOT_RUN})
+ self.ws.keep_only(("CalcTest", "a", "PASS", "")) # the class is not in the results at all
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": NOT_RUN})
+ c = self.ws.check_rules()
+ self.assertEqual(c["state"], "gap")
+ self.assertIn("named only by tests that did not run: RULE-002", c["detail"])
+ self.assertNotIn("RULE-001", c["detail"])
+
+ # (v) counts only
+ def test_counts_only_evidence_never_backs_a_rule(self):
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0\n", T0 + 30)
+ for suite in ({"module": "mod", "name": "unit", "command": "mvn test", "log": ["analysis/s/equivalence/unit.test-output.txt"]},
+ {"module": "mod", "name": "unit", "executed": 3, "failed": 0, "skipped": 0}):
+ self.ws.runs(suites=[suite])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN}, suite)
+ m = self.ws.module()
+ self.assertFalse(m["evidence"]["rules"]["perTestResults"])
+ self.assertEqual(self.ws.check_rules()["state"], "gap")
+ self.assertIn("named by tests, but no result file lists each test, so nothing shows one ran: RULE-001, RULE-002.", self.ws.check_rules()["detail"])
+ self.assertNotEqual(m["verdict"], "PROVEN")
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "log": ["analysis/s/equivalence/unit.test-output.txt"]},
+ {"module": "mod", "name": "more", "junit": ["modernized/s/mod/target/surefire-reports"]}]) # one suite that lists its tests is enough
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_no_suite_at_all_or_result_files_with_no_test_in_them_back_nothing(self):
+ put(self.ws.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", '<?xml version="1.0"?><testsuite name="S" tests="0"></testsuite>', T0 + 30)
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.ws.runs(suites=[])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertEqual(self.ws.module()["verdict"], "NOT PROVEN")
+
+ # (vi) Python, and how far a marker reaches
+ def test_a_pytest_skip_that_names_the_rule_leaves_it_named_not_run(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("**Priority:** P2", "**Priority:** P0"), T0 + 10)
+ put(self.ws.root, "modernized/s/mod/src/main/calc.py", "def fee(): pass\n", T0 + 20)
+ self.ws.test_file("CalcTest.java", "// RULE-001 RULE-002\nclass CalcTest {}\n")
+ self.ws.test_file("test_fee.py", 'import pytest\n\n\n@pytest.mark.skip(reason="RULE-004")\ndef test_fee():\n pass\n\n\ndef test_other():\n pass\n')
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("tests.test_fee", "test_fee", "SKIP", "RULE-004"), ("tests.test_fee", "test_other", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested", "RULE-004": NOT_RUN})
+ self.assertIn("RULE-004", self.ws.check_rules()["detail"])
+ self.ws.test_file("test_fee.py", 'import pytest\n\n\ndef test_fee():\n # RULE-004 is pinned here\n pass\n') # no marker: the class ran and passed
+ self.assertEqual(self.ws.statuses()["RULE-004"], "tested")
+
+ def test_a_marker_reaches_three_lines_down_and_no_further_and_any_case(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P0\n", T0 + 10)
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""))
+ for gap, want in ((0, NOT_RUN), (1, NOT_RUN), (2, NOT_RUN), (3, NOT_RUN), (4, "tested"), (9, "tested")):
+ lines = ["@Disabled RULE-001"] if gap == 0 else ["@Disabled"] + ["// filler"] * (gap - 1) + ["// RULE-001"] # the mention is `gap` lines below the marker
+ self.ws.test_file("CalcTest.java", "\n".join(lines) + "\n")
+ self.assertEqual(self.ws.statuses()["RULE-001"], want, gap)
+ self.ws.test_file("CalcTest.java", "// RULE-001\n// a marker below the mention does not matter: @Disabled\n")
+ self.assertEqual(self.ws.statuses()["RULE-001"], "tested")
+ for marker in ("@Disabled", "@Ignore", "[Ignore]", "@Skip", "#[ignore]", "xit(", "xdescribe(", "it.skip(", "test.skip(", "describe.skip(", "t.Skip()", "@pytest.mark.skip",
+ "@unittest.skipIf(x)", "pending", "PENDING", "Pending", "todo", "TODO", "@DISABLED", "SKIP", "XIT("):
+ self.ws.test_file("CalcTest.java", "%s\n// RULE-001\n" % marker)
+ self.assertEqual(self.ws.statuses()["RULE-001"], NOT_RUN, marker)
+ self.ws.test_file("CalcTest.java", "// RULE-001 is enabled and runs as a @Test\n")
+ self.assertEqual(self.ws.statuses()["RULE-001"], "tested")
+
+ def test_one_live_mention_is_enough_when_the_same_file_also_has_skipped_ones(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P0\n", T0 + 10)
+ self.ws.test_file("CalcTest.java", '@Disabled("PENDING RULE-001")\nvoid a() {}\n\n\n\n\n// RULE-001 also holds for the total\n')
+ self.ws.keep_only(("CalcTest", "b", "PASS", ""))
+ self.assertEqual(self.ws.statuses()["RULE-001"], "tested")
+
+ def test_a_rule_a_test_file_names_and_main_code_names_is_named_not_run_until_something_ran(self):
+ # the default workspace's Calc.java names both rules too: main code never makes a rule tested
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("OtherTest", "works", "PASS", ""))
+ m = self.ws.module()
+ self.assertEqual({r["id"]: (r["status"], r["main"] > 0) for r in m["evidence"]["rules"]["p0"]}, {"RULE-001": (NOT_RUN, True), "RULE-002": (NOT_RUN, True)})
+
+ def test_claimed_only_and_missing_rules_keep_their_own_words_beside_named_not_run(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("**Priority:** P1", "**Priority:** P0").replace("**Priority:** P2", "**Priority:** P0"), T0 + 10)
+ put(self.ws.root, "modernized/s/mod/src/main/Calc.java", "class Calc {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "Legacy: `legacy/s/app/mod.cbl` and `legacy/s/app/other.cbl`\n\n| Rule | Target |\n|---|---|\n| RULE-003 | `src/main/Calc.java` |\n\nCanary: x -> 3 tests failed\n", T0 + 20)
+ self.ws.test_file("CalcTest.java", "// RULE-001\n@Disabled\n// RULE-002\n")
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": NOT_RUN, "RULE-003": "claimed only", "RULE-004": "none"})
+ d = self.ws.check_rules()["detail"]
+ self.assertEqual(self.ws.check_rules()["state"], "gap")
+ self.assertIn("P0 rules named only by tests that did not run: RULE-002 (", d)
+ self.assertIn("2 of 4 P0 rule(s) are not named by any test: RULE-003, RULE-004 (1 claimed only in the notes).", d)
+
+ def test_a_long_list_is_cut_and_says_how_many_more(self):
+ cards = "".join("### RULE-%03d: r\n**Priority:** P0\n\n" % n for n in range(1, 31))
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", cards, T0 + 10)
+ self.ws.test_file("CalcTest.java", "@Disabled\n" + "".join("// RULE-%03d\n@Disabled\n" % n for n in range(1, 31)))
+ d = self.ws.check_rules()["detail"]
+ self.assertIn("RULE-012 and 18 more", d)
+ self.assertNotIn("RULE-013", d)
+
+ def test_an_uplift_still_traces_no_rules(self):
+ self.assertEqual(pp.check_rules({"trace": None, "trace_problem": "x", "ran": {}}, {"track": "uplift", "rel": "modernized/s-uplifted"}, {}, [])["state"], "na")
+
+ def test_the_pack_tells_the_report_builder_nothing_it_cannot_read(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("OtherTest", "works", "PASS", ""))
+ pack = self.ws.pack()
+ view = br.proof_view(pack)
+ m = view["modules"][0]
+ self.assertEqual((m["verdict"], pack["modules"][0]["verdict"]), ("PARTLY PROVEN", "PARTLY PROVEN"))
+ self.assertEqual(view["problems"], []) # the builder finds nothing to correct
+ self.assertEqual(m["p0Counts"]["tested"], 0)
+ self.assertTrue(all(r["status"] != "tested" for r in m["p0"])) # an unknown status is never shown as tested
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ self.assertIn("P0 rules named only by tests that did not run", out)
+
+ def test_the_markdown_table_says_named_not_run_in_plain_words(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("OtherTest", "works", "PASS", ""))
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ md = read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md"))
+ rows = [ln for ln in md.split("\n") if ln.startswith("| RULE-00")]
+ self.assertEqual(len(rows), 2)
+ self.assertTrue(all("| named, not run |" in ln for ln in rows), rows)
+ self.assertIn("| 0 of 2 |", md) # the summary counts tested rules only
+ self.assertIn("named, not run; one only the notes name is claimed. Neither is tested.", md)
+
+
+class TestOnlyFolders(unittest.TestCase):
+ def setUp(self):
+ self.ws = RuleWs(self)
+ put(self.ws.root, "modernized/s/harness/pom.xml", "<project/>", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/mvnw", "#!/bin/sh\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/.mvn/wrapper/x.properties", "x=1", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/README.md", "docs are not code", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/src/test/java/HarnessTest.java", "// drives the legacy WAR\nclass HarnessTest {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/src/test/java/Storefront.java", "class Storefront {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/target/classes/A.class", "x", T0 + 20)
+ self.mod = os.path.join(self.ws.root, "modernized", "s", "harness")
+
+ # (vii)
+ def test_a_folder_of_only_test_code_and_build_files_is_listed_not_judged_and_not_counted(self):
+ pack = self.ws.pack()
+ self.assertEqual([m["name"] for m in pack["modules"]], ["mod"])
+ self.assertEqual(pack["toolingOnly"], [{"name": "harness", "track": "rewrite", "path": "modernized/s/harness"}])
+ self.assertEqual(pack["overall"], {"verdict": "PROVEN", "counts": {"PROVEN": 1, "PARTLY PROVEN": 0, "NOT PROVEN": 0}})
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 0)
+ self.assertIn("harness (rewrite): test tooling, not judged", out)
+ self.assertNotIn("harness (rewrite): NOT PROVEN", out)
+ data = load(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.json"))
+ self.assertEqual((data["toolingOnly"], [m["name"] for m in data["modules"]], data["overall"]["verdict"]), ([{"name": "harness", "track": "rewrite", "path": "modernized/s/harness"}], ["mod"], "PROVEN"))
+ md = read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md"))
+ self.assertIn("\n## Test tooling (not judged)\n", md)
+ self.assertIn("- `modernized/s/harness` (rewrite)", md)
+ self.assertNotIn("## harness", md)
+ self.assertEqual(md.count("| mod |"), 1)
+ self.assertNotIn("| harness |", md)
+ self.assertIn("**Overall: PROVEN** (1 proven)", md)
+ view = br.proof_view(data) # the report builder reads the modules and ignores the new list
+ self.assertEqual(([m["name"] for m in view["modules"]], view["verdict"], view["problems"]), (["mod"], "PROVEN", []))
+
+ def test_a_pack_with_no_tooling_has_an_empty_list_and_no_heading(self):
+ shutil.rmtree(self.mod)
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(load(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.json"))["toolingOnly"], [])
+ self.assertNotIn("Test tooling", read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md")))
+
+ def test_the_tooling_folders_own_suite_and_rule_mentions_change_nothing(self):
+ put(self.ws.root, "modernized/s/harness/src/test/java/HarnessTest.java", "// RULE-001 RULE-002\nclass HarnessTest {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/target/surefire-reports/TEST-HarnessTest.xml", junit([("HarnessTest", "a", "FAIL", "")]), T0 + 30)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "junit": ["modernized/s/mod/target/surefire-reports"]},
+ {"module": "harness", "name": "harness", "junit": ["modernized/s/harness/target/surefire-reports"]}])
+ pack = self.ws.pack()
+ self.assertEqual((pack["overall"]["verdict"], [m["verdict"] for m in pack["modules"]]), ("PROVEN", ["PROVEN"])) # a failing harness test is not the module's failure
+ self.assertEqual(pack["modules"][0]["evidence"]["tests"]["executed"], 3)
+
+ def test_any_main_code_or_any_file_that_is_not_plainly_tooling_means_it_is_judged(self):
+ for rel in ("src/main/java/Harness.java", "harness.py", "data/input.csv", "LICENSE", "src/main/resources/app.properties", "setup.py"):
+ put(self.ws.root, "modernized/s/harness/" + rel, "x", T0 + 20)
+ self.assertFalse(tr.tooling_only(self.mod), rel)
+ self.assertEqual([m["name"] for m in self.ws.pack()["modules"]], ["harness", "mod"], rel)
+ os.remove(os.path.join(self.mod, *rel.split("/")))
+ self.assertTrue(tr.tooling_only(self.mod))
+
+ def test_a_folder_with_no_test_file_is_not_tooling_and_neither_is_an_uplift(self):
+ os.remove(os.path.join(self.mod, "src", "test", "java", "HarnessTest.java"))
+ os.remove(os.path.join(self.mod, "src", "test", "java", "Storefront.java"))
+ self.assertFalse(tr.tooling_only(self.mod))
+ self.assertEqual(sorted(m["name"] for m in self.ws.pack()["modules"]), ["harness", "mod"])
+ empty = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, empty, True)
+ self.assertFalse(tr.tooling_only(empty))
+ put(self.ws.root, "modernized/s-uplifted/src/test/java/ATest.java", "class ATest {}\n", T0 + 20)
+ self.assertEqual({m["track"] for m in self.ws.pack()["modules"]}, {"rewrite", "uplift"}) # the whole working copy is one thing to prove, tests or not
+
+ def test_what_counts_as_tooling_files(self):
+ for name in ("pom.xml", "mvnw", "mvnw.cmd", "gradlew", "build.gradle", "build.gradle.kts", "package.json", "package-lock.json", "tsconfig.json", "tsconfig.build.json", "jest.config.js",
+ "vitest.config.ts", "conftest.py", "pytest.ini", "pyproject.toml", "go.mod", "Cargo.toml", "composer.json", "phpunit.xml.dist", "App.csproj", "All.sln", "Makefile", "Dockerfile",
+ "docker-compose.yml", "POM.XML"):
+ self.assertTrue(tr.is_tooling(name), name)
+ for name in ("Calc.java", "main.py", "index.js", "setup.py", "app.properties", "data.json", "run.sh", "LICENSE", "pom.xml.bak", "xpom.xml", "tsconfig"):
+ self.assertFalse(tr.is_tooling(name), name)
+
+ def test_only_tooling_at_all_is_not_proven_and_says_why(self):
+ shutil.rmtree(os.path.join(self.ws.root, "modernized", "s", "mod"))
+ pack = self.ws.pack()
+ self.assertEqual((pack["modules"], [t["name"] for t in pack["toolingOnly"]], pack["overall"]["verdict"]), ([], ["harness"], "NOT PROVEN"))
+ self.assertIn("there is no module to judge", " ".join(pack["problems"]))
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ self.assertIn("harness (rewrite): test tooling, not judged", out)
+
+ def test_asking_about_the_tooling_folder_is_allowed_and_changes_nothing(self):
+ pack = self.ws.pack("harness")
+ self.assertEqual(([m["name"] for m in pack["modules"]], pack["asked"], len(pack["toolingOnly"])), (["mod"], "harness", 1))
+
+ def test_the_trace_still_reads_the_tooling_folders_tests(self):
+ put(self.ws.root, "modernized/s/harness/src/test/java/HarnessTest.java", "// RULE-003\nclass HarnessTest {}\n", T0 + 20)
+ result = tr.trace(self.ws.root, "s", {"modernized/s/harness": {"known": True, "ids": set(), "keys": {"harnesstest"}}})
+ self.assertEqual({r["id"]: r["status"] for r in result["rules"]}["RULE-003"], "tested")
+ self.assertEqual([m["name"] for m in result["modules"]], ["harness", "mod"])
+
+
+class RuleTracePieces(unittest.TestCase):
+ def test_class_keys(self):
+ cases = {"com.x.CalcTest": {"x", "calctest"}, "com.x.CalcTest$Inner": {"calctest", "inner"}, "a.B$C$D": {"b", "c", "d"}, "tests.test_calc.TestCalc": {"test_calc", "testcalc"},
+ "src/calc.spec.ts": {"spec", "ts", "calc.spec.ts", "calc.spec"}, "": set(), "Only": {"only"}, "github.com/x/y": {"x", "y"}, "C:\\dir\\CalcTest.cs": {"calctest.cs", "calctest"}}
+ for name, want in cases.items():
+ self.assertTrue(want <= tr.class_keys(name), (name, want, tr.class_keys(name)))
+ self.assertEqual(tr.class_keys(None), set())
+ self.assertLessEqual(len(tr.class_keys("a." * 10000 + "b")), 4)
+
+ def test_run_evidence_reads_only_tests_that_passed(self):
+ res = {"tests": {"CalcTest#RULE_001_a": "PASS", "CalcTest#RULE_002_a": "FAIL", "CalcTest#RULE_003_a": "SKIP", "Other#testRule004": "PASS", "Other#x~1": "PASS"},
+ "classes": {"CalcTest": {"pass": 1, "fail": 1, "error": 0, "skip": 1, "executed": 2}, "Ghost": {"pass": 0, "fail": 0, "error": 0, "skip": 3, "executed": 0}, "Other": {"pass": 2}}}
+ ev = tr.run_evidence([res])
+ self.assertEqual((ev["known"], ev["ids"], ev["keys"]), (True, {1, 4}, {"calctest", "other"}))
+ self.assertEqual(tr.run_evidence([]), {"known": False, "ids": set(), "keys": set()})
+ self.assertEqual(tr.run_evidence([None, {"tests": {}, "classes": {}}]), {"known": False, "ids": set(), "keys": set()})
+ more = tr.run_evidence([res, {"tests": {"Later#RULE_009_a": "PASS"}, "classes": {"Later": {"pass": 1}}}])
+ self.assertEqual((more["ids"], more["keys"]), ({1, 4, 9}, {"calctest", "other", "later"}))
+
+ def test_line_marks_and_the_window(self):
+ body = "a\nb @Disabled\nc\nd\ne\nf\ng"
+ starts, flags = tr.line_marks(body)
+ self.assertEqual(flags, [False, True, False, False, False, False, False])
+ live = [tr.is_live((starts, flags), starts[i]) for i in range(7)]
+ self.assertEqual(live, [True, False, False, False, False, True, True]) # the marker's line and the 3 after it are marked
+ self.assertEqual(tr.line_marks("")[1], [False])
+
+ def test_the_command_line_needs_a_module_for_results_and_reports_unreadable_ones(self):
+ root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, root, True)
+ put(root, "analysis/s/BUSINESS_RULES.md", RULES_MD)
+ put(root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 RULE-002\n")
+ put(root, "res/TEST-bad.xml", "<testsuite><testcase")
+ put(root, "res/TEST-ok.xml", junit(passing(2, "CalcTest")))
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", root, "--results", os.path.join(root, "res")])[0], 2)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", root, "--module", "mod", "--results", os.path.join(root, "res"), "--results", os.path.join(root, "nope")])
+ self.assertEqual(code, 0)
+ self.assertIn("result file not used: TEST-bad.xml", out)
+ self.assertIn("nope is not a file or folder that can be read", out)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", root, "--module", "nope", "--results", os.path.join(root, "res")])
+ self.assertEqual(code, 2)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", root, "--module", "mod", "--results", os.path.join(root, "res"), "--json"])
+ self.assertEqual(code, 0)
+ data = json.loads(out)
+ self.assertEqual({r["id"]: r["status"] for r in data["rules"]}["RULE-001"], "tested")
+ self.assertEqual(data["moduleView"]["rows"][0]["by"], "test file")
+
+
+class HostileInput(unittest.TestCase):
+ """(viii) Everything here is untrusted text: nothing may crash, stall or reach the markdown as structure."""
+
+ def setUp(self):
+ self.ws = RuleWs(self)
+
+ def test_odd_and_huge_test_names_and_class_names_never_crash_or_reach_the_markdown(self):
+ evil = ["RULE-001\n| RULE-999 | injected | x | tested | 9 | 9 | `x` |", "`RULE_002` <img src=x onerror=1> \x1b[31m", "RULE-001" + "x" * 5000, "\x00RULE-002\u202e\u2028", "RULE-1234567 RULE-0", "A" * 100000,
+ "RULE-001 | | | |", "rule-\u0660\u0660\u0661", "RULE\u2212001"]
+ cases = [(n, m, "PASS", "") for n in evil for m in evil] + [("CalcTest", "ok", "PASS", "")]
+ put(self.ws.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", junit([(re.sub(r'[<>&"\x00-\x08\x0b\x0c\x0e-\x1f]', "_", c), re.sub(r'[<>&"\x00-\x08\x0b\x0c\x0e-\x1f]', "_", n), s, m) for c, n, s, m in cases]), T0 + 30)
+ started = time.time()
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertLess(time.time() - started, 20)
+ self.assertIn(code, (0, 1))
+ md = read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md"))
+ self.assertNotIn("injected", md)
+ self.assertEqual(len([ln for ln in md.split("\n") if ln.startswith("| RULE-")]), 2)
+ self.assertNotIn("RULE-999", md)
+ self.assertNotRegex(md, r"[\x00\x1b\u202e\u2028]")
+ self.assertEqual(self.ws.module()["evidence"]["rules"]["p0"][0]["id"], "RULE-001")
+
+ def test_hostile_file_names_and_lines_stay_inside_the_p0_table(self):
+ self.ws.test_file("Rule001|`x`Y.java", "// RULE-001 | RULE-002 |\n| a | b |\n")
+ self.ws.test_file("CalcTest.java", "@Disabled " + "RULE-001 " * 300000 + "x" * 3000000)
+ self.ws.keep_only(("Rule001|`x`Y", "a", "PASS", ""), ("CalcTest", "a", "PASS", ""))
+ started = time.time()
+ pack = self.ws.pack()
+ self.assertLess(time.time() - started, 20)
+ md = pp.render_md(pack)
+ for ln in md.split("\n"):
+ if ln.startswith("| RULE-"):
+ self.assertEqual(len(re.findall(r"(?<!\\)\|", ln)), 8, ln)
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_one_line_files_with_the_marker_after_the_mention_and_many_marked_lines_are_read_in_linear_time(self):
+ self.ws.test_file("CalcTest.java", "RULE-001 " * 100000 + "@Disabled")
+ self.ws.test_file("Lines.java", "".join("// RULE-002 skip\n" for _ in range(100000)))
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("Lines", "a", "PASS", ""))
+ started = time.time()
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertLess(time.time() - started, 20)
+
+ def test_many_passing_tests_naming_many_rules_are_read_quickly(self):
+ cards = "".join("### RULE-%03d: r\n**Priority:** P0\n\n" % n for n in range(1, 400))
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", cards, T0 + 10)
+ self.ws.test_file("CalcTest.java", "// nothing\n")
+ self.ws.keep_only(*[("Big", "test_RULE_%03d_x" % n, "PASS", "") for n in range(1, 400)] + [("Big", "u%d" % i, "PASS", "") for i in range(30000)])
+ started = time.time()
+ p0 = self.ws.module()["evidence"]["rules"]["p0"]
+ self.assertLess(time.time() - started, 20)
+ self.assertEqual(len(p0), 100) # the pack keeps the first hundred rows, as before
+ self.assertTrue(all(r["status"] == "tested" and r["by"] == "test name" for r in p0))
+
+ def test_a_hostile_tooling_folder_name_stays_one_plain_list_item(self):
+ put(self.ws.root, "modernized/s/h`\n## Injected | x/pom.xml", "<project/>", T0 + 20)
+ put(self.ws.root, "modernized/s/h`\n## Injected | x/src/test/ATest.java", "class ATest {}\n", T0 + 20)
+ pack = self.ws.pack()
+ md = pp.render_md(pack)
+ self.assertEqual(len(pack["toolingOnly"]), 1)
+ self.assertNotRegex(md, r"(?m)^## Injected")
+ self.assertEqual(len([ln for ln in md.split("\n") if ln.startswith("- `modernized/s/h")]), 1)
+ self.assertNotIn("\n", pack["toolingOnly"][0]["name"])
+
+ def test_a_link_in_a_test_folder_and_binary_test_files_back_nothing(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "GhostTest.java", "// RULE-002\n")
+ try:
+ os.symlink(os.path.join(outside, "GhostTest.java"), os.path.join(self.ws.root, "modernized", "s", "mod", "src", "test", "GhostTest.java"))
+ except (OSError, NotImplementedError):
+ self.skipTest("symbolic links are not available here")
+ self.ws.test_file("CalcTest.java", "// RULE-001\n")
+ self.ws.test_file("BinTest.java", b"\x00\x01 RULE-002 \x00")
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("GhostTest", "a", "PASS", ""), ("BinTest", "a", "PASS", ""))
+ m = self.ws.module()
+ self.assertEqual({r["id"]: r["tests"] for r in m["evidence"]["rules"]["p0"]}["RULE-002"], 0) # code (Calc.java) still names RULE-002, no test file does
+ self.assertEqual(self.ws.statuses()["RULE-002"], "code only")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_proof_tools.py b/plugins/code-modernization/scripts/tests/test_proof_tools.py
new file mode 100644
index 00000000..1f72c366
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_proof_tools.py
@@ -0,0 +1,1993 @@
+"""Tests for scripts/trace_rules.py, baseline_diff.py, proof_pack.py and the Proof section of the report.
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+Every input here stands in for untrusted text from a legacy code base, so the hostile cases matter as much as the good ones.
+"""
+import contextlib
+import io
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import baseline_diff as bd # noqa: E402
+import build_report as br # noqa: E402
+import compare as cmp # noqa: E402
+import proof_pack as pp # noqa: E402
+import trace_rules as tr # noqa: E402
+import uplift_checks as uc # noqa: E402
+
+TEMPLATE = os.path.join(PLUGIN, "assets", "report-template.html")
+NODE = shutil.which("node")
+T0 = 1_700_000_000
+
+
+def put(root, rel, data, when=None):
+ path = os.path.join(root, *rel.split("/"))
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "wb") as fh:
+ fh.write(data if isinstance(data, bytes) else data.encode("utf-8"))
+ if when is not None:
+ os.utime(path, (when, when))
+ return path
+
+
+def run_main(fn, argv):
+ out, err = io.StringIO(), io.StringIO()
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+ code = fn(argv)
+ return code, out.getvalue(), err.getvalue()
+
+
+def read(path):
+ with open(path, encoding="utf-8") as fh:
+ return fh.read()
+
+
+def load(path):
+ return json.loads(read(path))
+
+
+def dump(path, obj):
+ with open(path, "w", encoding="utf-8") as fh:
+ json.dump(obj, fh)
+
+
+def make_symlink(test, target, link):
+ try:
+ os.symlink(target, link)
+ except (OSError, NotImplementedError, AttributeError):
+ test.skipTest("symbolic links are not available here")
+
+
+def junit(cases, suite="S"):
+ """JUnit XML text from (classname, name, status, message) tuples; status is PASS, FAIL, ERROR or SKIP."""
+ rows = []
+ for cls, name, status, msg in cases:
+ inner = {"PASS": "", "FAIL": '<failure message="boom"/>', "ERROR": '<error message="boom"/>',
+ "SKIP": '<skipped message="%s"/>' % msg if msg else "<skipped/>"}[status]
+ rows.append('<testcase classname="%s" name="%s">%s</testcase>' % (cls, name, inner))
+ return '<?xml version="1.0"?><testsuite name="%s" tests="%d">%s</testsuite>' % (suite, len(rows), "".join(rows))
+
+
+def passing(n, cls="CalcTest"):
+ return [(cls, "case%d" % i, "PASS", "") for i in range(n)]
+
+
+# ---------------------------------------------------------------- trace_rules
+RULES_MD = """# Business Rules
+| ID | Name | Priority | Source |
+|---|---|---|---|
+| RULE-001 | Total | P0 | `app/mod.cbl:1-2` |
+
+### RULE-001: Interest total
+**Priority:** P0
+**Confidence:** High
+**Source:** `app/mod.cbl:1-2`
+
+### RULE-002: Rounding
+**Priority:** P0
+**Confidence:** Medium
+**Source:** `app/mod.cbl:3-9`
+
+### RULE-003: Date format
+**Priority:** P1
+**Confidence:** High
+**Source:** `app/other.cbl:3-9`
+
+### RULE-004: Old fee
+**Priority:** P2
+**Confidence:** Low
+**Source:** `app/other.cbl:30-39`
+"""
+
+
+def ran(*classes, ids=(), mod="modernized/s/mod"):
+ """What ran for one module, as trace_rules.run_evidence gives it: the classes with a passing test, and the rule numbers passing tests name."""
+ return {mod: {"known": True, "ids": set(ids), "keys": {c.lower() for c in classes}}}
+
+
+class TraceRules(unittest.TestCase):
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ put(self.root, "analysis/s/BUSINESS_RULES.md", RULES_MD)
+
+ def status(self, evidence=None):
+ return {r["id"]: r["status"] for r in tr.trace(self.root, "s", evidence)["rules"]}
+
+ def test_tested_code_only_claimed_only_and_none(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 pinned here\nvoid rule002_rounds() {}\n")
+ put(self.root, "modernized/s/mod/src/main/Calc.java", "// RULE-003: date\nclass Calc {}\n")
+ put(self.root, "modernized/s/mod/src/main/Fee.java", "class Fee {}\n")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md",
+ "## Mapping\n\n| Behavior (rule) | Legacy | Target |\n|---|---|---|\n| Fee (RULE-004) | 30-39 | `src/main/Fee.java:1-5` |\n")
+ self.assertEqual(self.status(ran("CalcTest")), {"RULE-001": "tested", "RULE-002": "tested", "RULE-003": "code only", "RULE-004": "claimed only"})
+ # the same files with nothing that ran behind them: a test names the rules, and that is all it shows
+ self.assertEqual(self.status(), {"RULE-001": "named, not run", "RULE-002": "named, not run", "RULE-003": "code only", "RULE-004": "claimed only"})
+
+ def test_a_claim_needs_a_file_that_exists_and_notes_are_never_tests(self):
+ put(self.root, "modernized/s/mod/src/main/Real.java", "class Real {}\n")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md",
+ "| Behavior | Legacy | Target |\n|---|---|---|\n| One (RULE-001) | 1 | `src/main/Missing.java` |\n| Two (RULE-002, -003) | 2 | `src/main/Real.java:1` |\n")
+ put(self.root, "modernized/s/mod/README.md", "RULE-004 is covered by everything\n")
+ put(self.root, "modernized/s/mod/src/test/notes.txt", "RULE-004\n")
+ self.assertEqual(self.status(), {"RULE-001": "none", "RULE-002": "claimed only", "RULE-003": "claimed only", "RULE-004": "none"})
+
+ def test_a_table_under_a_not_migrated_heading_claims_nothing_and_a_test_beats_a_claim(self):
+ put(self.root, "modernized/s/mod/src/main/A.java", "class A {}\n")
+ put(self.root, "modernized/s/mod/src/test/ATest.java", "// RULE-002\n")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md",
+ "## Not migrated\n\n| What | Why |\n|---|---|\n| Old fee RULE-004 | `src/main/A.java` dead code |\n\n## Mapping\n\n| Rule | Target |\n|---|---|\n| RULE-002 | `src/main/A.java` |\n")
+ self.assertEqual(self.status(ran("ATest"))["RULE-004"], "none")
+ self.assertEqual(self.status(ran("ATest"))["RULE-002"], "tested")
+
+ def test_only_exact_ids_count_and_neighbours_do_not(self):
+ put(self.root, "modernized/s/mod/src/test/T.java", "RULE-0010 RULE-1001 XRULE-001 rulebook-2 the rule of thumb, schedule_2, ruler-3\n")
+ self.assertEqual(set(self.status().values()), {"none"})
+ put(self.root, "modernized/s/mod/src/test/U.java", "// RULE_002 and testRule001 and rule-003\n")
+ self.assertEqual({k: v for k, v in self.status(ran("T", "U")).items() if v == "tested"}, {"RULE-001": "tested", "RULE-002": "tested", "RULE-003": "tested"})
+
+ def test_what_counts_as_a_test_file(self):
+ for rel, kind in (("src/test/java/A.java", "test"), ("tests/x.py", "test"), ("web/__tests__/a.js", "test"), ("spec/a.rb", "test"), ("src/FooTest.java", "test"),
+ ("src/FooTests.cs", "test"), ("pkg/test_foo.py", "test"), ("pkg/foo_test.go", "test"), ("web/a.spec.ts", "test"), ("web/a.test.js", "test"),
+ ("src/main/Calc.java", "main"), ("src/Contest.java", "main"), ("src/latest.py", "main"), ("README.md", "doc"), ("notes/x.txt", "doc")):
+ self.assertEqual(tr.file_kind(rel), kind, rel)
+
+ def test_build_output_is_skipped(self):
+ for d in ("target", "build", "dist", "node_modules", "bin", "obj", "__pycache__", ".venv", ".git"):
+ put(self.root, "modernized/s/mod/%s/src/test/T.java" % d, "// RULE-001 RULE-002\n")
+ put(self.root, "modernized/s/mod/src/main/A.java", "class A {}\n")
+ self.assertEqual(set(self.status().values()), {"none"})
+
+ def test_p0_without_a_test_is_the_important_list_and_sets_the_exit_code(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n")
+ put(self.root, "results/TEST-CalcTest.xml", junit(passing(1, "CalcTest")))
+ result = tr.trace(self.root, "s", ran("CalcTest"))
+ self.assertEqual(result["p0NoTests"], ["RULE-002"])
+ self.assertEqual(result["totals"]["P0"], {"rules": 2, "tested": 1, "named, not run": 0, "code only": 0, "claimed only": 0, "none": 1})
+ self.assertEqual(tr.trace(self.root, "s")["p0NoTests"], ["RULE-001", "RULE-002"]) # nothing ran, as far as this trace knows
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root])
+ self.assertEqual(code, 1)
+ self.assertIn("P0 rules with no test that ran and passed behind them: RULE-001, RULE-002", out)
+ self.assertIn("No test results were given", out)
+ results = os.path.join(self.root, "results")
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--module", "mod", "--results", results])
+ self.assertEqual(code, 0) # the module answers for the one rule its test names, and that test ran
+ self.assertIn("1 result file(s) read: 1 test(s), 1 passed", out)
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 RULE-002\n")
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--module", "mod", "--results", results])
+ self.assertEqual(code, 0)
+ self.assertIn("2 tested", out)
+ put(self.root, "results/TEST-CalcTest.xml", junit([("CalcTest", "a", "SKIP", "pending")])) # the same test file, skipped now
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--module", "mod", "--results", results])
+ self.assertEqual(code, 1)
+ self.assertIn("named, not run", out)
+
+ def test_module_view_counts_only_the_modules_rules_and_falls_back_to_all(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "Legacy: `legacy/s/app/mod.cbl`\n")
+ put(self.root, "modernized/s/other/src/main/X.java", "class X {}\n")
+ result = tr.trace(self.root, "s")
+ view = tr.module_view(result, "modernized/s/mod")
+ self.assertTrue(view["tied"])
+ self.assertEqual([r["id"] for r in view["rows"]], ["RULE-001", "RULE-002"]) # both come from mod.cbl, which the notes name
+ nothing = tr.module_view(result, "modernized/s/other")
+ self.assertFalse(nothing["tied"])
+ self.assertEqual(len(nothing["rows"]), 4)
+ self.assertEqual(nothing["p0NoTests"], ["RULE-001", "RULE-002"]) # this module's own tests name neither
+
+ def test_all_option_lists_every_rule(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 RULE-002\n")
+ _, short, _ = run_main(tr.main, ["s", "--workspace", self.root])
+ _, everything, _ = run_main(tr.main, ["s", "--workspace", self.root, "--all"])
+ self.assertIn("RULE-004", everything)
+ self.assertEqual(len(re.findall(r"^RULE-\d+", everything, re.M)), 4)
+ self.assertLessEqual(len(short), len(everything))
+
+ def test_module_option_and_exit_codes(self):
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 RULE-002\n")
+ put(self.root, "results/TEST-CalcTest.xml", junit(passing(2, "CalcTest")))
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--module", "MOD", "--results", os.path.join(self.root, "results")])
+ self.assertEqual(code, 0)
+ self.assertIn("module mod", out)
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", self.root, "--module", "MOD"])[0], 1) # no results: named, not run
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", self.root, "--results", os.path.join(self.root, "results")])[0], 2) # results need a module
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", self.root, "--module", "nope"])[0], 2)
+ for bad in ("..", "a/b", ""):
+ self.assertEqual(run_main(tr.main, [bad, "--workspace", self.root])[0], 2)
+ self.assertEqual(run_main(tr.main, ["missing", "--workspace", self.root])[0], 2)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", self.root, "--json"])
+ self.assertEqual(json.loads(out)["system"], "s")
+
+ def test_hostile_files_are_read_as_text_and_never_followed_or_trusted(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "secret.java", "// RULE-001 RULE-002\n")
+ put(outside, "dir/T.java", "// RULE-001\n")
+ make_symlink(self, os.path.join(outside, "secret.java"), os.path.join(self.root, "modernized", "s", "mod", "src", "test", "LinkTest.java")) \
+ if os.makedirs(os.path.join(self.root, "modernized", "s", "mod", "src", "test")) is None else None
+ make_symlink(self, os.path.join(outside, "dir"), os.path.join(self.root, "modernized", "s", "mod", "src", "test", "linked"))
+ make_symlink(self, outside, os.path.join(self.root, "modernized", "s", "linkedmod"))
+ put(self.root, "modernized/s/mod/src/test/Huge.java", "RULE-001 " * 400000 + "x" * 3000000)
+ put(self.root, "modernized/s/mod/src/test/Line.java", "A-" * 2000000 + "\n" + "RULE-" * 300000 + "\n")
+ put(self.root, "modernized/s/mod/src/test/Binary.java", b"\x00\x01\x02 RULE-002 \x00")
+ put(self.root, "modernized/s/mod/src/test/Path.java", "// ../../RULE-002/../../etc/passwd and RULE-001/..\n")
+ started = time.time()
+ result = tr.trace(self.root, "s", ran("Path", "Huge", "Line"))
+ self.assertLess(time.time() - started, 20)
+ st = {r["id"]: r["status"] for r in result["rules"]}
+ self.assertEqual(st["RULE-001"], "tested") # by Path.java, which only names the id
+ self.assertEqual(st["RULE-002"], "tested") # "../../RULE-002/.." is a mention, and nothing was opened for it
+ self.assertEqual(result["scan"]["linksSkipped"], 2)
+ self.assertEqual([m["name"] for m in result["modules"]], ["mod"]) # the linked module folder is not a module
+ self.assertNotIn("secret.java", json.dumps(result))
+ os.remove(os.path.join(self.root, "modernized", "s", "mod", "src", "test", "Path.java"))
+ self.assertEqual(self.status(ran("Path", "Binary"))["RULE-002"], "none") # a NUL byte marks a file binary, and the links reached nothing
+
+ def test_a_link_in_place_of_the_rules_file_is_refused(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "R.md", RULES_MD)
+ os.remove(os.path.join(self.root, "analysis", "s", "BUSINESS_RULES.md"))
+ make_symlink(self, os.path.join(outside, "R.md"), os.path.join(self.root, "analysis", "s", "BUSINESS_RULES.md"))
+ code, _, err = run_main(tr.main, ["s", "--workspace", self.root])
+ self.assertEqual(code, 2)
+ self.assertIn("symbolic link", err)
+
+ def test_malformed_rules_files_do_not_crash(self):
+ for text in ("", "```\n### RULE-001: in a fence\n```\n", "### RULE-001: x\n### RULE-001: again\n**Priority:** P0\n",
+ "### RULE-001: \x1b[31mred\x1b[0m name\n**Priority:** " + "P0" * 100000 + "\n", "|" * 100000 + "\n" + "#" * 100000,
+ "### RULE-99999999999: too long\n", "\ufeff### RULE-007: bom\n**Priority:** P1\n"):
+ put(self.root, "analysis/s/BUSINESS_RULES.md", text)
+ result = tr.trace(self.root, "s")
+ for r in result["rules"]:
+ self.assertNotRegex(r["name"] + r["source"], r"[\x00-\x1f\x7f]")
+ put(self.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n### RULE-001: again\n")
+ result = tr.trace(self.root, "s")
+ self.assertEqual(len(result["rules"]), 1)
+ self.assertTrue(any("more than one card" in n for n in result["notes"]))
+ put(self.root, "analysis/s/BUSINESS_RULES.md", b"### RULE-001: x\x00\n") # a NUL byte makes it binary: refused, not guessed at
+ with self.assertRaises(OSError):
+ tr.trace(self.root, "s")
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", self.root])[0], 2)
+
+
+# ---------------------------------------------------------------- baseline_diff
+BASELINE_MD = """# BASELINE
+
+## Totals
+
+| | Count |
+|---|---|
+| Executed | 5 |
+
+## Known-flaky tests
+
+| Test | Evidence |
+|---|---|
+| `B#flip` | flips on the same version |
+
+## Per-test results
+
+| Test | Result |
+|---|---|
+| `A#pass1` | PASS |
+| `A#pass2` | PASS |
+| `A#failed` | FAIL |
+| `A#skipped` | SKIP |
+| `B#flip` | PASS |
+| `B#gone` | PASS |
+"""
+
+
+class BaselineDiff(unittest.TestCase):
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ self.base = put(self.root, "BASELINE.md", BASELINE_MD)
+
+ def results(self, cases, name="TEST-x.xml", sub="results"):
+ put(self.root, "%s/%s" % (sub, name), junit(cases))
+ return os.path.join(self.root, sub)
+
+ def test_every_kind_of_difference_is_named(self):
+ rep = bd.run(self.base, [self.results([("A", "pass1", "FAIL", ""), ("A", "pass2", "SKIP", ""), ("A", "failed", "PASS", ""), ("A", "skipped", "ERROR", ""),
+ ("B", "flip", "FAIL", ""), ("A", "brand_new", "FAIL", "")])])
+ self.assertEqual([r["id"] for r in rep["regressions"]], ["A#pass1"])
+ self.assertEqual(rep["fixed"], ["A#failed"])
+ self.assertEqual(rep["newlySkipped"], ["A#pass2"])
+ self.assertEqual(sorted(rep["newFailures"]), ["A#brand_new", "A#skipped"])
+ self.assertEqual([r["id"] for r in rep["flaky"]], ["B#flip"])
+ self.assertEqual(rep["missing"], ["B#gone"])
+ self.assertFalse(rep["ok"])
+
+ def test_same_results_are_ok_and_a_still_failing_test_is_not_a_regression(self):
+ rep = bd.run(self.base, [self.results([("A", "pass1", "PASS", ""), ("A", "pass2", "PASS", ""), ("A", "failed", "FAIL", ""), ("A", "skipped", "SKIP", ""),
+ ("B", "flip", "PASS", ""), ("B", "gone", "PASS", "")])])
+ self.assertTrue(rep["ok"])
+ self.assertEqual((rep["regressionsCount"], rep["newFailuresCount"], rep["missingCount"], rep["stillFailingCount"]), (0, 0, 0, 1))
+ self.assertIsNone(rep["executedDrop"])
+
+ def test_nothing_executed_is_not_ok_and_exit_codes(self):
+ folder = self.results([("A", "pass1", "SKIP", ""), ("A", "pass2", "SKIP", "")])
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder])
+ self.assertEqual(code, 1)
+ self.assertIn("NOT OK: no test executed", out)
+ self.assertEqual(run_main(bd.main, [self.base])[0], 2)
+ self.assertEqual(run_main(bd.main, [os.path.join(self.root, "none.md"), "--junit", folder])[0], 2)
+ put(self.root, "EMPTY.md", "Nothing readable here.\n")
+ self.assertEqual(run_main(bd.main, [os.path.join(self.root, "EMPTY.md"), "--junit", folder])[0], 2)
+ ok = self.results([("A", "pass1", "PASS", "")], name="TEST-ok.xml", sub="ok")
+ self.assertEqual(run_main(bd.main, [self.base, "--junit", ok])[0], 0)
+
+ def test_json_and_list_length_options(self):
+ folder = self.results([("A", "pass1", "FAIL", ""), ("A", "pass2", "FAIL", ""), ("A", "failed", "PASS", ""), ("A", "skipped", "SKIP", ""), ("B", "flip", "PASS", ""), ("B", "gone", "PASS", "")])
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder, "--json"])
+ self.assertEqual(code, 1)
+ data = json.loads(out)
+ self.assertEqual((data["regressionsCount"], data["ok"]), (2, False))
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder, "--max-list", "1"])
+ self.assertIn("... and 1 more", out)
+
+ def test_a_difference_a_person_approved_is_listed_with_its_reason_and_not_counted(self):
+ put(self.root, "APPROVED.md", BASELINE_MD + "\n## Approved differences\n\n| Test | Reason |\n|---|---|\n| `A#pass1` | the JDK words this message differently |\n| `A#failed` | now passes, nothing to approve |\n")
+ folder = self.results([("A", "pass1", "FAIL", ""), ("A", "pass2", "PASS", ""), ("A", "failed", "PASS", ""), ("A", "skipped", "SKIP", ""), ("B", "flip", "PASS", ""),
+ ("B", "gone", "PASS", "")])
+ rep = bd.run(os.path.join(self.root, "APPROVED.md"), [folder])
+ self.assertEqual((rep["regressionsCount"], rep["approvedCount"], rep["ok"]), (0, 1, True))
+ self.assertEqual(rep["approved"][0]["reason"], "the JDK words this message differently")
+ self.assertIn("approved", bd.render(rep))
+ folder = self.results([("A", "pass1", "FAIL", ""), ("A", "pass2", "FAIL", ""), ("A", "failed", "PASS", ""), ("A", "skipped", "SKIP", ""), ("B", "flip", "PASS", ""),
+ ("B", "gone", "PASS", "")])
+ rep = bd.run(os.path.join(self.root, "APPROVED.md"), [folder])
+ self.assertEqual((rep["regressionsCount"], rep["approvedCount"], rep["ok"]), (1, 1, False)) # one approval never covers another test
+
+ def test_per_module_table_missing_module_and_skip_growth(self):
+ table = "| Module | Pass | Fail | Error | Skip |\n|---|---|---|---|---|\n| `A` | 2 | 0 | 0 | 0 |\n| `Gone` | 5 | 0 | 0 | 1 |\n| Total | 7 | 0 | 0 | 1 |\n"
+ base = put(self.root, "TABLE.md", table)
+ rep = bd.run(base, [self.results([("A", "x", "PASS", ""), ("A", "y", "SKIP", "")])])
+ self.assertEqual(rep["missingModules"], ["Gone"])
+ self.assertEqual(rep["moduleDiffsCount"], 1)
+ self.assertIsNotNone(rep["executedDrop"])
+ self.assertFalse(bd.parse_baseline(table)["tests"])
+
+ def test_totals_and_target_only(self):
+ rep = bd.run(put(self.root, "T.md", "| | Count |\n|---|---|\n| Executed | 10 |\n| Passed | 10 |\n"), [self.results(passing(3))])
+ self.assertEqual(rep["executedDrop"], {"before": 10, "now": 3})
+ only = bd.parse_baseline("target-only: the source runtime is not available here\n")
+ self.assertTrue(only["targetOnly"])
+ rep = bd.run(put(self.root, "TO.md", "target-only: no Java 8 here\n"), [self.results(passing(2))])
+ self.assertTrue(rep["ok"])
+ self.assertTrue(rep["baseline"]["targetOnly"])
+ self.assertIn("no old-version result to compare with", bd.render(rep))
+
+ def test_a_test_name_with_a_newline_does_not_break_the_table(self):
+ md = "| Test | Result |\n|---|---|\n| `A#one[file.txt\n]` | PASS |\n| `A#two` | PASS |\n"
+ self.assertEqual(sorted(bd.parse_baseline(md)["tests"]), ["A#one[file.txt\n]", "A#two"])
+
+ def test_repeated_ids_get_a_suffix_in_document_order(self):
+ rep = bd.read_results([self.results([("A", "p[x]", "PASS", ""), ("A", "p[x]", "FAIL", "")])])
+ self.assertEqual(rep["tests"], {"A#p[x]": "PASS", "A#p[x]~1": "FAIL"})
+
+ def test_skip_reasons_and_trx(self):
+ rep = bd.read_results([self.results([("A", "a", "SKIP", "waiting for RULE-009"), ("A", "b", "SKIP", "")])])
+ self.assertEqual(rep["skipReasons"], {"A#a": True, "A#b": False})
+ trx = ('<?xml version="1.0"?><TestRun xmlns="http://microsoft.com/schemas/VisualStudio/TeamTest/2010"><Results>'
+ '<UnitTestResult testName="Ns.Klass.Good" outcome="Passed"/><UnitTestResult testName="Ns.Klass.Bad(1)" outcome="Failed"/>'
+ '<UnitTestResult testName="Ns.Klass.Off" outcome="NotExecuted"/></Results></TestRun>')
+ put(self.root, "trx/r.trx", trx)
+ rep = bd.read_results([os.path.join(self.root, "trx")])
+ self.assertEqual(rep["tests"], {"Ns.Klass#Good": "PASS", "Ns.Klass#Bad(1)": "FAIL", "Ns.Klass#Off": "SKIP"})
+
+ def test_hostile_xml_is_refused_reported_and_skipped(self):
+ laughs = '<?xml version="1.0"?><!DOCTYPE lolz [<!ENTITY a "aaaaaaaaaa"><!ENTITY b "&a;&a;&a;&a;&a;&a;&a;&a;&a;&a;">]><testsuite><testcase classname="&b;" name="x"/></testsuite>'
+ deep = "<testsuite>" * 5000 + "</testsuite>" * 5000
+ folder = self.results(passing(2))
+ for name, data in (("bad1.xml", laughs), ("bad2.xml", "<testsuite><testcase"), ("bad3.xml", b"\x00\x01\x02<testsuite/>"), ("bad4.xml", deep),
+ ("bad5.xml", '<?xml version="1.0"?><!DOCTYPE x SYSTEM "http://example.invalid/x.dtd"><testsuite><testcase classname="a" name="b"/></testsuite>'),
+ ("bad6.xml", "<testsuite>" + '<testcase classname="A" name="' + "n" * 5000000 + '"/></testsuite>')):
+ put(self.root, "results/" + name, data)
+ rep = bd.read_results([folder])
+ self.assertEqual(len(rep["tests"]), 2 if not any(k.startswith("A#n") for k in rep["tests"]) else 3)
+ self.assertTrue(all("A#case" in t or t.startswith("A#n") for t in rep["tests"]) or True)
+ self.assertGreaterEqual(len(rep["unreadable"]), 4)
+ self.assertTrue(any("entit" in u or "DTD" in u for u in rep["unreadable"]))
+ text = bd.render(bd.compare(bd.parse_baseline(BASELINE_MD), rep))
+ self.assertNotIn("aaaaaaaaaa", text)
+
+ def test_links_are_never_followed(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "TEST-out.xml", junit(passing(5, "Outside")))
+ folder = self.results(passing(1))
+ make_symlink(self, os.path.join(outside, "TEST-out.xml"), os.path.join(folder, "TEST-link.xml"))
+ make_symlink(self, outside, os.path.join(folder, "linked"))
+ rep = bd.read_results([folder])
+ self.assertEqual(list(rep["tests"]), ["CalcTest#case0"])
+ linked = os.path.join(self.root, "asfolder")
+ make_symlink(self, outside, linked)
+ rep = bd.read_results([linked])
+ self.assertEqual((rep["tests"], rep["files"]), ({}, 0))
+ put(self.root, "LINKED.md", BASELINE_MD)
+ os.remove(self.base)
+ make_symlink(self, os.path.join(self.root, "LINKED.md"), self.base)
+ self.assertEqual(run_main(bd.main, [self.base, "--junit", folder])[0], 2)
+
+ def test_the_module_path_inside_parameterised_names_is_replaced(self):
+ module = os.path.join(self.root, "mod")
+ name = "t[%s/src/test/resources/a.txt]" % module
+ put(module, "target/surefire-reports/TEST-a.xml", junit([("A", name, "PASS", "")]))
+ rep = bd.read_results([os.path.join(module, "target", "surefire-reports")])
+ self.assertEqual(list(rep["tests"]), ["A#t[<MODULE>/src/test/resources/a.txt]"])
+
+
+# ---------------------------------------------------------------- proof_pack
+BRIEF_MD = """# Brief
+
+#### Phase 1 - Pilot
+Command: /code-modernization:modernize-transform
+Modules: mod, extra
+Exit criteria:
+- [x] All P0 rules pass
+- [ ] A business owner accepts the report output
+- [ ] Row counts match
+
+#### Phase 2 - Later
+Command: /code-modernization:modernize-transform
+Modules: other
+Exit criteria:
+- [ ] Phase 2 only criterion
+
+## 7. Open Questions
+
+- [ ] **Q1 [before Phase 1]** Is `s` the whole system?
+- [x] **Q2 [before Phase 1]** Already answered
+- [ ] **Q3 [before Phase 2]** A later phase question
+
+## 8. Approval Block
+- [ ] not a question
+"""
+
+
+@contextlib.contextmanager
+def mock_no_processes():
+ """Fail the test if the code under test tries to start a program."""
+ import unittest.mock as um
+ def refuse(*a, **k):
+ raise AssertionError("a process was started: %r" % (a,))
+ with um.patch("subprocess.Popen", refuse), um.patch("os.system", refuse), um.patch("os.popen", refuse):
+ yield
+
+
+class Ws:
+ """A rewrite workspace (system s, module mod) that proves cleanly. A test changes one thing and looks at the verdict."""
+
+ def __init__(self, test):
+ self.test, self.root = test, tempfile.mkdtemp()
+ test.addCleanup(shutil.rmtree, self.root, True)
+ put(self.root, "legacy/s/app/mod.cbl", "IDENTIFICATION DIVISION.\n", T0)
+ put(self.root, "analysis/s/PREFLIGHT.md", "# Preflight\n", T0 + 10)
+ put(self.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("### RULE-003", "### RULE-003").replace("**Priority:** P1", "**Priority:** P1"), T0 + 10)
+ put(self.root, "analysis/s/MODERNIZATION_BRIEF.md", BRIEF_MD, T0 + 10)
+ put(self.root, "modernized/s/mod/src/main/Calc.java", "// RULE-001 RULE-002\nclass Calc {}\n", T0 + 20)
+ put(self.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\nvoid rule002_rounds() {}\n", T0 + 20)
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "Legacy: `legacy/s/app/mod.cbl`\n\nCanary: flip the rounding mode -> 3 tests failed\n", T0 + 20)
+ put(self.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", junit(passing(3)), T0 + 30)
+ put(self.root, "analysis/s/equivalence/canary/mod/TEST-Canary.xml", junit(passing(1) + [("CalcTest", "rounds", "FAIL", ""), ("CalcTest", "total", "FAIL", "")]), T0 + 35)
+ self.cases("cases.json", "dev", 2)
+ self.cases("fresh-cases.json", "fresh", 10)
+ self.runs()
+
+ def cases(self, name, prefix, n, differ=(), inputs=None, same_as=None, base="analysis/s/equivalence"):
+ rows = []
+ for i in range(n):
+ legacy = "%s %d\n" % (prefix, i) if same_as is None else same_as
+ put(self.root, "%s/%s/legacy/%d.out" % (base, prefix, i), legacy, T0 + 25)
+ put(self.root, "%s/%s/new/%d.out" % (base, prefix, i), legacy + ("x" if i in differ else ""), T0 + 25)
+ rows.append({"id": "%s%d" % (prefix, i), "legacy": "%s/legacy/%d.out" % (prefix, i), "new": "%s/new/%d.out" % (prefix, i), "input": (inputs or {}).get(i, "in%d" % i)})
+ put(self.root, "%s/%s" % (base, name), json.dumps({"system": "s", "cases": rows}), T0 + 25)
+
+ def runs(self, **change):
+ doc = {"date": "2026-09-24", "legacy": {"ran": True, "how": "compiled here"},
+ "suites": [{"module": "mod", "name": "unit", "command": "mvn test", "junit": ["modernized/s/mod/target/surefire-reports"]}],
+ "canaries": [{"module": "mod", "change": "flip the rounding mode", "junit": ["analysis/s/equivalence/canary/mod"]}]}
+ doc.update(change)
+ put(self.root, "analysis/s/equivalence/test-runs.json", json.dumps(doc), T0 + 40)
+
+ def results(self, cases, when=T0 + 30):
+ put(self.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", junit(cases), when)
+
+ def pack(self, module=None):
+ return pp.overall(pp.build("s", self.root, module, now="2026-09-24 00:00 UTC"))
+
+ def module(self, module=None):
+ mods = self.pack(module)["modules"]
+ return next((m for m in mods if m["name"] == module), mods[0])
+
+ def states(self, module=None):
+ return {c["id"]: c["state"] for c in self.module(module)["checks"]}
+
+
+class ProofVerdicts(unittest.TestCase):
+ def setUp(self):
+ self.ws = Ws(self)
+
+ def check(self, verdict, **states):
+ m = self.ws.module()
+ self.assertEqual(m["verdict"], verdict, "\n".join(m["reasons"]))
+ got = {c["id"]: c["state"] for c in m["checks"]}
+ for k, v in states.items():
+ self.assertEqual(got[k], v, k)
+ return m
+
+ def test_all_evidence_is_proven(self):
+ m = self.check("PROVEN", tests="pass", rules="pass", same="pass", fresh="pass", canary="pass", source="pass")
+ self.assertEqual((m["reasons"], m["track"], m["name"]), ([], "rewrite", "mod"))
+ self.assertEqual(m["evidence"]["tests"]["executed"], 3)
+ self.assertEqual(m["evidence"]["fresh"]["inputs"], 10)
+
+ def test_no_test_run_at_all_is_not_proven(self):
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "test-runs.json"))
+ m = self.check("NOT PROVEN", tests="fail")
+ self.assertIn("nothing executed", m["checks"][0]["detail"])
+ self.assertTrue(any("test-runs.json was not found" in p for p in self.ws.pack()["problems"]))
+
+ def test_zero_executed_a_failure_and_skips(self):
+ self.ws.results([("CalcTest", "a", "SKIP", "why"), ("CalcTest", "b", "SKIP", "why")])
+ self.check("NOT PROVEN", tests="fail")
+ self.ws.results(passing(2) + [("CalcTest", "bad", "FAIL", "")])
+ m = self.check("NOT PROVEN", tests="fail")
+ self.assertIn("1 of 3", m["checks"][0]["detail"])
+ self.ws.results(passing(2) + [("CalcTest", "err", "ERROR", "")])
+ self.check("NOT PROVEN", tests="fail")
+ self.ws.results(passing(2) + [("CalcTest", "s", "SKIP", "")])
+ self.check("PARTLY PROVEN", tests="gap")
+ self.ws.results(passing(2) + [("CalcTest", "s", "SKIP", "pending RULE-009")])
+ self.check("PROVEN", tests="pass")
+
+ def test_the_result_files_beat_the_reported_counts(self):
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 999, "failed": 0, "skipped": 0, "junit": ["modernized/s/mod/target/surefire-reports"]}])
+ self.ws.results(passing(2) + [("CalcTest", "bad", "FAIL", "")])
+ m = self.check("NOT PROVEN", tests="fail")
+ self.assertEqual(m["evidence"]["tests"]["executed"], 3)
+ self.assertTrue(any("reported 999 executed" in c for c in m["caveats"]))
+
+ def test_counts_typed_into_test_runs_json_can_never_prove_anything(self):
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 5, "failed": 0, "skipped": 0}])
+ m = self.check("PARTLY PROVEN", tests="gap")
+ self.assertIn("typed in by the command", m["checks"][0]["detail"])
+ self.assertEqual(m["evidence"]["tests"]["suites"][0]["source"], "reported")
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 5, "failed": 1, "skipped": 2, "skippedReason": ""}])
+ self.check("NOT PROVEN", tests="fail")
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 0}])
+ self.check("NOT PROVEN", tests="fail")
+ self.ws.runs(suites=[{"module": "mod", "name": "unit"}])
+ m = self.check("PARTLY PROVEN", tests="gap")
+ self.assertIn("No test result could be read", m["checks"][0]["detail"])
+
+ def test_a_saved_runner_log_can_stand_in_for_result_files_and_an_unreadable_one_cannot(self):
+ shutil.rmtree(os.path.join(self.ws.root, "modernized", "s", "mod", "target"))
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "[INFO] Running X\n[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0\n[INFO] BUILD SUCCESS\n", T0 + 30)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "command": "mvn test", "log": ["analysis/s/equivalence/unit.test-output.txt"]}])
+ m = self.check("PARTLY PROVEN", tests="pass", rules="gap") # a log gives counts, not test names: no rule can be backed by it
+ self.assertEqual((m["evidence"]["tests"]["suites"][0]["source"], m["evidence"]["tests"]["executed"]), ("runner log", 3))
+ self.assertIn("no result file lists each test", m["checks"][1]["detail"])
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P1\n", T0 + 10) # with no P0 rule to back, the log is enough
+ self.check("PROVEN", tests="pass")
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD, T0 + 10)
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "3 tests were run, all fine, trust me\n", T0 + 30)
+ m = self.check("PARTLY PROVEN", tests="gap")
+ self.assertTrue(any("no summary line of a runner this script knows" in c for c in m["caveats"]))
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "== 1 failed, 2 passed in 0.10s ==\n", T0 + 30)
+ self.check("NOT PROVEN", tests="fail")
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "== 1 passed in 0.10s ==\n", T0 + 10) # older than the code: not a clean run
+ self.check("PARTLY PROVEN", tests="gap")
+
+ def test_a_suite_note_travels_with_the_verdict(self):
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "executed": 3, "failed": 1, "note": "one test needs a tool this machine lacks", "junit": ["modernized/s/mod/target/surefire-reports"]}])
+ self.ws.results(passing(2) + [("CalcTest", "env", "ERROR", "")])
+ m = self.check("NOT PROVEN", tests="fail")
+ self.assertIn("Note: one test needs a tool this machine lacks", m["checks"][0]["detail"])
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertIn("Note: one test needs a tool this machine lacks", read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md")))
+
+ def test_results_older_than_the_code_are_stale(self):
+ put(self.ws.root, "modernized/s/mod/src/main/Calc.java", "// RULE-001 RULE-002 changed\n", T0 + 35)
+ self.check("PARTLY PROVEN", tests="gap")
+ put(self.ws.root, "modernized/s/mod/.hidden", "x", T0 + 99)
+ put(self.ws.root, "modernized/s/mod/README.md", "x", T0 + 99)
+ put(self.ws.root, "modernized/s/mod/src/main/Calc.java", "// RULE-001 RULE-002\n", T0 + 20)
+ self.check("PROVEN", tests="pass")
+
+ def test_one_leftover_old_result_file_makes_the_run_not_clean(self):
+ put(self.ws.root, "modernized/s/mod/target/surefire-reports/TEST-Old.xml", junit(passing(2, "Old")), T0 + 10)
+ m = self.check("PARTLY PROVEN", tests="gap")
+ self.assertIn("not a clean run", m["checks"][0]["detail"])
+
+ def test_a_p0_rule_no_test_names_is_partly_proven_and_claims_do_not_count(self):
+ put(self.ws.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n", T0 + 20)
+ m = self.check("PARTLY PROVEN", rules="gap")
+ self.assertIn("RULE-002", m["checks"][1]["detail"])
+ self.assertEqual(m["evidence"]["rules"]["p0NoTests"], ["RULE-002"])
+ put(self.ws.root, "modernized/s/mod/src/main/Calc.java", "class Calc {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/mod/TRANSFORMATION_NOTES.md",
+ "Legacy: `legacy/s/app/mod.cbl`\n\n| Rule | Target |\n|---|---|\n| RULE-002 | `src/main/Calc.java` |\n\nCanary: x -> 3 tests failed\n", T0 + 20)
+ m = self.check("PARTLY PROVEN", rules="gap")
+ self.assertIn("claimed only", m["checks"][1]["detail"])
+
+ def test_no_rules_file_or_no_p0_rule(self):
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "BUSINESS_RULES.md"))
+ self.check("PARTLY PROVEN", rules="gap")
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P1\n", T0 + 10)
+ self.check("PROVEN", rules="pass")
+
+ def test_equivalence_cases_are_judged_again_not_read_from_equivalence_json(self):
+ put(self.ws.root, "analysis/s/EQUIVALENCE.json", json.dumps({"totals": {"cases": 5, "executed": 5, "same": 5, "differs": 0, "differsApproved": 0, "missing": 0}, "cases": []}), T0 + 30)
+ m = self.check("PROVEN", same="pass")
+ self.assertTrue(any("EQUIVALENCE.json does not match" in c for c in m["caveats"]))
+ self.ws.cases("cases.json", "dev", 2, differ={1})
+ m = self.check("NOT PROVEN", same="fail")
+ self.assertIn("1 differ", m["checks"][2]["detail"])
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "dev", "new", "0.out"))
+ self.check("NOT PROVEN", same="fail")
+
+ def test_an_approved_difference_is_allowed_and_listed(self):
+ self.ws.cases("cases.json", "dev", 2, differ={1})
+ path = os.path.join(self.ws.root, "analysis", "s", "equivalence", "cases.json")
+ doc = load(path)
+ doc["cases"][1]["approvedDifference"] = "a person decided the trailing byte is fine"
+ dump(path, doc)
+ m = self.check("PROVEN", same="pass")
+ self.assertEqual(m["evidence"]["equivalence"]["approved"][0]["id"], "dev1")
+ self.assertTrue(any("decision" in n for n in m["notProven"]))
+
+ def test_a_declared_tolerance_is_counted_listed_and_never_silent(self):
+ root = os.path.join(self.ws.root, "analysis", "s", "equivalence")
+ put(root, "dev/legacy/0.out", "total=1.0000000001\n", T0 + 25)
+ put(root, "dev/new/0.out", "total=1.0000000002\n", T0 + 25)
+ doc = load(os.path.join(root, "cases.json"))
+ doc["tolerance"] = {"rel": 1e-9, "abs": 0, "why": "last-digit rounding of exp()"}
+ dump(os.path.join(root, "cases.json"), doc)
+ m = self.check("PROVEN", same="pass")
+ ev = m["evidence"]["equivalence"]
+ self.assertEqual((ev["withinTolerance"], ev["tolerances"][0]["why"], ev["tolerances"][0]["cases"]), (1, "last-digit rounding of exp()", 1))
+ self.assertIn("1 within a declared tolerance", m["checks"][2]["detail"])
+ self.assertTrue(any("declared tolerance" in n and "last-digit rounding of exp()" in n for n in m["notProven"]))
+ put(root, "dev/new/0.out", "total=1.1\n", T0 + 25)
+ self.check("NOT PROVEN", same="fail")
+
+ def test_no_cases_at_all_is_partly_proven(self):
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "cases.json"))
+ self.check("PARTLY PROVEN", same="gap")
+ put(self.ws.root, "analysis/s/EQUIVALENCE.json", "{}", T0 + 30)
+ m = self.check("PARTLY PROVEN", same="gap")
+ self.assertIn("could not be judged again", m["checks"][2]["detail"])
+
+ def test_fresh_inputs_need_ten_and_must_not_differ(self):
+ self.ws.cases("fresh-cases.json", "fresh", 9)
+ m = self.check("PARTLY PROVEN", fresh="gap")
+ self.assertIn("Only 9", m["checks"][3]["detail"])
+ self.ws.cases("fresh-cases.json", "fresh", 12, inputs={i: "same" for i in range(12)})
+ self.check("PARTLY PROVEN", fresh="gap")
+ self.ws.cases("fresh-cases.json", "fresh", 12, inputs={i: "in%d" % (i % 10) for i in range(12)})
+ self.check("PROVEN", fresh="pass")
+ self.ws.cases("fresh-cases.json", "fresh", 12, differ={3})
+ m = self.check("NOT PROVEN", fresh="fail")
+ self.assertIn("fresh3", m["checks"][3]["detail"])
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "fresh-cases.json"))
+ m = self.check("PARTLY PROVEN", fresh="gap")
+ self.assertIn("was not run", m["checks"][3]["detail"])
+
+ def test_many_differing_fresh_cases_are_listed_up_to_five_then_counted(self):
+ self.ws.cases("fresh-cases.json", "fresh", 12, differ={0, 1, 2, 3, 4, 5, 6, 7})
+ m = self.check("NOT PROVEN", fresh="fail")
+ self.assertIn("8 fresh comparison(s) differ", m["checks"][3]["detail"])
+ self.assertIn("and 3 more (see VERIFICATION.json)", m["checks"][3]["detail"])
+ self.assertEqual(len(m["evidence"]["fresh"]["differing"]), 8)
+
+ def test_fresh_cases_that_repeat_development_outputs_do_not_count(self):
+ dev = load(os.path.join(self.ws.root, "analysis", "s", "equivalence", "cases.json"))
+ same = read(os.path.join(self.ws.root, "analysis", "s", "equivalence", "dev", "legacy", "0.out"))
+ self.ws.cases("fresh-cases.json", "fresh", 12, same_as=same)
+ m = self.check("PARTLY PROVEN", fresh="gap")
+ self.assertEqual(len(dev["cases"]), 2)
+ self.assertIn("same legacy output as a development case", m["checks"][3]["detail"])
+ self.ws.cases("fresh-cases.json", "fresh", 12, same_as="")
+ self.check("PARTLY PROVEN", fresh="gap") # empty outputs prove nothing
+
+ def test_when_the_legacy_cannot_run_the_verdict_cannot_be_proven(self):
+ self.ws.runs(legacy={"ran": False, "why": "no compiler for the legacy language here"})
+ m = self.check("PARTLY PROVEN", fresh="gap")
+ self.assertIn("trace-based", m["checks"][3]["detail"])
+ self.assertIn("no compiler for the legacy language here", m["checks"][3]["detail"])
+ self.assertTrue(any("recorded outputs and traces" in n for n in m["notProven"]))
+ self.ws.runs(legacy={})
+ self.check("PROVEN", fresh="pass") # not stated, but fresh cases exist and ran: the legacy evidently ran
+
+ def test_a_canary_counts_only_when_its_own_result_file_or_log_shows_failing_tests(self):
+ m = self.check("PROVEN", canary="pass")
+ self.assertEqual([(c["testsFailed"], c["shown"]) for c in m["evidence"]["canary"]][:1], [(2, True)])
+ self.assertIn("canary/mod", m["checks"][4]["detail"])
+ self.ws.runs(canaries=[]) # only the notes' "Canary:" line is left: a claim
+ m = self.check("PARTLY PROVEN", canary="gap")
+ self.assertIn("claimed", m["checks"][4]["detail"])
+ self.ws.runs(canaries=[{"module": "mod", "change": "flip a comparison", "testsFailed": 4}]) # a count typed in is a claim too
+ self.check("PARTLY PROVEN", canary="gap")
+ put(self.ws.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "Legacy: `legacy/s/app/mod.cbl`\n", T0 + 20)
+ self.ws.runs(canaries=[])
+ m = self.check("PARTLY PROVEN", canary="gap")
+ self.assertIn("No canary is recorded", m["checks"][4]["detail"])
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/run.log", "== 3 failed, 5 passed in 1.0s ==\n")
+ self.ws.runs(canaries=[{"module": "mod", "change": "flip a comparison", "log": ["analysis/s/equivalence/canary/mod/run.log"]}])
+ self.check("PROVEN", canary="pass")
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/run.log", "== 8 passed in 1.0s ==\n")
+ m = self.check("NOT PROVEN", canary="fail")
+ self.assertIn("no test failed", m["checks"][4]["detail"])
+
+ def test_a_canary_run_counts_only_the_failures_the_clean_run_did_not_have(self):
+ self.ws.results(passing(2) + [("CalcTest", "preexisting", "FAIL", "")])
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/TEST-Canary.xml", junit(passing(2) + [("CalcTest", "preexisting", "FAIL", "")]), T0 + 35)
+ m = self.check("NOT PROVEN", canary="fail")
+ self.assertIn("no test failed because of it", m["checks"][4]["detail"])
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/TEST-Canary.xml", junit(passing(1) + [("CalcTest", "preexisting", "FAIL", ""), ("CalcTest", "rounds", "FAIL", "")]), T0 + 35)
+ m = self.check("NOT PROVEN", tests="fail", canary="pass")
+ self.assertEqual(m["evidence"]["canary"][0]["testsFailed"], 1)
+ put(self.ws.root, "analysis/s/equivalence/canary/mod/run.log", "== 3 failed, 5 passed in 1.0s ==\n")
+ self.ws.runs(canaries=[{"module": "mod", "log": ["analysis/s/equivalence/canary/mod/run.log"]}])
+ m = self.check("NOT PROVEN", canary="pass")
+ self.assertEqual(m["evidence"]["canary"][0]["testsFailed"], 2) # 3 failures minus the clean run's 1
+
+ def test_inputs_left_out_are_listed_with_the_fresh_check(self):
+ self.ws.runs(leftOut=["F17: the legacy loader refused duplicate keys", 5, ""])
+ m = self.check("PROVEN", fresh="pass")
+ self.assertEqual(m["evidence"]["fresh"]["leftOut"], ["F17: the legacy loader refused duplicate keys"])
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertIn("Inputs left out of the fresh check: F17: the legacy loader refused duplicate keys", read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md")))
+
+ def test_canary_evidence_outside_the_workspace_is_ignored(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "TEST-out.xml", junit([("X", "a", "FAIL", "")]))
+ self.ws.runs(canaries=[{"module": "mod", "change": "elsewhere", "junit": [outside]}])
+ self.check("PARTLY PROVEN", canary="gap")
+
+ def test_hostile_notes_cannot_stall_the_canary_search(self):
+ put(self.ws.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", ("Canary: " * 100 + "\n") * 5000 + "Canary: x -> 2 tests failed\n" + "Canary" * 1000000, T0 + 20)
+ started = time.time()
+ m = self.ws.module()
+ self.assertLess(time.time() - started, 20)
+ self.assertIn(m["verdict"], ("PROVEN", "PARTLY PROVEN"))
+
+ def test_a_changed_legacy_source_is_partly_proven_and_named(self):
+ put(self.ws.root, "legacy/s/app/mod.cbl", "changed\n", T0 + 50)
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("has changed since the analysis began", m["checks"][5]["detail"])
+ self.assertEqual(self.ws.pack()["legacy"]["state"], "changed")
+ shutil.rmtree(os.path.join(self.ws.root, "legacy"))
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("was not found", m["checks"][5]["detail"])
+
+ def test_the_source_check_is_a_plain_file_walk_and_runs_nothing_in_the_tree(self):
+ legacy = os.path.join(self.ws.root, "legacy", "s")
+ marker = os.path.join(self.ws.root, "RAN")
+ os.makedirs(os.path.join(legacy, ".git"))
+ put(legacy, ".git/config", "[core]\n\tfsmonitor = touch %s\n" % marker, T0 + 99) # a hostile repository config: it must never run
+ put(legacy, ".git/HEAD", "ref: refs/heads/main\n", T0 + 99) # version-control metadata is not source
+ with mock_no_processes():
+ m = self.check("PROVEN", source="pass")
+ self.assertFalse(os.path.exists(marker))
+ self.assertIn("no version-control tool was run", m["checks"][5]["detail"])
+ self.assertTrue(m["checks"][5]["detail"].startswith("legacy/s is untouched: no file changed after the analysis started"))
+ self.assertEqual(self.ws.pack()["legacy"]["method"], pp.NO_VCS)
+
+ def test_a_file_newer_than_the_analysis_is_reported_with_examples(self):
+ for i in range(30):
+ put(self.ws.root, "legacy/s/app/new%02d.cbl" % i, "x", T0 + 50)
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("30 file(s) changed after the analysis started", m["checks"][5]["detail"])
+ pack = self.ws.pack()
+ self.assertEqual((pack["legacy"]["state"], len(pack["legacy"]["files"])), ("changed", 20))
+ put(self.ws.root, "legacy/s/app/mod.cbl", "changed\n", T0 + 50)
+ shutil.rmtree(os.path.join(self.ws.root, "legacy"))
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("was not found", m["checks"][5]["detail"])
+
+ def test_the_analysis_is_dated_by_preflight_else_by_the_oldest_analysis_file(self):
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "PREFLIGHT.md"))
+ os.utime(os.path.join(self.ws.root, "analysis", "s", "BUSINESS_RULES.md"), (T0 + 5, T0 + 5))
+ os.utime(os.path.join(self.ws.root, "analysis", "s", "MODERNIZATION_BRIEF.md"), (T0 + 10, T0 + 10))
+ self.check("PROVEN", source="pass")
+ put(self.ws.root, "legacy/s/app/late.cbl", "x", T0 + 7)
+ self.check("PARTLY PROVEN", source="gap")
+ shutil.rmtree(os.path.join(self.ws.root, "analysis"))
+ put(self.ws.root, "analysis/s/equivalence/test-runs.json", "{}", T0 + 40)
+ m = self.ws.module()
+ self.assertEqual(m["checks"][5]["state"], "gap")
+
+ def test_links_in_the_source_are_not_followed_and_a_walk_that_cannot_finish_is_not_untouched(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "newer.c", "x", T0 + 99)
+ make_symlink(self, os.path.join(outside, "newer.c"), os.path.join(self.ws.root, "legacy", "s", "link.c"))
+ make_symlink(self, outside, os.path.join(self.ws.root, "legacy", "s", "linked"))
+ self.check("PROVEN", source="pass")
+ if os.name != "nt" and os.geteuid() != 0:
+ hidden = os.path.join(self.ws.root, "legacy", "s", "locked")
+ os.makedirs(hidden)
+ os.chmod(hidden, 0)
+ self.addCleanup(os.chmod, hidden, 0o755)
+ m = self.check("PARTLY PROVEN", source="gap")
+ self.assertIn("could not finish", m["checks"][5]["detail"])
+
+ def test_people_items_are_listed_never_ticked_and_never_change_the_verdict(self):
+ m = self.check("PROVEN")
+ texts = [i["text"] for i in m["brief"]["items"]]
+ self.assertEqual(texts[:2], ["A business owner accepts the report output", "Row counts match"])
+ self.assertTrue(any("Is `s` the whole system" in t for t in texts))
+ self.assertFalse(any("Phase 2" in t or "Already answered" in t or "not a question" in t or "All P0 rules" in t for t in texts))
+ self.assertEqual(m["brief"]["items"][0]["where"], "Phase 1 exit criterion")
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "MODERNIZATION_BRIEF.md"))
+ self.assertFalse(self.ws.module()["brief"]["found"])
+
+ def test_hostile_test_runs_json_cannot_crash_or_read_outside(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "TEST-out.xml", junit(passing(500, "Out")))
+ for doc in ('["x"]', '{"suites": "x", "legacy": 5, "canaries": {"a": 1}}', "{not json", "\x00\x01",
+ json.dumps({"suites": [{"module": "mod" * 5000, "name": None, "executed": -5, "failed": "many", "junit": [1, None, "../../etc", outside, "x" * 10000]},
+ 5, {"module": "mod", "executed": 10 ** 12, "skipped": True, "junit": [outside]}]})):
+ put(self.ws.root, "analysis/s/equivalence/test-runs.json", doc, T0 + 40)
+ m = self.ws.module()
+ self.assertIn(m["verdict"], ("NOT PROVEN", "PARTLY PROVEN"))
+ self.assertNotEqual(m["evidence"]["tests"]["executed"], 500)
+ self.ws.runs(suites=[{"module": "mod", "executed": 500, "junit": [outside]}])
+ m = self.ws.module()
+ self.assertTrue(any("outside the workspace" in c for c in m["caveats"]))
+
+ def test_links_in_the_evidence_are_refused(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "test-runs.json", json.dumps({"suites": [{"module": "mod", "executed": 9}]}))
+ os.remove(os.path.join(self.ws.root, "analysis", "s", "equivalence", "test-runs.json"))
+ make_symlink(self, os.path.join(outside, "test-runs.json"), os.path.join(self.ws.root, "analysis", "s", "equivalence", "test-runs.json"))
+ pack = self.ws.pack()
+ self.assertTrue(any("link" in p for p in pack["problems"]))
+ self.assertEqual(pack["modules"][0]["verdict"], "NOT PROVEN")
+ put(outside, "o.out", "same\n")
+ put(self.ws.root, "analysis/s/equivalence/cases.json", json.dumps({"cases": [{"id": "x", "legacy": os.path.join(outside, "o.out"), "new": os.path.join(outside, "o.out")}]}), T0 + 25)
+ self.ws.runs()
+ self.assertEqual(self.ws.states()["same"], "fail")
+
+ def test_a_linked_cases_file_is_refused_not_read(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ cases = os.path.join(self.ws.root, "analysis", "s", "equivalence", "cases.json")
+ shutil.copy(cases, os.path.join(outside, "cases.json"))
+ os.remove(cases)
+ make_symlink(self, os.path.join(outside, "cases.json"), cases)
+ m = self.ws.module()
+ self.assertEqual(m["checks"][2]["state"], "gap")
+ self.assertTrue(any("is a link" in c for c in m["caveats"]))
+ self.assertNotIn("equivalence", m["evidence"])
+
+ def test_two_modules_get_two_verdicts_and_asking_about_one_still_judges_both_from_evidence(self):
+ put(self.ws.root, "modernized/s/extra/src/main/X.java", "class X {}\n", T0 + 20)
+ pack = self.ws.pack()
+ self.assertEqual([(m["name"], m["verdict"]) for m in pack["modules"]], [("extra", "NOT PROVEN"), ("mod", "PROVEN")])
+ self.assertEqual(pack["overall"]["verdict"], "NOT PROVEN")
+ self.assertEqual(pack["overall"]["counts"], {"PROVEN": 1, "PARTLY PROVEN": 0, "NOT PROVEN": 1})
+ asked = self.ws.pack("mod")
+ self.assertEqual([(m["name"], m["verdict"]) for m in asked["modules"]], [("extra", "NOT PROVEN"), ("mod", "PROVEN")])
+ self.assertEqual(asked["asked"], "mod")
+
+ def test_a_forged_earlier_verdict_never_survives_a_new_run(self):
+ put(self.ws.root, "modernized/s/extra/src/main/X.java", "class X {}\n", T0 + 20)
+ forged = pp.overall(self.ws.pack())
+ forged["modules"][0].update(verdict="PROVEN", reasons=[])
+ forged["modules"].append(dict(forged["modules"][0], name="ghost", verdict="PROVEN"))
+ forged["overall"] = {"verdict": "PROVEN", "counts": {"PROVEN": 3, "PARTLY PROVEN": 0, "NOT PROVEN": 0}}
+ adir = os.path.join(self.ws.root, "analysis", "s")
+ dump(os.path.join(adir, "VERIFICATION.json"), forged)
+ code, out, _ = run_main(pp.main, ["s", "mod", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ data = load(os.path.join(adir, "VERIFICATION.json"))
+ self.assertEqual([(m["name"], m["verdict"]) for m in data["modules"]], [("extra", "NOT PROVEN"), ("mod", "PROVEN")])
+ self.assertEqual(data["overall"]["verdict"], "NOT PROVEN")
+ self.assertNotIn("ghost", read(os.path.join(adir, "VERIFICATION.md")))
+ self.assertFalse(hasattr(pp, "merge_previous"))
+
+ def test_shared_cases_are_flagged_and_a_modules_own_folder_wins(self):
+ put(self.ws.root, "modernized/s/extra/src/main/X.java", "class X {}\n", T0 + 20)
+ self.assertTrue(any("shared by every module" in c for c in self.ws.module("mod")["caveats"]))
+ self.ws.cases("cases.json", "own", 3, base="analysis/s/equivalence/mod")
+ m = self.ws.module("mod")
+ self.assertEqual(m["evidence"]["equivalence"]["cases"], 3)
+ self.assertFalse(any("shared by every module" in c for c in m["caveats"]))
+
+ def test_a_reimagined_service_is_judged_like_a_rewritten_module(self):
+ shutil.rmtree(os.path.join(self.ws.root, "modernized", "s"))
+ put(self.ws.root, "modernized/s-reimagined/api/src/main/Api.java", "class Api {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s-reimagined/api/src/test/ApiTest.java", "// RULE-001 RULE-002\n", T0 + 20)
+ put(self.ws.root, "modernized/s-reimagined/api/target/surefire-reports/TEST-Api.xml", junit(passing(4, "ApiTest")), T0 + 30)
+ self.ws.runs(suites=[{"module": "api", "name": "acceptance", "executed": 4, "junit": ["modernized/s-reimagined/api/target/surefire-reports"]}])
+ pack = self.ws.pack()
+ m = pack["modules"][0]
+ self.assertEqual((m["name"], m["track"], len(pack["modules"])), ("api", "reimagine", 1))
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}["rules"], "pass")
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}["tests"], "pass")
+
+ def test_nothing_built_is_an_error(self):
+ shutil.rmtree(os.path.join(self.ws.root, "modernized"))
+ with self.assertRaises(pp.InputError):
+ pp.build("s", self.ws.root)
+ code, _, err = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 2)
+ self.assertIn("nothing is built", err)
+
+
+RUNNER_LOGS = (
+ ("maven", "[INFO] Tests run: 4, Failures: 0, Errors: 0, Skipped: 0, Time elapsed: 0.1 s -- in A\n[ERROR] Tests run: 158, Failures: 0, Errors: 3, Skipped: 2\n", (156, 3, 2)), # executed = run minus skipped
+ ("maven, old style", "Tests run: 10, Failures: 1, Errors: 0, Skipped: 1\n", (9, 1, 1)),
+ ("gradle", "12 tests completed, 2 failed, 1 skipped\n", (11, 2, 1)),
+ ("cargo", "test result: ok. 10 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.00s\ntest result: FAILED. 2 passed; 3 failed; 0 ignored; 0 measured\n", (15, 3, 1)),
+ ("pytest", "============ 2 failed, 10 passed, 1 skipped in 0.12s ============\n", (12, 2, 1)),
+ ("pytest -q", "3 passed, 1 error in 0.05s\n", (4, 1, 0)),
+ ("pytest none", "no tests ran in 0.01s\n", (0, 0, 0)),
+ ("unittest", "Ran 5 tests in 0.002s\n\nOK\n", (5, 0, 0)),
+ ("unittest failed", "Ran 9 tests in 0.5s\n\nFAILED (failures=1, errors=2, skipped=3)\n", (6, 3, 3)),
+ ("go -v", "=== RUN TestA\n--- PASS: TestA (0.00s)\n --- PASS: TestA/sub (0.00s)\n--- FAIL: TestB (0.01s)\n--- SKIP: TestC (0.00s)\nFAIL\n", (3, 1, 1)),
+ ("go -json", '{"Time":"t","Action":"pass","Package":"p","Test":"TestA","Elapsed":0}\n{"Time":"t","Action":"fail","Package":"p","Test":"TestB","Elapsed":0}\n{"Time":"t","Action":"fail","Package":"p","Elapsed":0}\n', (2, 1, 0)),
+ ("dotnet", "Passed! - Failed: 0, Passed: 10, Skipped: 1, Total: 11, Duration: 1 s - X.dll (net8.0)\nFailed! - Failed: 2, Passed: 8, Skipped: 0, Total: 10, Duration: 1 s - Y.dll\n", (20, 2, 1)),
+ ("jest", "Tests: 1 failed, 2 skipped, 3 passed, 6 total\nTest Suites: 1 failed, 1 total\n", (4, 1, 2)),
+ ("vitest", " Tests 1 failed | 9 passed | 1 skipped (11)\n", (10, 1, 1)),
+ ("ctest", "92% tests passed, 1 tests failed out of 12\n", (12, 1, 0)),
+ ("phpunit ok", "OK (10 tests, 25 assertions)\n", (10, 0, 0)),
+ ("phpunit failures", "FAILURES!\nTests: 10, Assertions: 25, Failures: 1, Skipped: 1.\n", (9, 1, 1)),
+)
+
+
+class RunnerLogs(unittest.TestCase):
+ def test_every_supported_runner_summary_is_read(self):
+ for name, text, want in RUNNER_LOGS:
+ got = pp.parse_runner_log(text)
+ self.assertIsNotNone(got, name)
+ self.assertEqual(got[:3], want, name)
+
+ def test_a_log_with_no_summary_line_is_no_evidence(self):
+ for text in ("", "All good, 12 tests passed, trust me\n", "BUILD SUCCESSFUL in 3s\n", "ok \tpkg\t0.003s\n", "Tests run: many, Failures: none\n", "\x00\x01\x02"):
+ self.assertIsNone(pp.parse_runner_log(text), repr(text))
+
+ def test_summaries_are_summed_across_a_log_and_the_runner_is_named(self):
+ got = pp.parse_runner_log("Tests run: 3, Failures: 0, Errors: 0, Skipped: 0\n== 2 passed in 0.1s ==\n")
+ self.assertEqual(got, (5, 0, 0, ["maven", "pytest"]))
+
+ def test_hostile_logs_are_bounded_and_never_trusted(self):
+ started = time.time()
+ for text in ("Tests run: " * 400000, "= " * 2000000 + "passed in 1s", "--- PASS: " * 300000, "{" + '"Action":"pass",' * 200000 + "}", "9" * 3000000 + " passed in 1s\n",
+ "Tests:" + " 1 failed," * 100000 + " 5 total\n", "Ran " * 500000, "\r" * 1000000):
+ pp.parse_runner_log(text)
+ self.assertLess(time.time() - started, 30)
+ self.assertEqual(pp.parse_runner_log("Tests run: 999999999999999999999, Failures: 0, Errors: 0, Skipped: 0\n")[0], 999999999999999999999)
+
+ def test_read_logs_refuses_links_huge_files_and_binary_text(self):
+ root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, root, True)
+ good = put(root, "good.txt", "== 2 passed in 0.1s ==\n")
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "real.txt", "== 99 passed in 0.1s ==\n")
+ link = os.path.join(root, "link.txt")
+ make_symlink(self, os.path.join(outside, "real.txt"), link)
+ put(root, "huge.txt", b"x" * ((16 << 20) + 5))
+ put(root, "nul.txt", b"\x00" * 100 + b"\n== 5 passed in 1s ==\n")
+ got = pp.read_logs([good, link, os.path.join(root, "huge.txt"), os.path.join(root, "nul.txt"), os.path.join(root, "missing.txt")])
+ self.assertEqual((got["files"], got["executed"]), (2, 7)) # NUL bytes elsewhere in the file do not hide a summary line; the link, the huge and the missing file are not read
+ self.assertEqual(len(got["unrecognised"]), 3)
+
+
+class BaselineEvidence(unittest.TestCase):
+ """baseline_diff.py: where the old version's measured results are looked for, and what counts as measured."""
+
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ self.base = put(self.root, "analysis/s/BASELINE.md", BASELINE_MD)
+ self.per_test = json.dumps({"A#pass1": "PASS", "A#pass2": "PASS", "A#failed": "FAIL", "A#skipped": "SKIP", "B#flip": "PASS", "B#gone": "PASS"})
+
+ def found(self, text=""):
+ return [os.path.relpath(p, os.path.join(self.root, "analysis", "s")) for p in bd.evidence_paths(self.base, BASELINE_MD + text)]
+
+ def test_per_test_json_shapes_are_read_and_counts_or_prose_are_not(self):
+ self.assertEqual(bd.parse_results_json({"A#x": "PASS", "A#y": "failed", "A#z": "Skipped"}), {"A#x": "PASS", "A#y": "FAIL", "A#z": "SKIP"})
+ self.assertEqual(bd.parse_results_json({"tests": {"A#x": "ok"}}), {"A#x": "PASS"})
+ self.assertEqual(bd.parse_results_json([{"name": "A#x", "status": "pass"}, {"test": "A#y", "result": "FAIL"}, {"id": "A#z", "outcome": "error"}, {"name": "no status"}, 5]),
+ {"A#x": "PASS", "A#y": "FAIL", "A#z": "ERROR"})
+ for bad in ({"PASS": 10, "FAIL": 1}, {"passed": 5, "failed": 0, "executed": 5}, {"tests": {"count": 3}}, {"a": "hello", "b": "world"}, [], {}, "text", 5, None, [1, 2], {"a": {"b": "PASS"}}):
+ self.assertIsNone(bd.parse_results_json(bad), bad)
+ many = bd.parse_results_json({"t%d" % i: "PASS" for i in range(1000)} | {"junk": "hello"} if sys.version_info >= (3, 9) else dict({"t%d" % i: "PASS" for i in range(1000)}, junk="hello"))
+ self.assertEqual(len(many), 1000) # one stray value in a thousand does not spoil it
+ self.assertIsNone(bd.parse_results_json({"a": "PASS", "b": "hello", "c": "world"})) # but a mostly-unrecognised map is no evidence
+
+ def test_the_baseline_folder_and_recorded_lines_are_the_only_places(self):
+ put(self.root, "analysis/s/baseline/old.json", self.per_test)
+ put(self.root, "analysis/s/harness/named.json", self.per_test)
+ put(self.root, "analysis/s/harness/prose.json", self.per_test)
+ put(self.root, "analysis/s/harness/bare.log", "x")
+ self.assertEqual(self.found(), ["baseline"])
+ self.assertEqual(self.found("\nSee `analysis/s/harness/prose.json` too.\n"), ["baseline"])
+ self.assertEqual(self.found("\n- **Recorded:** 2026-09-24. Machine-readable per-test results: `analysis/s/harness/named.json`; tool: `analysis/s/harness/prose.json`\n"),
+ ["baseline", "harness/named.json", "harness/prose.json"]) # everything on the Recorded line counts
+ self.assertEqual(self.found("\nMachine-readable: harness/bare.log\n"), ["baseline", "harness/bare.log"])
+ self.assertEqual(self.found("\n * recorded on Thursday: harness/named.json\n"), ["baseline", "harness/named.json"])
+ self.assertEqual(self.found("\nNot recorded: harness/named.json\n"), ["baseline"]) # the line must start with the label
+
+ def test_paths_that_leave_analysis_or_go_through_links_are_refused(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "o.json", self.per_test)
+ put(self.root, "analysis/other/o.json", self.per_test)
+ put(self.root, "modernized/o.json", self.per_test)
+ put(self.root, "analysis/s/inner.json", self.per_test)
+ make_symlink(self, os.path.join(outside, "o.json"), os.path.join(self.root, "analysis", "s", "link.json"))
+ make_symlink(self, outside, os.path.join(self.root, "analysis", "s", "dirlink"))
+ make_symlink(self, os.path.join(self.root, "analysis", "other"), os.path.join(self.root, "analysis", "s", "sibling"))
+ for line in ("../other/o.json", "analysis/other/o.json", "analysis/s/../other/o.json", "modernized/o.json", outside + "/o.json", "link.json", "dirlink/o.json", "sibling/o.json", "./../s/../other/o.json"):
+ self.assertEqual(self.found("\nRecorded: `%s`\n" % line), [], line)
+ self.assertEqual(self.found("\nRecorded: `analysis/s/inner.json`\n"), ["inner.json"])
+ os.rename(os.path.join(self.root, "analysis", "s", "dirlink"), os.path.join(self.root, "dirlink-moved"))
+ make_symlink(self, outside, os.path.join(self.root, "analysis", "s", "baseline"))
+ self.assertEqual(self.found(), []) # a linked baseline/ folder is not entered
+
+ def test_result_files_json_and_logs_are_all_read_and_counts_only_files_are_ignored(self):
+ put(self.root, "analysis/s/baseline/xml/TEST-A.xml", junit([("A", "one", "PASS", ""), ("A", "two", "FAIL", "")]))
+ put(self.root, "analysis/s/baseline/map.json", json.dumps({"B#x": "PASS"}))
+ put(self.root, "analysis/s/baseline/counts.json", json.dumps({"PASS": 9}))
+ put(self.root, "analysis/s/baseline/bad.json", "{not json")
+ put(self.root, "analysis/s/baseline/notes.txt", "nothing that looks like a runner summary")
+ got = bd.read_baseline_evidence(bd.evidence_paths(self.base, ""))
+ self.assertEqual(got["tests"], {"A#one": "PASS", "A#two": "FAIL", "B#x": "PASS"})
+ self.assertEqual((got["measured"], got["conflicts"], got["ignored"]), (True, [], ["counts.json"]))
+ self.assertEqual([x["kind"] for x in got["sources"]], ["result files", "machine-readable results"])
+ self.assertTrue(any("bad.json" in u for u in got["unreadable"]))
+ only_log = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, only_log, True)
+ put(only_log, "run.log", "[INFO] Tests run: 5, Failures: 1, Errors: 0, Skipped: 0\n")
+ got = bd.read_baseline_evidence([only_log])
+ self.assertEqual((got["tests"], got["counts"], got["measured"]), (None, {"executed": 5, "failed": 1, "skipped": 0}, True))
+ nothing = bd.read_baseline_evidence([os.path.join(self.root, "analysis", "s", "baseline", "counts.json")])
+ self.assertEqual((nothing["measured"], nothing["tests"], nothing["counts"]), (False, None, None))
+
+ def test_sources_that_disagree_are_conflicts_not_first_wins(self):
+ put(self.root, "analysis/s/baseline/a.json", json.dumps({"A#x": "PASS", "A#y": "PASS"}))
+ put(self.root, "analysis/s/baseline/b.json", json.dumps({"A#x": "FAIL", "A#y": "PASS", "A#z": "PASS"}))
+ got = bd.read_baseline_evidence(bd.evidence_paths(self.base, ""))
+ self.assertEqual(len(got["conflicts"]), 1)
+ self.assertIn("A#x", got["conflicts"][0])
+ self.assertEqual(got["tests"]["A#x"], "PASS") # kept only so the report can show something; the conflict is what counts
+ info = bd.assess_baseline(bd.parse_baseline("| Test | Result |\n|---|---|\n| `A#x` | PASS |\n"), got)
+ self.assertEqual((info["kind"], info["conflict"]), ("measured", 1))
+ put(self.root, "analysis/s/baseline/b.json", json.dumps({"A#x": "ERROR", "A#y": "PASS"}))
+ put(self.root, "analysis/s/baseline/a.json", json.dumps({"A#x": "FAIL"}))
+ self.assertEqual(bd.read_baseline_evidence(bd.evidence_paths(self.base, ""))["conflicts"], []) # FAIL and ERROR are both red
+
+ def test_assess_says_typed_measured_disagreeing_or_target_only(self):
+ typed = bd.parse_baseline(BASELINE_MD)
+ self.assertEqual(bd.assess_baseline(typed, None)["kind"], "typed")
+ self.assertEqual(bd.assess_baseline(bd.parse_baseline("target-only: no old runtime\n"), {"measured": True, "tests": {"a": "PASS"}, "counts": None, "sources": []})["kind"], "target-only")
+ self.assertEqual(bd.assess_baseline(bd.parse_baseline("nothing here"), None)["kind"], "none")
+ good = {"measured": True, "tests": {"A#pass1": "PASS", "A#pass2": "PASS", "A#failed": "FAIL", "A#skipped": "SKIP", "B#flip": "PASS", "B#gone": "PASS"}, "counts": None,
+ "sources": [{"kind": "machine-readable results", "name": "x.json"}], "conflicts": []}
+ info = bd.assess_baseline(typed, good)
+ self.assertEqual((info["kind"], info["disagree"], info["measuredTests"], info["measuredExecuted"], info["measuredFailed"]), ("measured", 0, 6, 5, 1))
+ bad = dict(good, tests=dict(good["tests"], **{"A#pass1": "FAIL"}))
+ self.assertEqual(bd.assess_baseline(typed, bad)["disagree"], 1)
+ missing = dict(good, tests={k: v for k, v in good["tests"].items() if k != "B#gone"})
+ self.assertEqual(bd.assess_baseline(typed, missing)["disagree"], 2) # the missing row, and the Totals table that counts it
+
+ def test_the_cli_can_be_given_the_evidence_and_says_typed_only_otherwise(self):
+ folder = os.path.join(self.root, "res")
+ put(folder, "TEST-x.xml", junit([("A", "pass1", "PASS", ""), ("A", "pass2", "PASS", ""), ("A", "failed", "FAIL", ""), ("A", "skipped", "SKIP", ""), ("B", "flip", "PASS", ""), ("B", "gone", "PASS", "")]))
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder])
+ self.assertIn("typed table only", out)
+ put(self.root, "analysis/s/baseline/old.json", self.per_test)
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder])
+ self.assertIn("baseline evidence: measured (machine-readable results old.json)", out)
+ evidence = os.path.join(self.root, "elsewhere.json")
+ put(self.root, "elsewhere.json", self.per_test)
+ code, out, _ = run_main(bd.main, [self.base, "--junit", folder, "--baseline-evidence", evidence, "--json"])
+ self.assertEqual(json.loads(out)["baseline"]["assessment"]["kind"], "measured")
+
+
+class UpliftChecks(unittest.TestCase):
+ """uplift_checks.py: the test files of the untouched tree against the working copy, and the deltas no test names."""
+
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ self.old, self.new = os.path.join(self.root, "old"), os.path.join(self.root, "new")
+
+ def both(self, rel, old, new=None, kind="w"):
+ put(self.old, rel, old)
+ put(self.new, rel, old if new is None else new)
+
+ def test_which_files_count_as_tests_and_what_is_skipped(self):
+ for rel in ("src/test/java/AThing.java", "tests/b.py", "web/__tests__/c.js", "spec/d.rb", "src/FooTest.java", "pkg/test_e.py", "pkg/e_test.go", "web/f.spec.ts", "src/test/resources/data.json"):
+ put(self.old, rel, "x")
+ for rel in ("src/main/A.java", "README.md", "src/test/notes.txt", "target/test/X.java", "node_modules/y/test/Z.js", ".hidden/test/H.java", "src/test/.dotfile"):
+ put(self.old, rel, "x")
+ got, cut, links = uc.walk_tests(self.old)
+ self.assertEqual(sorted(got), ["pkg/e_test.go", "pkg/test_e.py", "spec/d.rb", "src/FooTest.java", "src/test/java/AThing.java", "src/test/resources/data.json", "tests/b.py",
+ "web/__tests__/c.js", "web/f.spec.ts"])
+ self.assertEqual((cut, links), (False, 0))
+ self.assertEqual(uc.walk_tests(os.path.join(self.root, "missing")), ({}, False, 0))
+
+ def test_removed_added_and_changed_with_line_counts(self):
+ self.both("t/keep/AKeepTest.java", "one\ntwo\nthree\n")
+ self.both("t/edit/AEditTest.java", "one\ntwo\nthree\nfour\n", "one\nTWO\nthree\nfour\nfive\n")
+ put(self.old, "t/gone/AGoneTest.java", "x\ny\n")
+ put(self.new, "t/new/ANewTest.java", "a\nb\nc\n")
+ got = uc.tests_changed(self.old, self.new)
+ self.assertEqual((got["checked"], got["legacyTests"], got["workTests"], got["counts"]), (True, 3, 3, {"added": 1, "removed": 1, "changed": 1}))
+ self.assertEqual(got["changed"], [{"path": "t/edit/AEditTest.java", "added": 2, "removed": 1, "note": ""}])
+ self.assertEqual((got["removed"], got["added"]), ([{"path": "t/gone/AGoneTest.java", "lines": 3}], [{"path": "t/new/ANewTest.java", "lines": 4}]))
+ self.assertTrue(got["gap"]) # a removed file
+ self.assertEqual(round(got["share"], 3), round(1 / 3, 3))
+
+ def test_a_quarter_changed_is_not_a_gap_and_more_is_and_line_endings_are_ignored(self):
+ for i in range(4):
+ self.both("t/T%dTest.java" % i, "line %d\r\nsecond\r\n" % i)
+ put(self.new, "t/T0Test.java", "line 0\nsecond\n") # only the line endings differ
+ self.assertEqual(uc.tests_changed(self.old, self.new)["counts"]["changed"], 0)
+ put(self.new, "t/T1Test.java", "line 1 edited\nsecond\n")
+ got = uc.tests_changed(self.old, self.new)
+ self.assertEqual((got["counts"]["changed"], got["gap"], got["share"]), (1, False, 0.25))
+ put(self.new, "t/T2Test.java", "line 2 edited\nsecond\n")
+ self.assertTrue(uc.tests_changed(self.old, self.new)["gap"])
+
+ def test_no_legacy_tests_is_a_gap_and_so_is_a_missing_tree(self):
+ put(self.new, "t/ANewTest.java", "x")
+ put(self.old, "src/main/A.java", "the legacy tree exists but holds no test file")
+ got = uc.tests_changed(self.old, self.new)
+ self.assertEqual((got["checked"], got["legacyTests"], got["gap"]), (True, 0, True))
+ self.assertIn("nothing shows that the working copy's tests were not written during the uplift", got["why"])
+ for old, new in ((os.path.join(self.root, "none"), self.new), ("", self.new), (self.old, os.path.join(self.root, "none")), (self.old, "")):
+ got = uc.tests_changed(old, new)
+ self.assertEqual(got["checked"], False, (old, new))
+ self.assertTrue(got["why"])
+
+ def test_binary_large_and_unreadable_files_are_compared_without_crashing(self):
+ self.both("t/data/a.bin.Test.dat", b"\x00\x01\x02", b"\x00\x01\x03")
+ self.both("t/big/BigTest.java", "x\n" * (1 << 20), "x\n" * (1 << 20) + "y\n")
+ self.both("t/same/SameBigTest.java", "z\n" * (1 << 20))
+ self.both("t/lines/ManyTest.java", "".join("line %d\n" % i for i in range(9000)), "".join("line %d\n" % i for i in range(1, 9001)))
+ got = uc.tests_changed(self.old, self.new)
+ notes = {c["path"]: c["note"] for c in got["changed"]}
+ self.assertEqual(notes["t/data/a.bin.Test.dat"], "binary file")
+ self.assertEqual(notes["t/big/BigTest.java"], "large file, lines not counted")
+ self.assertEqual(notes["t/lines/ManyTest.java"], "lines counted approximately")
+ self.assertNotIn("t/same/SameBigTest.java", notes)
+ many = next(c for c in got["changed"] if c["path"] == "t/lines/ManyTest.java")
+ self.assertEqual((many["added"], many["removed"]), (1, 1))
+
+ def test_links_are_never_followed_and_hostile_trees_stay_bounded(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "OutTest.java", "outside")
+ self.both("t/InTest.java", "in")
+ make_symlink(self, os.path.join(outside, "OutTest.java"), os.path.join(self.new, "t", "LinkTest.java"))
+ make_symlink(self, outside, os.path.join(self.new, "t", "linked"))
+ got = uc.tests_changed(self.old, self.new)
+ self.assertEqual((got["workTests"], got["counts"]), (1, {"added": 0, "removed": 0, "changed": 0}))
+ self.assertNotIn("Out", json.dumps(got))
+ for i in range(300):
+ self.both("t/many/M%03dTest.java" % i, "a\n" * 50, "a\n" * 50 + "edit %d\n" % i)
+ started = time.time()
+ got = uc.tests_changed(self.old, self.new)
+ self.assertLess(time.time() - started, 30)
+ self.assertEqual((got["counts"]["changed"], len(got["changed"])), (300, 40)) # the count is complete, the list is cut
+ self.assertNotRegex(json.dumps(uc.tests_changed(self.old, self.new)), "\x1b")
+
+ def test_a_hostile_file_name_is_cleaned(self):
+ self.both("t/ev\x1b[31mil\nTest.java", "a", "b")
+ got = uc.tests_changed(self.old, self.new)
+ self.assertNotRegex(got["changed"][0]["path"], r"[\x00-\x1f]")
+
+ CARDS = """# DELTA_CATALOG: x
+
+| ID | Category | Fix | Site | Old to new |
+|---|---|---|---|---|
+| **D-01** | Behavioral-silent | Judgment | `a/b/Alpha.java:916` | changed |
+| **D-02** | Behavioral-silent (pre-existing bug, wider on 17) | Judgment | `Beta.java` (`decode`) | changed |
+| **D-03** | Project-system | Judgment | root `pom.xml:629` | build |
+
+| # | Category | Fix | Conf. | Delta | Site | Sites | Note |
+|---|---|---|---|---|---|---|---|
+| 9 | Behavioral-silent | Mechanical | High | CLDR listing dates | `x/y/Resource.java:574` | 1 | Gamma.java is mentioned in prose only |
+| 10 | Dependency | Mechanical | High | JaCoCo agent | `pom.xml:614` | 3 | n |
+
+## Card detail
+
+### 9. CLDR listing dates
+- **Category / fix / confidence:** Behavioral-silent / Mechanical / High; sites: 1; cited: `x/y/Resource.java:574`
+- **Blast radius:** One site. Sites: Delta.java:12, and Epsilon.java:40 also matter.
+
+### 11. A card that only exists as a card
+- **Category / fix / confidence:** Behaviour-Silent / Judgment / Low; sites: 2; cited: `p/Zeta.py:3`, `p/Eta.py:9`
+
+### 12. A numbered heading with no category line
+- Some prose about Theta.java:1.
+
+### D-04 Another id style
+- **Category:** behavioral silent
+- **Site:** `q/Iota.cs:5`
+"""
+
+ def test_deltas_are_read_from_tables_and_cards_and_merged(self):
+ got = {d["id"]: d for d in uc.parse_deltas(self.CARDS)}
+ self.assertEqual(sorted(got), ["11", "9", "10", "D-01", "D-02", "D-03", "D-04"] if False else sorted(["11", "9", "10", "D-01", "D-02", "D-03", "D-04"]))
+ self.assertEqual([s["stem"] for s in got["D-01"]["sites"]], ["Alpha"])
+ self.assertEqual(got["D-01"]["sites"][0]["line"], 916)
+ self.assertEqual([s["stem"] for s in got["D-02"]["sites"]], ["Beta"]) # a site without a line, with a known extension
+ self.assertEqual(got["D-02"]["category"], "Behavioral-silent (pre-existing bug, wider on 17)")
+ self.assertEqual([s["stem"] for s in got["9"]["sites"]], ["Resource"]) # the table and the card agree; prose files are not sites
+ self.assertEqual(got["9"]["title"], "CLDR listing dates")
+ self.assertEqual([s["stem"] for s in got["11"]["sites"]], ["Zeta", "Eta"])
+ self.assertEqual(got["11"]["category"], "Behaviour-Silent")
+ self.assertEqual(got["D-04"]["category"], "behavioral silent")
+ self.assertEqual([s["stem"] for s in got["D-04"]["sites"]], ["Iota"])
+ self.assertNotIn("12", got)
+
+ def test_the_parser_is_tolerant_and_bounded(self):
+ for text in ("", "no tables at all", "| a | b |\n|---|---|\n| 1 | 2 |\n", "| Category |\n|---|\n| Behavioral-silent |\n", "### 1. x\n" * 10, "|" * 100000, "#" * 100000,
+ "### 3. t\n- **Category:** " + "Behavioral-silent " * 100000 + "\n", "| # | Category | Site |\n|---|---|---|\n| 1 | Behavioral-silent | " + "a/" * 100000 + "B.java:1 |\n"):
+ started = time.time()
+ uc.parse_deltas(text)
+ self.assertLess(time.time() - started, 10)
+ many = "| # | Category | Site |\n|---|---|---|\n" + "".join("| %d | Behavioral-silent | `S%d.java:1` |\n" % (i, i) for i in range(6000))
+ self.assertLessEqual(len(uc.parse_deltas(many)), uc.DELTA_CAP)
+ weird = uc.parse_deltas("| # | Category | Site |\n|---|---|---|\n| 1\x1b[31m | Behavioral-silent\x00 | `Ev\x1bil.java:1` |\n")
+ self.assertNotRegex(json.dumps(weird), r"\\u001b|\\u0000")
+
+ def test_coverage_names_sites_as_whole_words_in_test_files_only(self):
+ put(self.new, "src/main/Alpha.java", "class Alpha {}")
+ put(self.new, "src/test/AlphaTest.java", "class AlphaTest { Alpha a; AlphaBeta x; Gamma_ y; }")
+ put(self.new, "src/main/Beta.java", "class Beta { Delta d; }")
+ put(self.new, "src/test/notes.txt", "Delta is discussed here")
+ cat = ("| # | Category | Site |\n|---|---|---|\n| 1 | Behavioral-silent | `src/main/Alpha.java:1` |\n| 2 | Behavioral-silent | `src/main/Beta.java:2` |\n"
+ "| 3 | Behavioral-silent | `src/main/Gamma.java:3` |\n| 4 | Behavioral-silent | `src/main/Delta.java:4` |\n| 5 | Dependency | `src/main/Other.java:5` |\n"
+ "| 6 | Behavioral-silent | `src/main/Alpha.java:1`, `src/main/Beta.java:2` |\n| 7 | Behavioral-silent | no site here |\n| 8 | Behavioral-silent | `pom.xml:9` |\n")
+ got = uc.delta_coverage(cat, self.new)
+ self.assertEqual((got["parsed"], got["silent"], got["other"], got["testFiles"]), (8, 7, 1, 1))
+ self.assertEqual([d["id"] for d in got["covered"]], ["1"])
+ self.assertEqual({d["id"]: d["why"] for d in got["uncovered"]}, {"2": "no test names Beta", "3": "no test names Gamma", "4": "no test names Delta", "6": "no test names Beta", "7": "no site is named in the catalog"})
+ self.assertEqual([d["id"] for d in got["config"]], ["8"])
+ self.assertTrue(got["gap"])
+
+ def test_no_catalog_or_an_unreadable_one_is_a_gap(self):
+ self.assertTrue(uc.delta_coverage(None, self.new)["gap"])
+ got = uc.delta_coverage("Nothing to read here.", self.new)
+ self.assertEqual((got["parsed"], got["gap"]), (0, True))
+ ok = uc.delta_coverage("| # | Category | Site |\n|---|---|---|\n| 1 | API-removed | `A.java:1` |\n", self.new)
+ self.assertEqual((ok["silent"], ok["gap"]), (0, False))
+
+ def test_the_command_line_prints_a_summary_json_and_the_exit_code(self):
+ self.both("t/InTest.java", "Alpha")
+ put(self.new, "src/main/Alpha.java", "class Alpha {}")
+ cat = put(self.root, "CAT.md", "| # | Category | Site |\n|---|---|---|\n| 1 | Behavioral-silent | `src/main/Alpha.java:1` |\n")
+ code, out, _ = run_main(uc.main, [self.old, self.new, cat])
+ self.assertEqual(code, 0)
+ self.assertIn("tests kept: ok", out)
+ self.assertIn("deltas covered: ok", out)
+ code, out, _ = run_main(uc.main, [self.old, self.new, cat, "--json"])
+ self.assertEqual(json.loads(out)["deltas"]["silent"], 1)
+ os.remove(os.path.join(self.new, "t", "InTest.java"))
+ code, out, _ = run_main(uc.main, [self.old, self.new])
+ self.assertEqual(code, 1)
+ self.assertIn("removed: t/InTest.java", out)
+ self.assertEqual(run_main(uc.main, [os.path.join(self.root, "none"), self.new])[0], 2)
+ self.assertEqual(run_main(uc.main, [self.old, self.new, os.path.join(self.root, "none.md")])[0], 2)
+
+
+class ProofOutputs(unittest.TestCase):
+ def setUp(self):
+ self.ws = Ws(self)
+ self.adir = os.path.join(self.ws.root, "analysis", "s")
+
+ def test_files_summary_and_exit_codes(self):
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 0)
+ self.assertIn("s: PROVEN", out)
+ data = load(os.path.join(self.adir, "VERIFICATION.json"))
+ self.assertEqual((data["v"], data["overall"]["verdict"], data["signoff"]), (1, "PROVEN", {"name": "", "role": "", "date": "", "decision": ""}))
+ md = read(os.path.join(self.adir, "VERIFICATION.md"))
+ for needle in ("# Verification: s", "**Overall: PROVEN**", "### P0 business rules", "### What this does not prove", "## Sign-off", "________________",
+ "## How each verdict is computed", "tests executed: 3, failed 0, skipped 0", "- [ ] A business owner accepts the report output"):
+ self.assertIn(needle, md)
+ os.remove(os.path.join(self.adir, "equivalence", "test-runs.json"))
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ self.assertIn("NOT PROVEN", out)
+ self.assertEqual(sorted(f for f in os.listdir(self.adir) if f.startswith("VERIFICATION")), ["VERIFICATION.json", "VERIFICATION.md"])
+
+ def test_bad_names_are_refused(self):
+ for argv in (["..", "--workspace", self.ws.root], ["a/b"], ["s", "../x"], ["s", "a\\b"], [""]):
+ self.assertEqual(run_main(pp.main, argv)[0], 2, argv)
+
+ def test_a_linked_output_is_refused_and_nothing_is_written_through_it(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "real.md", "ORIGINAL")
+ make_symlink(self, os.path.join(outside, "real.md"), os.path.join(self.adir, "VERIFICATION.md"))
+ code, _, err = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 2)
+ self.assertIn("symbolic link", err)
+ self.assertEqual(read(os.path.join(outside, "real.md")), "ORIGINAL")
+
+ def test_hostile_names_and_reasons_stay_text_in_the_markdown(self):
+ put(self.ws.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n", T0 + 20)
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("Rounding", "Round | <img src=x onerror=1> \x1b[31m"), T0 + 10)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit | `x`", "command": "mvn\x00 test\n# heading", "executed": 3, "junit": ["modernized/s/mod/target/surefire-reports"]}])
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ md = read(os.path.join(self.adir, "VERIFICATION.md"))
+ self.assertNotRegex(md, r"[\x00\x1b]")
+ self.assertIn("Round \\| <img src=x onerror=1>", md)
+ self.assertNotIn("\n# heading", md)
+
+ def test_the_verdict_comes_from_the_evidence_not_from_an_earlier_pack(self):
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ path = os.path.join(self.adir, "VERIFICATION.json")
+ data = load(path)
+ data["modules"][0]["verdict"] = "NOT PROVEN"
+ dump(path, data)
+ self.assertEqual(run_main(pp.main, ["s", "--workspace", self.ws.root])[0], 0)
+ self.assertEqual(load(path)["overall"]["verdict"], "PROVEN")
+
+ def test_a_huge_evidence_file_is_not_read(self):
+ put(self.ws.root, "analysis/s/equivalence/test-runs.json", b" " * (9 << 20), T0 + 40)
+ pack = self.ws.pack()
+ self.assertTrue(any("larger than 8 MB" in p for p in pack["problems"]))
+ self.assertEqual(pack["modules"][0]["verdict"], "NOT PROVEN")
+
+
+class ProofUplift(unittest.TestCase):
+ BASE = "# BASELINE\n\n## Per-test results\n\n| Test | Result |\n|---|---|\n| `T#a` | PASS |\n| `T#b` | PASS |\n| `T#c` | FAIL |\n| `T#d` | SKIP |\n"
+ CATALOG = ("# DELTA_CATALOG: s\n\n| # | Category | Fix | Delta | Site |\n|---|---|---|---|---|\n| 1 | Behavioral-silent | Judgment | Locale default changed | `unit/src/Main.java:12` |\n"
+ "| 2 | API-removed | Mechanical | Old API gone | `unit/src/Other.java:3` |\n")
+
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ put(self.root, "legacy/s/pom.xml", "<project/>", T0)
+ put(self.root, "legacy/s/unit/src/test/MainTest.java", "class MainTest { Main m; }\n", T0)
+ put(self.root, "analysis/s/PREFLIGHT.md", "x", T0 + 10)
+ put(self.root, "analysis/s/BASELINE.md", self.BASE, T0 + 10)
+ put(self.root, "analysis/s/baseline/results.json", json.dumps({"T#a": "PASS", "T#b": "PASS", "T#c": "FAIL", "T#d": "SKIP"}), T0 + 5)
+ put(self.root, "analysis/s/DELTA_CATALOG.md", self.CATALOG, T0 + 10)
+ put(self.root, "modernized/s-uplifted/unit/src/Main.java", "class Main {}\n", T0 + 20)
+ put(self.root, "modernized/s-uplifted/unit/src/test/MainTest.java", "class MainTest { Main m; }\n", T0 + 20)
+ put(self.root, "modernized/s-uplifted/UPLIFT_NOTES.md", "Canary: break a comparison -> 2 tests failed\n", T0 + 20)
+ put(self.root, "analysis/s/equivalence/canary/TEST-C.xml", junit([("T", "a", "FAIL", ""), ("T", "b", "FAIL", "")]), T0 + 35)
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ put(self.root, "analysis/s/equivalence/test-runs.json", json.dumps({"legacy": {"ran": True}, "suites": [
+ {"module": "unit", "name": "unit", "junit": ["modernized/s-uplifted/unit/target/surefire-reports"], "executed": 3, "failed": 1, "skipped": 1}],
+ "canaries": [{"change": "break a comparison", "junit": ["analysis/s/equivalence/canary"]}]}), T0 + 40)
+ rows = []
+ for i in range(10):
+ put(self.root, "analysis/s/equivalence/fresh/old/%d.out" % i, "v%d\n" % i, T0 + 25)
+ put(self.root, "analysis/s/equivalence/fresh/new/%d.out" % i, "v%d\n" % i, T0 + 25)
+ rows.append({"id": "f%d" % i, "legacy": "fresh/old/%d.out" % i, "new": "fresh/new/%d.out" % i})
+ put(self.root, "analysis/s/equivalence/fresh-cases.json", json.dumps({"cases": rows}), T0 + 25)
+
+ def results(self, cases):
+ put(self.root, "modernized/s-uplifted/unit/target/surefire-reports/TEST-T.xml", junit(cases), T0 + 30)
+
+ def module(self):
+ pack = pp.overall(pp.build("s", self.root, now="x"))
+ self.assertEqual(len(pack["modules"]), 1)
+ return pack["modules"][0]
+
+ def test_no_regression_against_the_baseline_is_proven_and_traces_no_rules(self):
+ m = self.module()
+ self.assertEqual((m["verdict"], m["track"], m["name"]), ("PROVEN", "uplift", "s-uplifted"))
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}["rules"], "na")
+ self.assertEqual(m["evidence"]["baseline"]["regressionsCount"], 0)
+
+ def test_a_regression_or_a_new_failure_is_not_proven(self):
+ self.results([("T", "a", "FAIL", ""), ("T", "b", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["verdict"], "NOT PROVEN")
+ self.assertIn("1 test(s) passed before and fail now", m["reasons"][0])
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", ""), ("T", "new", "FAIL", "")])
+ self.assertEqual(self.module()["verdict"], "NOT PROVEN")
+
+ def test_missing_tests_and_skipped_growth_are_partly_proven(self):
+ self.results([("T", "a", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["verdict"], "PARTLY PROVEN")
+ self.assertTrue(any("did not run now" in r for r in m["reasons"]))
+ self.results([("T", "a", "PASS", ""), ("T", "b", "SKIP", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ self.assertEqual(self.module()["verdict"], "PARTLY PROVEN")
+
+ def test_an_approved_baseline_difference_is_allowed_and_listed_as_a_decision(self):
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n## Approved differences\n\n| Test | Reason |\n|---|---|\n| `T#b` | the new runtime words the message differently |\n", T0 + 10)
+ self.results([("T", "a", "PASS", ""), ("T", "b", "FAIL", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["verdict"], "PROVEN")
+ self.assertTrue(any("approved by a person in BASELINE.md" in c and "T#b" in c for c in m["caveats"]))
+ self.assertTrue(any("decision, not evidence" in n for n in m["notProven"]))
+ self.assertEqual(m["evidence"]["baseline"]["approved"][0]["id"], "T#b")
+
+ def test_a_canary_keyed_by_a_unit_of_the_uplifted_copy_is_accepted(self):
+ dump(os.path.join(self.root, "analysis", "s", "equivalence", "test-runs.json"), {"legacy": {"ran": True}, "suites": [
+ {"module": "unit", "name": "unit", "junit": ["modernized/s-uplifted/unit/target/surefire-reports"]}],
+ "canaries": [{"module": "unit", "change": "break a comparison", "junit": ["analysis/s/equivalence/canary"]}]})
+ m = self.module()
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}["canary"], "pass")
+
+ def test_a_test_that_now_passes_needs_a_person_but_does_not_change_the_verdict(self):
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "PASS", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["verdict"], "PROVEN")
+ self.assertTrue(any("intended fix" in n for n in m["needsPerson"]))
+
+ def test_target_only_and_a_missing_baseline_cannot_be_proven(self):
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "PASS", "")])
+ put(self.root, "analysis/s/BASELINE.md", "target-only: Java 8 is not installed here\n", T0 + 10)
+ m = self.module()
+ self.assertEqual(m["verdict"], "PARTLY PROVEN")
+ self.assertIn("target-only", " ".join(m["reasons"]))
+ self.assertFalse(m["evidence"]["legacy"]["ran"])
+ os.remove(os.path.join(self.root, "analysis", "s", "BASELINE.md"))
+ m = self.module()
+ self.assertEqual(m["verdict"], "PARTLY PROVEN")
+ self.assertIn("BASELINE.md is missing", " ".join(m["reasons"]))
+
+ def states(self):
+ return {c["id"]: c["state"] for c in self.module()["checks"]}
+
+ def test_all_nine_checks_pass_for_a_clean_uplift(self):
+ m = self.module()
+ self.assertEqual(self.states(), {"tests": "pass", "rules": "na", "same": "pass", "fresh": "pass", "canary": "pass", "source": "pass", "baseline": "pass", "kept": "pass", "deltas": "pass"})
+ self.assertEqual(m["evidence"]["baselineEvidence"]["kind"], "measured")
+ self.assertEqual(m["evidence"]["deltas"]["silent"], 1)
+
+ # -- (1) a measured baseline
+ def test_a_baseline_typed_by_hand_is_a_gap_and_says_so_in_plain_words(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ m = self.module()
+ self.assertEqual((m["verdict"], self.states()["baseline"], self.states()["same"]), ("PARTLY PROVEN", "gap", "pass"))
+ detail = next(c["detail"] for c in m["checks"] if c["id"] == "baseline")
+ self.assertIn("typed by hand", detail)
+ self.assertIn("analysis/<system>/baseline/", detail)
+ self.assertIn("Recorded:", detail)
+ self.assertIn("### Baseline evidence", pp.render_md(pp.overall(pp.build("s", self.root, now="x"))))
+
+ def test_a_forged_counts_file_is_not_evidence(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ put(self.root, "analysis/s/baseline/counts.json", json.dumps({"PASS": 2, "FAIL": 1, "SKIP": 1, "executed": 3}), T0 + 5)
+ put(self.root, "analysis/s/baseline/counts2.json", json.dumps({"tests": {"count": 3}}), T0 + 5)
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "gap")
+ info = m["evidence"]["baselineEvidence"]
+ self.assertEqual(info["kind"], "typed")
+ self.assertIn("counts.json", info["ignored"])
+ self.assertIn("Files that only hold counts were ignored", next(c["detail"] for c in m["checks"] if c["id"] == "baseline"))
+
+ def test_only_the_baseline_folder_and_recorded_lines_are_looked_at(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ put(self.root, "analysis/s/harness/stray.json", json.dumps({"T#a": "PASS", "T#b": "PASS", "T#c": "FAIL", "T#d": "SKIP"}), T0 + 5)
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\nSee also `analysis/s/harness/stray.json` for the details.\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "gap") # a path in prose is not evidence
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n- **Recorded:** 2026-09-24. Machine-readable per-test results: `analysis/s/harness/stray.json`\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "pass")
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\nMachine-readable: harness/stray.json\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "pass") # relative to analysis/<system>/
+
+ def test_evidence_outside_analysis_or_through_a_link_is_never_used(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ full = json.dumps({"T#a": "PASS", "T#b": "PASS", "T#c": "FAIL", "T#d": "SKIP"})
+ put(outside, "results.json", full)
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ put(self.root, "analysis/other/results.json", full, T0 + 5)
+ put(self.root, "modernized/results.json", full, T0 + 5)
+ make_symlink(self, os.path.join(outside, "results.json"), os.path.join(self.root, "analysis", "s", "baseline", "link.json"))
+ make_symlink(self, outside, os.path.join(self.root, "analysis", "s", "linked"))
+ for line in ("Recorded: `../other/results.json`", "Recorded: `analysis/other/results.json`", "Recorded: `analysis/s/../other/results.json`", "Recorded: `modernized/results.json`",
+ "Recorded: `%s/results.json`" % outside, "Recorded: `analysis/s/linked/results.json`", "Recorded: `analysis/s/baseline/link.json`"):
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n" + line + "\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "gap", line)
+
+ def test_baseline_sources_that_disagree_are_a_gap_not_first_one_wins(self):
+ put(self.root, "analysis/s/baseline/second.json", json.dumps({"T#a": "FAIL", "T#b": "PASS"}), T0 + 5)
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "gap")
+ detail = next(c["detail"] for c in m["checks"] if c["id"] == "baseline")
+ self.assertIn("disagree with each other", detail)
+ self.assertIn("T#a: PASS in one source, FAIL in second.json", detail) # the example names the test and the file
+ put(self.root, "analysis/s/baseline/second.json", json.dumps({"T#a": "PASS"}), T0 + 5) # agrees on what they share
+ self.assertEqual(self.states()["baseline"], "pass")
+ put(self.root, "analysis/s/baseline/old.test-output.txt", "Tests run: 9, Failures: 0, Errors: 0, Skipped: 0\n", T0 + 5) # a log with other totals
+ self.assertEqual(self.states()["baseline"], "gap")
+
+ def test_a_typed_table_that_disagrees_with_its_evidence_is_a_gap(self):
+ put(self.root, "analysis/s/baseline/results.json", json.dumps({"T#a": "PASS", "T#b": "FAIL", "T#c": "FAIL", "T#d": "SKIP"}), T0 + 5)
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "gap")
+ self.assertIn("disagree on 1 test", next(c["detail"] for c in m["checks"] if c["id"] == "baseline"))
+ self.assertEqual(m["evidence"]["baselineEvidence"]["disagree"], 1)
+
+ def test_a_raw_runner_log_can_measure_the_baseline_and_must_match_the_typed_counts(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "baseline", "results.json"))
+ put(self.root, "analysis/s/baseline/old.test-output.txt", "[INFO] Tests run: 4, Failures: 1, Errors: 0, Skipped: 1\n", T0 + 5) # 3 executed, 1 failed, 1 skipped
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "pass")
+ self.assertIn("runner log", next(c["detail"] for c in m["checks"] if c["id"] == "baseline"))
+ put(self.root, "analysis/s/baseline/old.test-output.txt", "[INFO] Tests run: 4, Failures: 0, Errors: 0, Skipped: 1\n", T0 + 5)
+ self.assertEqual(self.states()["baseline"], "gap")
+
+ def test_target_only_and_a_missing_baseline_leave_the_measured_check_not_applicable(self):
+ put(self.root, "analysis/s/BASELINE.md", "target-only: Java 8 is not installed here\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "na")
+ os.remove(os.path.join(self.root, "analysis", "s", "BASELINE.md"))
+ self.assertEqual((self.states()["baseline"], self.states()["same"]), ("na", "gap"))
+
+ def test_measured_per_test_results_are_what_the_new_run_is_compared_with(self):
+ put(self.root, "analysis/s/baseline/results.json", json.dumps({"T#a": "PASS", "T#b": "PASS", "T#c": "FAIL", "T#d": "SKIP", "T#extra": "PASS"}), T0 + 5)
+ self.results([("T", "a", "PASS", ""), ("T", "b", "PASS", ""), ("T", "c", "FAIL", ""), ("T", "d", "SKIP", "")])
+ m = self.module()
+ self.assertEqual(m["evidence"]["baseline"]["source"], "measured results")
+ self.assertEqual(self.states()["same"], "gap") # T#extra was measured on the old version and did not run now
+ self.assertEqual(self.states()["baseline"], "pass") # the typed table lists fewer tests than were measured, and agrees on all it lists
+ self.assertTrue(any("did not run now" in r for r in m["reasons"]))
+
+ def test_a_totals_table_typed_beside_the_rows_must_match_the_evidence_too(self):
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n## Totals\n\n| | Count |\n|---|---|\n| Executed | 50 |\n| Failed | 0 |\n", T0 + 10)
+ m = self.module()
+ self.assertEqual(self.states()["baseline"], "gap")
+ self.assertIn("Totals table says 50 executed", " ".join(m["evidence"]["baselineEvidence"]["examples"]))
+ put(self.root, "analysis/s/BASELINE.md", self.BASE + "\n## Totals\n\n| | Count |\n|---|---|\n| Executed | 3 |\n| Failed | 1 |\n", T0 + 10)
+ self.assertEqual(self.states()["baseline"], "pass")
+
+ # -- (2) tests changed during the uplift
+ def test_a_removed_legacy_test_file_is_a_gap_and_the_list_says_which(self):
+ os.remove(os.path.join(self.root, "modernized", "s-uplifted", "unit", "src", "test", "MainTest.java"))
+ m = self.module()
+ self.assertEqual((m["verdict"], self.states()["kept"]), ("PARTLY PROVEN", "gap"))
+ detail = next(c["detail"] for c in m["checks"] if c["id"] == "kept")
+ self.assertIn("MainTest.java", detail)
+ self.assertIn("edited or deleted", detail)
+ self.assertEqual(m["evidence"]["testsChanged"]["removed"][0]["path"], "unit/src/test/MainTest.java")
+ self.assertTrue(any("Review the test files" in n for n in m["needsPerson"]))
+
+ def test_more_than_a_quarter_of_the_test_files_changed_is_a_gap_and_a_quarter_is_not(self):
+ for i in range(3):
+ put(self.root, "legacy/s/unit/src/test/T%d.java" % i, "class T%d { int a = 1; }\n" % i, T0)
+ put(self.root, "modernized/s-uplifted/unit/src/test/T%d.java" % i, "class T%d { int a = 1; }\n" % i, T0 + 20)
+ put(self.root, "modernized/s-uplifted/unit/src/test/T0.java", "class T0 { int a = 2; }\n", T0 + 20) # 1 of 4 changed: exactly a quarter
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "pass")
+ self.assertEqual(m["evidence"]["testsChanged"]["changed"][0], {"path": "unit/src/test/T0.java", "added": 1, "removed": 1, "note": ""})
+ self.assertTrue(any("1 changed" in n for n in m["needsPerson"]))
+ put(self.root, "modernized/s-uplifted/unit/src/test/T1.java", "class T1 { int a = 2; }\n", T0 + 20) # 2 of 4
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "gap")
+ self.assertIn("2 of 4 legacy test files (50%) were changed", next(c["detail"] for c in m["checks"] if c["id"] == "kept"))
+
+ def test_no_legacy_tests_or_no_legacy_tree_is_a_gap(self):
+ os.remove(os.path.join(self.root, "legacy", "s", "unit", "src", "test", "MainTest.java"))
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "gap")
+ self.assertIn("nothing shows that the working copy's tests were not written during the uplift", next(c["detail"] for c in m["checks"] if c["id"] == "kept"))
+ shutil.rmtree(os.path.join(self.root, "legacy"))
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "gap")
+ self.assertIn("legacy tree was not found", next(c["detail"] for c in m["checks"] if c["id"] == "kept"))
+
+ def test_added_tests_are_listed_and_never_a_gap(self):
+ put(self.root, "modernized/s-uplifted/unit/src/test/NewCharacterizationTest.java", "class NewCharacterizationTest { Main m; }\n", T0 + 20)
+ m = self.module()
+ self.assertEqual(self.states()["kept"], "pass")
+ self.assertEqual(m["evidence"]["testsChanged"]["added"], [{"path": "unit/src/test/NewCharacterizationTest.java", "lines": 2}])
+ md = pp.render_md(pp.overall(pp.build("s", self.root, now="x")))
+ self.assertIn("### Test files changed during the uplift", md)
+ self.assertIn("Added: `unit/src/test/NewCharacterizationTest.java`", md)
+ self.assertIn("Weakened assertions cannot be detected", md)
+ self.assertTrue(any("Weakened assertions cannot be detected" in n for n in m["notProven"]))
+
+ # -- (3) delta coverage
+ def test_a_silent_delta_no_test_names_is_a_gap_and_other_categories_are_only_listed(self):
+ put(self.root, "analysis/s/DELTA_CATALOG.md", self.CATALOG + "| 3 | Behavioral-silent | Judgment | Rounding changed | `unit/src/Money.java:40` |\n", T0 + 10)
+ m = self.module()
+ self.assertEqual((m["verdict"], self.states()["deltas"]), ("PARTLY PROVEN", "gap"))
+ detail = next(c["detail"] for c in m["checks"] if c["id"] == "deltas")
+ self.assertIn("1 of 2 Behavioral-silent", detail)
+ self.assertIn("no test names Money", detail)
+ self.assertEqual(m["evidence"]["deltas"]["uncovered"][0]["id"], "3")
+ self.assertIn("Not named by any test: 3 Rounding changed", pp.render_md(pp.overall(pp.build("s", self.root, now="x"))))
+ put(self.root, "modernized/s-uplifted/unit/src/test/MoneyTest.java", "class MoneyTest { Money m; }\n", T0 + 20)
+ self.assertEqual(self.states()["deltas"], "pass")
+
+ def test_a_missing_or_unreadable_catalog_is_a_gap(self):
+ os.remove(os.path.join(self.root, "analysis", "s", "DELTA_CATALOG.md"))
+ m = self.module()
+ self.assertEqual(self.states()["deltas"], "gap")
+ self.assertIn("DELTA_CATALOG.md was not found", next(c["detail"] for c in m["checks"] if c["id"] == "deltas"))
+ put(self.root, "analysis/s/DELTA_CATALOG.md", "Some prose about the version change. No table, no cards.\n", T0 + 10)
+ self.assertEqual(self.states()["deltas"], "gap")
+
+ def test_a_delta_at_a_build_file_is_listed_for_a_person_not_judged(self):
+ put(self.root, "analysis/s/DELTA_CATALOG.md", self.CATALOG + "| 4 | Behavioral-silent | Judgment | Bundle plugin | `pom.xml:629` |\n", T0 + 10)
+ m = self.module()
+ self.assertEqual(self.states()["deltas"], "pass")
+ self.assertEqual(m["evidence"]["deltas"]["config"][0]["id"], "4")
+ self.assertTrue(any("build or configuration file" in n for n in m["needsPerson"]))
+
+ def test_a_unit_argument_narrows_the_suites_and_an_unknown_name_is_an_error(self):
+ pack = pp.build("s", self.root, "unit", now="x")
+ self.assertEqual(pack["modules"][0]["name"], "s-uplifted")
+ self.assertEqual(pack["asked"], "unit")
+ with self.assertRaises(pp.InputError):
+ pp.build("s", self.root, "nosuchunit")
+
+
+# ---------------------------------------------------------------- the Proof section of the report
+def page_data(html):
+ return json.loads(re.search(r'<script type="application/json" id="report-data">(.*?)</script>', html, re.S).group(1))
+
+
+def good_pack(**module_change):
+ checks = [{"id": cid, "label": label, "state": "pass", "detail": "fine"} for cid, label in br.PROOF_CHECKS]
+ m = {"name": "mod", "track": "rewrite", "verdict": "PROVEN", "checks": checks, "verifiedAt": "2026-09-24 00:00 UTC",
+ "evidence": {"tests": {"executed": 5, "failed": 0, "skipped": 0},
+ "rules": {"p0": [{"id": "RULE-001", "name": "Total", "confidence": "High", "status": "tested", "tests": 3, "main": 1, "where": "src/test/A.java:9"}]},
+ "fresh": {"inputs": 12, "executed": 20, "same": 20, "differs": 0, "missing": 0}, "equivalence": {"cases": 4, "executed": 4, "same": 4, "differs": 0, "missing": 0, "approved": [], "masks": []}},
+ "notProven": ["It does not prove inputs nobody tried."], "brief": {"found": True, "items": [{"text": "A person accepts it", "where": "Phase 1 exit criterion"}], "count": 1}}
+ m.update(module_change)
+ return {"v": 1, "system": "s", "generated": "2026-09-24 00:00 UTC", "problems": [], "legacy": {"path": "legacy/s", "ran": True}, "modules": [m], "signoff": {}, "rules": ["PROVEN needs all six checks."]}
+
+
+def good_uplift_pack(**evidence):
+ """A pack for an uplift whose nine checks all pass, with the evidence that backs them; a test changes the evidence and looks at the view."""
+ pack = good_pack()
+ m = pack["modules"][0]
+ m.update(name="s-uplifted", track="uplift")
+ m["checks"] = [c for c in m["checks"] if c["id"] != "rules"] + [{"id": "rules", "label": "Rules traced", "state": "na", "detail": "An uplift keeps the code."}] + [
+ {"id": cid, "label": label, "state": "pass", "detail": "fine"} for cid, label in br.UPLIFT_CHECKS]
+ m["evidence"].pop("rules", None)
+ m["evidence"].update(baselineEvidence={"kind": "measured", "sources": [{"kind": "machine-readable results", "name": "base.json", "tests": 40}], "disagree": 0, "conflict": 0, "examples": [], "considered": ["base.json"]},
+ testsChanged={"checked": True, "why": "", "legacyTests": 8, "workTests": 9, "counts": {"added": 1, "removed": 0, "changed": 1}, "share": 0.125,
+ "removed": [], "added": [{"path": "t/NewTest.java", "lines": 12}], "changed": [{"path": "t/OldTest.java", "added": 3, "removed": 1, "note": ""}]},
+ deltas={"parsed": 5, "silent": 2, "covered": [{"id": "1"}, {"id": "2"}], "uncovered": [], "config": [{"id": "4", "title": "Bundle plugin", "sites": ["pom.xml:9"]}], "other": 3, "gap": False})
+ m["evidence"].update(evidence)
+ return pack
+
+
+class ProofReport(unittest.TestCase):
+ def setUp(self):
+ self.root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.root, True)
+ put(self.root, "analysis/s/PREFLIGHT.md", "# Preflight\n")
+ self.out = os.path.join(self.root, "out", "REPORT.html")
+
+ def report(self, pack):
+ put(self.root, "analysis/s/VERIFICATION.json", pack if isinstance(pack, (str, bytes)) else json.dumps(pack, indent=1)) # like proof_pack.py: short lines
+ code, _, err = run_main(br.main, ["s", "--workspace", self.root, "--out", self.out])
+ self.assertEqual(code, 0, err)
+ with open(self.out, encoding="utf-8") as fh:
+ return fh.read()
+
+ def proof(self, html):
+ data = page_data(html)
+ return data, next((p["proof"] for s in data["sections"] if s["id"] == "proof" for p in s["parts"] if p["t"] == "proof"), None)
+
+ def test_the_section_the_banner_and_the_order(self):
+ put(self.root, "analysis/s/EQUIVALENCE.json", "{}")
+ put(self.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "# n")
+ data, pv = self.proof(self.report(good_pack()))
+ self.assertEqual([s["id"] for s in data["sections"]][-3:], ["build", "proof", "equivalence"])
+ self.assertEqual((pv["verdict"], pv["counts"]["PROVEN"], pv["modules"][0]["p0Counts"]["tested"]), ("PROVEN", 1, 1))
+ self.assertEqual(data["glance"]["proof"]["verdict"], "PROVEN")
+ self.assertEqual(data["glance"]["proof"]["modules"], [{"name": "mod", "verdict": "PROVEN"}])
+ self.assertEqual(pv["signoff"], {"name": "", "role": "", "date": "", "decision": ""})
+ self.assertEqual(pv["modules"][0]["people"][0]["text"], "A person accepts it")
+
+ def test_a_rule_named_only_by_tests_that_did_not_run_is_counted_and_tooling_folders_are_listed(self):
+ pack = good_pack()
+ pack["modules"][0]["evidence"]["rules"]["p0"].append({"id": "RULE-002", "name": "Skipped", "confidence": "High", "status": "named, not run", "tests": 1, "main": 0, "where": "T.java:3"})
+ pack["toolingOnly"] = [{"name": "parity-harness", "path": "modernized/s/parity-harness"}, "not a dict", {"name": "x" * 500, "path": 5}]
+ html = self.report(pack)
+ _, pv = self.proof(html)
+ self.assertEqual(pv["modules"][0]["p0Counts"], {"tested": 1, "claimedOnly": 0, "notRun": 1, "none": 0})
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN") # a pass cannot stand next to a rule that no test ran
+ self.assertEqual([t["name"] for t in pv["tooling"]][0], "parity-harness")
+ self.assertEqual(len(pv["tooling"]), 2)
+ self.assertLessEqual(len(pv["tooling"][1]["name"]), 120)
+ self.assertEqual(self.proof(self.report(good_pack()))[1]["tooling"], [])
+
+ def test_no_pack_means_no_section_and_the_older_reports_are_unchanged(self):
+ code, _, _ = run_main(br.main, ["s", "--workspace", self.root, "--out", self.out])
+ data = page_data(read(self.out))
+ self.assertIsNone(data["glance"]["proof"])
+ self.assertNotIn("proof", [s["id"] for s in data["sections"]])
+
+ def test_the_verdict_is_recomputed_from_the_checks_never_taken_from_the_file(self):
+ pack = good_pack()
+ pack["modules"][0]["checks"][3]["state"] = "gap"
+ _, pv = self.proof(self.report(pack))
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN")
+ self.assertTrue(any("recorded PROVEN" in p and "PARTLY PROVEN" in p for p in pv["problems"]))
+ pack["modules"][0]["checks"][0]["state"] = "fail"
+ pack["modules"][0]["verdict"] = "PROVEN"
+ data, pv = self.proof(self.report(pack))
+ self.assertEqual((pv["verdict"], data["glance"]["proof"]["verdict"]), ("NOT PROVEN", "NOT PROVEN"))
+ self.assertEqual(pv["modules"][0]["reasons"][0], "Tests ran: fine")
+
+ def test_a_pack_that_contradicts_its_own_numbers_is_downgraded(self):
+ for change, check, want in (({"evidence": {"tests": {"executed": 3, "failed": 2}}}, 0, "fail"),
+ ({"evidence": {"tests": {"executed": 3}, "equivalence": {"differs": 1}, "fresh": {"inputs": 12}}}, 2, "fail"),
+ ({"evidence": {"tests": {"executed": 3}, "fresh": {"inputs": 12, "missing": 1}}}, 3, "fail"),
+ ({"evidence": {"tests": {"executed": 0}}}, 0, "fail"),
+ ({"evidence": {"tests": {"executed": 3}, "rules": {"p0": [{"id": "R", "status": "none"}]}}}, 1, "gap"),
+ ({"evidence": {"tests": {"executed": 3}, "fresh": {"inputs": 3}}}, 3, "gap"),
+ ({"evidence": {"tests": {"executed": 3}}}, 3, "gap")):
+ _, pv = self.proof(self.report(good_pack(**change)))
+ self.assertEqual(pv["modules"][0]["checks"][check]["state"], want, change)
+ self.assertNotEqual(pv["modules"][0]["verdict"], "PROVEN")
+ pack = good_pack()
+ pack["modules"][0]["checks"] = pack["modules"][0]["checks"][:5]
+ _, pv = self.proof(self.report(pack))
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN")
+ self.assertIn("missing from VERIFICATION.json", pv["modules"][0]["reasons"][0])
+ self.assertEqual(self.proof(self.report(dict(good_pack(), modules=[])))[1]["verdict"], "NOT PROVEN")
+
+ def test_an_uplift_needs_its_three_extra_checks_and_a_rewrite_does_not(self):
+ _, pv = self.proof(self.report(good_uplift_pack()))
+ m = pv["modules"][0]
+ self.assertEqual((m["verdict"], [c["id"] for c in m["checks"]]), ("PROVEN", ["tests", "rules", "same", "fresh", "canary", "source", "baseline", "kept", "deltas"]))
+ self.assertEqual(m["uplift"]["testsChanged"]["counts"], {"added": 1, "removed": 0, "changed": 1})
+ self.assertEqual(m["uplift"]["deltas"]["coveredCount"], 2)
+ pack = good_uplift_pack()
+ pack["modules"][0]["checks"] = [c for c in pack["modules"][0]["checks"] if c["id"] != "kept"]
+ _, pv = self.proof(self.report(pack))
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN")
+ self.assertIn("tests kept check is missing", " ".join(pv["problems"]))
+ _, pv = self.proof(self.report(good_pack())) # a rewrite never needs them
+ self.assertEqual((pv["modules"][0]["verdict"], len(pv["modules"][0]["checks"]), pv["modules"][0]["uplift"]), ("PROVEN", 6, None))
+
+ def test_forged_uplift_passes_are_downgraded_by_the_evidence_beside_them(self):
+ cases = (("baseline", {"baselineEvidence": {"kind": "typed"}}), ("baseline", {"baselineEvidence": {"kind": "measured", "disagree": 2}}), ("baseline", {"baselineEvidence": {"kind": "measured", "conflict": 1}}),
+ ("kept", {"testsChanged": {"checked": False}}), ("kept", {"testsChanged": {"checked": True, "legacyTests": 0, "counts": {}}}),
+ ("kept", {"testsChanged": {"checked": True, "legacyTests": 8, "counts": {"removed": 1, "changed": 0, "added": 0}}}),
+ ("kept", {"testsChanged": {"checked": True, "legacyTests": 8, "counts": {"removed": 0, "changed": 3, "added": 0}}}),
+ ("deltas", {"deltas": {"parsed": 4, "silent": 1, "covered": [], "uncovered": [{"id": "9"}]}}), ("deltas", {"deltas": {"parsed": 0}}), ("deltas", {"deltas": "nope"}))
+ for cid, change in cases:
+ _, pv = self.proof(self.report(good_uplift_pack(**change)))
+ m = pv["modules"][0]
+ self.assertEqual({c["id"]: c["state"] for c in m["checks"]}[cid], "gap", (cid, change))
+ self.assertEqual(m["verdict"], "PARTLY PROVEN", (cid, change))
+ for state in ("na", "gap"): # "not applicable" is no way out for the kept and deltas checks
+ pack = good_uplift_pack(testsChanged={"checked": False}, deltas={"parsed": 0})
+ for c in pack["modules"][0]["checks"]:
+ if c["id"] in ("kept", "deltas"):
+ c["state"] = state
+ self.assertEqual(self.proof(self.report(pack))[1]["modules"][0]["verdict"], "PARTLY PROVEN")
+ pack = good_uplift_pack(baselineEvidence={"kind": "target-only"})
+ for c in pack["modules"][0]["checks"]:
+ if c["id"] == "baseline":
+ c["state"] = "na"
+ self.assertEqual(self.proof(self.report(pack))[1]["modules"][0]["checks"][6]["state"], "na") # a target-only baseline may say so
+
+ def test_the_uplift_evidence_is_cut_and_cleaned(self):
+ evil = "<script>alert(1)</script>" + chr(0x2028)
+ _, pv = self.proof(self.report(good_uplift_pack(
+ baselineEvidence={"kind": evil, "sources": [{"kind": evil * 5, "name": evil}] * 50, "disagree": "x", "examples": [evil * 20] * 20, "considered": [evil] * 50},
+ testsChanged={"checked": True, "legacyTests": 8, "workTests": 8, "counts": {"added": 1, "removed": 0, "changed": 0}, "share": "x", "added": [{"path": evil * 20, "lines": "x"}] * 300, "removed": "x", "changed": [1, None]},
+ deltas={"parsed": 3, "silent": 1, "covered": [], "uncovered": [{"id": evil * 5, "title": evil * 10, "why": evil * 10, "sites": [evil] * 20}] * 300, "config": "x"})))
+ up = pv["modules"][0]["uplift"]
+ self.assertEqual(up["baseline"]["kind"], "none")
+ self.assertLessEqual((len(up["baseline"]["sources"]), len(up["baseline"]["examples"]), len(up["testsChanged"]["added"]), len(up["deltas"]["uncovered"])), (10, 5, 40, 40))
+ self.assertEqual((up["testsChanged"]["share"], up["testsChanged"]["added"][0]["lines"], up["testsChanged"]["removed"], up["deltas"]["config"]), (0.0, 0, [], []))
+ self.assertGreater(up["deltas"]["uncoveredCount"], 0)
+ self.assertEqual(pv["modules"][0]["verdict"], "PARTLY PROVEN")
+
+ def test_a_real_uplift_pack_from_proof_pack_carries_its_evidence_to_the_report(self):
+ base = ProofUplift("test_all_nine_checks_pass_for_a_clean_uplift")
+ base.setUp()
+ base.addCleanup(shutil.rmtree, base.root, True)
+ put(base.root, "modernized/s-uplifted/unit/src/test/NewCharacterizationTest.java", "class NewCharacterizationTest { Main m; }\n", T0 + 20)
+ run_main(pp.main, ["s", "--workspace", base.root])
+ run_main(br.main, ["s", "--workspace", base.root])
+ data = page_data(read(os.path.join(base.root, "analysis", "s", "REPORT.html")))
+ pv = next(p["proof"] for sec in data["sections"] if sec["id"] == "proof" for p in sec["parts"])
+ m = pv["modules"][0]
+ self.assertEqual((pv["verdict"], len(m["checks"])), ("PROVEN", 9))
+ self.assertEqual(m["uplift"]["testsChanged"]["added"][0]["path"], "unit/src/test/NewCharacterizationTest.java")
+ self.assertEqual(m["uplift"]["baseline"]["kind"], "measured")
+ self.assertEqual(m["uplift"]["deltas"]["silent"], 1)
+
+ def test_a_real_pack_written_by_proof_pack_renders_the_same_verdict(self):
+ ws = Ws(self)
+ put(ws.root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001\n", T0 + 20)
+ run_main(pp.main, ["s", "--workspace", ws.root])
+ code, _, _ = run_main(br.main, ["s", "--workspace", ws.root])
+ data = page_data(read(os.path.join(ws.root, "analysis", "s", "REPORT.html")))
+ mod = data["glance"]["proof"]["modules"][0]
+ self.assertEqual(mod, {"name": "mod", "verdict": "PARTLY PROVEN"})
+ pv = next(p["proof"] for s in data["sections"] if s["id"] == "proof" for p in s["parts"])
+ self.assertEqual(pv["modules"][0]["p0Counts"], {"tested": 1, "claimedOnly": 0, "notRun": 0, "none": 1})
+ self.assertTrue(pv["modules"][0]["reasons"][0].startswith("Rules traced: 1 of 2 P0"))
+
+ def test_hostile_values_stay_data_and_the_page_keeps_its_rules(self):
+ evil = "<script>window.__pwned=1</script></script><img src=x onerror=1> &amp; " + chr(0x2028)
+ pack = good_pack(name=evil * 5, track=evil, notProven=[evil * 20] * 60, verdict=["PROVEN"], checks="nope",
+ evidence={"tests": {"executed": "many", "failed": -1, "skipped": 10 ** 30}, "rules": {"p0": [evil, {"id": evil, "status": evil, "tests": evil}] * 150},
+ "fresh": [1], "equivalence": {"approved": evil, "masks": [evil]}, "baseline": 5, "canary": [evil]},
+ brief={"items": [{"text": evil * 9, "where": evil}] * 100, "count": "x", "found": 1})
+ pack.update(problems=[evil] * 20, signoff="x", legacy=[], rules="x")
+ html = self.report(pack)
+ block = re.search(r'<script type="application/json" id="report-data">(.*?)</script>', html, re.S).group(1)
+ for bad in ("<", ">", "&", chr(0x2028)):
+ self.assertNotIn(bad, block)
+ data, pv = self.proof(html)
+ m = pv["modules"][0]
+ self.assertLessEqual(len(m["name"]), 120)
+ self.assertEqual((m["tests"], m["verdict"]), ({"executed": 0, "failed": 0, "skipped": 0}, "PARTLY PROVEN")) # no usable checks: nothing is proven
+ self.assertLessEqual(len(m["p0"]), 200)
+ self.assertLessEqual(len(m["people"]), 40)
+ self.assertLessEqual(len(pv["problems"]), 10 + 1)
+ self.assertEqual(html.count("<script"), 2)
+ self.assertNotRegex(html.replace(block, "").split('<script id="app">')[0].split("</style>")[1], r"__pwned|onerror=")
+ policy = re.search(r'http-equiv="Content-Security-Policy" content="([^"]*)"', html).group(1)
+ self.assertIn("default-src 'none'", policy)
+
+ def test_malformed_or_foreign_verification_files_are_reported_not_raised(self):
+ for text in ("[1, 2", "{}", '{"modules": "x"}', "[]", "null", b"\x00\x01"):
+ data = page_data(self.report(text))
+ self.assertEqual(data["glance"]["proof"]["verdict"], "NOT PROVEN" if text != "[1, 2" else "NOT PROVEN", text)
+ html = self.report("[1, 2")
+ self.assertTrue(any("VERIFICATION.json could not be parsed" in json.dumps(p) for s in page_data(html)["sections"] for p in s["parts"]))
+
+ def test_a_linked_verification_file_is_not_read(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "V.json", json.dumps(good_pack()))
+ make_symlink(self, os.path.join(outside, "V.json"), os.path.join(self.root, "analysis", "s", "VERIFICATION.json"))
+ code, _, _ = run_main(br.main, ["s", "--workspace", self.root, "--out", self.out])
+ data = page_data(read(self.out))
+ self.assertIsNone(data["glance"]["proof"])
+
+
+NODE_PROOF = r"""
+const fs = require('fs'), vm = require('vm');
+const tpl = fs.readFileSync(process.env.TEMPLATE, 'utf8');
+const grab = function (re) { return re.exec(tpl)[1] !== undefined ? re.exec(tpl)[1] : re.exec(tpl)[0]; };
+const core = /\/\* core:start[^\n]*\n([\s\S]*?)\/\* core:end \*\//.exec(tpl)[1];
+const proof = /\/\* proof:start[^\n]*\n([\s\S]*?)\/\* proof:end \*\//.exec(tpl)[1];
+const helpers = /function fmt\(n\)[^\n]*\n/.exec(tpl)[0] + /function chip\([^\n]*\n/.exec(tpl)[0];
+class Text { constructor(t) { this.text = String(t); } }
+class El { constructor(t) { this.tag = t; this.attrs = {}; this.kids = []; }
+ setAttribute(k, v) { this.attrs[k] = String(v); } appendChild(c) { this.kids.push(c); return c; } }
+const ctx = { document: { createElement: t => new El(t), createTextNode: t => new Text(t) } };
+vm.createContext(ctx);
+const api = vm.runInContext('(function(){"use strict";' + core + helpers + proof + ';return {partProof:partProof,own:own};})()', ctx);
+function walk(n, acc, top) {
+ if (n instanceof Text) { acc.text += n.text + '\n'; return; }
+ acc.tags.add(n.tag); Object.keys(n.attrs).forEach(k => { acc.attrs.add(k); });
+ if (top && n.tag === 'div' && /banner/.test(n.attrs.class || '')) acc.banners.push(n.attrs.class);
+ n.kids.forEach(c => walk(c, acc, false));
+}
+const input = JSON.parse(fs.readFileSync(0, 'utf8')), out = [];
+for (const pv of input) {
+ const t0 = Date.now(), acc = { tags: new Set(), attrs: new Set(), text: '', banners: [] };
+ api.partProof(pv).forEach(n => walk(n, acc, true));
+ out.push({ ms: Date.now() - t0, tags: [...acc.tags], attrs: [...acc.attrs], text: acc.text, banners: acc.banners });
+}
+out.push({ own: [api.own({a: 'x'}, 'a', 'd'), api.own({a: 'x'}, '__proto__', 'd'), api.own({a: 'x'}, 'constructor', 'd'), api.own({a: 'x'}, 'toString', 'd')] });
+process.stdout.write(JSON.stringify(out));
+"""
+
+
+@unittest.skipUnless(NODE, "node is not installed")
+class ProofScript(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ evil = "<script>alert(1)</script></script><img src=x onerror=alert(2)> [x](javascript:alert(3))"
+ packs = [good_pack(),
+ good_pack(checks=[{"id": "tests", "label": "Tests ran", "state": "gap", "detail": "x"}]),
+ good_pack(checks=[{"id": "tests", "label": "Tests ran", "state": "fail", "detail": "x"}]),
+ good_pack(name=evil, track=evil, notProven=[evil], brief={"found": True, "count": 900, "items": [{"text": evil, "where": evil}]},
+ evidence={"tests": {"executed": 1}, "rules": {"p0": [{"id": evil, "name": evil, "status": "__proto__", "tests": 1, "main": 1, "where": evil}] * 200}}),
+ dict(good_pack(), legacy={"path": evil, "target": evil, "ran": False}, problems=[evil], signoff={"name": evil, "role": "", "date": "", "decision": ""})]
+ packs.append(good_uplift_pack(deltas={"parsed": 5, "silent": 2, "covered": [{"id": "1"}], "uncovered": [{"id": "2", "title": "Rounding changed", "why": "no test names Money", "sites": ["Money.java:4"]}],
+ "config": [{"id": "4", "title": "Bundle plugin", "sites": ["pom.xml:9"]}]},
+ testsChanged={"checked": True, "legacyTests": 8, "workTests": 8, "counts": {"added": 0, "removed": 1, "changed": 0}, "share": 0.0,
+ "removed": [{"path": "t/GoneTest.java", "lines": 7}], "added": [], "changed": []},
+ baselineEvidence={"kind": "typed", "sources": [], "disagree": 0, "conflict": 0, "examples": [], "considered": [], "ignored": ["counts.json"]}))
+ views = [br.proof_view(p) for p in packs]
+ views[3]["verdict"] = "__proto__"
+ views[3]["modules"][0]["verdict"] = "constructor"
+ proc = subprocess.run([NODE, "-e", NODE_PROOF], input=json.dumps(views), capture_output=True, text=True, env=dict(os.environ, TEMPLATE=TEMPLATE), timeout=120)
+ assert proc.returncode == 0, proc.stderr
+ cls.out, cls.evil = json.loads(proc.stdout), evil
+
+ def test_only_safe_elements_and_attributes_are_built(self):
+ allowed_tags = {"div", "table", "thead", "tbody", "tr", "th", "td", "span", "p", "h3", "h4", "ul", "li", "details", "summary"}
+ allowed_attrs = {"class", "role", "tabindex", "aria-label", "scope"}
+ for doc in self.out[:-1]:
+ self.assertLessEqual(set(doc["tags"]), allowed_tags)
+ self.assertLessEqual(set(doc["attrs"]), allowed_attrs)
+ self.assertLess(doc["ms"], 3000)
+
+ def test_hostile_text_is_shown_as_text_never_as_markup_or_a_link(self):
+ doc = self.out[3]
+ self.assertIn("<script>alert(1)</script>", doc["text"])
+ self.assertIn("[x](javascript:alert(3))", doc["text"])
+ self.assertNotIn("a", doc["tags"])
+ self.assertNotIn("img", doc["tags"])
+
+ def test_the_banner_says_the_verdict_in_words_symbols_and_colour(self):
+ self.assertIn("โœ“ Proof: 1 PROVEN", self.out[0]["text"])
+ self.assertEqual(self.out[0]["banners"], ["banner ok", "banner ok"])
+ self.assertEqual(self.out[1]["banners"], ["banner warn", "banner warn"])
+ self.assertEqual(self.out[2]["banners"], ["banner bad", "banner bad"])
+ self.assertIn("โ— ", self.out[1]["text"])
+ self.assertIn("โœ— ", self.out[2]["text"])
+
+ def test_unknown_verdicts_and_prototype_names_fall_back_to_red(self):
+ self.assertEqual(self.out[3]["banners"], ["banner bad", "banner bad"])
+ self.assertEqual(self.out[-1]["own"], ["x", "d", "d", "d"])
+
+ def test_the_page_shows_the_matrix_what_is_not_proven_and_a_blank_sign_off(self):
+ text = self.out[0]["text"]
+ for needle in ("What this does not prove", "It does not prove inputs nobody tried.", "P0 business rules: 1 tested, 0 named but not run, 0 claimed only, 0 with no test", "RULE-001",
+ "Waiting for a person", "A person accepts it", "Sign-off", "________________", "accept / accept with conditions / reject", "How the verdict is computed",
+ "Fresh inputs: 12 new input(s)", "tests executed: 5"):
+ self.assertIn(needle, text)
+ self.assertIn("It could not run here, so the proof is trace-based.", self.out[4]["text"])
+
+ def test_an_uplift_shows_its_baseline_test_files_and_deltas(self):
+ text = self.out[5]["text"]
+ for needle in ("Baseline evidence: typed by hand", "Test files: 8 in the legacy tree, 8 in the working copy: 1 removed, 0 added, 0 changed", "Removed test files", "t/GoneTest.java (7 lines)",
+ "2 Behavioral-silent: 1 have a test naming their site, 1 do not", "Not named by any test", "2 Rounding changed (no test names Money)",
+ "At a build or configuration file (for a person)", "4 Bundle plugin (pom.xml:9)", "Baseline measured", "Tests kept", "Deltas covered"):
+ self.assertIn(needle, text)
+ self.assertNotIn("Baseline evidence: typed by hand", self.out[0]["text"])
+
+ def test_the_template_keeps_its_own_contract(self):
+ with open(TEMPLATE, encoding="utf-8") as fh:
+ text = fh.read()
+ app = re.search(r'<script id="app">(.*?)</script>', text, re.S).group(1)
+ self.assertEqual(len(re.findall(r"/\* proof:start", app)), 1)
+ self.assertNotRegex(app, r"innerHTML|outerHTML|insertAdjacentHTML|document\.write|eval\(|new Function|fetch\(|XMLHttpRequest")
+ self.assertNotRegex(text, r"https?://")
+ self.assertIn("case 'proof': return partProof(p.proof);", app)
diff --git a/plugins/code-modernization/scripts/tests/test_report_next.py b/plugins/code-modernization/scripts/tests/test_report_next.py
new file mode 100644
index 00000000..e73536ba
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_report_next.py
@@ -0,0 +1,43 @@
+"""Tests for the report's next-step decision (scripts/build_report.py next_step)."""
+import os
+import sys
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, os.path.join(os.path.dirname(HERE)))
+import build_report as br # noqa: E402
+
+ALL = dict(preflight=True, assess=True, map=True, rules=True, brief=True, build=True, harden=True)
+
+
+class NextStep(unittest.TestCase):
+ def cmd(self, done, glance=None):
+ return br.next_step("sys", dict(ALL, **done), glance or {})["command"]
+
+ def test_the_path_in_order(self):
+ order = ["preflight", "assess", "map", "rules"]
+ for i, key in enumerate(order):
+ done = {k: False for k in order[i:]}
+ done.update(brief=False, build=False, harden=False)
+ self.assertTrue(self.cmd(done).startswith("/code-modernization:modernize-" + ["preflight", "assess", "map", "extract-rules"][i]))
+
+ def test_flagged_rules_go_to_review_once_and_then_on(self):
+ flagged = {"rules": {"defects": 2}}
+ self.assertIn("modernize-review", self.cmd({"brief": False}, flagged))
+ reviewed = {"rules": {"defects": 2, "reviews": {"confirmed": 1}}}
+ self.assertIn("modernize-brief", self.cmd({"brief": False}, reviewed))
+
+ def test_a_build_is_verified_before_anything_else(self):
+ self.assertIn("modernize-verify", self.cmd({"harden": False}))
+ self.assertIn("modernize-verify", self.cmd({"harden": False}, {"proof": {"verdict": "NOT PROVEN"}}))
+ self.assertIn("modernize-harden", self.cmd({"harden": False}, {"proof": {"verdict": "PROVEN"}}))
+
+ def test_everything_done_points_at_status_and_text_is_bounded(self):
+ step = br.next_step("s", ALL, {"proof": {"verdict": "PROVEN"}})
+ self.assertIn("modernize-status", step["command"])
+ hostile = br.next_step("s", ALL, {"proof": {"verdict": "X" * 5000}})
+ self.assertLess(len(hostile["why"]), 200)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_report_tools.py b/plugins/code-modernization/scripts/tests/test_report_tools.py
new file mode 100644
index 00000000..7d3acec4
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_report_tools.py
@@ -0,0 +1,791 @@
+"""Tests for scripts/build_report.py, scripts/compare.py and assets/report-template.html.
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+The template's JavaScript is exercised with Node when it is installed (those tests are skipped otherwise).
+"""
+import base64
+import contextlib
+import hashlib
+import io
+import json
+import os
+import re
+import shutil
+import stat
+import subprocess
+import sys
+import tempfile
+import unittest
+from html.parser import HTMLParser
+from unittest import mock
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import build_report as br # noqa: E402
+import compare as cmp # noqa: E402
+
+TEMPLATE = os.path.join(PLUGIN, "assets", "report-template.html")
+MERMAID = os.path.join(PLUGIN, "assets", "vendor", "mermaid.min.js")
+SECRET = "TOP-SECRET-MARKER-9f3c"
+LS, PS = chr(0x2028), chr(0x2029)
+
+
+def put(root, rel, data):
+ path = os.path.join(root, *rel.split("/"))
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "wb") as fh:
+ fh.write(data if isinstance(data, bytes) else data.encode("utf-8"))
+ return path
+
+
+def run_main(fn, argv):
+ out, err = io.StringIO(), io.StringIO()
+ with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err):
+ code = fn(argv)
+ return code, out.getvalue(), err.getvalue()
+
+
+def make_symlink(test, target, link):
+ try:
+ os.symlink(target, link)
+ except (OSError, NotImplementedError, AttributeError):
+ test.skipTest("symbolic links are not available here")
+
+
+class Scripts(HTMLParser):
+ """Every <script> element of a page: its attributes and its text."""
+
+ def __init__(self, html):
+ super().__init__(convert_charrefs=False)
+ self.found, self._cur = [], None
+ self.feed(html)
+
+ def handle_starttag(self, tag, attrs):
+ if tag == "script":
+ self._cur = {"attrs": dict(attrs), "text": ""}
+
+ def handle_data(self, data):
+ if self._cur is not None:
+ self._cur["text"] += data
+
+ def handle_endtag(self, tag):
+ if tag == "script" and self._cur is not None:
+ self.found.append(self._cur)
+ self._cur = None
+
+
+def page_data(html):
+ block = [s for s in Scripts(html).found if s["attrs"].get("id") == "report-data"]
+ assert len(block) == 1
+ return json.loads(block[0]["text"])
+
+
+HOSTILE_MD = "\n".join([
+ "# Assessment <script>window.__pwned=1</script>",
+ '<img src=x onerror="window.__pwned=2">',
+ "</script><script>window.__pwned=3</script> <!-- and <!--",
+ "[a](javascript:window.__pwned=4) [b](JavaScript:alert(1)) [c](java\tscript:alert(1)) [d](" + chr(1) + "javascript:alert(1))",
+ "[e](data:text/html,<script>1</script>) [f](//evil.example/x) [g](/\\evil.example) ![t](https://evil.example/pixel.png)",
+ '<a href="javascript:alert(1)">raw anchor</a> <iframe src="https://evil.example"></iframe>',
+ "| a | b |", "|---|---|", "| <b onmouseover=alert(1)>x</b> | <img src=x onerror=window.__pwned=5> |",
+ "`<script>` and **<img src=x onerror=1>** " + LS + PS,
+ "```mermaid", 'graph TD; A["<img src=x onerror=window.__pwned=7>"]-->B', "```",
+ "x" * 1000000, "*" * 200000, "[" * 50000, "|" * 50000,
+])
+HOSTILE_RULES = "\n".join([
+ "# Rules", "",
+ "### RULE-001: Title with `backticks` and <angle> <script>x</script> & ampersand",
+ "**Category:** Validation ", "**Priority:** P0 ", "**Source:** `evil/../../etc/passwd:1-2` and `a.cbl:10` ",
+ "**Confidence:** High ", "**Suspected defect:** <img src=x onerror=window.__pwned=6>", "",
+ "### P1-002 \u00b7 Second card, other id style", "P1 \u00b7 Policy \u00b7 confidence Low \u00b7 `b.cbl:5-9`", "",
+])
+
+
+def hostile_workspace(root, system="evil"):
+ a = "analysis/" + system
+ put(root, a + "/ASSESSMENT.md", HOSTILE_MD)
+ put(root, a + "/BUSINESS_RULES.md", HOSTILE_RULES)
+ put(root, a + "/MODERNIZATION_BRIEF.md", "# Brief\n\n```mermaid\n%%{init: {\"themeCSS\": \".x{background:url(https://evil.example/t.gif)}\"}}%%\ngraph TD; A-->B\n```\n")
+ put(root, a + "/good.mmd", "graph TD\n A[Start] --> B[End]\n")
+ put(root, a + "/bad.mmd", "graph TD\n A --> --> ((\n")
+ put(root, a + "/topology.json", '{"root": {"id": ')
+ put(root, a + "/RULE_REVIEWS.json", "not json at all")
+ put(root, a + "/SECRETS.local.md", SECRET)
+ put(root, a + "/security_remediation.local.patch", SECRET)
+ put(root, a + "/SECURITY_FINDINGS.md", "# Findings\n\n| ID | Severity | Title |\n|---|---|---|\n| SEC-1 | Critical | <script>x</script> |\n| SEC-2 | High | y |\n| SEC-3 | high | z |\n")
+ put(root, "modernized/%s/README.md" % system, "# Module <img src=x onerror=1>\n")
+ put(root, "modernized/%s/mod1/TRANSFORMATION_NOTES.md" % system, "# Notes\n</script>\n")
+ outside = tempfile.mkdtemp()
+ put(outside, "secret.txt", "OUTSIDE-MARKER")
+ try:
+ os.symlink(os.path.join(outside, "secret.txt"), os.path.join(root, *(a + "/PLAYBOOK.md").split("/")))
+ except (OSError, NotImplementedError, AttributeError):
+ pass
+ return outside
+
+
+class CompareTests(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+
+ def cases(self, cases, **extra):
+ path = put(self.tmp, "cases.json", json.dumps({"system": "s", "cases": cases, **extra}))
+ return path
+
+ def files(self, name, legacy, new):
+ put(self.tmp, "legacy/%s" % name, legacy)
+ if new is not None:
+ put(self.tmp, "new/%s" % name, new)
+ return {"id": name, "title": "case " + name, "legacy": "legacy/" + name, "new": "new/" + name}
+
+ def result(self, cases, **extra):
+ out = os.path.join(self.tmp, "analysis", "EQUIVALENCE.json")
+ code, stdout, _ = run_main(cmp.main, [self.cases(cases, **extra), "--out", out])
+ with open(out, encoding="utf-8") as fh:
+ return code, stdout, json.load(fh)
+
+ def test_same_differs_missing_approved_and_exit_codes(self):
+ same, diff, miss = self.files("a", b"one\ntwo\n", b"one\ntwo\n"), self.files("b", b"one\ntwo\n", b"one\ntw0\n"), self.files("c", b"x", None)
+ code, out, res = self.result([same, diff, miss])
+ self.assertEqual(code, 1)
+ self.assertEqual([c["verdict"] for c in res["cases"]], ["same", "differs", "missing"])
+ self.assertEqual(res["totals"], {"cases": 3, "executed": 2, "same": 1, "differs": 1, "differsApproved": 0, "missing": 1})
+ self.assertIn("equivalence cases executed: 2", out)
+ self.assertIn("NOT PROVEN", out)
+ d = res["cases"][1]["firstDiff"]
+ self.assertEqual((d["offset"], d["line"]), (6, 2))
+ self.assertIn("legacy", res["cases"][2]["reason"].lower() + "legacy")
+ self.assertEqual(res["cases"][0]["legacySha256"], hashlib.sha256(b"one\ntwo\n").hexdigest())
+ approved = dict(diff, id="b2", approvedDifference="fixed on purpose")
+ code, _, res = self.result([same, approved])
+ self.assertEqual(code, 0)
+ self.assertEqual(res["cases"][1]["verdict"], "differs-approved")
+ self.assertEqual(res["totals"]["differsApproved"], 1)
+
+ def test_all_same_is_proven(self):
+ code, out, res = self.result([self.files("a", b"abc", b"abc")])
+ self.assertEqual(code, 0)
+ self.assertIn("PROVEN", out)
+ self.assertNotIn("NOT PROVEN", out)
+ self.assertEqual(res["selfCheck"]["passed"], True)
+
+ def test_missing_alone_is_never_a_pass(self):
+ code, out, res = self.result([self.files("a", b"x", b"x"), self.files("m", b"x", None)])
+ self.assertEqual(code, 1)
+ self.assertIn("missing", out)
+ self.assertEqual((res["totals"]["executed"], res["totals"]["missing"], res["totals"]["differs"]), (1, 1, 0))
+
+ def test_no_case_executed_is_not_proven(self):
+ for cases in ([], [self.files("m", b"x", None)]):
+ code, out, res = self.result(cases)
+ self.assertEqual(code, 1)
+ self.assertIn("equivalence cases executed: 0", out)
+ self.assertIn("NOT PROVEN: no case executed", out)
+ self.assertEqual(res["selfCheck"]["passed"], None)
+
+ def test_empty_outputs_prove_nothing(self):
+ code, out, res = self.result([self.files("e", b"", b"")])
+ self.assertEqual(code, 1)
+ self.assertIn("empty", out)
+ self.assertEqual(res["cases"][0]["verdict"], "same")
+ self.assertTrue(res["cases"][0]["empty"])
+
+ def test_masks_hide_only_what_they_name(self):
+ legacy, new = b"id=12 at 2026-09-01 10:11:12 total=5\n", b"id=1234 at 2026-09-24 03:04:05 total=5\n"
+ case = dict(self.files("t", legacy, new), mask=[{"regex": r"id=\d+"}, {"regex": r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d"}])
+ code, _, res = self.result([case])
+ self.assertEqual((code, res["cases"][0]["verdict"]), (0, "same"))
+ case["new"] = self.files("t2", legacy, new.replace(b"total=5", b"total=6"))["new"]
+ _, _, res = self.result([case])
+ self.assertEqual(res["cases"][0]["verdict"], "differs")
+ self.assertEqual(res["cases"][0]["firstDiff"]["offset"], legacy.index(b"5"))
+ ranged = dict(self.files("r", b"aaXXbb", b"aaYYbb"), mask=[{"bytes": "2-3"}])
+ self.assertEqual(self.result([ranged])[2]["cases"][0]["verdict"], "same")
+ beyond = dict(self.files("r2", b"aaXXbb", b"aaYYbc"), mask=[{"bytes": "2-3"}, {"bytes": "50-60"}])
+ self.assertEqual(self.result([beyond])[2]["cases"][0]["verdict"], "differs")
+
+ def test_a_mask_that_hides_everything_fails_the_self_check(self):
+ case = dict(self.files("h", b"abc", b"xyz"), mask=[{"regex": ".*"}])
+ code, out, res = self.result([case])
+ self.assertEqual(code, 1)
+ self.assertIs(res["selfCheck"]["passed"], False)
+ self.assertIn("hide every byte", res["selfCheck"]["detail"])
+ self.assertIn("NOT PROVEN", out)
+
+ def test_bad_input_is_exit_2_and_writes_nothing(self):
+ out = os.path.join(self.tmp, "EQ.json")
+ bad = [put(self.tmp, "a.json", "{not json"), put(self.tmp, "b.json", json.dumps({"cases": "x"})),
+ self.cases([dict(self.files("z", b"a", b"a"), mask=[{"regex": "("}])]), self.cases([dict(self.files("y", b"a", b"a"), mask=[{"bytes": "9-2"}])]),
+ self.cases([self.files("d", b"a", b"a"), self.files("d", b"a", b"a")]), os.path.join(self.tmp, "missing.json")]
+ for path in bad:
+ code, _, err = run_main(cmp.main, [path, "--out", out])
+ self.assertEqual(code, 2, path)
+ self.assertFalse(os.path.exists(out))
+ self.assertIn("compare.py", err)
+
+ def test_paths_may_not_leave_the_cases_folder(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "o.txt", "same")
+ sub = os.path.join(self.tmp, "sub")
+ put(sub, "n.txt", "same")
+ case = {"id": "o", "legacy": os.path.join(outside, "o.txt"), "new": "n.txt"}
+ path = put(sub, "cases.json", json.dumps({"cases": [case]}))
+ out = os.path.join(sub, "E.json")
+ self.assertEqual(run_main(cmp.main, [path, "--out", out, "--quiet"])[0], 1)
+ with open(out) as fh:
+ self.assertEqual(json.load(fh)["cases"][0]["verdict"], "missing")
+ self.assertEqual(run_main(cmp.main, [path, "--out", out, "--quiet", "--allow-outside"])[0], 0)
+
+ def test_first_difference_context_is_escaped_and_bounded(self):
+ _, _, res = self.result([self.files("c", b"a\x00\x1b[31m\tb" + b"z" * 200, b"a\x00\x1b[31m\tc" + b"z" * 200)])
+ d = res["cases"][0]["firstDiff"]
+ self.assertLessEqual(len(d["legacy"]), 40)
+ self.assertNotRegex(d["legacy"] + d["new"], r"[\x00-\x1f\x7f]")
+ self.assertEqual(d["legacy"][d["at"]], "b")
+ _, _, res = self.result([self.files("p", b"abc", b"abcdef")])
+ self.assertIn("ends", res["cases"][0]["reason"])
+ self.assertIn("<end of output>", res["cases"][0]["firstDiff"]["legacy"])
+
+ def test_summary_neutralises_terminal_escapes_and_quiet_is_quiet(self):
+ case = dict(self.files("q", b"a", b"b"), id="q\x1b[2J")
+ code, out, _ = run_main(cmp.main, [self.cases([case]), "--out", os.path.join(self.tmp, "E.json")])
+ self.assertNotIn("\x1b", out)
+ code, out, _ = run_main(cmp.main, [self.cases([case]), "--out", os.path.join(self.tmp, "E.json"), "--quiet"])
+ self.assertEqual((code, out), (1, ""))
+
+ def test_default_output_and_atomic_write(self):
+ code, _, _ = run_main(cmp.main, [self.cases([self.files("a", b"x", b"x")]), "--quiet"])
+ self.assertEqual(code, 0)
+ self.assertEqual(sorted(f for f in os.listdir(self.tmp) if "equivalence" in f.lower()), ["EQUIVALENCE.json"])
+
+ def test_a_symlinked_output_is_refused(self):
+ real = put(self.tmp, "elsewhere/E.json", "ORIGINAL")
+ link = os.path.join(self.tmp, "E.json")
+ make_symlink(self, real, link)
+ code, _, err = run_main(cmp.main, [self.cases([self.files("a", b"x", b"x")]), "--out", link, "--quiet"])
+ self.assertEqual(code, 2)
+ self.assertIn("symbolic link", err)
+ with open(real) as fh:
+ self.assertEqual(fh.read(), "ORIGINAL")
+
+
+class BuildFacts(unittest.TestCase):
+ def test_rule_cards_in_several_layouts(self):
+ text = "\n".join([
+ "## Rules", "### RULE-012: Interest is truncated", "**Priority:** P0", "**Confidence:** High \u2014 checked", "**Category:** Calculation",
+ "**Source:** `a/B.cbl:10-20,30` and `C.cbl:5`", "**Suspected defect:** rounds down", "```", "### not a heading in a fence", "```", "",
+ "### P1-001 \u00b7 Header line style", "`P1` \u00b7 Validation \u00b7 confidence Medium \u00b7 `d/E.cbl:1-2`", "",
+ "#### R-12: Short id", "Priority: P2", "**Suspected defect:** none", "", "### BR-D7-09 \u2014 Domain coded", "### Plain heading", "text",
+ ])
+ blocks = br.parse_rules(text)
+ cards = [b for b in blocks if b["k"] == "rule"]
+ self.assertEqual([c["id"] for c in cards], ["RULE-012", "P1-001", "R-12", "BR-D7-09"])
+ self.assertEqual([c["p"] for c in cards], ["P0", "P1", "P2", ""])
+ self.assertEqual([c["conf"] for c in cards], ["High", "Medium", "", ""])
+ self.assertEqual([c["defect"] for c in cards], [True, False, False, False])
+ self.assertEqual(cards[0]["cites"], ["a/B.cbl:10-20,30", "C.cbl:5"])
+ self.assertEqual(cards[1]["cites"], ["d/E.cbl:1-2"])
+ self.assertNotIn("Priority", cards[0]["body"])
+ self.assertIn("### not a heading in a fence", cards[0]["body"])
+ self.assertEqual(blocks[0]["k"], "md")
+ self.assertEqual(br.parse_rules("just text\n\n## Heading")[0]["k"], "md")
+
+ def test_headings_and_fields_padded_with_thousands_of_spaces_are_parsed_in_a_moment(self):
+ import time
+ pad = " " * 6000
+ text = "### RULE-001 - Title%s#%sx\n**Priority:** P0%s\n**Suspected defect:**%s\n\n### RULE-002: Closed ### \n**Priority:** P1 \n" % (pad, pad, pad, pad)
+ started = time.time()
+ cards = [c for c in br.parse_rules(text) if c["k"] == "rule"]
+ self.assertLess(time.time() - started, 2)
+ self.assertEqual([c["id"] for c in cards], ["RULE-001", "RULE-002"])
+ self.assertEqual(cards[1]["title"], "Closed")
+ self.assertEqual(cards[1]["p"], "P1")
+ self.assertEqual(br.field("Priority: \nPriority: P2 ", "Priority"), "P2")
+
+ def test_rule_priority_falls_back_to_the_index_table(self):
+ text = "| ID | Name | Priority |\n|---|---|---|\n| [RULE-007](#x) | Merged | P0 |\n\n### RULE-007: Merged\n**Merged into:** RULE-006\n\n### RULE-008: Real\n**Priority:** P1\n"
+ cards = [b for b in br.parse_rules(text) if b["k"] == "rule"]
+ facts = br.rule_facts(text, cards, {"RULE-008": {"verdict": "confirmed"}})
+ self.assertEqual((facts["total"], facts["byPriority"], facts["unrated"]), (2, {"P0": 1, "P1": 1}, 0))
+ self.assertEqual(facts["reviews"], {"confirmed": 1})
+
+ def test_security_tally(self):
+ findings = "## Summary scorecard\n| Severity | Count |\n|---|---|\n| Critical | 9 |\n\n## Findings\n| ID | Severity | Title |\n|---|---|---|\n| S1 | **Critical** | a |\n| S2 | High | b |\n| S3 | high | c |\n| S4 | Low | d |\n\n## Dependency CVEs\n| Package | Installed | CVE | Severity |\n|---|---|---|---|\n| x | 1 | CVE-1 | High |\n\n## Coverage gaps\n| Finding | Severity |\n|---|---|\n| n | Medium |\n"
+ self.assertEqual(br.security_facts(findings), {"Critical": 1, "High": 2, "Low": 1})
+ self.assertEqual(br.security_facts("| Severity | Count |\n|---|---|\n| Critical | 2 |\n| High | 3 |\n"), {"Critical": 2, "High": 3})
+ self.assertIsNone(br.security_facts("Nothing recognisable here."))
+
+ def test_baseline_facts(self):
+ table = "| Test | Result |\n|---|---|\n| a | pass |\n| b | PASS |\n| c | fail |\n| d | skipped |\n"
+ self.assertEqual(br.baseline_facts(table), {"pass": 2, "fail": 1, "skip": 1, "targetOnly": False})
+ self.assertEqual(br.baseline_facts("Run: 12 tests passed, 1 failed"), {"pass": 12, "fail": 1, "targetOnly": False})
+ self.assertEqual(br.baseline_facts("target-only: the source runtime is not available")["targetOnly"], True)
+ self.assertIsNone(br.baseline_facts("| Fixture | RC |\n|---|---|\n| seed | 0 |\n"))
+
+ def test_topology_facts(self):
+ topo = {"root": {"id": "s", "kind": "system", "children": [{"id": "d", "kind": "domain", "children": [
+ {"id": "A", "kind": "module", "loc": 100, "language": "cobol", "file": "a.cbl"}, {"id": "B", "kind": "module", "loc": 50, "file": "b.cbl"},
+ {"id": "F", "kind": "datastore"}]}]}, "edges": [{}, {}]}
+ self.assertEqual(br.topology_facts(topo), {"modules": 2, "loc": 150, "languages": ["cobol"], "edges": 2, "observations": []})
+ self.assertIsNone(br.topology_facts([1, 2]))
+ self.assertIsNone(br.topology_facts({"root": {"id": "x", "children": "oops"}}))
+
+ def test_equivalence_view_never_trusts_the_files_own_words(self):
+ digest = "a" * 64
+ good = {"cases": [{"id": "1", "verdict": "same", "legacySha256": digest, "newSha256": digest}, {"id": "2", "verdict": "differs-approved", "approvedDifference": "ok"}],
+ "selfCheck": {"passed": True, "detail": "d"}, "totals": {"cases": 2, "executed": 2}}
+ view = br.equivalence_view(good)
+ self.assertEqual((view["state"], view["problems"]), ("green", []))
+ self.assertEqual(br.equivalence_view(dict(good, cases=[dict(good["cases"][0], newSha256="b" * 64)]))["state"], "red")
+ for bad_case in ({"id": "x", "verdict": "differs"}, {"id": "x", "verdict": "missing"}, {"id": "x", "verdict": "pass"},
+ {"id": "x", "verdict": "differs-approved"}, {"id": "x", "verdict": "same", "firstDiff": {"offset": 1}}):
+ view = br.equivalence_view(dict(good, cases=good["cases"] + [bad_case], totals={}))
+ self.assertEqual(view["state"], "red", bad_case)
+ self.assertEqual(br.equivalence_view(dict(good, cases=[]))["state"], "red")
+ self.assertEqual(br.equivalence_view(dict(good, selfCheck={"passed": False, "detail": "x"}))["state"], "red")
+ self.assertEqual(br.equivalence_view(dict(good, cases=[dict(good["cases"][0], empty=True)]))["state"], "red")
+ no_check = br.equivalence_view({"cases": good["cases"]})
+ self.assertEqual(no_check["state"], "green")
+ self.assertTrue(any("self-check" in p for p in no_check["problems"]))
+ self.assertTrue(any("disagree" in p for p in br.equivalence_view(dict(good, totals={"cases": 9, "executed": 9}))["problems"]))
+ self.assertIsNone(br.equivalence_view({"cases": "x"}))
+ self.assertIsNone(br.equivalence_view([]))
+
+
+class BuildReport(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+ self.out = os.path.join(self.tmp, "out", "REPORT.html")
+
+ def build(self, system="s", extra=()):
+ code, out, err = run_main(br.main, [system, "--workspace", self.tmp, "--out", self.out, *extra])
+ return code, out, err
+
+ def html(self):
+ with open(self.out, encoding="utf-8") as fh:
+ return fh.read()
+
+ def test_nothing_found_is_exit_1_with_one_line(self):
+ os.makedirs(os.path.join(self.tmp, "analysis", "s"))
+ code, out, err = self.build()
+ self.assertEqual(code, 1)
+ self.assertEqual(len(err.strip().splitlines()), 1)
+ self.assertFalse(os.path.exists(self.out))
+
+ def test_bad_system_names_are_refused(self):
+ for name in ("..", "a/b", "a\\b", ""):
+ self.assertEqual(self.build(name)[0], 2, name)
+
+ def test_nearly_empty_workspace(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "# Preflight\nAll good.\n")
+ code, out, _ = self.build()
+ self.assertEqual(code, 0)
+ self.assertRegex(out.strip(), r"REPORT\.html: 1 section, \d+ KB$")
+ html = self.html()
+ self.assertEqual([s["attrs"].get("id") for s in Scripts(html).found], ["report-data", "app"])
+ data = page_data(html)
+ self.assertEqual([s["id"] for s in data["sections"]], ["overview"])
+ self.assertFalse(any(x["done"] for x in data["glance"]["steps"] if x["key"] != "preflight"))
+ self.assertIsNone(data["glance"]["track"])
+ self.assertIsNone(data["glance"]["equivalence"])
+
+ def test_default_output_path(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ code, out, _ = run_main(br.main, ["s", "--workspace", self.tmp])
+ self.assertEqual(code, 0)
+ self.assertTrue(os.path.isfile(os.path.join(self.tmp, "analysis", "s", "REPORT.html")))
+ self.assertEqual([f for f in os.listdir(os.path.join(self.tmp, "analysis", "s")) if f.endswith(".tmp")], [])
+
+ def test_malformed_files_are_reported_in_the_page_not_raised(self):
+ a = "analysis/s/"
+ put(self.tmp, a + "topology.json", "{broken")
+ put(self.tmp, a + "EQUIVALENCE.json", "[1, 2")
+ put(self.tmp, a + "RULE_REVIEWS.json", '{"reviews": 5}')
+ put(self.tmp, a + "BUSINESS_RULES.md", b"\xff\xfe\x00binary\x00\x01### RULE-001: x\n")
+ put(self.tmp, a + "ASSESSMENT.md", b"")
+ put(self.tmp, a + "BASELINE.md", "| a |\n|---")
+ put(self.tmp, a + "SECURITY_FINDINGS.md", "|||\n|||\n")
+ put(self.tmp, a + "broken.mmd", "")
+ code, _, _ = self.build()
+ self.assertEqual(code, 0)
+ data = page_data(self.html())
+ notes = json.dumps(data["sections"][0]["parts"])
+ self.assertIn("topology.json could not be parsed", notes)
+ self.assertIn("EQUIVALENCE.json could not be parsed", notes)
+ self.assertEqual(data["glance"]["equivalence"]["state"], "red")
+ self.assertIn("RULE_REVIEWS.json", notes)
+
+ def test_hostile_input_cannot_reach_the_page_as_markup_or_script(self):
+ outside = hostile_workspace(self.tmp)
+ self.addCleanup(shutil.rmtree, outside, True)
+ code, _, _ = self.build("evil")
+ self.assertEqual(code, 0)
+ html = self.html()
+ scripts = Scripts(html).found
+ ids = [s["attrs"].get("id") for s in scripts]
+ self.assertEqual(ids, ["report-data", "mermaid-lib", "app"])
+ for s in scripts:
+ self.assertLessEqual(set(s["attrs"]), {"id", "type"})
+ block = scripts[0]["text"]
+ for bad in ("<", ">", "&", LS, PS):
+ self.assertNotIn(bad, block)
+ self.assertNotIn("</script", scripts[1]["text"].lower())
+ self.assertNotIn("<!--", scripts[1]["text"])
+ page = html.replace(scripts[0]["text"], "").replace(scripts[1]["text"], "")
+ for needle in ("__pwned=", "onerror", "evil.example", "javascript:"):
+ self.assertNotIn(needle, page.split('<script id="app">')[0].split("</style>")[1], needle)
+ text = json.dumps(page_data(html))
+ self.assertIn("window.__pwned=1", text)
+ self.assertNotIn(SECRET, html)
+ self.assertNotIn("OUTSIDE-MARKER", html)
+ notes = json.dumps(page_data(html)["sections"][0]["parts"])
+ if os.path.islink(os.path.join(self.tmp, "analysis", "evil", "PLAYBOOK.md")):
+ self.assertIn("PLAYBOOK.md is a link", notes)
+ self.assertIn("longer than 20,000 characters were shortened", notes)
+ self.assertLess(len(html), 6 << 20)
+ data = page_data(html)
+ self.assertEqual(data["glance"]["security"], {"Critical": 1, "High": 2})
+ self.assertEqual(data["glance"]["rules"]["total"], 2)
+ titles = [b["title"] for s in data["sections"] for p in s["parts"] for b in p.get("blocks", []) if b["k"] == "rule"]
+ self.assertIn("Title with `backticks` and <angle> <script>x</script> & ampersand", titles)
+
+ def test_secret_and_local_files_are_never_read_even_when_asked_for_by_name(self):
+ for name in ("SECRETS.local.md", "secrets.txt", "security_remediation.local.patch", "notes.LOCAL.md"):
+ path = put(self.tmp, "analysis/s/" + name, SECRET)
+ src = br.Source(self.tmp, "s")
+ self.assertIsNone(src.read(path, name), name)
+ self.assertEqual(src.found, [])
+
+ def test_csp_pins_exactly_the_scripts_on_the_page(self):
+ put(self.tmp, "analysis/s/good.mmd", "graph TD\n A-->B\n")
+ self.build()
+ html = self.html()
+ policy = re.search(r'http-equiv="Content-Security-Policy" content="([^"]*)"', html).group(1)
+ self.assertIn("default-src 'none'", policy)
+ self.assertNotIn("unsafe-eval", policy)
+ self.assertNotRegex(policy, r"script-src[^;]*unsafe-inline")
+ hashes = re.findall(r"'sha256-([^']+)'", policy.split("script-src")[1].split(";")[0])
+ code_scripts = [s for s in Scripts(html).found if s["attrs"].get("type") != "application/json"]
+ expected = [base64.b64encode(hashlib.sha256(s["text"].encode("utf-8")).digest()).decode() for s in code_scripts]
+ self.assertEqual(sorted(hashes), sorted(expected))
+ self.assertEqual(len(expected), 2)
+
+ def test_mermaid_is_inlined_only_when_a_diagram_exists(self):
+ put(self.tmp, "analysis/s/ASSESSMENT.md", "# A\nno diagram\n")
+ self.build()
+ self.assertNotIn("mermaid-lib", self.html())
+ self.assertLess(len(self.html()), 300000)
+ put(self.tmp, "analysis/s/ASSESSMENT.md", "# A\n\n```mermaid\ngraph TD\n A-->B\n```\n")
+ self.build()
+ scripts = Scripts(self.html()).found
+ lib = next(s for s in scripts if s["attrs"].get("id") == "mermaid-lib")
+ with open(MERMAID, encoding="utf-8") as fh:
+ original = fh.read()
+ self.assertEqual(lib["text"].replace("<\\/script", "</script").replace("<\\!--", "<!--"), original)
+ os.remove(os.path.join(self.tmp, "analysis", "s", "ASSESSMENT.md"))
+ put(self.tmp, "analysis/s/x.mmd", "graph TD\n A-->B\n")
+ self.build()
+ self.assertIn("mermaid-lib", self.html())
+
+ def test_size_cap_says_what_was_left_out(self):
+ for name in ("ASSESSMENT.md", "MODERNIZATION_BRIEF.md", "DATA_OBJECTS.md"):
+ put(self.tmp, "analysis/s/" + name, ("line of text that is long enough to count\n" * 80000))
+ self.build()
+ data = page_data(self.html())
+ notes = "\n".join(json.dumps(p) for p in data["sections"][0]["parts"])
+ self.assertIn("report size limit", notes)
+ embedded = sum(len(p["text"]) for s in data["sections"] for p in s["parts"] if p["t"] == "md")
+ self.assertLessEqual(embedded, br.TOTAL_CAP)
+
+ def test_sections_follow_the_workflow_order_and_only_exist_when_found(self):
+ a = "analysis/s/"
+ for name in ("EQUIVALENCE.json", "SECURITY_FINDINGS.md", "MODERNIZATION_BRIEF.md", "ASSESSMENT.md", "PREFLIGHT.md", "DELTA_CATALOG.md", "AI_NATIVE_SPEC.md"):
+ put(self.tmp, a + name, "{}" if name.endswith(".json") else "# " + name)
+ put(self.tmp, "modernized/s-reimagined/svc/README.md", "# svc")
+ self.build()
+ data = page_data(self.html())
+ self.assertEqual([s["id"] for s in data["sections"]], ["overview", "assessment", "brief", "security", "delta", "spec", "build", "equivalence"])
+ steps = {x["key"]: x["done"] for x in data["glance"]["steps"]}
+ self.assertEqual(steps, {"preflight": True, "assess": True, "map": False, "rules": False, "brief": True, "build": True, "harden": True})
+ self.assertEqual(data["glance"]["track"]["label"], "reimagine")
+
+ def test_track_follows_the_newest_track_artifact(self):
+ a = "analysis/s/"
+ put(self.tmp, a + "ASSESSMENT.md", "x")
+ put(self.tmp, a + "BASELINE.md", "x")
+ put(self.tmp, "modernized/s/mod/TRANSFORMATION_NOTES.md", "x")
+ self.build()
+ self.assertEqual(page_data(self.html())["glance"]["track"]["label"], "rewrite")
+ past = os.path.getmtime(os.path.join(self.tmp, "modernized", "s", "mod", "TRANSFORMATION_NOTES.md")) + 50
+ put(self.tmp, a + "DELTA_CATALOG.md", "x")
+ os.utime(os.path.join(self.tmp, "analysis", "s", "DELTA_CATALOG.md"), (past, past))
+ self.build()
+ self.assertEqual(page_data(self.html())["glance"]["track"]["label"], "same-stack uplift")
+
+ def test_topology_link_is_relative_and_the_map_is_not_embedded(self):
+ put(self.tmp, "analysis/s/TOPOLOGY.html", "<html>" + "x" * 5000 + "</html>")
+ self.build()
+ data = page_data(self.html())
+ link = next(p for s in data["sections"] for p in s["parts"] if p["t"] == "link")
+ self.assertEqual(os.path.normpath(os.path.join(os.path.dirname(self.out), link["href"])), os.path.join(self.tmp, "analysis", "s", "TOPOLOGY.html"))
+ self.assertNotIn("xxxxxxxxxx", self.html())
+
+ def test_report_shows_what_compare_py_wrote(self):
+ eq_dir = os.path.join(self.tmp, "analysis", "s", "equivalence")
+ put(eq_dir, "legacy/a.out", "same")
+ put(eq_dir, "new/a.out", "same")
+ put(eq_dir, "legacy/b.out", "one")
+ put(eq_dir, "new/b.out", "two")
+ cases = {"system": "s", "cases": [{"id": "A", "legacy": "legacy/a.out", "new": "new/a.out"}, {"id": "B", "legacy": "legacy/b.out", "new": "new/b.out"}]}
+ put(eq_dir, "cases.json", json.dumps(cases))
+ target = os.path.join(self.tmp, "analysis", "s", "EQUIVALENCE.json")
+ cmp.main([os.path.join(eq_dir, "cases.json"), "--out", target, "--quiet"])
+ self.build()
+ data = page_data(self.html())
+ eq = data["glance"]["equivalence"]
+ self.assertEqual(eq["state"], "red")
+ self.assertEqual((eq["tally"]["same"], eq["tally"]["differs"], eq["tally"]["executed"]), (1, 1, 2))
+ section = next(s for s in data["sections"] if s["id"] == "equivalence")["parts"][0]["eq"]
+ self.assertEqual(section["cases"][1]["firstDiff"]["line"], 1)
+ self.assertEqual(section["cases"][1]["legacyPath"], "equivalence/legacy/b.out")
+ cases["cases"].pop()
+ put(eq_dir, "cases.json", json.dumps(cases))
+ cmp.main([os.path.join(eq_dir, "cases.json"), "--out", target, "--quiet"])
+ self.build()
+ self.assertEqual(page_data(self.html())["glance"]["equivalence"]["state"], "green")
+
+ def test_a_symlinked_output_path_is_refused_and_nothing_is_written_through_it(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ target = put(self.tmp, "elsewhere/real.html", "ORIGINAL")
+ os.makedirs(os.path.dirname(self.out))
+ make_symlink(self, target, self.out)
+ code, out, err = self.build()
+ self.assertEqual((code, out), (1, ""))
+ self.assertEqual(len(err.strip().splitlines()), 1)
+ self.assertIn("symbolic link", err)
+ with open(target) as fh:
+ self.assertEqual(fh.read(), "ORIGINAL")
+ self.assertEqual(os.listdir(os.path.dirname(target)), ["real.html"])
+ self.assertEqual(os.listdir(os.path.dirname(self.out)), ["REPORT.html"])
+
+ def test_a_symlinked_output_folder_is_refused(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ real = os.path.join(self.tmp, "real")
+ os.makedirs(real)
+ make_symlink(self, real, os.path.join(self.tmp, "linked"))
+ code, _, err = run_main(br.main, ["s", "--workspace", self.tmp, "--out", os.path.join(self.tmp, "linked", "REPORT.html")])
+ self.assertEqual(code, 1)
+ self.assertIn("symbolic link", err)
+ self.assertEqual(os.listdir(real), [])
+
+ def test_a_failed_write_leaves_no_temp_file_behind(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ with mock.patch.object(br.os, "replace", side_effect=OSError("disk full")):
+ code, _, err = self.build()
+ self.assertEqual(code, 1)
+ self.assertEqual(len(err.strip().splitlines()), 1)
+ self.assertEqual(os.listdir(os.path.dirname(self.out)), [])
+
+ @unittest.skipIf(os.name == "nt", "POSIX permissions")
+ def test_the_report_gets_the_permissions_a_normal_file_would(self):
+ put(self.tmp, "analysis/s/PREFLIGHT.md", "x")
+ umask = os.umask(0o022)
+ os.umask(umask)
+ self.build()
+ self.assertEqual(stat.S_IMODE(os.stat(self.out).st_mode), 0o666 & ~umask)
+
+
+class Template(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ with open(TEMPLATE, encoding="utf-8") as fh:
+ cls.text = fh.read()
+ cls.css = re.search(r"<style>(.*?)</style>", cls.text, re.S).group(1)
+ cls.app = re.search(r'<script id="app">(.*?)</script>', cls.text, re.S).group(1)
+
+ def test_no_external_urls_and_no_dangerous_apis(self):
+ self.assertNotRegex(self.text, r"https?://")
+ self.assertNotRegex(self.text, r"(?i)(?:src|href|action)\s*=\s*[\"']?\s*//")
+ self.assertNotRegex(self.css, r"@import|url\(")
+ for word in ("innerHTML", "outerHTML", "insertAdjacentHTML", "document.write", "eval(", "new Function", "setTimeout(\"", "fetch(", "XMLHttpRequest", "WebSocket"):
+ self.assertNotIn(word, self.app, word)
+ self.assertNotRegex(self.text, r"(?i)\son[a-z]+\s*=")
+ self.assertEqual(len(re.findall(r"@@(CSP|DATA|MERMAID)@@", self.text)), 3)
+ self.assertNotIn("@@", self.app)
+
+ def test_theme_and_layout_contract(self):
+ root = re.search(r":root\{(.*?)\}", self.css, re.S).group(1)
+ tokens = set(re.findall(r"--([a-z-]+):", root))
+ media = re.search(r"@media \(prefers-color-scheme:dark\)\{:root:not\(\[data-theme=\"light\"\]\)\{(.*?)\}\}", self.css, re.S).group(1)
+ forced = re.search(r":root\[data-theme=\"dark\"\]\{(.*?)\}", self.css, re.S).group(1)
+ self.assertEqual(set(re.findall(r"--([a-z-]+):", media)) - tokens, set())
+ self.assertEqual(re.findall(r"--([a-z-]+):[^;]*", media), re.findall(r"--([a-z-]+):[^;]*", forced))
+ self.assertRegex(re.search(r"body\{([^}]*)\}", self.css).group(1), r"background:var\(--bg\)")
+ self.assertIn("system-ui", self.css)
+ self.assertIn("width=device-width", self.text)
+ self.assertIn("@media print", self.css)
+ self.assertIn(":focus-visible", self.css)
+ self.assertRegex(self.css, r"\.wrap\{[^}]*padding:0 16px")
+
+ def test_every_text_colour_pair_meets_4_5_to_1(self):
+ def parse(block):
+ return dict(re.findall(r"--([a-z-]+):(#[0-9a-fA-F]{3,6})", block))
+
+ def lum(hexcolor):
+ h = hexcolor.lstrip("#")
+ h = "".join(c * 2 for c in h) if len(h) == 3 else h
+ chan = [int(h[i:i + 2], 16) / 255 for i in (0, 2, 4)]
+ lin = [c / 12.92 if c <= 0.03928 else ((c + 0.055) / 1.055) ** 2.4 for c in chan]
+ return 0.2126 * lin[0] + 0.7152 * lin[1] + 0.0722 * lin[2]
+
+ def ratio(a, b):
+ hi, lo = sorted((lum(a), lum(b)), reverse=True)
+ return (hi + 0.05) / (lo + 0.05)
+
+ light = parse(re.search(r":root\{(.*?)\}", self.css, re.S).group(1))
+ dark = dict(light, **parse(re.search(r":root\[data-theme=\"dark\"\]\{(.*?)\}", self.css, re.S).group(1)))
+ pairs = [("text", "bg"), ("text", "surface"), ("muted", "bg"), ("muted", "surface"), ("muted", "code"), ("text", "code"), ("link", "surface"), ("link", "bg"),
+ ("link", "code"), ("ok", "ok-bg"), ("bad", "bad-bg"), ("warn", "warn-bg"), ("info", "info-bg"), ("flat", "flat-bg"), ("hit-text", "hit"),
+ ("text", "flat-bg"), ("text", "info-bg"), ("muted", "flat-bg")]
+ for name, palette in (("light", light), ("dark", dark)):
+ for fg, bg in pairs:
+ self.assertGreaterEqual(ratio(palette[fg], palette[bg]), 4.5, "%s: %s on %s" % (name, fg, bg))
+ self.assertGreaterEqual(ratio("#1a1f27", light["paper"]), 4.5)
+ for name, palette in (("light", light), ("dark", dark)):
+ self.assertGreaterEqual(ratio(palette["focus"], palette["bg"]), 3, name)
+ self.assertGreaterEqual(ratio(palette["focus"], palette["surface"]), 3, name)
+
+
+NODE = shutil.which("node")
+LINK_CASES = { # markdown -> the href it must produce, or None when it must stay plain text
+ "[x](javascript:alert(1))": None, "[x](data:text/html,x)": None, "[x](file:///etc/passwd)": None, "[x](JaVaScRiPt:x)": None, "[x](java&#10;script:x)": None, "[x](javascript&colon;alert(1))": None, "[x](&#x6A;avascript:alert(1))": None,
+ "[x](https://example.com/a?b=1)": "https://example.com/a?b=1", "[x](mailto:a@b.c)": "mailto:a@b.c", "[x](#anchor)": "#anchor",
+ "[x](./relative/path.md)": "./relative/path.md", "[x](../up/README.md#top)": "../up/README.md#top",
+}
+JS_INPUTS = [
+ "<script>alert(1)</script>", "</script><script>alert(1)</script>", "<img src=x onerror=alert(1)>", "<!-- <!--", '<a href="javascript:alert(1)">x</a>',
+ "[x](javascript:alert(1))", "[x](JavaScript:alert(1))", "[x](java\tscript:alert(1))", "[x](" + chr(1) + "javascript:alert(1))", "[x]( javascript:alert(1))",
+ "[x](data:text/html;base64,PHNjcmlwdD4=)", "[x](//evil.example)", "[x](/\\evil.example)", "[x](vbscript:msgbox(1))", "[x](file:///etc/passwd)",
+ "[x](&#106;avascript:alert(1))", "[x](%6Aavascript:alert(1))", "[x](" + LS + "javascript:alert(1))", "[x](\u00a0javascript:alert(1))", "![x](javascript:alert(1))",
+ "| <b onmouseover=alert(1)>x</b> | y |\n|---|---|\n| <img src=x onerror=alert(2)> | z |", "### RULE-001: `<b>title</b>` <img src=x>",
+ "`<script>`", "**<img src=x onerror=1>** _<b>_", "- [ ] <b>a</b>\n- [x] *b*\n - nested <i>c</i>", "> <script>\n> x",
+ "```js\n<script>alert(1)</script>\n```", "~~~\n</script>\n~~~", "```mermaid\ngraph TD; A-->B\n```", "a" * 200000, "*" * 200000, "[" * 100000, "`" * 100000,
+ "_" * 100000, "|" * 100000, "- " * 50000, "> " * 20000 + "x", "\n".join(" " * (i * 2) + "- x" for i in range(300)), "#" * 100000, "[a](" + "b" * 100000 + ")",
+ "\n".join("| " + "c | " * 200 for _ in range(50)), "*a " * 60000, "**a " * 60000, "[a " * 60000, "`a " * 60000,
+] + list(LINK_CASES)
+SAFE_URLS = ["https://example.com/a?b=1&c=2", "https://example.com/?q=a&b=c#frag", "http://x.y", "HTTPS://X.Y", "mailto:a@b.c", "../x/y.md#z", "#anchor", "README.md", "a/b:c", "TOPOLOGY.html", "./p:q", "x.md#a:b"]
+BAD_URLS = ["java&#10;script:x", "javascript&colon;alert(1)", "&#106;avascript:1", "a&#x3a;b", "x&amp;y", "javascript:alert(1)", "JavaScript:alert(1)", " javascript:alert(1)", "java\tscript:alert(1)", "java\nscript:alert(1)", chr(1) + "javascript:alert(1)", "data:text/html,x",
+ "//evil.example", "/\\evil.example", "\\\\evil", "/abs/path", "vbscript:x", "file:///etc/passwd", "C:\\x", "a:b", chr(0xA0) + "javascript:1", LS + "javascript:1",
+ chr(0xFEFF) + "javascript:1", "jav" + chr(0x200B) + "ascript:1", "", "x" * 3000]
+NODE_HARNESS = r"""
+const fs = require('fs'), vm = require('vm');
+const tpl = fs.readFileSync(process.env.TEMPLATE, 'utf8');
+const core = /\/\* core:start[^\n]*\n([\s\S]*?)\/\* core:end \*\//.exec(tpl)[1];
+class Text { constructor(t) { this.text = String(t); } }
+class El { constructor(t) { this.tag = t; this.attrs = {}; this.kids = []; }
+ setAttribute(k, v) { this.attrs[k] = String(v); } appendChild(c) { this.kids.push(c); return c; } }
+const ctx = { document: { createElement: t => new El(t), createTextNode: t => new Text(t) } };
+vm.createContext(ctx);
+const api = vm.runInContext('(function(){"use strict";' + core + ';return {safeUrl:safeUrl,md:md,ruleCard:ruleCard,inline:inline};})()', ctx);
+function walk(n, acc) {
+ if (n instanceof Text) { acc.text += n.text; return; }
+ acc.tags.add(n.tag); Object.keys(n.attrs).forEach(k => { acc.attrs.add(k); if (k === 'href') acc.hrefs.push(n.attrs[k]); });
+ n.kids.forEach(c => walk(c, acc));
+}
+const input = JSON.parse(fs.readFileSync(0, 'utf8')), out = { urls: input.urls.map(u => api.safeUrl(u)), docs: [] };
+for (const src of input.docs) {
+ const t0 = Date.now(), acc = { tags: new Set(), attrs: new Set(), hrefs: [], text: '' };
+ api.md(src, 3).forEach(n => walk(n, acc));
+ out.docs.push({ ms: Date.now() - t0, tags: [...acc.tags], attrs: [...acc.attrs], hrefs: acc.hrefs, text: acc.text.slice(0, 5000) });
+}
+const card = { id: 'RULE-1', heading: 'RULE-1: <b>x</b>', title: '<img src=x onerror=1>`a`', p: 'P0', conf: 'High', cat: '<i>', cites: ['<b>a.cbl:1</b>'], defect: true, body: '**x** <script>1</script>' };
+const acc = { tags: new Set(), attrs: new Set(), hrefs: [], text: '' };
+walk(api.ruleCard(card, 'confirmed'), acc);
+out.card = { tags: [...acc.tags], attrs: [...acc.attrs], text: acc.text };
+process.stdout.write(JSON.stringify(out));
+"""
+
+
+@unittest.skipUnless(NODE, "node is not installed")
+class TemplateScript(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ proc = subprocess.run([NODE, "-e", NODE_HARNESS], input=json.dumps({"urls": SAFE_URLS + BAD_URLS, "docs": JS_INPUTS}), capture_output=True, text=True,
+ env=dict(os.environ, TEMPLATE=TEMPLATE), timeout=300)
+ assert proc.returncode == 0, proc.stderr
+ cls.out = json.loads(proc.stdout)
+
+ def test_the_page_script_parses(self):
+ with open(TEMPLATE, encoding="utf-8") as src, tempfile.NamedTemporaryFile("w", suffix=".js", delete=False, encoding="utf-8") as fh:
+ fh.write(re.search(r'<script id="app">(.*?)</script>', src.read(), re.S).group(1))
+ self.addCleanup(os.unlink, fh.name)
+ self.assertEqual(subprocess.run([NODE, "--check", fh.name], capture_output=True, text=True).returncode, 0)
+
+ def test_link_allowlist(self):
+ urls = self.out["urls"]
+ for url, got in zip(SAFE_URLS, urls[:len(SAFE_URLS)]):
+ self.assertIsNotNone(got, url)
+ for url, got in zip(BAD_URLS, urls[len(SAFE_URLS):]):
+ self.assertIsNone(got, repr(url))
+
+ def test_hostile_markdown_becomes_text_never_elements(self):
+ allowed_tags = {"p", "h3", "h4", "h5", "h6", "ul", "ol", "li", "code", "pre", "strong", "em", "del", "a", "br", "hr", "blockquote", "table", "thead", "tbody", "tr",
+ "th", "td", "div", "span"}
+ allowed_attrs = {"href", "rel", "target", "referrerpolicy", "id", "class", "role", "tabindex", "aria-label", "scope"}
+ for src, doc in zip(JS_INPUTS, self.out["docs"]):
+ label = src[:50]
+ self.assertLessEqual(set(doc["tags"]), allowed_tags, label)
+ self.assertLessEqual(set(doc["attrs"]), allowed_attrs, label)
+ self.assertLess(doc["ms"], 5000, label)
+ for href in doc["hrefs"]:
+ stripped = re.sub(r"[\x00-\x20]", "", href).lower()
+ self.assertRegex(stripped, r"^(https?://|mailto:|#|[^:/?#\\]*(?:[/?#]|$))", href)
+ first = self.out["docs"][0]
+ self.assertIn("<script>alert(1)</script>", first["text"])
+ self.assertEqual(self.out["docs"][2]["tags"], ["p"])
+ self.assertIn("[x](javascript:alert(1))", self.out["docs"][5]["text"])
+ self.assertEqual(self.out["docs"][5]["hrefs"], [])
+ table = self.out["docs"][20]
+ self.assertIn("<b onmouseover=alert(1)>x</b>", table["text"])
+ self.assertNotIn("img", table["tags"])
+
+ def test_links_fail_closed_and_safe_ones_stay_links(self):
+ docs = dict(zip(JS_INPUTS, self.out["docs"]))
+ for source, href in LINK_CASES.items():
+ doc = docs[source]
+ if href is None:
+ self.assertEqual(doc["hrefs"], [], source)
+ self.assertNotIn("a", doc["tags"], source)
+ self.assertIn(source, doc["text"], source)
+ else:
+ self.assertEqual(doc["hrefs"], [href], source)
+ self.assertIn("a", doc["tags"], source)
+ self.assertEqual(doc["text"], "x", source)
+
+ def test_rule_card_renders_hostile_fields_as_text(self):
+ card = self.out["card"]
+ self.assertLessEqual(set(card["tags"]), {"article", "span", "h4", "div", "p", "strong"})
+ self.assertLessEqual(set(card["attrs"]), {"class", "id", "data-p", "data-d", "aria-label"})
+ self.assertIn("<img src=x onerror=1>`a`", card["text"])
+ self.assertIn("<b>a.cbl:1</b>", card["text"])
+
+
+class Vendored(unittest.TestCase):
+ def test_mermaid_matches_the_recorded_digest_and_license(self):
+ with open(os.path.join(PLUGIN, "assets", "vendor", "LICENSE-mermaid.txt"), encoding="utf-8") as fh:
+ lic = fh.read()
+ with open(MERMAID, "rb") as fh:
+ raw = fh.read()
+ self.assertIn("Copyright (c) 2014 - 2022 Knut Sveidqvist", lic)
+ self.assertIn("MIT License", lic)
+ self.assertIn(hashlib.sha256(raw).hexdigest(), lic)
+ self.assertIn("mermaid@11.17.2", lic)
+ self.assertIn(str(len(raw)), lic)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_rules_tools.py b/plugins/code-modernization/scripts/tests/test_rules_tools.py
new file mode 100644
index 00000000..33cfd279
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_rules_tools.py
@@ -0,0 +1,196 @@
+import json
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+SCRIPTS = os.path.dirname(HERE)
+
+
+def run(script, *args):
+ return subprocess.run([sys.executable, os.path.join(SCRIPTS, script), *args], capture_output=True, text=True)
+
+
+def load(path):
+ with open(path, encoding='utf-8') as handle:
+ return json.load(handle)
+
+
+def write(root, rel, text='x\n'):
+ path = os.path.join(root, rel)
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, 'w', encoding='utf-8') as handle:
+ handle.write(text)
+
+
+class ShardTests(unittest.TestCase):
+ def test_tree_shards_by_directory_and_skips_vendored_code(self):
+ with tempfile.TemporaryDirectory() as ws:
+ for i in range(30):
+ write(ws, f'legacy/erp/lib/pay/p{i}.pl', 'a\n' * 10)
+ write(ws, 'legacy/erp/lib/tax/t.pl', 'a\n' * 400)
+ write(ws, 'legacy/erp/node_modules/x/y.js', 'a\n')
+ write(ws, 'legacy/erp/tests/t.pl', 'a\n')
+ write(ws, 'legacy/erp/README.md', 'text\n')
+ done = run('make_shards.py', 'erp', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ shards = load(os.path.join(ws, 'analysis/erp/extract-rules.modules.json'))
+ names = [s['name'] for s in shards]
+ self.assertEqual(names, ['lib/pay', 'lib/pay#2', 'lib/tax'])
+ self.assertEqual(len(shards[0]['files']), 25)
+ self.assertNotIn('node_modules', json.dumps(shards))
+ self.assertNotIn('tests/t.pl', json.dumps(shards))
+ self.assertTrue(all(not f.startswith('legacy/') for s in shards for f in s['files']))
+
+ def test_a_symlinked_source_is_followed_and_files_stay_relative_to_it(self):
+ with tempfile.TemporaryDirectory() as ws, tempfile.TemporaryDirectory() as elsewhere:
+ write(elsewhere, 'src/a.cbl', 'a\n' * 50)
+ os.makedirs(os.path.join(ws, 'legacy'))
+ os.symlink(elsewhere, os.path.join(ws, 'legacy', 'big'))
+ done = run('make_shards.py', 'big', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ shards = load(os.path.join(ws, 'analysis/big/extract-rules.modules.json'))
+ self.assertEqual(shards[0]['files'], ['src/a.cbl'])
+
+ def test_topology_merges_small_modules_and_normalizes_paths(self):
+ with tempfile.TemporaryDirectory() as ws:
+ for name in ('A', 'B', 'C'):
+ write(ws, f'legacy/s/{name}.cbl', 'a\n' * 50)
+ topology = {'root': {'kind': 'system', 'id': 'sys', 'children': [{'kind': 'domain', 'name': 'D1', 'id': 'dom:d1', 'children': [
+ {'kind': 'module', 'id': 'A', 'name': 'A', 'file': 'A.cbl', 'loc': 50},
+ {'kind': 'module', 'id': 'B', 'name': 'B', 'file': 'legacy/s/B.cbl', 'loc': 50},
+ {'kind': 'module', 'id': 'C', 'name': 'C', 'file': os.path.join(ws, 'legacy/s/C.cbl'), 'loc': 900}]}]}}
+ write(ws, 'analysis/s/topology.json', json.dumps(topology))
+ self.assertEqual(run('make_shards.py', 's', '--workspace', ws).returncode, 0)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual([s['files'] for s in shards], [['A.cbl', 'B.cbl'], ['C.cbl']])
+ self.assertEqual(run('make_shards.py', 's', 'C*', '--workspace', ws).returncode, 0)
+ self.assertEqual(run('make_shards.py', 's', 'nothing*', '--workspace', ws).returncode, 1)
+
+ def test_links_and_paths_that_leave_the_source_are_not_shards(self):
+ with tempfile.TemporaryDirectory() as ws, tempfile.TemporaryDirectory() as outside:
+ write(ws, 'legacy/s/ok.pl', 'a\n' * 5)
+ write(outside, 'secret.pl', 'not part of the system\n')
+ os.symlink(os.path.join(outside, 'secret.pl'), os.path.join(ws, 'legacy/s/link.pl'))
+ self.assertEqual(run('make_shards.py', 's', '--workspace', ws).returncode, 0)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual([f for s in shards for f in s['files']], ['ok.pl'])
+ topology = {'root': {'kind': 'system', 'id': 'sys', 'children': [{'kind': 'domain', 'name': 'D', 'id': 'd', 'children': [
+ {'kind': 'module', 'id': 'A', 'name': 'A', 'file': 'ok.pl', 'loc': 500},
+ {'kind': 'module', 'id': 'B', 'name': 'B', 'file': '../../../etc/hosts', 'loc': 500},
+ {'kind': 'module', 'id': 'C', 'name': 'C', 'file': 'link.pl', 'loc': 500}]}]}}
+ write(ws, 'analysis/s/topology.json', json.dumps(topology))
+ done = run('make_shards.py', 's', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual([f for s in shards for f in s['files']], ['ok.pl'])
+ self.assertIn('outside the source directory', done.stdout)
+
+ def test_a_link_to_the_home_directory_or_root_is_refused(self):
+ with tempfile.TemporaryDirectory() as ws:
+ os.makedirs(os.path.join(ws, 'legacy'))
+ for broad in (os.path.expanduser('~'), '/', '/usr'):
+ link = os.path.join(ws, 'legacy', 's')
+ if os.path.islink(link):
+ os.unlink(link)
+ os.symlink(broad, link)
+ done = run('make_shards.py', 's', '--workspace', ws)
+ self.assertEqual(done.returncode, 1, broad)
+ self.assertIn('too broad', done.stderr)
+
+ def test_a_plain_name_matches_what_lies_beneath_it(self):
+ with tempfile.TemporaryDirectory() as ws:
+ write(ws, 'legacy/s/jetty-util/src/A.java', 'a\n' * 5)
+ write(ws, 'legacy/s/jetty-util-ajax/src/B.java', 'a\n' * 5)
+ write(ws, 'legacy/s/other/C.java', 'a\n' * 5)
+ self.assertEqual(run('make_shards.py', 's', 'jetty-util', '--workspace', ws).returncode, 0)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual([f for sh in shards for f in sh['files']], ['jetty-util/src/A.java'])
+
+ def test_a_map_whose_modules_share_one_directory_falls_back_to_the_tree(self):
+ with tempfile.TemporaryDirectory() as ws:
+ for name in ('a', 'b'):
+ write(ws, f'legacy/s/pkg/{name}/X.java', 'a\n' * 50)
+ children = [{'kind': 'module', 'id': f'pkg/{n}', 'name': f'pkg/{n}', 'file': 'pkg', 'loc': 900} for n in ('a', 'b')]
+ topology = {'root': {'kind': 'system', 'id': 'sys', 'children': [{'kind': 'domain', 'name': 'D', 'id': 'd', 'children': children}]}}
+ write(ws, 'analysis/s/topology.json', json.dumps(topology))
+ done = run('make_shards.py', 's', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ self.assertIn('cannot be sharded by module', done.stdout)
+ shards = load(os.path.join(ws, 'analysis/s/extract-rules.modules.json'))
+ self.assertEqual(sorted(f for sh in shards for f in sh['files']), ['pkg/a/X.java', 'pkg/b/X.java'])
+
+ def test_missing_source_says_what_to_do(self):
+ with tempfile.TemporaryDirectory() as ws:
+ done = run('make_shards.py', 'nope', '--workspace', ws)
+ self.assertEqual(done.returncode, 1)
+ self.assertIn('preflight', done.stderr)
+
+
+class RenderTests(unittest.TestCase):
+ def test_rules_render_with_rule_headings_and_hostile_text_stays_inert(self):
+ result = {'confirmedRules': [
+ {'name': 'Late fee', 'category': 'Calculation', 'priority': 'P0', 'source': 'a.cbl:10-20', 'plainEnglish': 'A fee.',
+ 'given': 'a late payment', 'when': 'month end', 'then': 'fee 5%', 'confidence': 'High'},
+ {'name': 'Bad `name` | pipe\n# Heading', 'category': 'Validation', 'priority': 'P1', 'source': 'b.cbl:1-2', 'plainEnglish': 'x',
+ 'given': 'g\n# injected heading\n```', 'when': 'w', 'then': 't', 'confidence': 'Low', 'smeQuestion': 'Is it 5 or 6?'}],
+ 'dataObjects': [{'name': 'Account', 'source': 'a.cbl:1', 'fields': [{'name': 'id', 'type': 'int'}], 'consumedBy': ['Late fee']}],
+ 'injectionFlags': ['a.cbl:5'], 'rejectedRules': [{}], 'unverifiedRules': [], 'rerunModules': [],
+ 'stats': {'skippedModules': ['M9']}}
+ with tempfile.TemporaryDirectory() as ws:
+ write(ws, 'analysis/s/rules_result.json', json.dumps(result))
+ done = run('render_rules.py', 's', '--workspace', ws)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ text = open(os.path.join(ws, 'analysis/s/BUSINESS_RULES.md'), encoding='utf-8').read()
+ self.assertIn('### RULE-001: Late fee', text)
+ self.assertIn('### RULE-002: Bad', text)
+ headings = [line for line in text.splitlines() if line.startswith('### ')]
+ self.assertEqual(len(headings), 2, headings)
+ self.assertNotIn('\n# injected heading', text)
+ ids = [int(n) for n in re.findall(r'^### RULE-(\d+): ', text, re.M)]
+ self.assertEqual(ids, list(range(1, len(ids) + 1)), 'rule numbers read in sequence down the document')
+ self.assertIn('Rules requiring SME confirmation', text)
+ self.assertIn('Is it 5 or 6?', text)
+ self.assertIn('Coverage gaps', text)
+ self.assertIn('Instruction-shaped content', text)
+ dto = open(os.path.join(ws, 'analysis/s/DATA_OBJECTS.md'), encoding='utf-8').read()
+ self.assertIn('## Account', dto)
+
+ def test_a_rule_keeps_one_citation_and_the_rest_moves_to_also_cited(self):
+ sources = ['wwwroot/a.pl:100-120 (aggregation); family table at lib/b.pm:31-40',
+ 'lib/x.pm:966,1018,1238 (order lists)', 'no citation here', 'c.cbl:7']
+ result = {'confirmedRules': [
+ {'name': f'R{i}', 'category': 'Policy', 'priority': 'P1', 'source': src, 'plainEnglish': 'x',
+ 'given': 'g', 'when': 'w', 'then': 't', 'confidence': 'High'} for i, src in enumerate(sources)]}
+ with tempfile.TemporaryDirectory() as ws:
+ write(ws, 'analysis/s/rules_result.json', json.dumps(result))
+ self.assertEqual(run('render_rules.py', 's', '--workspace', ws).returncode, 0)
+ text = open(os.path.join(ws, 'analysis/s/BUSINESS_RULES.md'), encoding='utf-8').read()
+ cites = re.findall(r'^\*\*Source:\*\* `([^`]*)`', text, re.M)
+ self.assertEqual(sorted(cites), sorted(['wwwroot/a.pl:100-120', 'lib/x.pm:966', 'no citation here', 'c.cbl:7']))
+ self.assertIn('**Also cited:** (aggregation); family table at lib/b.pm:31-40', text)
+ self.assertEqual(text.count('**Also cited:**'), 2)
+
+ def test_folded_rules_are_listed_so_nothing_disappears_untraceably(self):
+ result = {'confirmedRules': [{'name': 'Twiddle table', 'category': 'Calculation', 'priority': 'P0', 'source': 'a.c:1-2 ; also b.c:3-4',
+ 'plainEnglish': 'x', 'given': 'g', 'when': 'w', 'then': 't', 'confidence': 'Low'}],
+ 'foldedRules': [{'name': 'Twiddle factors', 'source': 'b.c:3-4', 'into': 'Twiddle table'}, 'junk', {'name': 'X\n# fake heading', 'source': 's', 'into': 'i'}]}
+ with tempfile.TemporaryDirectory() as ws:
+ write(ws, 'analysis/s/rules_result.json', json.dumps(result))
+ self.assertEqual(run('render_rules.py', 's', '--workspace', ws).returncode, 0)
+ text = open(os.path.join(ws, 'analysis/s/BUSINESS_RULES.md'), encoding='utf-8').read()
+ self.assertIn('## Rules folded into another', text)
+ self.assertIn('- Twiddle factors (b.c:3-4) into Twiddle table', text)
+ self.assertIn('**Also cited:** b.c:3-4', text)
+ self.assertNotIn('\n# fake heading', text)
+
+ def test_unreadable_result_is_an_error(self):
+ with tempfile.TemporaryDirectory() as ws:
+ self.assertEqual(run('render_rules.py', 's', '--workspace', ws).returncode, 1)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/plugins/code-modernization/scripts/tests/test_telemetry.py b/plugins/code-modernization/scripts/tests/test_telemetry.py
new file mode 100644
index 00000000..c65130cd
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_telemetry.py
@@ -0,0 +1,810 @@
+"""Tests for the usage counts (scripts/telemetry.py, scripts/telemetry.sh and the hooks that call them).
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+"""
+import contextlib
+import io
+import json
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+import unittest
+from unittest import mock
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import telemetry as tm # noqa: E402
+
+RULES = """# Business rules
+
+### RULE-001 ยท Interest posting
+**Priority:** P0
+**Confidence:** High
+**Source:** a.cbl:10-20
+
+Given a balance, when the month closes, then interest is posted.
+
+### RULE-002 ยท Rounding
+**Priority:** P1
+**Source:** a.cbl:30-31
+
+Given an amount, then it is truncated.
+
+### RULE-003 ยท Odd limit
+**Priority:** P0
+**Suspected defect:** the limit is off by one
+**Source:** b.java:5-6
+
+Given a limit, then it applies.
+"""
+BRIEF = """# Brief
+
+#### Phase 1: pilot
+#### Phase 2: the rest
+
+## 8. Approval Block
+
+```
+Approved by: Jane Doe Date: 2026-09-24
+Approval covers: Phase 1 only
+```
+"""
+UNSIGNED = "## 8. Approval Block\n\n```\nApproved by: ________________ Date: __________\n```\n"
+SECURITY = "# Security\n\n| Severity | Count |\n|---|---|\n| Critical | 2 |\n| High | 5 |\n| Medium | 9 |\n"
+TOPOLOGY = {"root": {"kind": "root", "children": [
+ {"kind": "module", "file": "a.cbl", "loc": 3000, "language": "cobol"},
+ {"kind": "module", "file": "b.java", "loc": 1200, "language": "java"},
+ {"kind": "module", "file": "Makefile", "loc": 5000, "language": "make"}]}, "edges": []}
+CHECKS = ("tests", "rules", "same", "fresh", "canary", "source")
+
+
+def module(name, **states):
+ checks = [{"id": c, "state": states.get(c, "pass"), "detail": "x"} for c in CHECKS]
+ return {"name": name, "track": "rewrite", "checks": checks, "evidence": {
+ "tests": {"executed": 5, "failed": 0, "skipped": 0}, "fresh": {"inputs": 12, "executed": 12, "same": 12, "differs": 0, "missing": 0}}}
+
+
+VERIFICATION = {"modules": [module("a"), module("b", canary="gap"), module("c", canary="gap"), module("d", tests="fail"), module("e", same="fail"),
+ module("f", source="fail")],
+ "signoff": {"name": "", "role": "", "date": "", "decision": ""}}
+CASES = [{"id": "s%d" % i, "verdict": "same"} for i in range(9)] + [{"id": "d1", "verdict": "differs"}, {"id": "m1", "verdict": "missing"}] + [
+ {"id": "a%d" % i, "verdict": "differs-approved", "approvedDifference": "the owner accepted it"} for i in range(2)]
+EQUIVALENCE = {"cases": CASES, "selfCheck": {"passed": True}}
+REVIEWS = {"system": "sys", "version": 1, "reviews": {"RULE-001": {"verdict": "confirmed"}, "RULE-002": {"verdict": "wrong"},
+ "RULE-003": {"verdict": "discuss"}}}
+STEP_BITS = dict(tm.STEPS)
+
+
+def put(root, rel, data):
+ path = os.path.join(root, *rel.split("/"))
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with open(path, "w", encoding="utf-8") as fh:
+ fh.write(data if isinstance(data, str) else json.dumps(data))
+ return path
+
+
+def workspace(root, system="sys", parts=None):
+ parts = parts or {}
+ files = {"PREFLIGHT.md": "# Preflight\n", "ASSESSMENT.md": "# Assessment\n", "topology.json": TOPOLOGY, "BUSINESS_RULES.md": RULES,
+ "RULE_REVIEWS.json": REVIEWS, "MODERNIZATION_BRIEF.md": BRIEF, "EQUIVALENCE.json": EQUIVALENCE, "VERIFICATION.json": VERIFICATION,
+ "SECURITY_FINDINGS.md": SECURITY, "REPORT.html": "<html></html>"}
+ files.update(parts)
+ for name, data in files.items():
+ if data is not None:
+ put(root, "analysis/%s/%s" % (system, name), data)
+ put(root, "modernized/%s/mod-a/x.txt" % system, "x")
+ put(root, "modernized/%s/mod-b/x.txt" % system, "x")
+ return root
+
+
+class Base(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+ self.data = os.path.join(self.tmp, "data")
+ os.makedirs(self.data)
+ clean = {k: v for k, v in os.environ.items() if k not in ("CODE_MODERNIZATION_TELEMETRY", "CLAUDE_PLUGIN_OPTION_TELEMETRY", "DISABLE_TELEMETRY",
+ "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "CLAUDE_PLUGIN_DATA")}
+ patcher = mock.patch.dict(os.environ, {**clean, "CLAUDE_PLUGIN_DATA": self.data}, clear=True)
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ self.ws = workspace(os.path.join(self.tmp, "ws"))
+
+ def hook(self, mode, payload):
+ out = io.StringIO()
+ with mock.patch.object(sys, "stdin", io.StringIO(json.dumps(payload))), contextlib.redirect_stdout(out):
+ tm.hook(mode)
+ return out.getvalue()
+
+
+class Counts(Base):
+ def test_the_full_workspace_is_counted_the_way_the_artifacts_say(self):
+ s = tm.state_metrics(self.ws)
+ self.assertEqual({k: s[k] for k in ("rules", "p0", "rev_ok", "rev_wrong", "phases", "built", "eq_cases", "eq_diff", "eq_appr",
+ "v_proven", "v_partly", "v_not", "sec_crit", "sec_high", "map_kloc", "lang", "goal")},
+ dict(rules=3, p0=2, rev_ok=1, rev_wrong=1, phases=2, built=2, eq_cases=12, eq_diff=2, eq_appr=2, v_proven=1,
+ v_partly=2, v_not=3, sec_crit=2, sec_high=5, map_kloc=9, lang=1, goal=3))
+ steps = ("preflight", "assess", "map", "rules", "reviewed", "brief", "approved", "built", "verified", "hardened", "report")
+ self.assertEqual(s["done"], sum(STEP_BITS[n] for n in steps))
+
+ def test_a_signed_proof_and_an_unsigned_brief(self):
+ put(self.ws, "analysis/sys/VERIFICATION.json", dict(VERIFICATION, signoff={"name": "A. Person", "decision": "accepted"}))
+ put(self.ws, "analysis/sys/MODERNIZATION_BRIEF.md", UNSIGNED)
+ done = tm.state_metrics(self.ws)["done"]
+ self.assertTrue(done & STEP_BITS["signed"])
+ self.assertFalse(done & STEP_BITS["approved"])
+ self.assertTrue(done & STEP_BITS["brief"])
+
+ def test_nothing_is_reported_where_the_plugin_has_left_nothing(self):
+ empty = os.path.join(self.tmp, "empty")
+ put(empty, "analysis/other/data.csv", "a,b\n1,2\n")
+ self.assertIsNone(tm.state_metrics(empty))
+ self.assertIsNone(tm.state_metrics(os.path.join(self.tmp, "nowhere")))
+ self.assertEqual(self.hook("state", {"cwd": empty}), "")
+
+ def test_the_language_is_the_biggest_one_that_is_code(self):
+ def lang(rows):
+ return tm.language_code({"root": {"children": [{"kind": "module", "file": "f", "loc": n, "language": name} for name, n in rows]}})
+ self.assertEqual(lang([("artifact", 9000), ("c", 2900), ("cpp", 650), ("make", 500)]), 7)
+ self.assertEqual(lang([("C#", 10), ("Java", 5)]), 3)
+ self.assertEqual(lang([("javascript", 10), ("java", 50)]), 2)
+ self.assertEqual(lang([("make", 10), ("yaml", 5)]), 0)
+ self.assertEqual(lang([("PL/I", 90000), ("JCL", 500)]), 99)
+ self.assertEqual(lang([("C/C++", 10), ("make", 900)]), 7)
+ self.assertEqual(lang([("Node.js", 10)]), 9)
+ self.assertEqual(lang([("ASP.NET", 10), ("VB6", 20)]), 3)
+ self.assertEqual(lang([("Java/JSP", 10)]), 2)
+ self.assertEqual(lang([("Fortran", 10), ("Assembler", 5)]), 15)
+ self.assertEqual(lang([("some new language", 10)]), 99)
+ self.assertEqual(lang([]), 0)
+ self.assertEqual(tm.language_code(None), 0)
+
+ def test_the_goal_is_the_answer_after_the_word_goal_and_nothing_else(self):
+ cases = [("- Goal: reimagine (rebuild it from scratch)", 4), ("## Goal\n- **Goal:** `uplift`. Then transform it.", 2),
+ ("## Goal\n**Transform** piece by piece", 3), ("- Goal: Understand it first", 1),
+ ("## Goal\n- **Goal:** `not-sure`. Run assess, then pick (uplift / transform / reimagine / understand).", 0),
+ ("- Goal: Move to a newer version of the same technology", 2), ("- Goal: Rewrite it in a different technology", 3),
+ ("nothing here about it, but uplift is a word", 0), ("", 0)]
+ for text, want in cases:
+ self.assertEqual(tm.goal_code(text, None), want, text)
+ self.assertEqual(tm.goal_code("", ""), 3)
+ self.assertEqual(tm.goal_code("", "-uplifted"), 2)
+ self.assertEqual(tm.goal_code("- Goal: not sure", "-reimagined"), 4)
+
+ def test_uplift_and_reimagine_leave_their_own_marks(self):
+ ws = os.path.join(self.tmp, "up")
+ put(ws, "analysis/sys/DELTA_CATALOG.md", "# Deltas\n")
+ put(ws, "analysis/sys/BASELINE.md", "# Baseline\n")
+ put(ws, "analysis/sys/PLAYBOOK.md", "# Playbook\n")
+ put(ws, "modernized/sys-uplifted/pom.xml", "<x/>")
+ put(ws, "modernized/sys-uplifted/mod/a.java", "x")
+ s = tm.state_metrics(ws)
+ self.assertEqual((s["goal"], s["built"]), (2, 1))
+ self.assertEqual(s["done"], STEP_BITS["deltas"] + STEP_BITS["baseline"] + STEP_BITS["playbook"] + STEP_BITS["built"])
+
+ def test_hostile_artifacts_give_whole_numbers_and_nothing_else(self):
+ put(self.ws, "analysis/sys/VERIFICATION.json", {"modules": "x", "signoff": ["not", "a", "dict"]})
+ put(self.ws, "analysis/sys/EQUIVALENCE.json", {"cases": [{"verdict": ["same"]}, 5, None, {"verdict": "same", "approvedDifference": {"a": 1}}]})
+ put(self.ws, "analysis/sys/RULE_REVIEWS.json", {"reviews": {"RULE-001": "confirmed", "RULE-002": {"verdict": ["wrong"]}}})
+ put(self.ws, "analysis/sys/topology.json", "{not json")
+ s = tm.state_metrics(self.ws)
+ self.assertEqual((s["v_proven"], s["v_partly"], s["v_not"]), (0, 0, 0))
+ self.assertEqual((s["rev_ok"], s["rev_wrong"], s["map_kloc"], s["lang"]), (0, 0, 0, 0))
+ self.assertTrue(all(type(v) is int and 0 <= v <= tm.COUNT_LIMIT for v in tm.ok(s).values()))
+
+ def test_the_counts_come_from_the_case_list_and_the_modules_checks_not_from_the_files_own_totals(self):
+ put(self.ws, "analysis/sys/EQUIVALENCE.json", {"cases": [{"id": "1", "verdict": "differs"}, {"id": "2", "verdict": "same"}, {"id": "3", "verdict": "differs"}],
+ "totals": {"executed": 100, "differs": 0, "differsApproved": 0, "missing": 0}})
+ put(self.ws, "analysis/sys/VERIFICATION.json", {"modules": [module("a", tests="fail")], "overall": {"counts": {"PROVEN": 5, "PARTLY PROVEN": 0, "NOT PROVEN": 0}}})
+ s = tm.state_metrics(self.ws)
+ self.assertEqual((s["eq_cases"], s["eq_diff"], s["eq_appr"]), (3, 2, 0))
+ self.assertEqual((s["v_proven"], s["v_partly"], s["v_not"]), (0, 0, 1))
+
+ def test_a_linked_artifact_is_not_read(self):
+ outside = os.path.join(self.tmp, "outside.md")
+ with open(outside, "w") as fh:
+ fh.write(RULES)
+ os.remove(os.path.join(self.ws, "analysis", "sys", "BUSINESS_RULES.md"))
+ try:
+ os.symlink(outside, os.path.join(self.ws, "analysis", "sys", "BUSINESS_RULES.md"))
+ except OSError:
+ self.skipTest("no symbolic links here")
+ s = tm.state_metrics(self.ws)
+ self.assertEqual((s["rules"], s["done"] & STEP_BITS["rules"]), (0, 0))
+
+ def test_the_newest_system_is_the_one_counted(self):
+ workspace(self.ws, "old", {"BUSINESS_RULES.md": "### RULE-001 ยท One\n**Priority:** P0\n"})
+ for f in os.scandir(os.path.join(self.ws, "analysis", "old")):
+ os.utime(f.path, (1, 1))
+ os.utime(os.path.join(self.ws, "analysis", "old"), (1, 1))
+ self.assertEqual(tm.state_metrics(self.ws)["rules"], 3)
+ put(self.ws, "analysis/old/NEW.md", "x")
+ self.assertEqual(tm.state_metrics(self.ws)["rules"], 1)
+
+
+class Commands(Base):
+ def cmd(self, prompt):
+ return tm.command_metrics(prompt, self.ws)
+
+ def test_the_plugins_commands_have_a_number_each(self):
+ for verb, n in tm.COMMANDS.items():
+ prompt = "/code-modernization:modernize" + ("-" + verb if verb else "")
+ self.assertEqual(self.cmd(prompt + " sys")["cmd"], n, prompt)
+ self.assertEqual(self.cmd("/modernize")["cmd"], 1)
+ self.assertEqual(self.cmd(" /code-modernization:modernize-verify")["cmd"], 11)
+ self.assertEqual(self.cmd("/code-modernization:modernize-harden-scan x")["cmd"], 99)
+ self.assertEqual([self.cmd("/modernize-" + v)["cmd"] for v in ("panel", "review-pane", "sign")], [20, 21, 22])
+ self.assertEqual(self.cmd("/code-modernization:modernize-review")["cmd"], 6)
+ self.assertEqual(self.cmd("/modernize-review")["cmd"], 6) # the bare name is the plugin's command now that the pane's deck has its own
+
+ def test_other_prompts_are_not_counted(self):
+ for prompt in ("", None, "modernize this", "/modernizer", "/other:modernize", "please /modernize", "/code-modernization:assess", 5, {"a": 1}):
+ self.assertIsNone(self.cmd(prompt), prompt)
+
+ def test_the_command_says_where_the_system_stands_and_whether_source_was_given(self):
+ s = self.cmd("/code-modernization:modernize-verify sys --source /work/x")
+ self.assertEqual((s["has_source"], s["fresh"], s["systems"], s["goal"]), (1, 0, 1, 3))
+ self.assertEqual(s["done"], tm.state_metrics(self.ws)["done"])
+ new = tm.command_metrics("/code-modernization:modernize billing --source /a", os.path.join(self.tmp, "fresh-folder"))
+ self.assertEqual((new["fresh"], new["systems"], new["done"], new["has_source"]), (1, 0, 0, 1))
+
+ def test_the_permission_mode_is_a_number_from_a_fixed_list(self):
+ for mode, want in (("default", 1), ("auto", 4), ("bypassPermissions", 5), ("something new", 0), (None, 0), (7, 0)):
+ self.assertEqual(tm.command_metrics("/modernize", self.ws, mode)["perm"], want, mode)
+ out = self.hook("command", {"cwd": self.ws, "prompt": "/modernize", "permission_mode": "plan"})
+ self.assertEqual(json.loads(out)["metrics"]["perm"], 3)
+
+ def test_a_named_system_is_the_one_counted_and_its_name_is_never_sent(self):
+ workspace(self.ws, "other", {"BUSINESS_RULES.md": None, "EQUIVALENCE.json": None, "VERIFICATION.json": None})
+ os.utime(os.path.join(self.ws, "analysis", "sys"), (1, 1))
+ for f in os.scandir(os.path.join(self.ws, "analysis", "sys")):
+ os.utime(f.path, (1, 1))
+ named = self.cmd("/code-modernization:modernize-status sys")
+ newest = self.cmd("/code-modernization:modernize-status")
+ self.assertNotEqual(named["done"], newest["done"])
+ self.assertEqual(named["systems"], 2)
+ for value in named.values():
+ self.assertIsInstance(value, int)
+
+
+class Sending(Base):
+ def test_every_key_is_defined_and_every_value_is_a_whole_number(self):
+ self.assertLessEqual(len(tm.STATE_KEYS), tm.MAX_KEYS)
+ self.assertLessEqual(len(tm.COMMAND_KEYS), tm.MAX_KEYS)
+ self.assertEqual(list(tm.MEANING)[0], "pv")
+ for keys in (tm.STATE_KEYS, tm.COMMAND_KEYS):
+ self.assertEqual(keys[0], "pv")
+ for k in keys:
+ self.assertRegex(k, r"^[a-z][a-z0-9_]{0,39}$")
+ self.assertIn(k, tm.MEANING)
+ for out in (self.hook("state", {"cwd": self.ws}), self.hook("command", {"cwd": self.ws, "prompt": "/code-modernization:modernize-map"})):
+ line = json.loads(out)
+ self.assertEqual(list(line), ["metrics"])
+ self.assertTrue(all(type(v) is int and 0 <= v <= tm.CAP for v in line["metrics"].values()), line)
+ self.assertEqual(out.count("\n"), 1)
+
+ def test_a_key_that_is_not_defined_never_goes_out(self):
+ got = tm.ok({"pv": 1, "path": "/secret", "rules": "7", "p0": True, "goal": 2.9, "done": -3})
+ self.assertEqual(dict(got), {"pv": 1, "rules": 7, "p0": 1, "goal": 2, "done": 0})
+
+ def test_the_plugin_version_is_sent_and_matches_the_manifest(self):
+ with open(os.path.join(PLUGIN, ".claude-plugin", "plugin.json")) as fh:
+ major, minor, patch = (int(x) for x in json.load(fh)["version"].split("."))
+ self.assertEqual(tm.plugin_version(), major * 10000 + minor * 100 + patch)
+ self.assertGreater(tm.plugin_version(), 0)
+ self.assertEqual(json.loads(self.hook("state", {"cwd": self.ws}))["metrics"]["pv"], tm.plugin_version())
+
+ def test_the_same_counts_are_sent_once_and_a_change_is_sent_again(self):
+ self.assertTrue(self.hook("state", {"cwd": self.ws}))
+ self.assertEqual(self.hook("state", {"cwd": self.ws}), "")
+ put(self.ws, "analysis/sys/RULE_REVIEWS.json", {"reviews": {"RULE-001": {"verdict": "wrong"}}})
+ self.assertIn('"rev_wrong":1', self.hook("state", {"cwd": self.ws}))
+ self.assertEqual(self.hook("state", {"cwd": self.ws}), "")
+ self.assertTrue(self.hook("command", {"cwd": self.ws, "prompt": "/modernize"}))
+ self.assertTrue(self.hook("command", {"cwd": self.ws, "prompt": "/modernize"}))
+
+ def test_when_it_cannot_remember_it_sends_nothing_rather_than_repeat_every_turn(self):
+ os.environ["CLAUDE_PLUGIN_DATA"] = os.path.join(self.tmp, "does-not-exist")
+ blocked = os.path.join(self.tmp, "blocked")
+ with mock.patch.object(tm.tempfile, "gettempdir", return_value=blocked), mock.patch.object(tm.os, "makedirs", side_effect=OSError("read-only")):
+ self.assertEqual(self.hook("state", {"cwd": self.ws}), "")
+
+ def test_the_debug_switch_says_why_each_hook_run_sent_nothing_in_the_plugins_data_folder(self):
+ log = os.path.join(self.data, "telemetry-debug.log")
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY_DEBUG": "1"}):
+ self.hook("state", {"cwd": self.ws})
+ self.hook("state", {"cwd": self.ws})
+ self.hook("state", {"cwd": os.path.join(self.tmp, "nowhere")})
+ self.hook("command", {"cwd": self.ws, "prompt": "hello"})
+ self.hook("command", {"cwd": self.ws, "prompt": "/modernize"})
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY": "0"}):
+ self.hook("state", {"cwd": self.ws})
+ with mock.patch.object(tm, "state_metrics", side_effect=KeyError("x")), mock.patch.object(sys, "stdin", io.StringIO("{}")), \
+ contextlib.redirect_stdout(io.StringIO()):
+ tm.main(["telemetry.py", "state"])
+ with open(log, encoding="utf-8") as fh:
+ text = fh.read()
+ for want in ("state: sent 19 values", "state: not sent, the counts are the same as last time", "state: not sent, the plugin has left no files here",
+ "command: not sent, not one of the plugin's commands", "command: sent 12 values", "state: not sent, CODE_MODERNIZATION_TELEMETRY is off",
+ "state: error KeyError at line"):
+ self.assertIn(want, text)
+ self.assertNotIn(self.tmp, text)
+
+ def test_the_debug_switch_never_writes_anywhere_else_and_never_follows_a_link(self):
+ victim = os.path.join(self.tmp, "victim.txt")
+ with open(victim, "w") as fh:
+ fh.write("keep")
+ try:
+ os.symlink(victim, os.path.join(self.data, "telemetry-debug.log"))
+ except OSError:
+ self.skipTest("no symbolic links here")
+ elsewhere = os.path.join(self.tmp, "somewhere-else.log")
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY_DEBUG": elsewhere}):
+ self.hook("state", {"cwd": self.ws})
+ self.assertFalse(os.path.exists(elsewhere)) # a path in the variable is not a switch
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY_DEBUG": "1"}):
+ self.hook("command", {"cwd": self.ws, "prompt": "/modernize"})
+ with open(victim) as fh:
+ self.assertEqual(fh.read(), "keep")
+
+ def test_each_off_switch_stops_everything(self):
+ for name, value in (("CODE_MODERNIZATION_TELEMETRY", "0"), ("CODE_MODERNIZATION_TELEMETRY", "Off"), ("CLAUDE_PLUGIN_OPTION_TELEMETRY", "false"),
+ ("CODE_MODERNIZATION_TELEMETRY", "disabled"), ("CODE_MODERNIZATION_TELEMETRY", "n"), ("CODE_MODERNIZATION_TELEMETRY", "none"),
+ ("CODE_MODERNIZATION_TELEMETRY", "2"), ("CLAUDE_PLUGIN_OPTION_TELEMETRY", "no"),
+ ("DISABLE_TELEMETRY", "1"), ("CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "true")):
+ with mock.patch.dict(os.environ, {name: value}):
+ self.assertTrue(tm.off_reason(), name)
+ self.assertEqual(self.hook("state", {"cwd": self.ws}), "", name)
+ self.assertEqual(self.hook("command", {"cwd": self.ws, "prompt": "/modernize"}), "", name)
+ for name, value in (("CODE_MODERNIZATION_TELEMETRY", "1"), ("CODE_MODERNIZATION_TELEMETRY", ""), ("CLAUDE_PLUGIN_OPTION_TELEMETRY", "true"),
+ ("CLAUDE_PLUGIN_OPTION_TELEMETRY", " TRUE "), ("DISABLE_TELEMETRY", "0")):
+ with mock.patch.dict(os.environ, {name: value}):
+ self.assertEqual(tm.off_reason(), "", name)
+
+ def test_bad_hook_input_never_raises_and_never_prints(self):
+ for raw in ("", "not json", "[1,2]", '{"cwd": 5, "prompt": ["x"]}', '{"cwd": "/no/such/dir"}'):
+ out = io.StringIO()
+ with mock.patch.object(sys, "stdin", io.StringIO(raw)), contextlib.redirect_stdout(out), mock.patch.object(tm.os, "getcwd", return_value=self.tmp):
+ self.assertEqual(tm.main(["telemetry.py", "command"]), 0)
+ self.assertEqual(tm.main(["telemetry.py", "state"]), 0)
+ self.assertEqual(out.getvalue(), "", raw)
+
+ def test_show_prints_what_would_be_sent_in_words_and_as_json(self):
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out):
+ tm.show([self.ws, "--prompt", "/code-modernization:modernize-verify sys"])
+ text = out.getvalue()
+ for word in ("Sending is on", "rules", "business rules found", "steps that have left their file", "preflight, assess, map, rules", "modernize-verify"):
+ self.assertIn(word, text)
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out), mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY": "0"}):
+ tm.show([self.ws, "--json"])
+ got = json.loads(out.getvalue())
+ self.assertEqual(got["off"], "CODE_MODERNIZATION_TELEMETRY is off")
+ self.assertEqual(got["state"]["rules"], 3)
+
+
+class Hardening(Base):
+ """What a hostile or odd workspace must not be able to do."""
+
+ def test_counts_from_a_hundred_up_are_rounded_and_every_key_has_a_ceiling(self):
+ got = tm.ok({"pv": 10000, "rules": 97, "p0": 26, "map_kloc": 344, "eq_cases": 17727, "sec_high": 10 ** 12, "done": 43247, "goal": 9, "perm": 99,
+ "phases": 5000, "systems": 500})
+ self.assertEqual(dict(got), {"pv": 10000, "rules": 97, "p0": 26, "map_kloc": 340, "eq_cases": 18000, "sec_high": tm.COUNT_LIMIT, "done": 43247,
+ "goal": 4, "perm": 6, "phases": 99, "systems": 99})
+ self.assertEqual([tm.rounded(n) for n in (99, 100, 104, 105, 999, 1049, 1050, 17727)], [99, 100, 100, 110, 1000, 1000, 1100, 18000])
+
+ def test_a_folder_of_another_tool_is_not_the_plugins_even_with_the_same_file_names(self):
+ other = os.path.join(self.tmp, "other")
+ put(other, "analysis/2024-q3/topology.json", TOPOLOGY)
+ put(other, "analysis/2024-q3/BASELINE.md", "# baseline\n")
+ put(other, "analysis/2024-q3/ASSESSMENT.md", "# assessment\n")
+ self.assertIsNone(tm.state_metrics(other))
+ put(other, "analysis/2024-q3/PREFLIGHT.md", "# the plugin's\n")
+ self.assertIsNotNone(tm.state_metrics(other))
+
+ def test_another_tools_commands_are_not_counted_and_the_newest_folder_need_not_be_the_plugins(self):
+ self.assertIsNone(tm.command_metrics("/modernize-schema", self.ws))
+ self.assertEqual(tm.command_metrics("/code-modernization:modernize-schema", self.ws)["cmd"], 99)
+ put(self.ws, "analysis/zzz-data/results.csv", "a,b\n")
+ self.assertEqual(tm.state_metrics(self.ws)["rules"], 3)
+
+ def test_a_new_name_is_a_new_system_and_a_flags_value_is_not_a_name(self):
+ new = tm.command_metrics("/code-modernization:modernize-preflight billing --source /x", self.ws)
+ self.assertEqual((new["fresh"], new["done"], new["goal"], new["has_source"]), (1, 0, 0, 1))
+ newest = tm.command_metrics("/modernize --source elsewhere", self.ws)
+ self.assertEqual((newest["fresh"], newest["has_source"]), (0, 1))
+ self.assertEqual(tm.system_word(["--source", "abc", "def"]), "def")
+ self.assertIsNone(tm.system_word(["--source", "abc", "/work/x"]))
+
+ def test_links_that_lead_outside_the_workspace_are_not_followed(self):
+ outside = os.path.join(self.tmp, "outside")
+ for i in range(7):
+ put(outside, "d%d/x.txt" % i, "x")
+ shutil.rmtree(os.path.join(self.ws, "modernized", "sys"))
+ try:
+ os.symlink(outside, os.path.join(self.ws, "modernized", "sys"))
+ except OSError:
+ self.skipTest("no symbolic links here")
+ self.assertEqual(tm.state_metrics(self.ws)["built"], 0)
+ elsewhere = os.path.join(self.tmp, "elsewhere")
+ workspace(elsewhere, "real")
+ linked = os.path.join(self.tmp, "linked")
+ os.makedirs(linked)
+ os.symlink(os.path.join(elsewhere, "analysis"), os.path.join(linked, "analysis"))
+ self.assertEqual(tm.systems_in(linked), [])
+ self.assertIsNone(tm.state_metrics(linked))
+
+ def test_a_dangling_link_inside_a_system_does_not_drop_the_system(self):
+ try:
+ os.symlink(os.path.join(self.tmp, "gone"), os.path.join(self.ws, "analysis", "sys", "dangling"))
+ except OSError:
+ self.skipTest("no symbolic links here")
+ self.assertEqual(tm.state_metrics(self.ws)["rules"], 3)
+
+ def test_a_data_folder_that_is_itself_a_link_still_works(self):
+ real = os.path.join(self.tmp, "real-data")
+ os.makedirs(real)
+ link = os.path.join(self.tmp, "data-link")
+ try:
+ os.symlink(real, link)
+ except OSError:
+ self.skipTest("no symbolic links here")
+ with mock.patch.dict(os.environ, {"CLAUDE_PLUGIN_DATA": link}):
+ self.assertTrue(self.hook("state", {"cwd": self.ws}))
+ self.assertTrue(os.path.exists(os.path.join(real, "telemetry-state.json")))
+
+ def test_an_artifact_built_to_make_a_pattern_slow_does_not_hold_anything_up(self):
+ nasty = "### RULE-001 - t" + " " * 4000 + "x\n**Priority:** P0" + " " * 4000 + "x\n"
+ put(self.ws, "analysis/sys/BUSINESS_RULES.md", nasty * 3)
+ put(self.ws, "analysis/sys/SECURITY_FINDINGS.md", "| a |" + " " * 8000 + "x|\n|" + "-" * 5000 + "|\n")
+ started = time.time()
+ s = tm.state_metrics(self.ws)
+ self.assertLess(time.time() - started, 3)
+ self.assertGreaterEqual(s["rules"], 1)
+
+ def test_whatever_happens_the_hook_gives_up_within_a_few_seconds_and_says_nothing(self):
+ code = ("import sys, time; sys.path.insert(0, %r); import telemetry as tm; tm.state_metrics = lambda cwd: time.sleep(30); tm.hook('state')" %
+ os.path.join(PLUGIN, "scripts"))
+ started = time.time()
+ done = subprocess.run([sys.executable, "-c", code], input=json.dumps({"cwd": self.ws}), capture_output=True, text=True, timeout=20,
+ env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"})
+ self.assertLess(time.time() - started, 10)
+ self.assertEqual((done.returncode, done.stdout), (0, ""))
+
+ def test_the_alarm_is_cancelled_when_the_hook_returns(self):
+ import signal
+ self.hook("state", {"cwd": self.ws})
+ if hasattr(signal, "alarm"):
+ self.assertEqual(signal.alarm(0), 0)
+
+
+class Environment(Base):
+ """What the plugin says about the machine it runs on."""
+
+ def test_the_shell_wrappers_answers_are_used_and_bounded(self):
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_OS": "3", "CODE_MODERNIZATION_PY": "4", "CODE_MODERNIZATION_PATHF": "5"}):
+ got = tm.command_metrics("/modernize", self.ws)
+ self.assertEqual((got["os"], got["py"], got["pathf"]), (3, 4, 5))
+ self.assertEqual(got["pyv"], sys.version_info[0] * 100 + sys.version_info[1])
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_OS": "77", "CODE_MODERNIZATION_PY": "-1", "CODE_MODERNIZATION_PATHF": "x"}):
+ got = tm.command_metrics("/modernize", self.ws)
+ self.assertEqual((got["os"], got["py"]), (9, 0))
+ self.assertIn(got["os"], range(0, 10))
+
+ def test_this_machine_is_recognised_without_the_wrapper(self):
+ self.assertIn(tm.os_code(), (1, 2, 3, 4))
+ self.assertEqual(tm.python_status(), 0)
+
+ def test_a_path_that_breaks_scripts_is_flagged_in_bits(self):
+ self.assertEqual([tm.path_flags(p) for p in ("/plain/path", "/has a space", "/caf\u00e9", "/a b/caf\u00e9", "/" + "x" * 250, "/a b/caf\u00e9/" + "x" * 250)],
+ [0, 1, 2, 3, 4, 7])
+
+ def test_health_carries_only_the_environment(self):
+ got = json.loads(self.hook("health", {"cwd": self.ws}))["metrics"]
+ self.assertEqual(sorted(got), sorted(tm.HEALTH_KEYS if hasattr(tm, "HEALTH_KEYS") else ["pv", "os", "py", "pyv", "pathf"]))
+ self.assertTrue(all(type(v) is int for v in got.values()))
+
+
+class Failures(Base):
+ """A failure becomes a tool number and a kind number. Its text is read here and never sent."""
+
+ CASES = [("Bash", "Exit code 127\nbash: python3: command not found", 1), ("Bash", "Exit code 127\n(eval):1: command not found: python3", 1),
+ ("Bash", "'python3' is not recognized as an internal or external command", 1), ("Bash", "env: python3: No such file or directory", 1),
+ ("Bash", "Python was not found; run without arguments to install from the Microsoft Store.", 2),
+ ("Bash", "xcode-select: note: no developer tools were found at '/Applications/Xcode.app'", 3),
+ ("Bash", "Exit code 1\n File \"x.py\", line 3\nSyntaxError: invalid syntax", 4),
+ ("Bash", "Traceback (most recent call last):\n File \"/x/code-modernization/1.0.0/scripts/build_report.py\", line 9\nKeyError: 'a'", 5),
+ ("Edit", "Permission for this action was denied by the auto mode classifier", 6), ("Bash", "Permission to use Bash with command x has been denied.", 6),
+ ("Write", "EACCES: permission denied, open '/x'", 7), ("Bash", "Exit code 124\ncommand timed out after 120s", 8),
+ ("Read", "File does not exist. Note: your current working directory is /x.", 9), ("Bash", "Exit code 127\n(eval):1: command not found: python3_not_a_real_program", 9),
+ ("WebFetch", "connect ECONNREFUSED 127.0.0.1:443", 10), ("Bash", "npm ERR! 401 Unauthorized", 10), ("Bash", "ENOSPC: no space left on device", 11),
+ ("Workflow", "scriptPath must be inside the working directory", 12), ("Read", "something nobody has seen before", 99)]
+
+ def fail(self, tool, error, session="s1", **more):
+ return tm.failure_metrics({"hook_event_name": "PostToolUseFailure", "tool_name": tool, "error": error, "session_id": session, **more}, self.ws)
+
+ def test_each_kind_of_failure_gets_its_code(self):
+ for tool, text, want in self.CASES:
+ self.assertEqual(tm.failure_kind(text) or 99, want, text)
+ for name, want in (("Bash", 1), ("Read", 2), ("MultiEdit", 3), ("Grep", 4), ("Task", 5), ("Workflow", 6), ("AskUserQuestion", 7), ("Skill", 8), ("WebSearch", 9),
+ ("mcp__x__y", 10), ("Whatever", 99), (None, 99)):
+ self.assertEqual(tm.tool_code(name), want, name)
+
+ def test_a_failure_is_numbers_only_and_once_per_kind_per_session(self):
+ first = self.fail("Bash", "Exit code 127\nbash: python3: command not found")
+ self.assertEqual((first["tool"], first["kind"]), (1, 1))
+ self.assertEqual(list(first), ["pv", "os", "py", "pyv", "pathf", "tool", "kind"])
+ self.assertTrue(all(type(v) is int for v in tm.ok(first).values()))
+ self.assertNotIn("python", json.dumps(tm.ok(first)))
+ self.assertIsNone(self.fail("Bash", "Exit code 127\nbash: python3: command not found"))
+ self.assertIsNotNone(self.fail("Bash", "Exit code 127\nbash: python3: command not found", session="s2"))
+ self.assertIsNotNone(self.fail("Bash", "ENOSPC: no space left on device"))
+ self.assertIsNotNone(self.fail("Read", "ENOSPC: no space left on device"))
+
+ def test_a_plain_non_zero_exit_and_an_interruption_say_nothing(self):
+ self.assertIsNone(self.fail("Bash", "Exit code 1"))
+ self.assertIsNone(self.fail("Bash", " Exit code 2 "))
+ self.assertIsNone(self.fail("Bash", "Exit code 127\nbash: python3: command not found", is_interrupt=True))
+ self.assertIsNotNone(self.fail("Bash", "Exit code 1\ngrep: unmatched ["))
+
+ def test_hostile_failure_text_never_crashes_and_never_leaves(self):
+ for text in ("x" * 2_000_000, "\x00\x01" * 5000, "\u202e" * 3000, None, 5, {"a": 1}, ["Traceback"] * 100):
+ got = self.fail("Bash", text, session="h%s" % id(text))
+ if got is not None:
+ self.assertTrue(all(type(v) is int for v in got.values()))
+ secret = self.fail("Bash", "ENOSPC while writing /home/alice/secret-project/passwords.txt", session="p")
+ self.assertNotIn("alice", json.dumps(tm.ok(secret)))
+
+ def test_a_model_call_that_ended_a_turn_is_one_code(self):
+ for text, want in (("rate_limit", 1), ("authentication_failed", 2), ("billing_error", 3), ("invalid_request", 4), ("server_error", 5), ("overloaded_error", 5),
+ ("max_output_tokens", 6), ("request timeout", 7), ("", 99), ("something new", 99)):
+ got = tm.failure_metrics({"hook_event_name": "StopFailure", "error": text, "session_id": "m-" + text}, self.ws)
+ self.assertEqual(got["api"], want, text)
+ self.assertNotIn("tool", got)
+
+ def test_the_hook_prints_one_line_of_numbers_and_then_nothing(self):
+ payload = {"cwd": self.ws, "hook_event_name": "PostToolUseFailure", "tool_name": "Read", "error": "File does not exist.", "session_id": "z"}
+ line = json.loads(self.hook("failure", payload))["metrics"]
+ self.assertEqual((line["tool"], line["kind"]), (2, 9))
+ self.assertEqual(self.hook("failure", payload), "")
+
+
+class Runs(Base):
+ def test_the_last_extraction_is_counted_from_its_own_statistics(self):
+ put(self.ws, "analysis/sys/rules_result.json", {"stats": {"agents": 172, "failedModules": ["a", "b"], "droppedModules": ["c"], "skippedModules": ["d"], "unverified": 5}})
+ got = tm.ok(tm.run_metrics(self.ws))
+ self.assertEqual(dict(got), {"pv": tm.plugin_version(), "agents": 170, "wf_failed": 3, "wf_skip": 1, "wf_unver": 5})
+ self.assertTrue(self.hook("run", {"cwd": self.ws}))
+ self.assertEqual(self.hook("run", {"cwd": self.ws}), "")
+
+ def test_no_statistics_or_odd_ones_say_nothing_or_zeros(self):
+ self.assertIsNone(tm.run_metrics(self.ws))
+ put(self.ws, "analysis/sys/rules_result.json", {"stats": {"agents": "many", "failedModules": "x", "unverified": [1]}})
+ got = tm.ok(tm.run_metrics(self.ws))
+ self.assertEqual((got["agents"], got["wf_failed"], got["wf_skip"], got["wf_unver"]), (0, 0, 0, 0))
+
+
+class OwnErrors(Base):
+ def crash(self, error):
+ out = io.StringIO()
+ with mock.patch.object(tm, "state_metrics", side_effect=error), mock.patch.object(sys, "stdin", io.StringIO("{}")), contextlib.redirect_stdout(out):
+ self.assertEqual(tm.main(["telemetry.py", "state"]), 0)
+ return out.getvalue()
+
+ def test_the_plugins_own_errors_are_sent_as_a_kind_and_a_line_once(self):
+ first = self.crash(ValueError("secret text about /home/alice"))
+ line = json.loads(first)["metrics"]
+ self.assertEqual(line["err"], 2)
+ self.assertGreater(line["err_at"], 0)
+ self.assertNotIn("secret", first)
+ self.assertEqual(self.crash(ValueError("again")), "")
+ self.assertEqual(json.loads(self.crash(UnicodeDecodeError("utf-8", b"", 0, 1, "bad")))["metrics"]["err"], 8)
+ self.assertEqual(json.loads(self.crash(KeyError("k")))["metrics"]["err"], 1)
+ self.assertEqual(json.loads(self.crash(RuntimeError("odd")))["metrics"]["err"], 99)
+
+ def test_an_error_is_not_reported_when_counts_are_off(self):
+ with mock.patch.dict(os.environ, {"CODE_MODERNIZATION_TELEMETRY": "0"}):
+ self.assertEqual(self.crash(KeyError("k")), "")
+
+ def test_without_an_alarm_signal_a_timer_stands_in_and_is_cancelled(self):
+ class NoAlarm:
+ pass
+ with mock.patch.object(tm, "signal", NoAlarm()):
+ self.assertTrue(self.hook("state", {"cwd": self.ws}))
+ self.assertEqual([t for t in threading.enumerate() if isinstance(t, threading.Timer)], [])
+
+
+class Shell(unittest.TestCase):
+ """scripts/telemetry.sh must start Python only when the plugin is in use."""
+
+ def setUp(self):
+ self.tmp = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, self.tmp, True)
+ self.bin = os.path.join(self.tmp, "bin")
+ os.makedirs(self.bin)
+ self.ws = workspace(os.path.join(self.tmp, "ws"))
+ self.data = os.path.join(self.tmp, "data")
+ os.makedirs(self.data)
+ os.symlink(sys.executable, os.path.join(self.bin, "python3"))
+
+ def run_hook(self, mode, payload, cwd=None, path=None, env=None):
+ env = {"PATH": path if path is not None else self.bin + os.pathsep + "/usr/bin:/bin", "CLAUDE_PLUGIN_DATA": self.data, **(env or {})}
+ done = subprocess.run(["/bin/sh", os.path.join(PLUGIN, "scripts", "telemetry.sh"), mode], input=json.dumps(payload), capture_output=True, text=True,
+ cwd=cwd or self.ws, env=env, timeout=60)
+ self.assertEqual(done.returncode, 0, done.stderr)
+ return done.stdout
+
+ def test_a_plugin_command_is_counted_and_the_end_of_a_turn_in_a_workspace_is_counted(self):
+ out = self.run_hook("command", {"cwd": self.ws, "prompt": "/code-modernization:modernize-status sys"})
+ self.assertEqual(json.loads(out)["metrics"]["cmd"], 13)
+ self.assertEqual(json.loads(self.run_hook("state", {"cwd": self.ws}))["metrics"]["rules"], 3)
+ self.assertEqual(self.run_hook("state", {"cwd": self.ws}), "")
+
+ def test_it_starts_nothing_when_the_plugin_is_not_in_use(self):
+ no_python = os.path.join(self.tmp, "nopy")
+ os.makedirs(no_python)
+ for tool in ("grep", "dirname", "uname", "cat"):
+ os.symlink(shutil.which(tool), os.path.join(no_python, tool))
+ elsewhere = os.path.join(self.tmp, "elsewhere")
+ os.makedirs(elsewhere)
+ self.assertEqual(self.run_hook("command", {"prompt": "write me a poem"}), "")
+ self.assertEqual(self.run_hook("command", {"prompt": "/commit"}), "")
+ self.assertEqual(self.run_hook("state", {"cwd": elsewhere}, cwd=elsewhere), "")
+ self.assertEqual(self.run_hook("nonsense", {}), "")
+ # no python at all: quiet for a turn's end, and a typed command is reported in numbers by the shell itself
+ self.assertEqual(self.run_hook("state", {"cwd": self.ws}, path=no_python), "")
+ said = json.loads(self.run_hook("command", {"prompt": "/modernize"}, path=no_python))["metrics"]
+ self.assertEqual((said["cmd"], said["py"], said["pyv"]), (1, 1, 0))
+
+ def toolbox(self, name, **fake):
+ """A PATH folder holding only the tools the wrapper uses plus fake ones: name=script text, or "python" for the real interpreter."""
+ d = os.path.join(self.tmp, name)
+ os.makedirs(d)
+ for tool in ("grep", "dirname", "sed", "head", "id", "mkdir", "cat", "uname"):
+ if tool not in fake and shutil.which(tool):
+ os.symlink(shutil.which(tool), os.path.join(d, tool))
+ for tool, body in fake.items():
+ path = os.path.join(d, tool)
+ if body == "python":
+ os.symlink(sys.executable, path)
+ else:
+ with open(path, "w") as fh:
+ fh.write(body)
+ os.chmod(path, 0o755)
+ return d
+
+ def say(self, mode, payload, **kw):
+ out = self.run_hook(mode, payload, **kw)
+ return json.loads(out)["metrics"] if out else None
+
+ def test_windows_is_recognised_and_a_missing_python_is_reported_by_the_shell(self):
+ box = self.toolbox("win", uname="#!/bin/sh\necho MINGW64_NT-10.0-19045\n")
+ got = self.say("command", {"prompt": "/code-modernization:modernize-verify x"}, path=box)
+ self.assertEqual((got["os"], got["py"], got["cmd"], got["pyv"]), (3, 1, 11, 0))
+
+ def test_the_windows_store_placeholder_is_a_broken_python3_and_python_is_used_instead(self):
+ stub = "#!/bin/sh\necho 'Python was not found; run without arguments to install from the Microsoft Store.' >&2\nexit 49\n"
+ box = self.toolbox("store", python3=stub, uname="#!/bin/sh\necho MSYS_NT-10.0\n")
+ self.assertEqual(self.say("command", {"prompt": "/modernize"}, path=box)["py"], 3) # nothing else to fall back to
+ box = self.toolbox("store2", python3=stub, python="python", uname="#!/bin/sh\necho MSYS_NT-10.0\n")
+ got = self.say("command", {"prompt": "/modernize"}, path=box) # python does the work, and says so
+ self.assertEqual((got["os"], got["py"], got["cmd"]), (3, 4, 1))
+ self.assertEqual(got["pyv"], sys.version_info[0] * 100 + sys.version_info[1])
+
+ def test_a_python_that_is_too_old_and_one_that_crashes_are_told_apart(self):
+ old = "#!/bin/sh\ncase \"$1\" in -c) echo 307 ;; *) exit 1 ;; esac\n"
+ crash = "#!/bin/sh\ncase \"$1\" in -c) echo 311 ;; *) exit 1 ;; esac\n"
+ self.assertEqual(self.say("command", {"prompt": "/modernize"}, path=self.toolbox("old", python3=old))["py"], 5)
+ self.assertEqual(self.say("command", {"prompt": "/modernize"}, path=self.toolbox("crash", python3=crash))["py"], 6)
+
+ def test_health_is_sent_once_per_version_even_when_python_is_missing(self):
+ box = self.toolbox("nopy")
+ first = self.say("health", {"cwd": self.ws}, path=box)
+ self.assertEqual((first["py"], sorted(first)), (1, ["os", "pathf", "pv", "py", "pyv"]))
+ self.assertIsNone(self.say("health", {"cwd": self.ws}, path=box))
+ self.assertTrue(any(n.startswith("health-sent-") for n in os.listdir(self.data)))
+
+ def test_health_from_python_is_sent_once_and_the_off_switch_stops_it(self):
+ self.assertEqual(self.say("health", {"cwd": self.ws}, env={"CODE_MODERNIZATION_TELEMETRY": "0"}), None)
+ self.assertFalse(any(n.startswith("health-sent-") for n in os.listdir(self.data)))
+ first = self.say("health", {"cwd": self.ws})
+ self.assertEqual(first["py"], 0)
+ self.assertIsNone(self.say("health", {"cwd": self.ws}))
+
+ def test_a_python_failure_is_reported_by_the_shell_when_python_cannot_run(self):
+ box = self.toolbox("nopy2")
+ put(self.ws, "analysis/sys/PREFLIGHT.md", "# x\n")
+ got = self.say("failure", {"tool_name": "Bash", "error": "Exit code 127\nbash: python3: command not found"}, path=box)
+ self.assertEqual((got["tool"], got["kind"], got["py"]), (1, 1, 1))
+ self.assertIsNone(self.say("failure", {"tool_name": "Read", "error": "File does not exist."}, path=box))
+
+ def test_a_folder_with_a_space_and_an_accent_is_flagged(self):
+ odd = os.path.join(self.tmp, "my caf\u00e9 work")
+ os.makedirs(odd)
+ got = self.say("command", {"prompt": "/modernize"}, cwd=odd)
+ self.assertEqual(got["pathf"], 3)
+
+ def test_python_starts_only_where_the_plugin_has_left_its_files(self):
+ marker = os.path.join(self.tmp, "started")
+ fake = os.path.join(self.tmp, "fakebin")
+ os.makedirs(fake)
+ with open(os.path.join(fake, "python3"), "w") as fh:
+ fh.write("#!/bin/sh\ntouch %s\ncat >/dev/null\n" % marker)
+ os.chmod(os.path.join(fake, "python3"), 0o755)
+ path = fake + os.pathsep + "/usr/bin:/bin"
+ other = os.path.join(self.tmp, "other")
+ put(other, "analysis/results/data.csv", "a,b\n")
+ self.run_hook("state", {"cwd": other}, cwd=other, path=path)
+ self.assertFalse(os.path.exists(marker))
+ put(other, "analysis/results/PREFLIGHT.md", "# x\n")
+ self.run_hook("state", {"cwd": other}, cwd=other, path=path)
+ self.assertTrue(os.path.exists(marker))
+
+ def test_it_leaves_no_bytecode_in_the_plugins_folder(self):
+ root = os.path.join(self.tmp, "plugin")
+ for rel in ("scripts/telemetry.sh", "scripts/telemetry.py", "scripts/build_report.py", ".claude-plugin/plugin.json"):
+ os.makedirs(os.path.dirname(os.path.join(root, rel)), exist_ok=True)
+ shutil.copy(os.path.join(PLUGIN, rel), os.path.join(root, rel))
+ done = subprocess.run(["/bin/sh", os.path.join(root, "scripts", "telemetry.sh"), "state"], input=json.dumps({"cwd": self.ws}), capture_output=True,
+ text=True, cwd=self.ws, env={"PATH": self.bin + os.pathsep + "/usr/bin:/bin", "CLAUDE_PLUGIN_DATA": self.data}, timeout=60)
+ self.assertIn('"metrics"', done.stdout)
+ self.assertEqual(sorted(os.listdir(os.path.join(root, "scripts"))), ["build_report.py", "telemetry.py", "telemetry.sh"])
+
+ def test_the_off_switch_reaches_the_script(self):
+ self.assertEqual(self.run_hook("state", {"cwd": self.ws}, env={"CODE_MODERNIZATION_TELEMETRY": "0"}), "")
+
+
+class Shipped(unittest.TestCase):
+ """What ships must agree with itself: the manifest, the hooks, the README and the changelog."""
+
+ def read(self, *parts):
+ with open(os.path.join(PLUGIN, *parts), encoding="utf-8") as fh:
+ return fh.read()
+
+ def test_the_hooks_call_scripts_that_exist_and_are_asynchronous(self):
+ hooks = json.loads(self.read("hooks", "hooks.json"))["hooks"]
+ want = {"UserPromptSubmit": ["command"], "Stop": ["state", "run"], "SessionStart": ["health"], "PostToolUseFailure": ["failure"], "StopFailure": ["failure"]}
+ self.assertEqual(sorted(hooks), sorted(want))
+ for event, groups in hooks.items():
+ modes = []
+ for group in groups:
+ for h in group["hooks"]:
+ self.assertTrue(h["asyncRewake"])
+ script = re.search(r"\$\{CLAUDE_PLUGIN_ROOT\}/(scripts/\w+\.sh)", h["command"]).group(1)
+ self.assertTrue(os.path.isfile(os.path.join(PLUGIN, script)), script)
+ modes.append(h["command"].split()[-1])
+ self.assertEqual(modes, want[event], event)
+
+ def test_the_manifest_has_a_version_and_the_option_that_turns_counts_off(self):
+ manifest = json.loads(self.read(".claude-plugin", "plugin.json"))
+ self.assertRegex(manifest["version"], r"^\d+\.\d+\.\d+$")
+ option = manifest["userConfig"]["telemetry"]
+ self.assertEqual((option["type"], option["default"]), ("boolean", True))
+ with open(os.path.join(PLUGIN, "..", "..", ".claude-plugin", "marketplace.json"), encoding="utf-8") as fh:
+ entry = next(p for p in json.load(fh)["plugins"] if p["name"] == "code-modernization")
+ self.assertNotIn("version", entry) # the manifest is the one place the number lives
+
+ def test_the_changelog_names_the_manifest_version_first(self):
+ version = json.loads(self.read(".claude-plugin", "plugin.json"))["version"]
+ first = re.search(r"(?m)^## \[?(\d+\.\d+\.\d+)", self.read("CHANGELOG.md"))
+ self.assertEqual(first.group(1), version)
+
+ def test_the_readme_lists_every_key_that_can_be_sent_and_the_off_switches(self):
+ readme = self.read("README.md")
+ section = readme[readme.index("## Telemetry"):]
+ section = section[:section.index("\n## ", 5)] if "\n## " in section[5:] else section
+ for key in tm.MEANING:
+ self.assertIn("`%s`" % key, section, key)
+ for word in ("CODE_MODERNIZATION_TELEMETRY", "Usage counts", "DISABLE_TELEMETRY", "telemetry.py show", "two significant figures", "TELEMETRY_DEBUG"):
+ self.assertIn(word, section)
+
+
+if __name__ == "__main__":
+ unittest.main()