summaryrefslogtreecommitdiff
path: root/plugins/code-modernization/scripts/tests/test_proof_rules_run.py
diff options
context:
space:
mode:
authorClaude <noreply@anthropic.com>2026-11-06 00:54:48 +0000
committerClaude <noreply@anthropic.com>2026-11-06 00:54:48 +0000
commit7aba3deae3ff17d032c37e07d2929d1426b34278 (patch)
tree9c482f9acda1599bd2f9bed95a6a46b75e634e80 /plugins/code-modernization/scripts/tests/test_proof_rules_run.py
parent5472f5f56e0070feeeb464318b316714fe3e9105 (diff)
parentb8331dd69ab3e132bbf8cd6e89b5d8c1747e21b1 (diff)
Merge remote-tracking branch 'origin/main' into add-plugin/xcodeadd-plugin/xcode
Diffstat (limited to 'plugins/code-modernization/scripts/tests/test_proof_rules_run.py')
-rw-r--r--plugins/code-modernization/scripts/tests/test_proof_rules_run.py506
1 files changed, 506 insertions, 0 deletions
diff --git a/plugins/code-modernization/scripts/tests/test_proof_rules_run.py b/plugins/code-modernization/scripts/tests/test_proof_rules_run.py
new file mode 100644
index 00000000..0eb3560c
--- /dev/null
+++ b/plugins/code-modernization/scripts/tests/test_proof_rules_run.py
@@ -0,0 +1,506 @@
+"""Tests for the "Rules traced" check: a rule is tested only when something that ran and passed backs it, and test-only folders are not judged.
+
+Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests
+A test that names a rule is not a test that ran: skipped and pending tests, and files whose class never ran, leave the rule "named, not run".
+"""
+import json
+import os
+import re
+import shutil
+import sys
+import tempfile
+import time
+import unittest
+
+HERE = os.path.dirname(os.path.abspath(__file__))
+PLUGIN = os.path.dirname(os.path.dirname(HERE))
+sys.path.insert(0, HERE)
+sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
+import build_report as br # noqa: E402
+import proof_pack as pp # noqa: E402
+import trace_rules as tr # noqa: E402
+from test_proof_tools import RULES_MD, T0, Ws, junit, load, passing, put, read, run_main # noqa: E402
+
+SKIPPED_TEST = '''package p;
+
+class CalcTest {
+ @Test
+ @Disabled("PENDING RULE-001")
+ void RULE_001_totals() {}
+
+ @Test
+ @Disabled("PENDING RULE-002")
+ void RULE_002_rounds() {}
+}
+'''
+NOT_RUN = "named, not run"
+
+
+class RuleWs(Ws):
+ """The workspace of the proof tests (module mod, P0 rules RULE-001 and RULE-002), with helpers to change what the tests are and what ran."""
+
+ def module(self, module="mod"):
+ return super().module(module)
+
+ def p0(self):
+ return {r["id"]: r for r in self.module()["evidence"]["rules"]["p0"]}
+
+ def statuses(self):
+ return {r["id"]: r["status"] for r in self.module()["evidence"]["rules"]["p0"]}
+
+ def check_rules(self):
+ return next(c for c in self.module()["checks"] if c["id"] == "rules")
+
+ def test_file(self, rel, text):
+ put(self.root, "modernized/s/mod/src/test/" + rel, text, T0 + 20)
+
+ def keep_only(self, *cases):
+ self.results(list(cases))
+
+
+class RulesNeedExecutedTests(unittest.TestCase):
+ def setUp(self):
+ self.ws = RuleWs(self)
+
+ def test_the_default_workspace_is_tested_by_files_whose_class_ran(self):
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+ self.assertEqual(self.ws.check_rules()["state"], "pass")
+ self.assertIn("backed by a test that ran and passed", self.ws.check_rules()["detail"])
+ self.assertEqual(self.ws.module()["verdict"], "PROVEN")
+
+ # (i) the hole that was found: @Disabled("PENDING RULE-001") and a result file that shows it skipped
+ def test_a_disabled_pending_test_that_the_results_show_skipped_is_named_not_run(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("CalcTest", "RULE_001_totals", "SKIP", "PENDING RULE-001"), ("CalcTest", "RULE_002_rounds", "SKIP", "PENDING RULE-002"), ("OtherTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ c = self.ws.check_rules()
+ self.assertEqual(c["state"], "gap")
+ self.assertIn("P0 rules named only by tests that did not run: RULE-001, RULE-002 (skipped, pending, failing or missing from the results).", c["detail"])
+ m = self.ws.module()
+ self.assertEqual(m["verdict"], "PARTLY PROVEN")
+ self.assertEqual(m["evidence"]["rules"]["namedNotRun"], ["RULE-001", "RULE-002"])
+ self.assertEqual(m["evidence"]["rules"]["p0NoTests"], ["RULE-001", "RULE-002"])
+ self.assertTrue(m["evidence"]["rules"]["perTestResults"])
+ self.assertTrue(any(r.startswith("Rules traced: P0 rules named only by tests that did not run") for r in m["reasons"]))
+ self.assertEqual(m["evidence"]["tests"]["skipped"], 2)
+
+ def test_the_same_test_without_a_skip_marker_but_skipped_at_run_time_is_named_not_run_too(self):
+ self.ws.test_file("CalcTest.java", "class CalcTest {\n // RULE-001 and RULE-002 need a database\n void a() {}\n}\n")
+ self.ws.keep_only(("CalcTest", "a", "SKIP", "no database here"), ("OtherTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertEqual({r["by"] for r in self.ws.p0().values()}, {""})
+
+ def test_a_test_that_ran_and_failed_backs_nothing(self):
+ self.ws.test_file("CalcTest.java", "class CalcTest {\n // RULE-001 RULE-002\n void a() {}\n}\n")
+ self.ws.keep_only(("CalcTest", "RULE_001_totals", "FAIL", ""), ("CalcTest", "RULE_002_rounds", "ERROR", ""), ("OtherTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertEqual(self.ws.module()["verdict"], "NOT PROVEN") # the failures fail the tests check as they always did
+ self.assertEqual(self.ws.check_rules()["state"], "gap")
+
+ # (ii) an executed, passing testcase that carries the id in its name
+ def test_a_passing_test_whose_name_carries_the_id_backs_the_rule_in_any_spelling(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST) # the only file that names the rules is all skipped
+ for name in ("RULE-001 totals", "RULE_001_totals", "rule001", "testRule001Totals", "Rule001", "totals[RULE-001]"):
+ self.ws.keep_only(("ChecksTest", name, "PASS", ""), ("ChecksTest", "test_RULE_002_rounds", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"}, name)
+ p0 = self.ws.p0()
+ self.assertEqual((p0["RULE-001"]["by"], p0["RULE-002"]["by"]), ("test name", "test name"))
+ self.assertEqual(self.ws.check_rules()["state"], "pass")
+ self.assertEqual(self.ws.module()["verdict"], "PROVEN")
+
+ def test_the_class_name_may_carry_the_id_and_the_shorthand_names_both_rules(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("com.x.Rule001Test", "a", "PASS", ""), ("com.x.OtherTest", "RULE-002/003", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_an_id_in_the_name_of_a_test_that_did_not_pass_or_in_another_modules_results_backs_nothing(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("ChecksTest", "RULE_001_a", "SKIP", "x"), ("ChecksTest", "RULE_002_b", "FAIL", ""), ("ChecksTest", "works", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ put(self.ws.root, "modernized/s/extra/src/main/X.java", "class X {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/extra/target/surefire-reports/TEST-X.xml", junit([("XTest", "RULE_001_a", "PASS", ""), ("XTest", "RULE_002_a", "PASS", "")]), T0 + 30)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "junit": ["modernized/s/mod/target/surefire-reports"]},
+ {"module": "extra", "name": "unit", "junit": ["modernized/s/extra/target/surefire-reports"]}])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN}) # what extra ran does not count for mod
+
+ # (iii) a plain comment in a test file whose class ran
+ def test_a_plain_comment_in_a_test_file_whose_class_ran_backs_the_rule(self):
+ self.ws.test_file("CalcTest.java", "class CalcTest {\n // RULE-001 is pinned by these\n void a() {}\n void rule002_rounds() {}\n}\n")
+ self.ws.keep_only(("com.x.CalcTest", "a", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+ p0 = self.ws.p0()
+ self.assertEqual(p0["RULE-001"]["by"], "test file")
+ self.assertTrue(p0["RULE-001"]["where"].endswith("src/test/CalcTest.java:2"), p0["RULE-001"]["where"])
+ self.assertEqual(self.ws.module()["verdict"], "PROVEN")
+
+ def test_the_where_points_at_the_mention_that_backs_the_rule_not_at_a_skipped_one(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST) # sorts before Live.java and names both rules, skipped
+ self.ws.test_file("Live.java", "// RULE-001 RULE-002 are checked by these\nclass Live {}\n")
+ self.ws.keep_only(("Live", "a", "PASS", ""), ("CalcTest", "RULE_001_totals", "SKIP", "x"))
+ p0 = self.ws.p0()
+ self.assertTrue(p0["RULE-001"]["where"].endswith("src/test/Live.java:1"), p0["RULE-001"]["where"])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_the_class_name_may_be_nested_dotted_a_python_module_or_a_path(self):
+ same = (("CalcTest.java", "com.x.CalcTest$Inner"), ("CalcTest.java", "com.x.Outer$Deep$CalcTest"), ("CalcTest.java", "com.x.CalcTest$A$B"), ("test_calc.py", "tests.test_calc.TestCalc"),
+ ("test_calc.py", "tests.test_calc"), ("CalcTests.cs", "Ns.CalcTests"), ("calc.spec.ts", "src/calc.spec.ts"), ("CalcTest.java", "calctest"), ("CalcTest.java", "CALCTEST"))
+ other = (("CalcTest.java", "com.x.OtherCalcTest"), ("CalcTest.java", "com.calc.test.Other"), ("test_calc.py", "tests.test_other.TestCalc"), ("CalcTest.java", "Calc"), ("CalcTest.java", ""))
+ for rel, cls in same + other:
+ self.ws.test_file(rel, "// RULE-001 RULE-002\n")
+ self.ws.keep_only((cls, "a", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"} if (rel, cls) in same else {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN}, (rel, cls))
+ os.remove(os.path.join(self.ws.root, "modernized", "s", "mod", "src", "test", rel))
+ self.ws.test_file("CalcTest.java", "// nothing here\n")
+
+ # (iv) a file none of whose tests ran
+ def test_a_rule_named_only_in_a_file_none_of_whose_tests_ran_is_named_not_run(self):
+ self.ws.test_file("CalcTest.java", "// RULE-001\nclass CalcTest {}\n")
+ self.ws.test_file("GhostTest.java", "// RULE-002 is pinned here, and nothing marks it skipped\nclass GhostTest {}\n")
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("GhostTest", "b", "SKIP", "no docker"))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": NOT_RUN})
+ self.ws.keep_only(("CalcTest", "a", "PASS", "")) # the class is not in the results at all
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": NOT_RUN})
+ c = self.ws.check_rules()
+ self.assertEqual(c["state"], "gap")
+ self.assertIn("named only by tests that did not run: RULE-002", c["detail"])
+ self.assertNotIn("RULE-001", c["detail"])
+
+ # (v) counts only
+ def test_counts_only_evidence_never_backs_a_rule(self):
+ put(self.ws.root, "analysis/s/equivalence/unit.test-output.txt", "[INFO] Tests run: 3, Failures: 0, Errors: 0, Skipped: 0\n", T0 + 30)
+ for suite in ({"module": "mod", "name": "unit", "command": "mvn test", "log": ["analysis/s/equivalence/unit.test-output.txt"]},
+ {"module": "mod", "name": "unit", "executed": 3, "failed": 0, "skipped": 0}):
+ self.ws.runs(suites=[suite])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN}, suite)
+ m = self.ws.module()
+ self.assertFalse(m["evidence"]["rules"]["perTestResults"])
+ self.assertEqual(self.ws.check_rules()["state"], "gap")
+ self.assertIn("named by tests, but no result file lists each test, so nothing shows one ran: RULE-001, RULE-002.", self.ws.check_rules()["detail"])
+ self.assertNotEqual(m["verdict"], "PROVEN")
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "log": ["analysis/s/equivalence/unit.test-output.txt"]},
+ {"module": "mod", "name": "more", "junit": ["modernized/s/mod/target/surefire-reports"]}]) # one suite that lists its tests is enough
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_no_suite_at_all_or_result_files_with_no_test_in_them_back_nothing(self):
+ put(self.ws.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", '<?xml version="1.0"?><testsuite name="S" tests="0"></testsuite>', T0 + 30)
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.ws.runs(suites=[])
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertEqual(self.ws.module()["verdict"], "NOT PROVEN")
+
+ # (vi) Python, and how far a marker reaches
+ def test_a_pytest_skip_that_names_the_rule_leaves_it_named_not_run(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("**Priority:** P2", "**Priority:** P0"), T0 + 10)
+ put(self.ws.root, "modernized/s/mod/src/main/calc.py", "def fee(): pass\n", T0 + 20)
+ self.ws.test_file("CalcTest.java", "// RULE-001 RULE-002\nclass CalcTest {}\n")
+ self.ws.test_file("test_fee.py", 'import pytest\n\n\n@pytest.mark.skip(reason="RULE-004")\ndef test_fee():\n pass\n\n\ndef test_other():\n pass\n')
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("tests.test_fee", "test_fee", "SKIP", "RULE-004"), ("tests.test_fee", "test_other", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested", "RULE-004": NOT_RUN})
+ self.assertIn("RULE-004", self.ws.check_rules()["detail"])
+ self.ws.test_file("test_fee.py", 'import pytest\n\n\ndef test_fee():\n # RULE-004 is pinned here\n pass\n') # no marker: the class ran and passed
+ self.assertEqual(self.ws.statuses()["RULE-004"], "tested")
+
+ def test_a_marker_reaches_three_lines_down_and_no_further_and_any_case(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P0\n", T0 + 10)
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""))
+ for gap, want in ((0, NOT_RUN), (1, NOT_RUN), (2, NOT_RUN), (3, NOT_RUN), (4, "tested"), (9, "tested")):
+ lines = ["@Disabled RULE-001"] if gap == 0 else ["@Disabled"] + ["// filler"] * (gap - 1) + ["// RULE-001"] # the mention is `gap` lines below the marker
+ self.ws.test_file("CalcTest.java", "\n".join(lines) + "\n")
+ self.assertEqual(self.ws.statuses()["RULE-001"], want, gap)
+ self.ws.test_file("CalcTest.java", "// RULE-001\n// a marker below the mention does not matter: @Disabled\n")
+ self.assertEqual(self.ws.statuses()["RULE-001"], "tested")
+ for marker in ("@Disabled", "@Ignore", "[Ignore]", "@Skip", "#[ignore]", "xit(", "xdescribe(", "it.skip(", "test.skip(", "describe.skip(", "t.Skip()", "@pytest.mark.skip",
+ "@unittest.skipIf(x)", "pending", "PENDING", "Pending", "todo", "TODO", "@DISABLED", "SKIP", "XIT("):
+ self.ws.test_file("CalcTest.java", "%s\n// RULE-001\n" % marker)
+ self.assertEqual(self.ws.statuses()["RULE-001"], NOT_RUN, marker)
+ self.ws.test_file("CalcTest.java", "// RULE-001 is enabled and runs as a @Test\n")
+ self.assertEqual(self.ws.statuses()["RULE-001"], "tested")
+
+ def test_one_live_mention_is_enough_when_the_same_file_also_has_skipped_ones(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", "### RULE-001: x\n**Priority:** P0\n", T0 + 10)
+ self.ws.test_file("CalcTest.java", '@Disabled("PENDING RULE-001")\nvoid a() {}\n\n\n\n\n// RULE-001 also holds for the total\n')
+ self.ws.keep_only(("CalcTest", "b", "PASS", ""))
+ self.assertEqual(self.ws.statuses()["RULE-001"], "tested")
+
+ def test_a_rule_a_test_file_names_and_main_code_names_is_named_not_run_until_something_ran(self):
+ # the default workspace's Calc.java names both rules too: main code never makes a rule tested
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("OtherTest", "works", "PASS", ""))
+ m = self.ws.module()
+ self.assertEqual({r["id"]: (r["status"], r["main"] > 0) for r in m["evidence"]["rules"]["p0"]}, {"RULE-001": (NOT_RUN, True), "RULE-002": (NOT_RUN, True)})
+
+ def test_claimed_only_and_missing_rules_keep_their_own_words_beside_named_not_run(self):
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", RULES_MD.replace("**Priority:** P1", "**Priority:** P0").replace("**Priority:** P2", "**Priority:** P0"), T0 + 10)
+ put(self.ws.root, "modernized/s/mod/src/main/Calc.java", "class Calc {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/mod/TRANSFORMATION_NOTES.md", "Legacy: `legacy/s/app/mod.cbl` and `legacy/s/app/other.cbl`\n\n| Rule | Target |\n|---|---|\n| RULE-003 | `src/main/Calc.java` |\n\nCanary: x -> 3 tests failed\n", T0 + 20)
+ self.ws.test_file("CalcTest.java", "// RULE-001\n@Disabled\n// RULE-002\n")
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""))
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": NOT_RUN, "RULE-003": "claimed only", "RULE-004": "none"})
+ d = self.ws.check_rules()["detail"]
+ self.assertEqual(self.ws.check_rules()["state"], "gap")
+ self.assertIn("P0 rules named only by tests that did not run: RULE-002 (", d)
+ self.assertIn("2 of 4 P0 rule(s) are not named by any test: RULE-003, RULE-004 (1 claimed only in the notes).", d)
+
+ def test_a_long_list_is_cut_and_says_how_many_more(self):
+ cards = "".join("### RULE-%03d: r\n**Priority:** P0\n\n" % n for n in range(1, 31))
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", cards, T0 + 10)
+ self.ws.test_file("CalcTest.java", "@Disabled\n" + "".join("// RULE-%03d\n@Disabled\n" % n for n in range(1, 31)))
+ d = self.ws.check_rules()["detail"]
+ self.assertIn("RULE-012 and 18 more", d)
+ self.assertNotIn("RULE-013", d)
+
+ def test_an_uplift_still_traces_no_rules(self):
+ self.assertEqual(pp.check_rules({"trace": None, "trace_problem": "x", "ran": {}}, {"track": "uplift", "rel": "modernized/s-uplifted"}, {}, [])["state"], "na")
+
+ def test_the_pack_tells_the_report_builder_nothing_it_cannot_read(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("OtherTest", "works", "PASS", ""))
+ pack = self.ws.pack()
+ view = br.proof_view(pack)
+ m = view["modules"][0]
+ self.assertEqual((m["verdict"], pack["modules"][0]["verdict"]), ("PARTLY PROVEN", "PARTLY PROVEN"))
+ self.assertEqual(view["problems"], []) # the builder finds nothing to correct
+ self.assertEqual(m["p0Counts"]["tested"], 0)
+ self.assertTrue(all(r["status"] != "tested" for r in m["p0"])) # an unknown status is never shown as tested
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ self.assertIn("P0 rules named only by tests that did not run", out)
+
+ def test_the_markdown_table_says_named_not_run_in_plain_words(self):
+ self.ws.test_file("CalcTest.java", SKIPPED_TEST)
+ self.ws.keep_only(("OtherTest", "works", "PASS", ""))
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ md = read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md"))
+ rows = [ln for ln in md.split("\n") if ln.startswith("| RULE-00")]
+ self.assertEqual(len(rows), 2)
+ self.assertTrue(all("| named, not run |" in ln for ln in rows), rows)
+ self.assertIn("| 0 of 2 |", md) # the summary counts tested rules only
+ self.assertIn("named, not run; one only the notes name is claimed. Neither is tested.", md)
+
+
+class TestOnlyFolders(unittest.TestCase):
+ def setUp(self):
+ self.ws = RuleWs(self)
+ put(self.ws.root, "modernized/s/harness/pom.xml", "<project/>", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/mvnw", "#!/bin/sh\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/.mvn/wrapper/x.properties", "x=1", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/README.md", "docs are not code", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/src/test/java/HarnessTest.java", "// drives the legacy WAR\nclass HarnessTest {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/src/test/java/Storefront.java", "class Storefront {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/target/classes/A.class", "x", T0 + 20)
+ self.mod = os.path.join(self.ws.root, "modernized", "s", "harness")
+
+ # (vii)
+ def test_a_folder_of_only_test_code_and_build_files_is_listed_not_judged_and_not_counted(self):
+ pack = self.ws.pack()
+ self.assertEqual([m["name"] for m in pack["modules"]], ["mod"])
+ self.assertEqual(pack["toolingOnly"], [{"name": "harness", "track": "rewrite", "path": "modernized/s/harness"}])
+ self.assertEqual(pack["overall"], {"verdict": "PROVEN", "counts": {"PROVEN": 1, "PARTLY PROVEN": 0, "NOT PROVEN": 0}})
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 0)
+ self.assertIn("harness (rewrite): test tooling, not judged", out)
+ self.assertNotIn("harness (rewrite): NOT PROVEN", out)
+ data = load(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.json"))
+ self.assertEqual((data["toolingOnly"], [m["name"] for m in data["modules"]], data["overall"]["verdict"]), ([{"name": "harness", "track": "rewrite", "path": "modernized/s/harness"}], ["mod"], "PROVEN"))
+ md = read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md"))
+ self.assertIn("\n## Test tooling (not judged)\n", md)
+ self.assertIn("- `modernized/s/harness` (rewrite)", md)
+ self.assertNotIn("## harness", md)
+ self.assertEqual(md.count("| mod |"), 1)
+ self.assertNotIn("| harness |", md)
+ self.assertIn("**Overall: PROVEN** (1 proven)", md)
+ view = br.proof_view(data) # the report builder reads the modules and ignores the new list
+ self.assertEqual(([m["name"] for m in view["modules"]], view["verdict"], view["problems"]), (["mod"], "PROVEN", []))
+
+ def test_a_pack_with_no_tooling_has_an_empty_list_and_no_heading(self):
+ shutil.rmtree(self.mod)
+ run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(load(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.json"))["toolingOnly"], [])
+ self.assertNotIn("Test tooling", read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md")))
+
+ def test_the_tooling_folders_own_suite_and_rule_mentions_change_nothing(self):
+ put(self.ws.root, "modernized/s/harness/src/test/java/HarnessTest.java", "// RULE-001 RULE-002\nclass HarnessTest {}\n", T0 + 20)
+ put(self.ws.root, "modernized/s/harness/target/surefire-reports/TEST-HarnessTest.xml", junit([("HarnessTest", "a", "FAIL", "")]), T0 + 30)
+ self.ws.runs(suites=[{"module": "mod", "name": "unit", "junit": ["modernized/s/mod/target/surefire-reports"]},
+ {"module": "harness", "name": "harness", "junit": ["modernized/s/harness/target/surefire-reports"]}])
+ pack = self.ws.pack()
+ self.assertEqual((pack["overall"]["verdict"], [m["verdict"] for m in pack["modules"]]), ("PROVEN", ["PROVEN"])) # a failing harness test is not the module's failure
+ self.assertEqual(pack["modules"][0]["evidence"]["tests"]["executed"], 3)
+
+ def test_any_main_code_or_any_file_that_is_not_plainly_tooling_means_it_is_judged(self):
+ for rel in ("src/main/java/Harness.java", "harness.py", "data/input.csv", "LICENSE", "src/main/resources/app.properties", "setup.py"):
+ put(self.ws.root, "modernized/s/harness/" + rel, "x", T0 + 20)
+ self.assertFalse(tr.tooling_only(self.mod), rel)
+ self.assertEqual([m["name"] for m in self.ws.pack()["modules"]], ["harness", "mod"], rel)
+ os.remove(os.path.join(self.mod, *rel.split("/")))
+ self.assertTrue(tr.tooling_only(self.mod))
+
+ def test_a_folder_with_no_test_file_is_not_tooling_and_neither_is_an_uplift(self):
+ os.remove(os.path.join(self.mod, "src", "test", "java", "HarnessTest.java"))
+ os.remove(os.path.join(self.mod, "src", "test", "java", "Storefront.java"))
+ self.assertFalse(tr.tooling_only(self.mod))
+ self.assertEqual(sorted(m["name"] for m in self.ws.pack()["modules"]), ["harness", "mod"])
+ empty = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, empty, True)
+ self.assertFalse(tr.tooling_only(empty))
+ put(self.ws.root, "modernized/s-uplifted/src/test/java/ATest.java", "class ATest {}\n", T0 + 20)
+ self.assertEqual({m["track"] for m in self.ws.pack()["modules"]}, {"rewrite", "uplift"}) # the whole working copy is one thing to prove, tests or not
+
+ def test_what_counts_as_tooling_files(self):
+ for name in ("pom.xml", "mvnw", "mvnw.cmd", "gradlew", "build.gradle", "build.gradle.kts", "package.json", "package-lock.json", "tsconfig.json", "tsconfig.build.json", "jest.config.js",
+ "vitest.config.ts", "conftest.py", "pytest.ini", "pyproject.toml", "go.mod", "Cargo.toml", "composer.json", "phpunit.xml.dist", "App.csproj", "All.sln", "Makefile", "Dockerfile",
+ "docker-compose.yml", "POM.XML"):
+ self.assertTrue(tr.is_tooling(name), name)
+ for name in ("Calc.java", "main.py", "index.js", "setup.py", "app.properties", "data.json", "run.sh", "LICENSE", "pom.xml.bak", "xpom.xml", "tsconfig"):
+ self.assertFalse(tr.is_tooling(name), name)
+
+ def test_only_tooling_at_all_is_not_proven_and_says_why(self):
+ shutil.rmtree(os.path.join(self.ws.root, "modernized", "s", "mod"))
+ pack = self.ws.pack()
+ self.assertEqual((pack["modules"], [t["name"] for t in pack["toolingOnly"]], pack["overall"]["verdict"]), ([], ["harness"], "NOT PROVEN"))
+ self.assertIn("there is no module to judge", " ".join(pack["problems"]))
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertEqual(code, 1)
+ self.assertIn("harness (rewrite): test tooling, not judged", out)
+
+ def test_asking_about_the_tooling_folder_is_allowed_and_changes_nothing(self):
+ pack = self.ws.pack("harness")
+ self.assertEqual(([m["name"] for m in pack["modules"]], pack["asked"], len(pack["toolingOnly"])), (["mod"], "harness", 1))
+
+ def test_the_trace_still_reads_the_tooling_folders_tests(self):
+ put(self.ws.root, "modernized/s/harness/src/test/java/HarnessTest.java", "// RULE-003\nclass HarnessTest {}\n", T0 + 20)
+ result = tr.trace(self.ws.root, "s", {"modernized/s/harness": {"known": True, "ids": set(), "keys": {"harnesstest"}}})
+ self.assertEqual({r["id"]: r["status"] for r in result["rules"]}["RULE-003"], "tested")
+ self.assertEqual([m["name"] for m in result["modules"]], ["harness", "mod"])
+
+
+class RuleTracePieces(unittest.TestCase):
+ def test_class_keys(self):
+ cases = {"com.x.CalcTest": {"x", "calctest"}, "com.x.CalcTest$Inner": {"calctest", "inner"}, "a.B$C$D": {"b", "c", "d"}, "tests.test_calc.TestCalc": {"test_calc", "testcalc"},
+ "src/calc.spec.ts": {"spec", "ts", "calc.spec.ts", "calc.spec"}, "": set(), "Only": {"only"}, "github.com/x/y": {"x", "y"}, "C:\\dir\\CalcTest.cs": {"calctest.cs", "calctest"}}
+ for name, want in cases.items():
+ self.assertTrue(want <= tr.class_keys(name), (name, want, tr.class_keys(name)))
+ self.assertEqual(tr.class_keys(None), set())
+ self.assertLessEqual(len(tr.class_keys("a." * 10000 + "b")), 4)
+
+ def test_run_evidence_reads_only_tests_that_passed(self):
+ res = {"tests": {"CalcTest#RULE_001_a": "PASS", "CalcTest#RULE_002_a": "FAIL", "CalcTest#RULE_003_a": "SKIP", "Other#testRule004": "PASS", "Other#x~1": "PASS"},
+ "classes": {"CalcTest": {"pass": 1, "fail": 1, "error": 0, "skip": 1, "executed": 2}, "Ghost": {"pass": 0, "fail": 0, "error": 0, "skip": 3, "executed": 0}, "Other": {"pass": 2}}}
+ ev = tr.run_evidence([res])
+ self.assertEqual((ev["known"], ev["ids"], ev["keys"]), (True, {1, 4}, {"calctest", "other"}))
+ self.assertEqual(tr.run_evidence([]), {"known": False, "ids": set(), "keys": set()})
+ self.assertEqual(tr.run_evidence([None, {"tests": {}, "classes": {}}]), {"known": False, "ids": set(), "keys": set()})
+ more = tr.run_evidence([res, {"tests": {"Later#RULE_009_a": "PASS"}, "classes": {"Later": {"pass": 1}}}])
+ self.assertEqual((more["ids"], more["keys"]), ({1, 4, 9}, {"calctest", "other", "later"}))
+
+ def test_line_marks_and_the_window(self):
+ body = "a\nb @Disabled\nc\nd\ne\nf\ng"
+ starts, flags = tr.line_marks(body)
+ self.assertEqual(flags, [False, True, False, False, False, False, False])
+ live = [tr.is_live((starts, flags), starts[i]) for i in range(7)]
+ self.assertEqual(live, [True, False, False, False, False, True, True]) # the marker's line and the 3 after it are marked
+ self.assertEqual(tr.line_marks("")[1], [False])
+
+ def test_the_command_line_needs_a_module_for_results_and_reports_unreadable_ones(self):
+ root = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, root, True)
+ put(root, "analysis/s/BUSINESS_RULES.md", RULES_MD)
+ put(root, "modernized/s/mod/src/test/CalcTest.java", "// RULE-001 RULE-002\n")
+ put(root, "res/TEST-bad.xml", "<testsuite><testcase")
+ put(root, "res/TEST-ok.xml", junit(passing(2, "CalcTest")))
+ self.assertEqual(run_main(tr.main, ["s", "--workspace", root, "--results", os.path.join(root, "res")])[0], 2)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", root, "--module", "mod", "--results", os.path.join(root, "res"), "--results", os.path.join(root, "nope")])
+ self.assertEqual(code, 0)
+ self.assertIn("result file not used: TEST-bad.xml", out)
+ self.assertIn("nope is not a file or folder that can be read", out)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", root, "--module", "nope", "--results", os.path.join(root, "res")])
+ self.assertEqual(code, 2)
+ code, out, _ = run_main(tr.main, ["s", "--workspace", root, "--module", "mod", "--results", os.path.join(root, "res"), "--json"])
+ self.assertEqual(code, 0)
+ data = json.loads(out)
+ self.assertEqual({r["id"]: r["status"] for r in data["rules"]}["RULE-001"], "tested")
+ self.assertEqual(data["moduleView"]["rows"][0]["by"], "test file")
+
+
+class HostileInput(unittest.TestCase):
+ """(viii) Everything here is untrusted text: nothing may crash, stall or reach the markdown as structure."""
+
+ def setUp(self):
+ self.ws = RuleWs(self)
+
+ def test_odd_and_huge_test_names_and_class_names_never_crash_or_reach_the_markdown(self):
+ evil = ["RULE-001\n| RULE-999 | injected | x | tested | 9 | 9 | `x` |", "`RULE_002` <img src=x onerror=1> \x1b[31m", "RULE-001" + "x" * 5000, "\x00RULE-002\u202e\u2028", "RULE-1234567 RULE-0", "A" * 100000,
+ "RULE-001 | | | |", "rule-\u0660\u0660\u0661", "RULE\u2212001"]
+ cases = [(n, m, "PASS", "") for n in evil for m in evil] + [("CalcTest", "ok", "PASS", "")]
+ put(self.ws.root, "modernized/s/mod/target/surefire-reports/TEST-CalcTest.xml", junit([(re.sub(r'[<>&"\x00-\x08\x0b\x0c\x0e-\x1f]', "_", c), re.sub(r'[<>&"\x00-\x08\x0b\x0c\x0e-\x1f]', "_", n), s, m) for c, n, s, m in cases]), T0 + 30)
+ started = time.time()
+ code, out, _ = run_main(pp.main, ["s", "--workspace", self.ws.root])
+ self.assertLess(time.time() - started, 20)
+ self.assertIn(code, (0, 1))
+ md = read(os.path.join(self.ws.root, "analysis", "s", "VERIFICATION.md"))
+ self.assertNotIn("injected", md)
+ self.assertEqual(len([ln for ln in md.split("\n") if ln.startswith("| RULE-")]), 2)
+ self.assertNotIn("RULE-999", md)
+ self.assertNotRegex(md, r"[\x00\x1b\u202e\u2028]")
+ self.assertEqual(self.ws.module()["evidence"]["rules"]["p0"][0]["id"], "RULE-001")
+
+ def test_hostile_file_names_and_lines_stay_inside_the_p0_table(self):
+ self.ws.test_file("Rule001|`x`Y.java", "// RULE-001 | RULE-002 |\n| a | b |\n")
+ self.ws.test_file("CalcTest.java", "@Disabled " + "RULE-001 " * 300000 + "x" * 3000000)
+ self.ws.keep_only(("Rule001|`x`Y", "a", "PASS", ""), ("CalcTest", "a", "PASS", ""))
+ started = time.time()
+ pack = self.ws.pack()
+ self.assertLess(time.time() - started, 20)
+ md = pp.render_md(pack)
+ for ln in md.split("\n"):
+ if ln.startswith("| RULE-"):
+ self.assertEqual(len(re.findall(r"(?<!\\)\|", ln)), 8, ln)
+ self.assertEqual(self.ws.statuses(), {"RULE-001": "tested", "RULE-002": "tested"})
+
+ def test_one_line_files_with_the_marker_after_the_mention_and_many_marked_lines_are_read_in_linear_time(self):
+ self.ws.test_file("CalcTest.java", "RULE-001 " * 100000 + "@Disabled")
+ self.ws.test_file("Lines.java", "".join("// RULE-002 skip\n" for _ in range(100000)))
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("Lines", "a", "PASS", ""))
+ started = time.time()
+ self.assertEqual(self.ws.statuses(), {"RULE-001": NOT_RUN, "RULE-002": NOT_RUN})
+ self.assertLess(time.time() - started, 20)
+
+ def test_many_passing_tests_naming_many_rules_are_read_quickly(self):
+ cards = "".join("### RULE-%03d: r\n**Priority:** P0\n\n" % n for n in range(1, 400))
+ put(self.ws.root, "analysis/s/BUSINESS_RULES.md", cards, T0 + 10)
+ self.ws.test_file("CalcTest.java", "// nothing\n")
+ self.ws.keep_only(*[("Big", "test_RULE_%03d_x" % n, "PASS", "") for n in range(1, 400)] + [("Big", "u%d" % i, "PASS", "") for i in range(30000)])
+ started = time.time()
+ p0 = self.ws.module()["evidence"]["rules"]["p0"]
+ self.assertLess(time.time() - started, 20)
+ self.assertEqual(len(p0), 100) # the pack keeps the first hundred rows, as before
+ self.assertTrue(all(r["status"] == "tested" and r["by"] == "test name" for r in p0))
+
+ def test_a_hostile_tooling_folder_name_stays_one_plain_list_item(self):
+ put(self.ws.root, "modernized/s/h`\n## Injected | x/pom.xml", "<project/>", T0 + 20)
+ put(self.ws.root, "modernized/s/h`\n## Injected | x/src/test/ATest.java", "class ATest {}\n", T0 + 20)
+ pack = self.ws.pack()
+ md = pp.render_md(pack)
+ self.assertEqual(len(pack["toolingOnly"]), 1)
+ self.assertNotRegex(md, r"(?m)^## Injected")
+ self.assertEqual(len([ln for ln in md.split("\n") if ln.startswith("- `modernized/s/h")]), 1)
+ self.assertNotIn("\n", pack["toolingOnly"][0]["name"])
+
+ def test_a_link_in_a_test_folder_and_binary_test_files_back_nothing(self):
+ outside = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, outside, True)
+ put(outside, "GhostTest.java", "// RULE-002\n")
+ try:
+ os.symlink(os.path.join(outside, "GhostTest.java"), os.path.join(self.ws.root, "modernized", "s", "mod", "src", "test", "GhostTest.java"))
+ except (OSError, NotImplementedError):
+ self.skipTest("symbolic links are not available here")
+ self.ws.test_file("CalcTest.java", "// RULE-001\n")
+ self.ws.test_file("BinTest.java", b"\x00\x01 RULE-002 \x00")
+ self.ws.keep_only(("CalcTest", "a", "PASS", ""), ("GhostTest", "a", "PASS", ""), ("BinTest", "a", "PASS", ""))
+ m = self.ws.module()
+ self.assertEqual({r["id"]: r["tests"] for r in m["evidence"]["rules"]["p0"]}["RULE-002"], 0) # code (Calc.java) still names RULE-002, no test file does
+ self.assertEqual(self.ws.statuses()["RULE-002"], "code only")
+
+
+if __name__ == "__main__":
+ unittest.main()