diff options
Diffstat (limited to 'plugins/code-modernization/scripts/tests/test_compare_tolerance.py')
| -rw-r--r-- | plugins/code-modernization/scripts/tests/test_compare_tolerance.py | 151 |
1 files changed, 151 insertions, 0 deletions
diff --git a/plugins/code-modernization/scripts/tests/test_compare_tolerance.py b/plugins/code-modernization/scripts/tests/test_compare_tolerance.py new file mode 100644 index 00000000..1bc1300a --- /dev/null +++ b/plugins/code-modernization/scripts/tests/test_compare_tolerance.py @@ -0,0 +1,151 @@ +"""Tests for the numeric tolerance in scripts/compare.py. + +Run: python3 -m unittest discover -s plugins/code-modernization/scripts/tests +""" +import json +import os +import shutil +import sys +import tempfile +import unittest +from unittest import mock + +HERE = os.path.dirname(os.path.abspath(__file__)) +PLUGIN = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, os.path.join(PLUGIN, "scripts")) +import compare as cmp # noqa: E402 + +TOL = {"rel": 1e-9, "abs": 0, "why": "last-digit rounding of exp() between libm versions"} + + +class ToleranceTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + + def case(self, name, legacy, new, **extra): + for side, data in (("legacy", legacy), ("new", new)): + os.makedirs(os.path.join(self.tmp, side), exist_ok=True) + with open(os.path.join(self.tmp, side, name), "wb") as fh: + fh.write(data) + return {"id": name, "legacy": "legacy/" + name, "new": "new/" + name, **extra} + + def run_cases(self, cases, **top): + path = os.path.join(self.tmp, "cases.json") + with open(path, "w") as fh: + json.dump({"system": "s", "cases": cases, **top}, fh) + return cmp.run(path, os.path.join(self.tmp, "EQUIVALENCE.json")) + + def test_a_tiny_float_difference_is_same_within_the_declared_tolerance_and_says_so(self): + res = self.run_cases([self.case("a", b"total 1234.5678901234\nrate 0.05\n", b"total 1234.5678901239\nrate 0.05\n", tolerance=TOL)]) + rec = res["cases"][0] + self.assertEqual(rec["verdict"], "same") + self.assertEqual(rec["withinTolerance"]["differing"], 1) + self.assertLess(rec["withinTolerance"]["maxRelativeDifference"], 1e-9) + self.assertIn("declared tolerance", rec["reason"]) + self.assertIn("libm", rec["reason"]) + self.assertEqual(res["totals"]["sameWithinTolerance"], 1) + self.assertIsNone(cmp.verdict_of(res)) + self.assertTrue(res["selfCheck"]["passed"], res["selfCheck"]) + + def test_without_a_tolerance_the_same_outputs_differ_and_no_new_key_appears(self): + res = self.run_cases([self.case("a", b"1234.5678901234\n", b"1234.5678901239\n")]) + self.assertEqual(res["cases"][0]["verdict"], "differs") + self.assertNotIn("sameWithinTolerance", res["totals"]) + + def test_a_difference_outside_the_tolerance_differs(self): + res = self.run_cases([self.case("a", b"total 100.0\n", b"total 100.5\n", tolerance=TOL)]) + rec = res["cases"][0] + self.assertEqual(rec["verdict"], "differs") + self.assertIn("beyond the declared tolerance", rec["reason"]) + self.assertIn("NOT PROVEN" if cmp.verdict_of(res) else "", "NOT PROVEN") + + def test_integers_and_text_must_match_exactly_even_with_a_tolerance(self): + loose = {"rel": 0.01, "abs": 0, "why": "test"} + for legacy, new in ((b"count 1000000\n", b"count 1000001\n"), (b"name alpha 1.5\n", b"name alphb 1.5\n"), + (b"1.5 2.5\n", b"1.5\n"), (b"x 10\n", b"x 10.0\n")): + res = self.run_cases([self.case("a", legacy, new, tolerance=loose)]) + self.assertEqual(res["cases"][0]["verdict"], "differs", (legacy, new)) + + def test_exponent_forms_compare_by_value(self): + res = self.run_cases([self.case("a", b"v 1.5e3\n", b"v 1500.0000000000002\n", tolerance=TOL)]) + self.assertEqual(res["cases"][0]["verdict"], "same") + + def test_a_tolerance_needs_a_reason_and_a_sane_size(self): + legacy = self.case("a", b"1.0\n", b"1.0\n") + for bad in ({"rel": 1e-9}, {"rel": 1e-9, "why": " "}, {"rel": 0.5, "why": "x"}, {"why": "x"}, {"rel": -1, "why": "x"}, + {"rel": "big", "why": "x"}, {"rel": float("nan"), "why": "x"}, "1e-9"): + with self.assertRaises(cmp.InputError, msg=repr(bad)): + self.run_cases([dict(legacy, tolerance=bad)]) + + def test_a_default_tolerance_applies_to_every_case_and_a_case_can_switch_it_off(self): + a = self.case("a", b"1.00000000000\n", b"1.00000000001\n") + b = self.case("b", b"1.00000000000\n", b"1.00000000001\n", tolerance=None) + res = self.run_cases([a, b], tolerance=TOL) + self.assertEqual([r["verdict"] for r in res["cases"]], ["same", "differs"]) + + def test_an_absolute_tolerance_is_capped_like_a_relative_one(self): + for bad in ({"abs": 1e-3, "why": "x"}, {"abs": 100, "why": "x"}): + with self.assertRaises(cmp.InputError, msg=repr(bad)): + self.run_cases([self.case("a", b"5.5\n", b"5.5\n", tolerance=bad)]) + ok = self.run_cases([self.case("a", b"x 0.0000000000000001\n", b"x 0.0\n", tolerance={"abs": 1e-9, "why": "a value that is zero to rounding"})]) + self.assertEqual(ok["cases"][0]["verdict"], "same") + + def test_the_self_check_still_fails_if_a_loose_tolerance_ever_got_through(self): + loose = {"abs": 100, "why": "absurdly loose"} + with mock.patch.object(cmp, "MAX_ABS_TOLERANCE", 1000.0): + res = self.run_cases([self.case("a", b"5.5\n", b"5.5\n", tolerance=loose)]) + self.assertFalse(res["selfCheck"]["passed"]) + self.assertIn("tolerance may be too loose", res["selfCheck"]["detail"]) + self.assertIn("self-check failed", cmp.verdict_of(res)) + + def test_dotted_identifiers_are_text_and_must_match_exactly(self): + for legacy, new in ((b"version 1.2.3\n", b"version 1.2.4\n"), (b"host 10.0.0.1\n", b"host 10.0.0.2\n"), + (b"build 2026.09.24 ok\n", b"build 2026.09.25 ok\n")): + res = self.run_cases([self.case("a", legacy, new, tolerance={"rel": 0.01, "why": "test"})]) + self.assertEqual(res["cases"][0]["verdict"], "differs", (legacy, new)) + same = self.run_cases([self.case("a", b"version 1.2.3 total 1.0000000001\n", b"version 1.2.3 total 1.0000000002\n", tolerance=TOL)]) + self.assertEqual(same["cases"][0]["verdict"], "same") + + def test_the_comparison_is_exact_decimal_arithmetic_not_doubles(self): + # 1e-22 apart: equal as doubles, but far outside a declared 1e-30 relative tolerance + tight = {"rel": 1e-30, "why": "essentially exact"} + res = self.run_cases([self.case("a", b"1.0000000000000000000001\n", b"1.0000000000000000000002\n", tolerance=tight)]) + self.assertEqual(res["cases"][0]["verdict"], "differs") + wide = {"rel": 1e-21, "why": "test"} + res = self.run_cases([self.case("a", b"1.0000000000000000000001\n", b"1.0000000000000000000002\n", tolerance=wide)]) + self.assertEqual(res["cases"][0]["verdict"], "same") + + def test_absurd_numbers_do_not_crash_or_pass(self): + for legacy, new in ((b"v 1e999999999999\n", b"v 1e999999999998\n"), (b"v 1." + b"1" * 200 + b"\n", b"v 1." + b"1" * 199 + b"2\n")): + res = self.run_cases([self.case("a", legacy, new, tolerance=TOL)]) + self.assertEqual(res["cases"][0]["verdict"], "differs") + + def test_an_input_can_be_approved_once_for_every_case_that_shares_its_label(self): + cases = [self.case("F23-a", b"one\n", b"two\n", input="F23"), self.case("F23-b", b"x\n", b"y\n", input="F23"), + self.case("F24-a", b"p\n", b"q\n", input="F24"), self.case("F25-a", b"same\n", b"same\n", input="F25")] + res = self.run_cases(cases, approvedInputs={"F23": "system owner accepts deviation 4", "F25": " ", "F99": "no such input"}) + verdicts = {r["id"]: r["verdict"] for r in res["cases"]} + self.assertEqual(verdicts, {"F23-a": "differs-approved", "F23-b": "differs-approved", "F24-a": "differs", "F25-a": "same"}) + self.assertEqual(res["totals"]["differsApproved"], 2) + self.assertIn("system owner accepts deviation 4", res["cases"][0]["reason"]) + self.assertEqual(res["cases"][2]["approvedDifference"], "") # a blank reason approves nothing + # a case's own reason wins, and a per-input approval never turns a missing file into a pass + own = self.run_cases([self.case("G1", b"a\n", b"b\n", input="G", approvedDifference="own reason"), dict(self.case("G2", b"a\n", b"a\n", input="G"), new="new/nothing")], + approvedInputs={"G": "input reason"}) + self.assertEqual(own["cases"][0]["approvedDifference"], "own reason") + self.assertEqual(own["cases"][1]["verdict"], "missing") + + def test_masks_and_tolerance_work_together(self): + res = self.run_cases([self.case("a", b"at 2026-09-24 10:00:00 value 2.0000000001\n", b"at 2027-01-01 11:11:11 value 2.0000000002\n", + mask=[{"regex": r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d", "why": "run time"}], tolerance=TOL)]) + self.assertEqual(res["cases"][0]["verdict"], "same") + + def test_large_outputs_are_not_tolerance_compared(self): + big = b"1.0 " * (cmp.MAX_TOLERANT_BYTES // 4 + 10) + rec = self.run_cases([self.case("a", big, big[:-3] + b"2.0", tolerance=TOL)])["cases"][0] + self.assertEqual(rec["verdict"], "differs") + + +if __name__ == "__main__": + unittest.main() |
