summaryrefslogtreecommitdiff
path: root/plugins/code-modernization/scripts/tests/test_compare_tolerance.py
blob: 1bc1300a94ef48c30d7bca261db20b27af9fa192 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
"""Tests for the numeric tolerance in scripts/compare.py.

Run:  python3 -m unittest discover -s plugins/code-modernization/scripts/tests
"""
import json
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

HERE = os.path.dirname(os.path.abspath(__file__))
PLUGIN = os.path.dirname(os.path.dirname(HERE))
sys.path.insert(0, os.path.join(PLUGIN, "scripts"))
import compare as cmp  # noqa: E402

TOL = {"rel": 1e-9, "abs": 0, "why": "last-digit rounding of exp() between libm versions"}


class ToleranceTests(unittest.TestCase):
    def setUp(self):
        self.tmp = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, self.tmp, True)

    def case(self, name, legacy, new, **extra):
        for side, data in (("legacy", legacy), ("new", new)):
            os.makedirs(os.path.join(self.tmp, side), exist_ok=True)
            with open(os.path.join(self.tmp, side, name), "wb") as fh:
                fh.write(data)
        return {"id": name, "legacy": "legacy/" + name, "new": "new/" + name, **extra}

    def run_cases(self, cases, **top):
        path = os.path.join(self.tmp, "cases.json")
        with open(path, "w") as fh:
            json.dump({"system": "s", "cases": cases, **top}, fh)
        return cmp.run(path, os.path.join(self.tmp, "EQUIVALENCE.json"))

    def test_a_tiny_float_difference_is_same_within_the_declared_tolerance_and_says_so(self):
        res = self.run_cases([self.case("a", b"total 1234.5678901234\nrate 0.05\n", b"total 1234.5678901239\nrate 0.05\n", tolerance=TOL)])
        rec = res["cases"][0]
        self.assertEqual(rec["verdict"], "same")
        self.assertEqual(rec["withinTolerance"]["differing"], 1)
        self.assertLess(rec["withinTolerance"]["maxRelativeDifference"], 1e-9)
        self.assertIn("declared tolerance", rec["reason"])
        self.assertIn("libm", rec["reason"])
        self.assertEqual(res["totals"]["sameWithinTolerance"], 1)
        self.assertIsNone(cmp.verdict_of(res))
        self.assertTrue(res["selfCheck"]["passed"], res["selfCheck"])

    def test_without_a_tolerance_the_same_outputs_differ_and_no_new_key_appears(self):
        res = self.run_cases([self.case("a", b"1234.5678901234\n", b"1234.5678901239\n")])
        self.assertEqual(res["cases"][0]["verdict"], "differs")
        self.assertNotIn("sameWithinTolerance", res["totals"])

    def test_a_difference_outside_the_tolerance_differs(self):
        res = self.run_cases([self.case("a", b"total 100.0\n", b"total 100.5\n", tolerance=TOL)])
        rec = res["cases"][0]
        self.assertEqual(rec["verdict"], "differs")
        self.assertIn("beyond the declared tolerance", rec["reason"])
        self.assertIn("NOT PROVEN" if cmp.verdict_of(res) else "", "NOT PROVEN")

    def test_integers_and_text_must_match_exactly_even_with_a_tolerance(self):
        loose = {"rel": 0.01, "abs": 0, "why": "test"}
        for legacy, new in ((b"count 1000000\n", b"count 1000001\n"), (b"name alpha 1.5\n", b"name alphb 1.5\n"),
                            (b"1.5 2.5\n", b"1.5\n"), (b"x 10\n", b"x 10.0\n")):
            res = self.run_cases([self.case("a", legacy, new, tolerance=loose)])
            self.assertEqual(res["cases"][0]["verdict"], "differs", (legacy, new))

    def test_exponent_forms_compare_by_value(self):
        res = self.run_cases([self.case("a", b"v 1.5e3\n", b"v 1500.0000000000002\n", tolerance=TOL)])
        self.assertEqual(res["cases"][0]["verdict"], "same")

    def test_a_tolerance_needs_a_reason_and_a_sane_size(self):
        legacy = self.case("a", b"1.0\n", b"1.0\n")
        for bad in ({"rel": 1e-9}, {"rel": 1e-9, "why": "  "}, {"rel": 0.5, "why": "x"}, {"why": "x"}, {"rel": -1, "why": "x"},
                    {"rel": "big", "why": "x"}, {"rel": float("nan"), "why": "x"}, "1e-9"):
            with self.assertRaises(cmp.InputError, msg=repr(bad)):
                self.run_cases([dict(legacy, tolerance=bad)])

    def test_a_default_tolerance_applies_to_every_case_and_a_case_can_switch_it_off(self):
        a = self.case("a", b"1.00000000000\n", b"1.00000000001\n")
        b = self.case("b", b"1.00000000000\n", b"1.00000000001\n", tolerance=None)
        res = self.run_cases([a, b], tolerance=TOL)
        self.assertEqual([r["verdict"] for r in res["cases"]], ["same", "differs"])

    def test_an_absolute_tolerance_is_capped_like_a_relative_one(self):
        for bad in ({"abs": 1e-3, "why": "x"}, {"abs": 100, "why": "x"}):
            with self.assertRaises(cmp.InputError, msg=repr(bad)):
                self.run_cases([self.case("a", b"5.5\n", b"5.5\n", tolerance=bad)])
        ok = self.run_cases([self.case("a", b"x 0.0000000000000001\n", b"x 0.0\n", tolerance={"abs": 1e-9, "why": "a value that is zero to rounding"})])
        self.assertEqual(ok["cases"][0]["verdict"], "same")

    def test_the_self_check_still_fails_if_a_loose_tolerance_ever_got_through(self):
        loose = {"abs": 100, "why": "absurdly loose"}
        with mock.patch.object(cmp, "MAX_ABS_TOLERANCE", 1000.0):
            res = self.run_cases([self.case("a", b"5.5\n", b"5.5\n", tolerance=loose)])
        self.assertFalse(res["selfCheck"]["passed"])
        self.assertIn("tolerance may be too loose", res["selfCheck"]["detail"])
        self.assertIn("self-check failed", cmp.verdict_of(res))

    def test_dotted_identifiers_are_text_and_must_match_exactly(self):
        for legacy, new in ((b"version 1.2.3\n", b"version 1.2.4\n"), (b"host 10.0.0.1\n", b"host 10.0.0.2\n"),
                            (b"build 2026.09.24 ok\n", b"build 2026.09.25 ok\n")):
            res = self.run_cases([self.case("a", legacy, new, tolerance={"rel": 0.01, "why": "test"})])
            self.assertEqual(res["cases"][0]["verdict"], "differs", (legacy, new))
        same = self.run_cases([self.case("a", b"version 1.2.3 total 1.0000000001\n", b"version 1.2.3 total 1.0000000002\n", tolerance=TOL)])
        self.assertEqual(same["cases"][0]["verdict"], "same")

    def test_the_comparison_is_exact_decimal_arithmetic_not_doubles(self):
        # 1e-22 apart: equal as doubles, but far outside a declared 1e-30 relative tolerance
        tight = {"rel": 1e-30, "why": "essentially exact"}
        res = self.run_cases([self.case("a", b"1.0000000000000000000001\n", b"1.0000000000000000000002\n", tolerance=tight)])
        self.assertEqual(res["cases"][0]["verdict"], "differs")
        wide = {"rel": 1e-21, "why": "test"}
        res = self.run_cases([self.case("a", b"1.0000000000000000000001\n", b"1.0000000000000000000002\n", tolerance=wide)])
        self.assertEqual(res["cases"][0]["verdict"], "same")

    def test_absurd_numbers_do_not_crash_or_pass(self):
        for legacy, new in ((b"v 1e999999999999\n", b"v 1e999999999998\n"), (b"v 1." + b"1" * 200 + b"\n", b"v 1." + b"1" * 199 + b"2\n")):
            res = self.run_cases([self.case("a", legacy, new, tolerance=TOL)])
            self.assertEqual(res["cases"][0]["verdict"], "differs")

    def test_an_input_can_be_approved_once_for_every_case_that_shares_its_label(self):
        cases = [self.case("F23-a", b"one\n", b"two\n", input="F23"), self.case("F23-b", b"x\n", b"y\n", input="F23"),
                 self.case("F24-a", b"p\n", b"q\n", input="F24"), self.case("F25-a", b"same\n", b"same\n", input="F25")]
        res = self.run_cases(cases, approvedInputs={"F23": "system owner accepts deviation 4", "F25": "  ", "F99": "no such input"})
        verdicts = {r["id"]: r["verdict"] for r in res["cases"]}
        self.assertEqual(verdicts, {"F23-a": "differs-approved", "F23-b": "differs-approved", "F24-a": "differs", "F25-a": "same"})
        self.assertEqual(res["totals"]["differsApproved"], 2)
        self.assertIn("system owner accepts deviation 4", res["cases"][0]["reason"])
        self.assertEqual(res["cases"][2]["approvedDifference"], "")   # a blank reason approves nothing
        # a case's own reason wins, and a per-input approval never turns a missing file into a pass
        own = self.run_cases([self.case("G1", b"a\n", b"b\n", input="G", approvedDifference="own reason"), dict(self.case("G2", b"a\n", b"a\n", input="G"), new="new/nothing")],
                             approvedInputs={"G": "input reason"})
        self.assertEqual(own["cases"][0]["approvedDifference"], "own reason")
        self.assertEqual(own["cases"][1]["verdict"], "missing")

    def test_masks_and_tolerance_work_together(self):
        res = self.run_cases([self.case("a", b"at 2026-09-24 10:00:00 value 2.0000000001\n", b"at 2027-01-01 11:11:11 value 2.0000000002\n",
                                        mask=[{"regex": r"\d{4}-\d\d-\d\d \d\d:\d\d:\d\d", "why": "run time"}], tolerance=TOL)])
        self.assertEqual(res["cases"][0]["verdict"], "same")

    def test_large_outputs_are_not_tolerance_compared(self):
        big = b"1.0 " * (cmp.MAX_TOLERANT_BYTES // 4 + 10)
        rec = self.run_cases([self.case("a", big, big[:-3] + b"2.0", tolerance=TOL)])["cases"][0]
        self.assertEqual(rec["verdict"], "differs")


if __name__ == "__main__":
    unittest.main()