summaryrefslogtreecommitdiff
path: root/plugins/code-modernization/scripts/compare.py
diff options
context:
space:
mode:
Diffstat (limited to 'plugins/code-modernization/scripts/compare.py')
-rw-r--r--plugins/code-modernization/scripts/compare.py487
1 files changed, 487 insertions, 0 deletions
diff --git a/plugins/code-modernization/scripts/compare.py b/plugins/code-modernization/scripts/compare.py
new file mode 100644
index 00000000..4a1e4429
--- /dev/null
+++ b/plugins/code-modernization/scripts/compare.py
@@ -0,0 +1,487 @@
+#!/usr/bin/env python3
+"""Deterministic judge of "does the new system give the same output as the legacy one".
+
+ python3 compare.py <cases.json> [--out EQUIVALENCE.json] [--quiet] [--allow-outside]
+
+The verdict comes from comparing bytes, never from a model's opinion. Input:
+
+ {"system": "carddemo",
+ "legacy": {"label": "COBOL, GnuCOBOL 3.2", "command": "optional free text"},
+ "new": {"label": "Java 17", "command": "optional free text"},
+ "cases": [{"id": "C01", "title": "Interest posting, normal account",
+ "legacy": "out/legacy/C01.out", "new": "out/new/C01.out",
+ "mask": [{"bytes": "278-330", "why": "run timestamp"},
+ {"regex": "\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}"}],
+ "approvedDifference": "optional: why a person accepted a difference",
+ "note": "optional"}]}
+
+Paths are relative to the cases file's folder and may not leave it unless --allow-outside is
+given. `bytes` ranges are 0-based and inclusive; `regex` runs on the bytes decoded as latin-1.
+Every masked span, on both sides, is replaced by one fixed marker, so masks never hide the
+position of a difference and a variable-length match still compares equal. A case is:
+ same identical after masking
+ differs different; the first difference is recorded (a person may approve it: differs-approved)
+ missing a file is absent, unreadable or outside the cases folder. Never a pass.
+
+A person who accepts a difference records it on the case (`"approvedDifference": "why"`), or once for every
+case that shares an `input` label with a top-level `"approvedInputs": {"F23": "why"}`. A reason is required either way, and an approved difference is still listed as a
+difference in the result.
+
+Numbers that a run legitimately prints a little differently (floating-point results from another
+compiler or math library) can be compared within a declared tolerance, per case or for all cases:
+
+ "tolerance": {"rel": 1e-9, "abs": 0, "why": "last-digit rounding of the exp() in the interest formula"}
+
+The reason is required and is recorded. Only numbers written with a decimal point or an exponent
+may differ, and only within the tolerance (compared as exact decimals); every other byte, every integer and every
+dotted run like 1.2.3 must match exactly. The case is `same`, with the largest relative difference
+recorded; anything outside the tolerance is `differs`. A relative tolerance above 1% or an absolute one
+above 1e-6 is refused: that is a different result, not rounding.
+
+Writes EQUIVALENCE.json (default: next to the cases file). Exit 0 only when at least one case
+executed, none is missing or differs (approved differences are allowed), the self-check did not
+fail and some compared output was not empty; 1 otherwise; 2 for unusable input.
+"""
+import argparse
+import bisect
+import datetime
+import hashlib
+import decimal
+import json
+import math
+import os
+import re
+import sys
+import tempfile
+
+MARK = b"\x00<MASK>\x00"
+MAX_BYTES = 256 * 1024 * 1024
+MAX_TOLERANT_BYTES = 32 * 1024 * 1024
+# A dotted run such as 1.2.3 or 10.0.0.1 is an identifier, not a number: it is left inside the text around it.
+NUMBER = re.compile(rb"(?P<dotted>\d+(?:\.\d+){2,})|(?P<num>[-+]?(?:\d+\.\d*|\.\d+|\d+)(?:[eE][-+]?\d+)?)")
+MAX_ABS_TOLERANCE = 1e-6
+MAX_NUMBER_CHARS = 64
+
+
+class InputError(Exception):
+ pass
+
+
+def clip(value, limit):
+ return value[:limit] if isinstance(value, str) else ""
+
+
+def parse_masks(specs, cid):
+ """[{"bytes": "0-3,10"} | {"regex": "..."}] -> [(kind, ranges-or-pattern, label)]."""
+ masks = []
+ for spec in specs if isinstance(specs, list) else []:
+ if not isinstance(spec, dict) or ("bytes" in spec) == ("regex" in spec):
+ raise InputError("case %s: each mask needs exactly one of 'bytes' or 'regex'" % cid)
+ why = " (%s)" % clip(spec.get("why"), 120) if spec.get("why") else ""
+ if "bytes" in spec:
+ ranges = []
+ for part in str(spec["bytes"]).split(","):
+ m = re.fullmatch(r"\s*(\d+)\s*(?:-\s*(\d+))?\s*", part)
+ if not m or (m.group(2) and int(m.group(2)) < int(m.group(1))):
+ raise InputError("case %s: bad byte range %r" % (cid, part))
+ ranges.append((int(m.group(1)), int(m.group(2) or m.group(1))))
+ masks.append(("bytes", ranges, "bytes %s%s" % (spec["bytes"], why)))
+ else:
+ try:
+ masks.append(("regex", re.compile(str(spec["regex"])), "regex %s%s" % (clip(str(spec["regex"]), 120), why)))
+ except re.error as err:
+ raise InputError("case %s: bad regex %r: %s" % (cid, spec["regex"], err))
+ return masks
+
+
+def parse_tolerance(spec, cid):
+ """{"rel": 1e-9, "abs": 0, "why": "..."} -> a validated dict, or None when there is none."""
+ if spec is None:
+ return None
+ if not isinstance(spec, dict):
+ raise InputError("case %s: 'tolerance' must be an object with rel, abs and why" % cid)
+ try:
+ rel, ab = float(spec.get("rel", 0)), float(spec.get("abs", 0))
+ except (TypeError, ValueError):
+ raise InputError("case %s: the tolerance's rel and abs must be numbers" % cid)
+ if not (math.isfinite(rel) and math.isfinite(ab)) or rel < 0 or ab < 0 or (rel == 0 and ab == 0):
+ raise InputError("case %s: give a positive rel or abs tolerance" % cid)
+ if rel > 0.01:
+ raise InputError("case %s: a relative tolerance above 1%% is a different result, not rounding" % cid)
+ if ab > MAX_ABS_TOLERANCE:
+ raise InputError("case %s: an absolute tolerance above %g is a different result, not rounding" % (cid, MAX_ABS_TOLERANCE))
+ why = clip(spec.get("why"), 300).strip()
+ if not why:
+ raise InputError("case %s: a tolerance needs a 'why' saying what legitimately differs" % cid)
+ return {"rel": rel, "abs": ab, "why": why}
+
+
+def split_numbers(data):
+ """-> [(is_number, bytes, offset)] covering `data` end to end."""
+ parts, pos = [], 0
+ for m in NUMBER.finditer(data):
+ if m.group("dotted") is not None:
+ continue
+ if m.start() > pos:
+ parts.append((False, data[pos:m.start()], pos))
+ parts.append((True, m.group(), m.start()))
+ pos = m.end()
+ if pos < len(data):
+ parts.append((False, data[pos:], pos))
+ return parts
+
+
+def is_float(token):
+ return any(c in token for c in b".eE")
+
+
+def within_tolerance(a, b, tol):
+ """Compare two masked outputs number by number. -> (True, stats) or (False, offset of the first real difference)."""
+ if len(a) > MAX_TOLERANT_BYTES or len(b) > MAX_TOLERANT_BYTES:
+ return False, first_diff(a, b) or 0
+ ta, tb = split_numbers(a), split_numbers(b)
+ if len(ta) != len(tb):
+ return False, first_diff(a, b) or 0
+ allow_abs, allow_rel = decimal.Decimal(repr(tol["abs"])), decimal.Decimal(repr(tol["rel"]))
+ differing, worst = 0, 0.0
+ for (na, xa, sa), (nb, xb, _) in zip(ta, tb):
+ if xa == xb:
+ continue
+ if na != nb:
+ return False, sa
+ if not na:
+ return False, sa + (first_diff(xa, xb) or 0)
+ if not (is_float(xa) and is_float(xb)) or max(len(xa), len(xb)) > MAX_NUMBER_CHARS:
+ return False, sa # an integer, or a very long number, must match exactly
+ try: # exact decimal arithmetic: no double-precision blind spot
+ x, y = decimal.Decimal(xa.decode("ascii")), decimal.Decimal(xb.decode("ascii"))
+ if not (x.is_finite() and y.is_finite()):
+ return False, sa
+ scale, diff = max(abs(x), abs(y)), abs(x - y)
+ if diff > max(allow_abs, allow_rel * scale):
+ return False, sa
+ if scale:
+ worst = max(worst, float(diff / scale))
+ except (ArithmeticError, ValueError):
+ return False, sa
+ differing += 1
+ return True, {"differing": differing, "maxRelativeDifference": worst}
+
+
+def mask_spans(data, masks):
+ """Merged (start, end) spans of `data` that the masks hide."""
+ spans, text = [], None
+ for kind, spec, _ in masks:
+ if kind == "bytes":
+ spans += [(s, min(e + 1, len(data))) for s, e in spec if s < len(data)]
+ else:
+ text = data.decode("latin-1") if text is None else text
+ spans += [m.span() for m in spec.finditer(text) if m.end() > m.start()]
+ merged = []
+ for s, e in sorted(spans):
+ if merged and s <= merged[-1][1]:
+ merged[-1] = (merged[-1][0], max(e, merged[-1][1]))
+ else:
+ merged.append((s, e))
+ return merged
+
+
+def apply_masks(data, masks):
+ """-> (masked bytes, cuts, hidden byte count); cuts map masked offsets back to file offsets."""
+ out, pos, cuts, hidden, at = [], 0, [], 0, 0
+ for s, e in mask_spans(data, masks):
+ out.append(data[pos:s])
+ at += s - pos
+ cuts.append((at, s, e))
+ out.append(MARK)
+ at += len(MARK)
+ hidden += e - s
+ pos = e
+ out.append(data[pos:])
+ return b"".join(out), cuts, hidden
+
+
+def original_offset(cuts, masked_offset):
+ shift = 0
+ for at, s, e in cuts:
+ if masked_offset < at:
+ break
+ if masked_offset < at + len(MARK):
+ return s
+ shift += (e - s) - len(MARK)
+ return masked_offset + shift
+
+
+def first_diff(a, b):
+ n, step, i = min(len(a), len(b)), 1 << 16, 0
+ while i < n:
+ j = min(i + step, n)
+ if a[i:j] != b[i:j]:
+ return next(k for k in range(i, j) if a[k] != b[k])
+ i = j
+ return n if len(a) != len(b) else None
+
+
+def escape(chunk):
+ out = []
+ for byte in chunk:
+ if byte == 0x5C:
+ out.append("\\\\")
+ elif 0x20 <= byte < 0x7F:
+ out.append(chr(byte))
+ else:
+ out.append({10: "\\n", 13: "\\r", 9: "\\t"}.get(byte, "\\x%02x" % byte))
+ return "".join(out).replace("\\x00<MASK>\\x00", "[masked]")
+
+
+def context(masked, off):
+ before = escape(masked[max(0, off - 12):off])[-12:]
+ after = escape(masked[off:off + 28])[:28] if off < len(masked) else "<end of output>"
+ return before + after, len(before)
+
+
+def read_file(rel, base, allow_outside):
+ """-> (bytes, None) or (None, plain-English problem)."""
+ if not isinstance(rel, str) or not rel:
+ return None, "no path was given"
+ path = os.path.join(base, rel)
+ try:
+ real, root = os.path.realpath(path), os.path.realpath(base)
+ if not allow_outside and os.path.commonpath([root, real]) != root:
+ return None, "the path leaves the cases folder (pass --allow-outside if that is intended)"
+ if not os.path.exists(path):
+ return None, "the file does not exist"
+ if os.path.getsize(path) > MAX_BYTES:
+ return None, "the file is larger than %d MB" % (MAX_BYTES >> 20)
+ with open(path, "rb") as fh:
+ return fh.read(), None
+ except (OSError, ValueError) as err:
+ return None, "the file could not be read (%s)" % err.__class__.__name__
+
+
+def rel_to(path, folder):
+ try:
+ return os.path.relpath(path, folder).replace(os.sep, "/")
+ except ValueError:
+ return path
+
+
+def judge(case, index, base, out_dir, allow_outside, default_tolerance=None, approved_inputs=None):
+ """-> (case record, legacy bytes or None, masks)."""
+ cid = clip(str(case.get("id") or "C%02d" % (index + 1)), 80)
+ masks = parse_masks(case.get("mask"), cid)
+ tol = parse_tolerance(case["tolerance"] if "tolerance" in case else default_tolerance, cid)
+ rec = {"id": cid, "title": clip(case.get("title"), 300), "verdict": "missing", "reason": "",
+ "legacyPath": "", "newPath": "", "legacySha256": "", "newSha256": "",
+ "masked": [m[2] for m in masks],
+ "approvedDifference": clip(case.get("approvedDifference") or (approved_inputs or {}).get(str(case.get("input", "")), ""), 500),
+ "note": clip(case.get("note"), 500)}
+ data, problems = {}, []
+ for side in ("legacy", "new"):
+ given = case.get(side)
+ rec[side + "Path"] = rel_to(os.path.join(base, given), out_dir) if isinstance(given, str) and given else ""
+ data[side], why = read_file(given, base, allow_outside)
+ if why:
+ problems.append("The %s output was not compared: %s (%s)." % (side, why, clip(str(given), 200)))
+ if problems:
+ rec["reason"] = " ".join(problems)
+ return rec, None, masks, tol
+ legacy, new = data["legacy"], data["new"]
+ rec["legacySha256"], rec["newSha256"] = hashlib.sha256(legacy).hexdigest(), hashlib.sha256(new).hexdigest()
+ a, cuts, hid_a = apply_masks(legacy, masks)
+ b, _, hid_b = apply_masks(new, masks)
+ rec["bytes"], rec["maskedBytes"] = {"legacy": len(legacy), "new": len(new)}, {"legacy": hid_a, "new": hid_b}
+ rec["empty"] = not legacy and not new
+ at = first_diff(a, b)
+ if at is None:
+ rec["verdict"] = "same"
+ rec["reason"] = ("Both outputs are empty, so nothing was compared." if rec["empty"] else
+ "Identical after masking: %d of %d bytes were hidden by %d mask(s)." % (hid_a, len(legacy), len(masks)) if masks else
+ "Identical, %d bytes." % len(legacy))
+ return rec, legacy, masks, tol
+ seen, seen_at = context(a, at)
+ got, _ = context(b, at)
+ line = a.count(b"\n", 0, at) + 1
+ rec["firstDiff"] = {"offset": original_offset(cuts, at), "line": line, "legacy": seen, "new": got, "at": seen_at}
+ where = "line %d, byte %d" % (line, rec["firstDiff"]["offset"])
+ if at >= len(a) or at >= len(b):
+ rec["reason"] = "The %s output ends at %s while the %s output continues." % (
+ "legacy" if at >= len(a) else "new", where, "new" if at >= len(a) else "legacy")
+ else:
+ rec["reason"] = "The outputs first differ at %s." % where
+ if tol:
+ rec["tolerance"] = dict(tol)
+ ok, info = within_tolerance(a, b, tol)
+ if ok:
+ rec["verdict"] = "same"
+ rec["withinTolerance"] = info
+ rec["reason"] = ("Identical apart from %d number(s) that differ within the declared tolerance "
+ "(relative %g, absolute %g; largest relative difference %.3g). Declared reason: %s"
+ % (info["differing"], tol["rel"], tol["abs"], info["maxRelativeDifference"], clip(tol["why"], 200)))
+ return rec, legacy, masks, tol
+ line = a.count(b"\n", 0, info) + 1
+ rec["reason"] = "A number or text differs beyond the declared tolerance at line %d, byte %d." % (line, original_offset(cuts, info))
+ if rec["approvedDifference"].strip():
+ rec["verdict"] = "differs-approved"
+ rec["reason"] += " A person approved this difference: %s" % clip(rec["approvedDifference"], 200)
+ else:
+ rec["verdict"] = "differs"
+ return rec, legacy, masks, tol
+
+
+def differs_under(changed, data, masks, tol):
+ """Would the comparator report `changed` as different from `data`?"""
+ a, b = apply_masks(data, masks)[0], apply_masks(changed, masks)[0]
+ return a != b and not (tol and within_tolerance(a, b, tol)[0])
+
+
+def self_check(items):
+ """Prove the comparator can fail: change one unmasked byte of each legacy output; it must differ."""
+ tested = 0
+ for cid, data, masks, tol in items:
+ if not data:
+ continue
+ tested += 1
+ free, pos = [], 0
+ for s, e in mask_spans(data, masks) + [(len(data), len(data))]:
+ if s > pos:
+ free.append((pos, s))
+ pos = e
+ if not free:
+ return False, "case %s: the masks hide every byte, so no difference could ever be seen" % cid
+ biggest = max(free, key=lambda f: f[1] - f[0])
+ flip = lambda p: data[:p] + bytes([data[p] ^ 1]) + data[p + 1:]
+ spots = [free[0][0], free[-1][1] - 1, (biggest[0] + biggest[1]) // 2]
+ caught = any(differs_under(flip(p), data, masks, tol) for p in spots)
+ if tol and caught:
+ # A tolerance must never hide a change to the numbers it covers: changing the leading digit of every
+ # sampled float that is not masked has to be reported, or the tolerance is too loose to prove anything.
+ for m in list(NUMBER.finditer(data[:1 << 20]))[:200]:
+ token = m.group("num")
+ if token is None or not is_float(token) or not any(a <= m.start() < b for a, b in free):
+ continue
+ at = m.start() + next(i for i, c in enumerate(token) if 48 <= c <= 57)
+ if not differs_under(flip(at), data, masks, tol):
+ caught = False
+ break
+ if not caught:
+ return False, "case %s: a one-byte change to the legacy output was not reported as different%s" % (
+ cid, " (the tolerance may be too loose)" if tol else "")
+ if not tested:
+ return None, "not applicable: no executed case had bytes to test"
+ return True, "each legacy output was compared with a copy changed by one byte and reported as different"
+
+
+def totals(records, executed, count):
+ t = {"cases": len(records), "executed": executed, "same": count("same"), "differs": count("differs"),
+ "differsApproved": count("differs-approved"), "missing": count("missing")}
+ tolerant = sum(1 for r in records if r.get("withinTolerance"))
+ if tolerant: # additive: absent unless a declared tolerance was used
+ t["sameWithinTolerance"] = tolerant
+ return t
+
+
+def run(cases_path, out_path=None, allow_outside=False):
+ """-> the EQUIVALENCE record. Raises InputError for a cases file that cannot be used."""
+ try:
+ with open(cases_path, "rb") as fh:
+ spec = json.loads(fh.read().decode("utf-8-sig"))
+ except (OSError, ValueError) as err:
+ raise InputError("cannot read %s: %s" % (cases_path, err))
+ if not isinstance(spec, dict) or not isinstance(spec.get("cases"), list):
+ raise InputError("%s must be a JSON object with a 'cases' list" % cases_path)
+ base = os.path.dirname(os.path.abspath(cases_path))
+ out_dir = os.path.dirname(os.path.abspath(out_path)) if out_path else base
+ records, items, ids = [], [], set()
+ approved = {str(k): v for k, v in spec["approvedInputs"].items() if isinstance(v, str) and v.strip()} if isinstance(spec.get("approvedInputs"), dict) else {}
+ for i, case in enumerate(spec["cases"]):
+ if not isinstance(case, dict):
+ raise InputError("case %d is not an object" % (i + 1))
+ rec, data, masks, tol = judge(case, i, base, out_dir, allow_outside, spec.get("tolerance"), approved)
+ if rec["id"] in ids:
+ raise InputError("duplicate case id %r" % rec["id"])
+ ids.add(rec["id"])
+ records.append(rec)
+ if data is not None:
+ items.append((rec["id"], data, masks, tol))
+ passed, detail = self_check(items)
+ count = lambda v: sum(1 for r in records if r["verdict"] == v)
+ ends = lambda key: {k: clip(spec.get(key, {}).get(k), 300) for k in ("label", "command")} if isinstance(spec.get(key), dict) else {"label": "", "command": ""}
+ return {"system": clip(spec.get("system"), 200), "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+ "legacy": ends("legacy"), "new": ends("new"), "selfCheck": {"passed": passed, "detail": detail},
+ "totals": totals(records, len(items), count), "cases": records}
+
+
+def verdict_of(result):
+ """-> None when proven, else the reason it is not."""
+ t = result["totals"]
+ if t["executed"] == 0:
+ return "no case executed"
+ if t["missing"] or t["differs"]:
+ return " and ".join(filter(None, ["%d case(s) differ" % t["differs"] if t["differs"] else "",
+ "%d case(s) are missing" % t["missing"] if t["missing"] else ""]))
+ if result["selfCheck"]["passed"] is False:
+ return "the self-check failed: " + result["selfCheck"]["detail"]
+ if all(r.get("empty") for r in result["cases"] if r["verdict"] != "missing"):
+ return "every compared output was empty"
+ return None
+
+
+def write_atomic(path, text):
+ path = os.path.abspath(path)
+ folder = os.path.dirname(path)
+ for p in (path, folder):
+ if os.path.islink(p):
+ raise InputError("refusing to write through a symbolic link: %s" % p)
+ os.makedirs(folder, exist_ok=True)
+ fd, tmp = tempfile.mkstemp(dir=folder, prefix=".equivalence-", suffix=".tmp")
+ try:
+ with os.fdopen(fd, "wb") as fh:
+ fh.write(text.encode("utf-8"))
+ umask = os.umask(0)
+ os.umask(umask)
+ os.chmod(tmp, 0o666 & ~umask)
+ os.replace(tmp, path)
+ except BaseException:
+ if os.path.lexists(tmp):
+ os.unlink(tmp)
+ raise
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description="Compare legacy and new outputs byte for byte.")
+ ap.add_argument("cases", help="cases.json (see the module docstring for the schema)")
+ ap.add_argument("--out", help="where to write EQUIVALENCE.json (default: next to the cases file)")
+ ap.add_argument("--quiet", action="store_true", help="write the file, print nothing")
+ ap.add_argument("--allow-outside", action="store_true", help="allow case paths that leave the cases folder")
+ args = ap.parse_args(argv)
+ out = args.out or os.path.join(os.path.dirname(os.path.abspath(args.cases)), "EQUIVALENCE.json")
+ try:
+ result = run(args.cases, out, args.allow_outside)
+ except InputError as err:
+ print("compare.py: %s" % err, file=sys.stderr)
+ return 2
+ try:
+ write_atomic(out, json.dumps(result, indent=2, ensure_ascii=True) + "\n")
+ except InputError as err:
+ print("compare.py: %s" % err, file=sys.stderr)
+ return 2
+ problem = verdict_of(result)
+ if not args.quiet:
+ t, plain = result["totals"], lambda s: re.sub(r"[\x00-\x1f\x7f-\x9f]", "?", s)
+ print("equivalence cases executed: %d" % t["executed"])
+ print("same %d%s | differs %d | differs but approved %d | missing %d | of %d cases" %
+ (t["same"], " (%d within a declared tolerance)" % t["sameWithinTolerance"] if t.get("sameWithinTolerance") else "",
+ t["differs"], t["differsApproved"], t["missing"], t["cases"]))
+ sc = result["selfCheck"]
+ print("self-check: %s - %s" % ({True: "passed", False: "FAILED", None: "not applicable"}[sc["passed"]], plain(sc["detail"])))
+ for r in result["cases"]:
+ if r["verdict"] != "same" or r.get("withinTolerance"):
+ print(" %s %s: %s" % (plain(r["id"]), r["verdict"], plain(r["reason"])[:200]))
+ print(("NOT PROVEN: %s" % problem) if problem else "PROVEN: %d case(s) executed, none differ" % t["executed"])
+ print("wrote %s" % out)
+ return 1 if problem else 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())