summaryrefslogtreecommitdiff
path: root/plugins/code-modernization/scripts/compare.py
blob: 4a1e4429add5f2744bd8ca32ecf5241e417825f7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
#!/usr/bin/env python3
"""Deterministic judge of "does the new system give the same output as the legacy one".

    python3 compare.py <cases.json> [--out EQUIVALENCE.json] [--quiet] [--allow-outside]

The verdict comes from comparing bytes, never from a model's opinion. Input:

    {"system": "carddemo",
     "legacy": {"label": "COBOL, GnuCOBOL 3.2", "command": "optional free text"},
     "new":    {"label": "Java 17",             "command": "optional free text"},
     "cases": [{"id": "C01", "title": "Interest posting, normal account",
                "legacy": "out/legacy/C01.out", "new": "out/new/C01.out",
                "mask": [{"bytes": "278-330", "why": "run timestamp"},
                         {"regex": "\\d{4}-\\d{2}-\\d{2}[ T]\\d{2}:\\d{2}:\\d{2}"}],
                "approvedDifference": "optional: why a person accepted a difference",
                "note": "optional"}]}

Paths are relative to the cases file's folder and may not leave it unless --allow-outside is
given. `bytes` ranges are 0-based and inclusive; `regex` runs on the bytes decoded as latin-1.
Every masked span, on both sides, is replaced by one fixed marker, so masks never hide the
position of a difference and a variable-length match still compares equal. A case is:
  same      identical after masking
  differs   different; the first difference is recorded (a person may approve it: differs-approved)
  missing   a file is absent, unreadable or outside the cases folder. Never a pass.

A person who accepts a difference records it on the case (`"approvedDifference": "why"`), or once for every
case that shares an `input` label with a top-level `"approvedInputs": {"F23": "why"}`. A reason is required either way, and an approved difference is still listed as a
difference in the result.

Numbers that a run legitimately prints a little differently (floating-point results from another
compiler or math library) can be compared within a declared tolerance, per case or for all cases:

    "tolerance": {"rel": 1e-9, "abs": 0, "why": "last-digit rounding of the exp() in the interest formula"}

The reason is required and is recorded. Only numbers written with a decimal point or an exponent
may differ, and only within the tolerance (compared as exact decimals); every other byte, every integer and every
dotted run like 1.2.3 must match exactly. The case is `same`, with the largest relative difference
recorded; anything outside the tolerance is `differs`. A relative tolerance above 1% or an absolute one
above 1e-6 is refused: that is a different result, not rounding.

Writes EQUIVALENCE.json (default: next to the cases file). Exit 0 only when at least one case
executed, none is missing or differs (approved differences are allowed), the self-check did not
fail and some compared output was not empty; 1 otherwise; 2 for unusable input.
"""
import argparse
import bisect
import datetime
import hashlib
import decimal
import json
import math
import os
import re
import sys
import tempfile

MARK = b"\x00<MASK>\x00"
MAX_BYTES = 256 * 1024 * 1024
MAX_TOLERANT_BYTES = 32 * 1024 * 1024
# A dotted run such as 1.2.3 or 10.0.0.1 is an identifier, not a number: it is left inside the text around it.
NUMBER = re.compile(rb"(?P<dotted>\d+(?:\.\d+){2,})|(?P<num>[-+]?(?:\d+\.\d*|\.\d+|\d+)(?:[eE][-+]?\d+)?)")
MAX_ABS_TOLERANCE = 1e-6
MAX_NUMBER_CHARS = 64


class InputError(Exception):
    pass


def clip(value, limit):
    return value[:limit] if isinstance(value, str) else ""


def parse_masks(specs, cid):
    """[{"bytes": "0-3,10"} | {"regex": "..."}] -> [(kind, ranges-or-pattern, label)]."""
    masks = []
    for spec in specs if isinstance(specs, list) else []:
        if not isinstance(spec, dict) or ("bytes" in spec) == ("regex" in spec):
            raise InputError("case %s: each mask needs exactly one of 'bytes' or 'regex'" % cid)
        why = " (%s)" % clip(spec.get("why"), 120) if spec.get("why") else ""
        if "bytes" in spec:
            ranges = []
            for part in str(spec["bytes"]).split(","):
                m = re.fullmatch(r"\s*(\d+)\s*(?:-\s*(\d+))?\s*", part)
                if not m or (m.group(2) and int(m.group(2)) < int(m.group(1))):
                    raise InputError("case %s: bad byte range %r" % (cid, part))
                ranges.append((int(m.group(1)), int(m.group(2) or m.group(1))))
            masks.append(("bytes", ranges, "bytes %s%s" % (spec["bytes"], why)))
        else:
            try:
                masks.append(("regex", re.compile(str(spec["regex"])), "regex %s%s" % (clip(str(spec["regex"]), 120), why)))
            except re.error as err:
                raise InputError("case %s: bad regex %r: %s" % (cid, spec["regex"], err))
    return masks


def parse_tolerance(spec, cid):
    """{"rel": 1e-9, "abs": 0, "why": "..."} -> a validated dict, or None when there is none."""
    if spec is None:
        return None
    if not isinstance(spec, dict):
        raise InputError("case %s: 'tolerance' must be an object with rel, abs and why" % cid)
    try:
        rel, ab = float(spec.get("rel", 0)), float(spec.get("abs", 0))
    except (TypeError, ValueError):
        raise InputError("case %s: the tolerance's rel and abs must be numbers" % cid)
    if not (math.isfinite(rel) and math.isfinite(ab)) or rel < 0 or ab < 0 or (rel == 0 and ab == 0):
        raise InputError("case %s: give a positive rel or abs tolerance" % cid)
    if rel > 0.01:
        raise InputError("case %s: a relative tolerance above 1%% is a different result, not rounding" % cid)
    if ab > MAX_ABS_TOLERANCE:
        raise InputError("case %s: an absolute tolerance above %g is a different result, not rounding" % (cid, MAX_ABS_TOLERANCE))
    why = clip(spec.get("why"), 300).strip()
    if not why:
        raise InputError("case %s: a tolerance needs a 'why' saying what legitimately differs" % cid)
    return {"rel": rel, "abs": ab, "why": why}


def split_numbers(data):
    """-> [(is_number, bytes, offset)] covering `data` end to end."""
    parts, pos = [], 0
    for m in NUMBER.finditer(data):
        if m.group("dotted") is not None:
            continue
        if m.start() > pos:
            parts.append((False, data[pos:m.start()], pos))
        parts.append((True, m.group(), m.start()))
        pos = m.end()
    if pos < len(data):
        parts.append((False, data[pos:], pos))
    return parts


def is_float(token):
    return any(c in token for c in b".eE")


def within_tolerance(a, b, tol):
    """Compare two masked outputs number by number. -> (True, stats) or (False, offset of the first real difference)."""
    if len(a) > MAX_TOLERANT_BYTES or len(b) > MAX_TOLERANT_BYTES:
        return False, first_diff(a, b) or 0
    ta, tb = split_numbers(a), split_numbers(b)
    if len(ta) != len(tb):
        return False, first_diff(a, b) or 0
    allow_abs, allow_rel = decimal.Decimal(repr(tol["abs"])), decimal.Decimal(repr(tol["rel"]))
    differing, worst = 0, 0.0
    for (na, xa, sa), (nb, xb, _) in zip(ta, tb):
        if xa == xb:
            continue
        if na != nb:
            return False, sa
        if not na:
            return False, sa + (first_diff(xa, xb) or 0)
        if not (is_float(xa) and is_float(xb)) or max(len(xa), len(xb)) > MAX_NUMBER_CHARS:
            return False, sa                      # an integer, or a very long number, must match exactly
        try:                                      # exact decimal arithmetic: no double-precision blind spot
            x, y = decimal.Decimal(xa.decode("ascii")), decimal.Decimal(xb.decode("ascii"))
            if not (x.is_finite() and y.is_finite()):
                return False, sa
            scale, diff = max(abs(x), abs(y)), abs(x - y)
            if diff > max(allow_abs, allow_rel * scale):
                return False, sa
            if scale:
                worst = max(worst, float(diff / scale))
        except (ArithmeticError, ValueError):
            return False, sa
        differing += 1
    return True, {"differing": differing, "maxRelativeDifference": worst}


def mask_spans(data, masks):
    """Merged (start, end) spans of `data` that the masks hide."""
    spans, text = [], None
    for kind, spec, _ in masks:
        if kind == "bytes":
            spans += [(s, min(e + 1, len(data))) for s, e in spec if s < len(data)]
        else:
            text = data.decode("latin-1") if text is None else text
            spans += [m.span() for m in spec.finditer(text) if m.end() > m.start()]
    merged = []
    for s, e in sorted(spans):
        if merged and s <= merged[-1][1]:
            merged[-1] = (merged[-1][0], max(e, merged[-1][1]))
        else:
            merged.append((s, e))
    return merged


def apply_masks(data, masks):
    """-> (masked bytes, cuts, hidden byte count); cuts map masked offsets back to file offsets."""
    out, pos, cuts, hidden, at = [], 0, [], 0, 0
    for s, e in mask_spans(data, masks):
        out.append(data[pos:s])
        at += s - pos
        cuts.append((at, s, e))
        out.append(MARK)
        at += len(MARK)
        hidden += e - s
        pos = e
    out.append(data[pos:])
    return b"".join(out), cuts, hidden


def original_offset(cuts, masked_offset):
    shift = 0
    for at, s, e in cuts:
        if masked_offset < at:
            break
        if masked_offset < at + len(MARK):
            return s
        shift += (e - s) - len(MARK)
    return masked_offset + shift


def first_diff(a, b):
    n, step, i = min(len(a), len(b)), 1 << 16, 0
    while i < n:
        j = min(i + step, n)
        if a[i:j] != b[i:j]:
            return next(k for k in range(i, j) if a[k] != b[k])
        i = j
    return n if len(a) != len(b) else None


def escape(chunk):
    out = []
    for byte in chunk:
        if byte == 0x5C:
            out.append("\\\\")
        elif 0x20 <= byte < 0x7F:
            out.append(chr(byte))
        else:
            out.append({10: "\\n", 13: "\\r", 9: "\\t"}.get(byte, "\\x%02x" % byte))
    return "".join(out).replace("\\x00<MASK>\\x00", "[masked]")


def context(masked, off):
    before = escape(masked[max(0, off - 12):off])[-12:]
    after = escape(masked[off:off + 28])[:28] if off < len(masked) else "<end of output>"
    return before + after, len(before)


def read_file(rel, base, allow_outside):
    """-> (bytes, None) or (None, plain-English problem)."""
    if not isinstance(rel, str) or not rel:
        return None, "no path was given"
    path = os.path.join(base, rel)
    try:
        real, root = os.path.realpath(path), os.path.realpath(base)
        if not allow_outside and os.path.commonpath([root, real]) != root:
            return None, "the path leaves the cases folder (pass --allow-outside if that is intended)"
        if not os.path.exists(path):
            return None, "the file does not exist"
        if os.path.getsize(path) > MAX_BYTES:
            return None, "the file is larger than %d MB" % (MAX_BYTES >> 20)
        with open(path, "rb") as fh:
            return fh.read(), None
    except (OSError, ValueError) as err:
        return None, "the file could not be read (%s)" % err.__class__.__name__


def rel_to(path, folder):
    try:
        return os.path.relpath(path, folder).replace(os.sep, "/")
    except ValueError:
        return path


def judge(case, index, base, out_dir, allow_outside, default_tolerance=None, approved_inputs=None):
    """-> (case record, legacy bytes or None, masks)."""
    cid = clip(str(case.get("id") or "C%02d" % (index + 1)), 80)
    masks = parse_masks(case.get("mask"), cid)
    tol = parse_tolerance(case["tolerance"] if "tolerance" in case else default_tolerance, cid)
    rec = {"id": cid, "title": clip(case.get("title"), 300), "verdict": "missing", "reason": "",
           "legacyPath": "", "newPath": "", "legacySha256": "", "newSha256": "",
           "masked": [m[2] for m in masks],
           "approvedDifference": clip(case.get("approvedDifference") or (approved_inputs or {}).get(str(case.get("input", "")), ""), 500),
           "note": clip(case.get("note"), 500)}
    data, problems = {}, []
    for side in ("legacy", "new"):
        given = case.get(side)
        rec[side + "Path"] = rel_to(os.path.join(base, given), out_dir) if isinstance(given, str) and given else ""
        data[side], why = read_file(given, base, allow_outside)
        if why:
            problems.append("The %s output was not compared: %s (%s)." % (side, why, clip(str(given), 200)))
    if problems:
        rec["reason"] = " ".join(problems)
        return rec, None, masks, tol
    legacy, new = data["legacy"], data["new"]
    rec["legacySha256"], rec["newSha256"] = hashlib.sha256(legacy).hexdigest(), hashlib.sha256(new).hexdigest()
    a, cuts, hid_a = apply_masks(legacy, masks)
    b, _, hid_b = apply_masks(new, masks)
    rec["bytes"], rec["maskedBytes"] = {"legacy": len(legacy), "new": len(new)}, {"legacy": hid_a, "new": hid_b}
    rec["empty"] = not legacy and not new
    at = first_diff(a, b)
    if at is None:
        rec["verdict"] = "same"
        rec["reason"] = ("Both outputs are empty, so nothing was compared." if rec["empty"] else
                         "Identical after masking: %d of %d bytes were hidden by %d mask(s)." % (hid_a, len(legacy), len(masks)) if masks else
                         "Identical, %d bytes." % len(legacy))
        return rec, legacy, masks, tol
    seen, seen_at = context(a, at)
    got, _ = context(b, at)
    line = a.count(b"\n", 0, at) + 1
    rec["firstDiff"] = {"offset": original_offset(cuts, at), "line": line, "legacy": seen, "new": got, "at": seen_at}
    where = "line %d, byte %d" % (line, rec["firstDiff"]["offset"])
    if at >= len(a) or at >= len(b):
        rec["reason"] = "The %s output ends at %s while the %s output continues." % (
            "legacy" if at >= len(a) else "new", where, "new" if at >= len(a) else "legacy")
    else:
        rec["reason"] = "The outputs first differ at %s." % where
    if tol:
        rec["tolerance"] = dict(tol)
        ok, info = within_tolerance(a, b, tol)
        if ok:
            rec["verdict"] = "same"
            rec["withinTolerance"] = info
            rec["reason"] = ("Identical apart from %d number(s) that differ within the declared tolerance "
                             "(relative %g, absolute %g; largest relative difference %.3g). Declared reason: %s"
                             % (info["differing"], tol["rel"], tol["abs"], info["maxRelativeDifference"], clip(tol["why"], 200)))
            return rec, legacy, masks, tol
        line = a.count(b"\n", 0, info) + 1
        rec["reason"] = "A number or text differs beyond the declared tolerance at line %d, byte %d." % (line, original_offset(cuts, info))
    if rec["approvedDifference"].strip():
        rec["verdict"] = "differs-approved"
        rec["reason"] += " A person approved this difference: %s" % clip(rec["approvedDifference"], 200)
    else:
        rec["verdict"] = "differs"
    return rec, legacy, masks, tol


def differs_under(changed, data, masks, tol):
    """Would the comparator report `changed` as different from `data`?"""
    a, b = apply_masks(data, masks)[0], apply_masks(changed, masks)[0]
    return a != b and not (tol and within_tolerance(a, b, tol)[0])


def self_check(items):
    """Prove the comparator can fail: change one unmasked byte of each legacy output; it must differ."""
    tested = 0
    for cid, data, masks, tol in items:
        if not data:
            continue
        tested += 1
        free, pos = [], 0
        for s, e in mask_spans(data, masks) + [(len(data), len(data))]:
            if s > pos:
                free.append((pos, s))
            pos = e
        if not free:
            return False, "case %s: the masks hide every byte, so no difference could ever be seen" % cid
        biggest = max(free, key=lambda f: f[1] - f[0])
        flip = lambda p: data[:p] + bytes([data[p] ^ 1]) + data[p + 1:]
        spots = [free[0][0], free[-1][1] - 1, (biggest[0] + biggest[1]) // 2]
        caught = any(differs_under(flip(p), data, masks, tol) for p in spots)
        if tol and caught:
            # A tolerance must never hide a change to the numbers it covers: changing the leading digit of every
            # sampled float that is not masked has to be reported, or the tolerance is too loose to prove anything.
            for m in list(NUMBER.finditer(data[:1 << 20]))[:200]:
                token = m.group("num")
                if token is None or not is_float(token) or not any(a <= m.start() < b for a, b in free):
                    continue
                at = m.start() + next(i for i, c in enumerate(token) if 48 <= c <= 57)
                if not differs_under(flip(at), data, masks, tol):
                    caught = False
                    break
        if not caught:
            return False, "case %s: a one-byte change to the legacy output was not reported as different%s" % (
                cid, " (the tolerance may be too loose)" if tol else "")
    if not tested:
        return None, "not applicable: no executed case had bytes to test"
    return True, "each legacy output was compared with a copy changed by one byte and reported as different"


def totals(records, executed, count):
    t = {"cases": len(records), "executed": executed, "same": count("same"), "differs": count("differs"),
         "differsApproved": count("differs-approved"), "missing": count("missing")}
    tolerant = sum(1 for r in records if r.get("withinTolerance"))
    if tolerant:  # additive: absent unless a declared tolerance was used
        t["sameWithinTolerance"] = tolerant
    return t


def run(cases_path, out_path=None, allow_outside=False):
    """-> the EQUIVALENCE record. Raises InputError for a cases file that cannot be used."""
    try:
        with open(cases_path, "rb") as fh:
            spec = json.loads(fh.read().decode("utf-8-sig"))
    except (OSError, ValueError) as err:
        raise InputError("cannot read %s: %s" % (cases_path, err))
    if not isinstance(spec, dict) or not isinstance(spec.get("cases"), list):
        raise InputError("%s must be a JSON object with a 'cases' list" % cases_path)
    base = os.path.dirname(os.path.abspath(cases_path))
    out_dir = os.path.dirname(os.path.abspath(out_path)) if out_path else base
    records, items, ids = [], [], set()
    approved = {str(k): v for k, v in spec["approvedInputs"].items() if isinstance(v, str) and v.strip()} if isinstance(spec.get("approvedInputs"), dict) else {}
    for i, case in enumerate(spec["cases"]):
        if not isinstance(case, dict):
            raise InputError("case %d is not an object" % (i + 1))
        rec, data, masks, tol = judge(case, i, base, out_dir, allow_outside, spec.get("tolerance"), approved)
        if rec["id"] in ids:
            raise InputError("duplicate case id %r" % rec["id"])
        ids.add(rec["id"])
        records.append(rec)
        if data is not None:
            items.append((rec["id"], data, masks, tol))
    passed, detail = self_check(items)
    count = lambda v: sum(1 for r in records if r["verdict"] == v)
    ends = lambda key: {k: clip(spec.get(key, {}).get(k), 300) for k in ("label", "command")} if isinstance(spec.get(key), dict) else {"label": "", "command": ""}
    return {"system": clip(spec.get("system"), 200), "generated": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
            "legacy": ends("legacy"), "new": ends("new"), "selfCheck": {"passed": passed, "detail": detail},
            "totals": totals(records, len(items), count), "cases": records}


def verdict_of(result):
    """-> None when proven, else the reason it is not."""
    t = result["totals"]
    if t["executed"] == 0:
        return "no case executed"
    if t["missing"] or t["differs"]:
        return " and ".join(filter(None, ["%d case(s) differ" % t["differs"] if t["differs"] else "",
                                         "%d case(s) are missing" % t["missing"] if t["missing"] else ""]))
    if result["selfCheck"]["passed"] is False:
        return "the self-check failed: " + result["selfCheck"]["detail"]
    if all(r.get("empty") for r in result["cases"] if r["verdict"] != "missing"):
        return "every compared output was empty"
    return None


def write_atomic(path, text):
    path = os.path.abspath(path)
    folder = os.path.dirname(path)
    for p in (path, folder):
        if os.path.islink(p):
            raise InputError("refusing to write through a symbolic link: %s" % p)
    os.makedirs(folder, exist_ok=True)
    fd, tmp = tempfile.mkstemp(dir=folder, prefix=".equivalence-", suffix=".tmp")
    try:
        with os.fdopen(fd, "wb") as fh:
            fh.write(text.encode("utf-8"))
        umask = os.umask(0)
        os.umask(umask)
        os.chmod(tmp, 0o666 & ~umask)
        os.replace(tmp, path)
    except BaseException:
        if os.path.lexists(tmp):
            os.unlink(tmp)
        raise


def main(argv=None):
    ap = argparse.ArgumentParser(description="Compare legacy and new outputs byte for byte.")
    ap.add_argument("cases", help="cases.json (see the module docstring for the schema)")
    ap.add_argument("--out", help="where to write EQUIVALENCE.json (default: next to the cases file)")
    ap.add_argument("--quiet", action="store_true", help="write the file, print nothing")
    ap.add_argument("--allow-outside", action="store_true", help="allow case paths that leave the cases folder")
    args = ap.parse_args(argv)
    out = args.out or os.path.join(os.path.dirname(os.path.abspath(args.cases)), "EQUIVALENCE.json")
    try:
        result = run(args.cases, out, args.allow_outside)
    except InputError as err:
        print("compare.py: %s" % err, file=sys.stderr)
        return 2
    try:
        write_atomic(out, json.dumps(result, indent=2, ensure_ascii=True) + "\n")
    except InputError as err:
        print("compare.py: %s" % err, file=sys.stderr)
        return 2
    problem = verdict_of(result)
    if not args.quiet:
        t, plain = result["totals"], lambda s: re.sub(r"[\x00-\x1f\x7f-\x9f]", "?", s)
        print("equivalence cases executed: %d" % t["executed"])
        print("same %d%s | differs %d | differs but approved %d | missing %d | of %d cases" %
              (t["same"], " (%d within a declared tolerance)" % t["sameWithinTolerance"] if t.get("sameWithinTolerance") else "",
               t["differs"], t["differsApproved"], t["missing"], t["cases"]))
        sc = result["selfCheck"]
        print("self-check: %s - %s" % ({True: "passed", False: "FAILED", None: "not applicable"}[sc["passed"]], plain(sc["detail"])))
        for r in result["cases"]:
            if r["verdict"] != "same" or r.get("withinTolerance"):
                print("  %s  %s: %s" % (plain(r["id"]), r["verdict"], plain(r["reason"])[:200]))
        print(("NOT PROVEN: %s" % problem) if problem else "PROVEN: %d case(s) executed, none differ" % t["executed"])
        print("wrote %s" % out)
    return 1 if problem else 0


if __name__ == "__main__":
    sys.exit(main())