diff options
Diffstat (limited to 'plugins/code-modernization/scripts/uplift_checks.py')
| -rw-r--r-- | plugins/code-modernization/scripts/uplift_checks.py | 356 |
1 files changed, 356 insertions, 0 deletions
diff --git a/plugins/code-modernization/scripts/uplift_checks.py b/plugins/code-modernization/scripts/uplift_checks.py new file mode 100644 index 00000000..33af819c --- /dev/null +++ b/plugins/code-modernization/scripts/uplift_checks.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +"""Two checks that only an uplift needs, for the proof pack (proof_pack.py imports this; it also runs on its own). + + python3 uplift_checks.py <legacy folder> <working copy> [DELTA_CATALOG.md] + +An uplift keeps the code and the tests and changes the version underneath. A green test run then proves less than it seems to in two ways: + + Tests changed during the uplift. The test files of the untouched legacy tree are compared with those of the working copy by walking + files only (no process is run, no link followed, build output skipped). A test file is a test file by the rules trace_rules.py uses. + Removed, added and changed files are listed with a line-diff count. A removed test file, more than 25% of the legacy test files + changed, or a legacy tree with no test files to compare against, is a gap: passing edited or deleted tests proves less, so a person reviews the list. Weakened assertions cannot be detected, + only that files changed. + + Deltas covered. DELTA_CATALOG.md lists the version changes that hit this code. For every delta whose category says Behavioral-silent + (the kind nothing reports when it changes behavior) the file of each listed site must be named, as a whole word, by some test file of + the working copy. A name is not proof that the test exercises the changed behavior; a silent delta no test even names is untested. + Other categories are listed, not judged. + +Everything read is untrusted text: files are only read, links are not followed, sizes and counts are capped, and no pattern here can be made +slow by a hostile line. Exit 0 when neither check finds a gap, 1 when one does, 2 for input that cannot be used. +""" +import argparse +import difflib +import hashlib +import json +import os +import re +import stat +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.dont_write_bytecode = True +sys.path.insert(0, HERE) +import baseline_diff # noqa: E402 +import trace_rules # noqa: E402 + +LIST_CAP, FILE_CAP, HASH_CAP, MAX_FILES, TEXT_CAP = 40, 2 << 20, 64 << 20, 60000, 400 << 20 +DIFF_LINES, DIFF_BUDGET, SHARE_LIMIT = 4000, 300000000, 0.25 +DELTA_CAP, CATALOG_CAP, LINE_CAP = 2000, 8 << 20, 4000 +SILENT = re.compile(r"behaviou?ral[\s_-]*silent|silent[\s_-]*behaviou?ral", re.I) +CONFIG_EXT = {"xml", "yml", "yaml", "json", "properties", "gradle", "toml", "ini", "cfg"} # build and configuration files: a test cannot name them +CODE_EXT = {"java", "kt", "kts", "scala", "groovy", "cs", "vb", "fs", "py", "js", "jsx", "ts", "tsx", "mjs", "cjs", "go", "rs", "rb", "php", "c", "h", "cc", "cpp", "hpp", "swift", + "m", "sql", "cbl", "cob", "cpy", "jcl", "pl", "pm", "sh", "ps1", "xml", "yml", "yaml", "json", "properties", "gradle", "toml", "ini", "cfg"} +# path/Stem.ext:line, or path/Stem.ext for a known extension; every quantifier is bounded, so a hostile line cannot make it slow +SITE = re.compile(r"((?:[\w.$@+-]{1,80}/){0,12})([A-Za-z_$][\w$-]{0,80})((?:\.[A-Za-z][A-Za-z0-9]{0,7}){1,3})(?::(\d{1,7}))?") +WORD = re.compile(r"[A-Za-z_$][\w$]{0,200}") +DASHED = re.compile(r"[A-Za-z0-9_$]{1,100}(?:-[A-Za-z0-9_$]{1,100}){1,8}") +clean = trace_rules.clean + + +# ---------------------------------------------------------------- walking test files +def walk_tests(root): + """({relative path: absolute path}, cut short, links skipped) for every regular test file under `root`. + A test file is one trace_rules.file_kind calls a test. Links, hidden folders and build output are never entered.""" + out, links = {}, 0 + if os.path.islink(root) or not os.path.isdir(root): + return out, False, 0 + for base, dirs, names in os.walk(root, followlinks=False): + keep = [] + for d in sorted(dirs): + if d in trace_rules.SKIP_DIRS or d.startswith("."): + continue + if os.path.islink(os.path.join(base, d)): + links += 1 + continue + keep.append(d) + dirs[:] = keep + for n in sorted(names): + full = os.path.join(base, n) + if n.startswith("."): + continue + try: + st = os.lstat(full) + except OSError: + continue + if stat.S_ISLNK(st.st_mode): + links += 1 + continue + rel = os.path.relpath(full, root).replace(os.sep, "/") + if stat.S_ISREG(st.st_mode) and trace_rules.file_kind(rel) == "test": + out[rel] = full + if len(out) >= MAX_FILES: + return out, True, links + return out, False, links + + +def read_head(path, cap=FILE_CAP): + """Up to cap+1 bytes of a file, or None.""" + try: + with open(path, "rb") as fh: + return fh.read(cap + 1) + except OSError: + return None + + +def digest(path): + """sha256 of a file's bytes (read in blocks, at most HASH_CAP), or None.""" + h, n = hashlib.sha256(), 0 + try: + with open(path, "rb") as fh: + while n < HASH_CAP: + block = fh.read(1 << 20) + if not block: + break + h.update(block) + n += len(block) + except OSError: + return None + return h.hexdigest() + + +def count_lines(path): + """Number of lines of a text file (0 for a binary or unreadable one).""" + raw = read_head(path) + return len(as_lines(raw)) if raw is not None and b"\x00" not in raw[:8192] else 0 + + +def as_lines(raw): + return raw.decode("utf-8", "replace").replace("\r\n", "\n").replace("\r", "\n").split("\n") + + +def line_diff(a, b, budget): + """(lines added, lines removed) turning list a into list b. difflib for files of ordinary size while the budget lasts, else a count of + lines that appear more often on one side (order is ignored, which is the price of being fast). -> (added, removed, exact, budget left).""" + if len(a) <= DIFF_LINES and len(b) <= DIFF_LINES and len(a) * len(b) <= budget: + added = removed = 0 + for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(None, a, b, autojunk=False).get_opcodes(): + if tag in ("replace", "delete"): + removed += i2 - i1 + if tag in ("replace", "insert"): + added += j2 - j1 + return added, removed, True, budget - len(a) * len(b) + ca, cb = {}, {} + for line in a: + ca[line] = ca.get(line, 0) + 1 + for line in b: + cb[line] = cb.get(line, 0) + 1 + return (sum(max(0, n - ca.get(k, 0)) for k, n in cb.items()), sum(max(0, n - cb.get(k, 0)) for k, n in ca.items()), False, budget) + + +def compare_files(old, new, budget): + """None when the two files hold the same content (line endings aside), else ({"added", "removed", "note"}, budget left).""" + ra, rb = read_head(old), read_head(new) + if ra is None or rb is None: + return {"added": 0, "removed": 0, "note": "could not be read"}, budget + if len(ra) > FILE_CAP or len(rb) > FILE_CAP: + da, db = digest(old), digest(new) + return (None, budget) if da is not None and da == db else ({"added": 0, "removed": 0, "note": "large file, lines not counted"}, budget) + if b"\x00" in ra[:8192] or b"\x00" in rb[:8192]: + return (None, budget) if ra == rb else ({"added": 0, "removed": 0, "note": "binary file"}, budget) + la, lb = as_lines(ra), as_lines(rb) + if la == lb: + return None, budget + added, removed, exact, budget = line_diff(la, lb, budget) + return {"added": added, "removed": removed, "note": "" if exact else "lines counted approximately"}, budget + + +# ---------------------------------------------------------------- tests changed during the uplift +def tests_changed(legacy_root, work_root): + """Compare the test files of the untouched legacy tree with those of the working copy. + -> {"checked", "why", "legacyTests", "workTests", "counts": {added, removed, changed}, "added", "removed", "changed" (lists cut to 40), "share", "gap", "cut"}.""" + out = {"checked": False, "why": "", "legacyTests": 0, "workTests": 0, "counts": {"added": 0, "removed": 0, "changed": 0}, "added": [], "removed": [], "changed": [], + "share": 0.0, "gap": False, "cut": False} + if not legacy_root or not os.path.isdir(legacy_root): + out["why"] = "the legacy tree was not found, so its test files could not be compared with the working copy's" + return out + if not work_root or not os.path.isdir(work_root): + out["why"] = "the working copy was not found" + return out + old, cut_a, _ = walk_tests(legacy_root) + new, cut_b, _ = walk_tests(work_root) + out.update({"checked": True, "legacyTests": len(old), "workTests": len(new), "cut": cut_a or cut_b}) + removed = sorted(set(old) - set(new)) + added = sorted(set(new) - set(old)) + changed, budget = [], DIFF_BUDGET + for rel in sorted(set(old) & set(new)): + got, budget = compare_files(old[rel], new[rel], budget) + if got is not None: + changed.append(dict(got, path=clean(rel, 300))) + out["counts"] = {"added": len(added), "removed": len(removed), "changed": len(changed)} + changed.sort(key=lambda c: (-(c["added"] + c["removed"]), c["path"])) + out["removed"] = [{"path": clean(r, 300), "lines": count_lines(old[r])} for r in removed[:LIST_CAP]] + out["added"] = [{"path": clean(r, 300), "lines": count_lines(new[r])} for r in added[:LIST_CAP]] + out["changed"] = changed[:LIST_CAP] + out["share"] = round(len(changed) / len(old), 4) if old else 0.0 + out["gap"] = bool(removed) or not old or len(changed) > SHARE_LIMIT * len(old) + if not old: + out["why"] = "the legacy tree has no test files, so nothing shows that the working copy's tests were not written during the uplift" + return out + + +# ---------------------------------------------------------------- reading DELTA_CATALOG.md +def norm_id(text): + return re.sub(r"[\s*`#.]+", "", text or "").lower()[:40] + + +def sites_in(text): + """[{"path", "stem", "line"}] for every file mention in `text` that is a file:line, or a file with a known extension.""" + out = [] + for m in SITE.finditer(text[:LINE_CAP]): + ext = m.group(3).lstrip(".").split(".")[-1].lower() + stem = m.group(2) + if len(stem) < 2 or (m.group(4) is None and ext not in CODE_EXT): + continue + site = {"path": clean((m.group(1) + stem + m.group(3)), 200), "stem": stem, "line": int(m.group(4)) if m.group(4) else 0, "ext": ext} + if not any(o["path"] == site["path"] for o in out): + out.append(site) + return out[:20] + + +def add_delta(found, did, title, category, sites): + key = norm_id(did) + if not key or (len(found) >= DELTA_CAP and key not in found): + return + d = found.setdefault(key, {"id": clean(did.replace("*", "").strip(), 40), "title": "", "category": "", "sites": []}) + d["title"] = d["title"] or clean(title, 200) + d["category"] = d["category"] or clean(category, 80) + d["sites"] += [s for s in sites if not any(o["path"] == s["path"] for o in d["sites"])] + d["sites"] = d["sites"][:20] + + +def parse_deltas(text): + """Deltas from DELTA_CATALOG.md, tolerantly: pipe tables with a Category column (an id in the first or an ID/# column, sites in a + Site column), and cards (`### 9. title` or `### D-01 title`, with a `Category ...:` line and a `cited:` or `Site:` line). + Rows and cards of the same id are merged. -> [{"id", "title", "category", "sites": [{"path", "stem", "line"}]}].""" + found = {} + for _heading, header, rows in baseline_diff.md_tables(text[:CATALOG_CAP]): + low = [h.lower() for h in header] + cat = next((i for i, h in enumerate(low) if re.search(r"\bcat", h)), None) + if cat is None: + continue + idc = next((i for i, h in enumerate(low) if re.fullmatch(r"(?:id|#|no\.?|delta|card|ref)", h.strip())), 0) + titlec = next((i for i, h in enumerate(low) if re.search(r"delta|title|name|change|summary|\bold\b", h) and i != idc), None) + sitec = [i for i, h in enumerate(low) if re.search(r"\bsites?\b|location|cited|file", h)] + for row in rows[:5000]: + if cat >= len(row) or idc >= len(row) or not row[idc].strip(): + continue + cells = [row[i] for i in sitec if i < len(row) and not re.fullmatch(r"\s*\d+\s*", row[i])] + sites = sites_in(" ".join(cells)) if sitec else sites_in(" ".join(c for i, c in enumerate(row) if i not in (cat, idc))) + add_delta(found, row[idc], row[titlec] if titlec is not None and titlec < len(row) else "", row[cat], sites) + lines, i = [ln[:LINE_CAP] for ln in text[:CATALOG_CAP].split("\n")], 0 + while i < len(lines): + m = re.match(r"#{2,6}[ \t]+(?:\*\*)?(D-\d{1,4}|\d{1,4})(?:\*\*)?[.):\s][ \t]*(.*)$", lines[i]) + i += 1 + if not m: + continue + category, sites = "", [] + while i < len(lines) and not lines[i].startswith("#"): + line = lines[i] + c = re.search(r"categor[^:\n]{0,40}:\**[ \t]*(.{1,200})", line, re.I) + if c and not category: + category = re.split(r"[/;]", c.group(1))[0].strip(" *`") + cited = re.search(r"\bcited\s*:", line, re.I) or re.match(r"^[ \t>*-]*\**(?:sites?|location|source)\b[^:\n]{0,30}:", line, re.I) + if cited: # only the site the card cites, never files that its prose merely mentions + sites += [s for s in sites_in(line[cited.end():]) if not any(o["path"] == s["path"] for o in sites)] + i += 1 + if category or norm_id(m.group(1)) in found: # a numbered heading with no Category line is not a delta card + add_delta(found, m.group(1), m.group(2), category, sites) + return list(found.values()) + + +# ---------------------------------------------------------------- deltas covered by tests +def test_words(work_root): + """(the set of words that appear in the working copy's test files, number of test files read). Files are read as text, up to 2 MB each.""" + words, total, files = set(), 0, 0 + tests, _cut, _links = walk_tests(work_root) + for rel in sorted(tests): + raw = read_head(tests[rel]) + if raw is None or b"\x00" in raw[:8192]: + continue + total += len(raw) + if total > TEXT_CAP: + break + text = raw[:FILE_CAP].decode("utf-8", "replace") + words.update(WORD.findall(text)) + words.update(DASHED.findall(text)) + files += 1 + return words, files + + +def delta_coverage(catalog_text, work_root): + """Which Behavioral-silent deltas have their site's file named by a test in the working copy? A delta whose only sites are build or + configuration files cannot be named by a test: it is listed for a person, not judged. + -> {"parsed", "silent", "covered": [...], "uncovered": [...], "config": [...], "other": n, "testFiles", "gap", "why"}.""" + out = {"parsed": 0, "silent": 0, "covered": [], "uncovered": [], "config": [], "other": 0, "testFiles": 0, "gap": False, "why": ""} + if catalog_text is None: + out.update(gap=True, why="DELTA_CATALOG.md was not found, so it is not known which version changes hit this code") + return out + deltas = parse_deltas(catalog_text) + out["parsed"] = len(deltas) + if not deltas: + out.update(gap=True, why="no delta could be read from DELTA_CATALOG.md (a table with a Category column, or cards with a Category line)") + return out + words, out["testFiles"] = test_words(work_root) + for d in deltas: + if not SILENT.search(d["category"]): + out["other"] += 1 + continue + out["silent"] += 1 + code = [s for s in d["sites"] if s["ext"] not in CONFIG_EXT] + rec = {"id": d["id"], "title": d["title"], "sites": [s["path"] + (":%d" % s["line"] if s["line"] else "") for s in d["sites"][:5]]} + if d["sites"] and not code: + out["config"].append(rec) + continue + missing = [s["stem"] for s in code if s["stem"] not in words] + if code and not missing: + out["covered"].append(rec) + else: + rec["why"] = "no site is named in the catalog" if not code else "no test names " + ", ".join(clean(x, 60) for x in list(dict.fromkeys(missing))[:5]) + out["uncovered"].append(rec) + out["gap"] = bool(out["uncovered"]) + return out + + +# ---------------------------------------------------------------- command line +def main(argv=None): + ap = argparse.ArgumentParser(description="Uplift-only checks: test files changed since the legacy tree, and silent deltas no test names.") + ap.add_argument("legacy", help="the untouched legacy folder (legacy/<system>)") + ap.add_argument("work", help="the uplifted working copy (modernized/<system>-uplifted)") + ap.add_argument("catalog", nargs="?", help="analysis/<system>/DELTA_CATALOG.md") + ap.add_argument("--json", action="store_true", help="print the full result as JSON") + args = ap.parse_args(argv) + for path in (args.legacy, args.work): + if not os.path.isdir(path): + print("uplift_checks.py: %s is not a folder" % path, file=sys.stderr) + return 2 + text = None + if args.catalog: + try: + text = baseline_diff.read_capped(args.catalog, CATALOG_CAP).decode("utf-8", "replace") + except baseline_diff.InputError as err: + print("uplift_checks.py: %s" % err, file=sys.stderr) + return 2 + changed = tests_changed(os.path.realpath(args.legacy), os.path.realpath(args.work)) + deltas = delta_coverage(text, os.path.realpath(args.work)) if args.catalog else None + if args.json: + print(json.dumps({"testsChanged": changed, "deltas": deltas}, indent=1, ensure_ascii=True)) + else: + c = changed["counts"] + print("test files: %d in the legacy tree, %d in the working copy; %d removed, %d added, %d changed (%.0f%% of the legacy test files)" % ( + changed["legacyTests"], changed["workTests"], c["removed"], c["added"], c["changed"], 100 * changed["share"])) + for key in ("removed", "changed", "added"): + for e in changed[key][:10]: + print(" %s: %s%s" % (key, e["path"], " (+%d -%d lines)" % (e["added"], e["removed"]) if key == "changed" else "")) + print("tests kept: " + ("GAP" if changed["gap"] else "ok") + ("" if not changed["why"] else " (" + changed["why"] + ")")) + if deltas is not None: + print("deltas: %d parsed, %d behavioral-silent, %d covered, %d not named by any test, %d at a build or configuration file" % ( + deltas["parsed"], deltas["silent"], len(deltas["covered"]), len(deltas["uncovered"]), len(deltas["config"]))) + for d in deltas["uncovered"][:10]: + print(" uncovered: %s %s (%s)" % (d["id"], d["title"], d.get("why", ""))) + print("deltas covered: " + ("GAP" if deltas["gap"] else "ok") + ("" if not deltas["why"] else " (" + deltas["why"] + ")")) + return 1 if changed["gap"] or (deltas is not None and deltas["gap"]) else 0 + + +if __name__ == "__main__": + sys.exit(main()) |
