summaryrefslogtreecommitdiff
path: root/plugins/code-modernization/scripts/baseline_diff.py
diff options
context:
space:
mode:
Diffstat (limited to 'plugins/code-modernization/scripts/baseline_diff.py')
-rw-r--r--plugins/code-modernization/scripts/baseline_diff.py809
1 files changed, 809 insertions, 0 deletions
diff --git a/plugins/code-modernization/scripts/baseline_diff.py b/plugins/code-modernization/scripts/baseline_diff.py
new file mode 100644
index 00000000..93686737
--- /dev/null
+++ b/plugins/code-modernization/scripts/baseline_diff.py
@@ -0,0 +1,809 @@
+#!/usr/bin/env python3
+"""Compare a fresh test run with the baseline recorded before a same-stack uplift.
+
+ python3 baseline_diff.py <BASELINE.md> --junit DIR_OR_FILE [--junit ...] [--json] [--max-list N]
+
+BASELINE.md (analysis/<system>/BASELINE.md) records how the tests behaved on the OLD version. This
+script reads it and the fresh results, and reports what changed. Nothing is judged by a model.
+
+Baseline, any of these (the most detailed one found is used):
+ a per-test table | Test | Result | one row per test, PASS / FAIL / ERROR / SKIP
+ a per-module table | Module | Pass | Fail | Error | Skip | (the first column may be a module,
+ project or test class; column names are matched, their order does not matter)
+ a totals table | Executed | 946 |, | Passed | 945 |, | Failed | 1 | ...
+ a line target-only: <why the old version could not run here> (nothing to compare)
+
+Measured or typed: a baseline typed by hand proves nothing about the old version. The old version's own evidence is looked for in
+analysis/<system>/baseline/ and at the paths on a "Recorded:" or "Machine-readable:" line of BASELINE.md inside analysis/<system>/ (JUnit or .trx XML, a per-test JSON map of test ids to
+outcomes, or a raw runner log with a known summary line; a file that only holds counts is ignored, and sources that disagree are a conflict). When it exists, it is what the fresh results are compared with,
+and the typed table must agree with it; when it does not, the report says "typed table only".
+
+Fresh results: JUnit-style XML (Maven surefire, Gradle, pytest --junitxml, Ant, jest-junit ...) or
+Visual Studio .trx files. Give a file or a folder; a folder is searched for result files, links are
+never followed, and a file that declares a DTD or entity is refused. A test id is
+"<classname>#<name>"; a repeated id in one file gets a "~N" suffix in document order (the first
+stays plain); the module's own absolute path in a parameterized name is replaced by <MODULE>.
+
+Reported: regressions (passed before, fails now), new failures (a test the baseline never listed),
+tests that still fail as they did, tests that now pass, tests that were skipped and now run, tests
+that no longer run, missing modules, skipped growth and a drop in executed tests. When test names
+drifted inside a class but its counts did not get worse, that is counted as renamed, not missing.
+
+Exit 0 when nothing regressed and something ran; 1 on any regression or new failure, or when no test
+executed; 2 for input that cannot be used. Standard library only.
+"""
+import argparse
+import json
+import os
+import re
+import stat
+import sys
+from xml.parsers import expat
+
+BAD = ("FAIL", "ERROR")
+MAX_FILE, MAX_FILES, MAX_CASES, MAX_DEPTH, KEEP = 200 << 20, 20000, 2000000, 60, 200
+SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv"}
+STATUS = {"pass": "PASS", "passed": "PASS", "ok": "PASS", "success": "PASS", "green": "PASS", "fail": "FAIL", "failed": "FAIL", "failure": "FAIL",
+ "red": "FAIL", "error": "ERROR", "errors": "ERROR", "errored": "ERROR", "skip": "SKIP", "skipped": "SKIP", "ignored": "SKIP",
+ "pending": "SKIP", "disabled": "SKIP"}
+COLUMNS = (("pass", r"pass|passed|passing|ok|successes?"), ("fail", r"fail|failed|failing|failures?"), ("error", r"errors?|errored"),
+ ("skip", r"skip|skipped|ignored|pending|disabled"))
+TOTALS = {"executed": "executed", "test cases": "cases", "passed": "pass", "pass": "pass", "failed": "fail", "failures": "fail", "errors": "error",
+ "skipped": "skip", "ignored": "skip"}
+
+
+def clean(value, limit=200):
+ text = " ".join(re.sub(r"[\x00-\x1f\x7f-\x9f\u2028\u2029]", " ", str(value if value is not None else "")).split())
+ return text if len(text) <= limit else text[:limit - 1] + "…"
+
+
+class InputError(Exception):
+ pass
+
+
+def read_capped(path, cap):
+ """Bytes of a regular file inside the size cap; raises InputError with a plain reason for a link, a huge file or an unreadable one."""
+ try:
+ st = os.lstat(path)
+ if not stat.S_ISREG(st.st_mode):
+ raise InputError("%s is a link or not a regular file" % clean(os.path.basename(path), 80))
+ if st.st_size > cap:
+ raise InputError("%s is larger than %d MB" % (clean(os.path.basename(path), 80), cap >> 20))
+ with open(path, "rb") as fh:
+ return fh.read()
+ except OSError as err:
+ raise InputError("cannot read %s (%s)" % (clean(os.path.basename(path), 80), err.__class__.__name__))
+
+
+# ---------------------------------------------------------------- the baseline
+def cell(text):
+ text = text.replace("\\|", "|").strip()
+ if len(text) > 4 and text.startswith("**") and text.endswith("**"):
+ text = text[2:-2].strip()
+ if len(text) >= 2 and text.startswith("`") and text.endswith("`"):
+ text = text[1:-1]
+ return text
+
+
+def split_row(line):
+ s = line.strip()
+ s = s[1:] if s.startswith("|") else s
+ s = s[:-1] if s.endswith("|") and not s.endswith("\\|") else s
+ return [cell(c) for c in re.split(r"(?<!\\)\|", s[:20000])]
+
+
+def join_cut_rows(lines):
+ """A test name that contains a newline cuts its table row in two; put the row back together (at most 3 extra lines)."""
+ out, i = [], 0
+ while i < len(lines):
+ line, extra = lines[i], 0
+ while (line.lstrip().startswith("|") and not line.rstrip().endswith("|") and extra < 3 and i + 1 < len(lines)
+ and lines[i + 1].strip() and not lines[i + 1].lstrip().startswith(("|", "#"))):
+ i += 1
+ extra += 1
+ line += "\n" + lines[i]
+ out.append(line)
+ i += 1
+ return out
+
+
+def md_tables(text):
+ """(nearest heading, header cells, [row cells]) for every pipe table."""
+ lines, heading = join_cut_rows(text.split("\n")), ""
+ i = 0
+ while i < len(lines):
+ h = re.match(r"#{1,6}[ \t]+(.*)", lines[i][:2000])
+ heading = h.group(1) if h else heading
+ if "|" in lines[i] and i + 1 < len(lines) and re.fullmatch(r"\s*\|?\s*:?-+:?\s*(\|\s*:?-+:?\s*)*\|?\s*", lines[i + 1][:5000]):
+ rows, j = [], i + 2
+ while j < len(lines) and "|" in lines[j] and lines[j].strip():
+ rows.append(split_row(lines[j]))
+ j += 1
+ yield heading, split_row(lines[i]), rows
+ i = j
+ else:
+ i += 1
+
+
+def to_int(text):
+ t = text.strip().replace(",", "")
+ if t in ("", "-", "\u2013", "\u2014"):
+ return 0
+ return int(t) if re.fullmatch(r"\d{1,12}", t) else None
+
+
+def parse_baseline(text):
+ """-> {targetOnly, why, modules {name: counts}, tests {id: status}, totals {name: n}}."""
+ text = text.replace("\r\n", "\n").replace("\r", "\n")
+ m = re.search(r"(?im)^\W*target-only:[ \t]*(.*)$", text)
+ out = {"targetOnly": bool(m), "why": clean(m.group(1), 300) if m else "", "modules": {}, "tests": {}, "totals": {}, "flaky": [], "approved": {}}
+ for heading, header, rows in md_tables(text):
+ low = [h.lower() for h in header]
+ if re.search(r"(?i)flak", heading): # tests the baseline itself says flip on the same version
+ out["flaky"] += [clean(r[0], 400) for r in rows if r and r[0]]
+ continue
+ if re.search(r"(?i)approved", heading): # differences a person accepted: | Test | Reason |
+ for r in rows:
+ if r and r[0]:
+ out["approved"].setdefault(clean(r[0], 400), clean(r[1] if len(r) > 1 else "", 300))
+ continue
+ cols = {key: next((i for i, h in enumerate(low) if re.fullmatch(rx, h)), None) for key, rx in COLUMNS}
+ result = next((i for i, h in enumerate(low) if re.fullmatch(r"result|status|outcome|verdict", h)), None)
+ if cols["pass"] is not None and any(cols[k] is not None for k in ("fail", "error", "skip")):
+ for row in rows:
+ name, nums = row[0] if row else "", {k: to_int(row[i]) if i is not None and i < len(row) else 0 for k, i in cols.items()}
+ if name and not re.fullmatch(r"(?i)totals?|all|sum", name) and None not in nums.values():
+ out["modules"][name[:300]] = nums
+ elif result is not None:
+ name_col = next((i for i, h in enumerate(low) if re.fullmatch(r"tests?|test name|test id|id|name|cases?|test case", h)), 0)
+ for row in rows:
+ status = STATUS.get(row[result].lower()) if result < len(row) else None
+ if status and name_col < len(row) and row[name_col] and result != name_col:
+ out["tests"][row[name_col][:400]] = status
+ elif len(header) >= 2:
+ for row in rows:
+ label, n = (row[0].lower() if row else ""), to_int(row[1]) if len(row) > 1 else None
+ if label in TOTALS and n is not None:
+ out["totals"][TOTALS[label]] = n
+ if not (out["modules"] or out["tests"] or out["totals"]):
+ counts = {}
+ for word, key in (("pass(?:ed|ing)", "pass"), ("fail(?:ed|ing|ures?)", "fail"), ("skipped", "skip"), ("errors?", "error")):
+ hit = re.search(r"(?i)(\d{1,9})\s+(?:tests?\s+|cases?\s+)?" + word, text)
+ if hit:
+ counts[key] = int(hit.group(1))
+ out["totals"] = counts
+ return out
+
+
+def tally(statuses):
+ c = {"pass": 0, "fail": 0, "error": 0, "skip": 0}
+ for s in statuses:
+ c[s.lower()] += 1
+ c["executed"] = c["pass"] + c["fail"] + c["error"]
+ return c
+
+
+def baseline_counts(base):
+ """(counts, where they came from)."""
+ if base["tests"]:
+ return tally(base["tests"].values()), "per-test table"
+ if base["modules"]:
+ c = {k: sum(v[k] for v in base["modules"].values()) for k in ("pass", "fail", "error", "skip")}
+ c["executed"] = c["pass"] + c["fail"] + c["error"]
+ return c, "per-module table"
+ t = base["totals"]
+ if t:
+ c = {k: t.get(k, 0) for k in ("pass", "fail", "error", "skip")}
+ c["executed"] = t["executed"] if "executed" in t else c["pass"] + c["fail"] + c["error"]
+ return c, "totals"
+ return None, "nothing"
+
+
+# ---------------------------------------------------------------- the fresh results
+def refuse(*_):
+ raise ValueError("it declares a DTD or an entity, which test results never do")
+
+
+def parse_results(path):
+ """[(classname, name, status, skip message given, suite name)] from one JUnit or TRX file. Raises ValueError."""
+ cases, suites, cur, depth = [], [], [], [0]
+ trx = {"Passed": "PASS", "Failed": "FAIL", "Error": "ERROR", "Timeout": "ERROR", "Aborted": "ERROR"}
+
+ def start(tag, attrs):
+ depth[0] += 1
+ if depth[0] > MAX_DEPTH:
+ raise ValueError("the XML is nested more than %d levels deep" % MAX_DEPTH)
+ if tag == "testsuite":
+ suites.append(attrs.get("name", "")[:300])
+ elif tag == "testcase":
+ if len(cases) >= MAX_CASES:
+ raise ValueError("more than %d test cases" % MAX_CASES)
+ flag = (attrs.get("status") or attrs.get("result") or "").lower()
+ cur[:] = [(attrs.get("classname") or (suites[-1] if suites else ""))[:300], attrs.get("name", "")[:400],
+ "SKIP" if flag in ("skipped", "ignored", "notrun", "notexecuted", "disabled") else "PASS", False]
+ elif cur and tag in ("failure", "error", "skipped"):
+ if tag == "skipped":
+ cur[3] = bool(attrs.get("message", "").strip())
+ new = {"failure": "FAIL", "error": "ERROR", "skipped": "SKIP"}[tag]
+ if cur[2] == "PASS" or (cur[2] == "SKIP" and new != "SKIP"):
+ cur[2] = new
+ elif tag == "UnitTestResult":
+ name = attrs.get("testName", "")[:400]
+ head, sep, tail = name.partition("(")
+ klass, dot, method = head.rpartition(".")
+ outcome = attrs.get("outcome", "")
+ cases.append((klass[:300] if dot else "", (method + sep + tail) if dot else name, trx.get(outcome, "SKIP"), False, ""))
+
+ def end(tag):
+ depth[0] -= 1
+ if tag == "testcase" and cur:
+ cases.append((cur[0], cur[1], cur[2], cur[3], suites[-1] if suites else ""))
+ cur.clear()
+ elif tag == "testsuite" and suites:
+ suites.pop()
+
+ parser = expat.ParserCreate()
+ parser.StartDoctypeDeclHandler = refuse
+ parser.EntityDeclHandler = refuse
+ parser.StartElementHandler = start
+ parser.EndElementHandler = end
+ try:
+ with open(path, "rb") as fh:
+ parser.ParseFile(fh)
+ except expat.ExpatError as err:
+ raise ValueError("the XML is not well formed (%s)" % expat.ErrorString(err.code))
+ return cases
+
+
+def result_files(paths):
+ """([files], [problems]) from files and folders. Links are never followed."""
+ found, problems = [], []
+ for p in paths:
+ if os.path.isfile(p) and not os.path.islink(p):
+ found.append(p)
+ elif os.path.isdir(p) and not os.path.islink(p):
+ for base, dirs, names in os.walk(p):
+ dirs[:] = [d for d in sorted(dirs) if d not in SKIP_DIRS and not os.path.islink(os.path.join(base, d))]
+ for n in sorted(names):
+ full = os.path.join(base, n)
+ if os.path.islink(full) or not n.lower().endswith((".xml", ".trx")):
+ continue
+ try:
+ with open(full, "rb") as fh:
+ head = fh.read(4096)
+ except OSError:
+ continue
+ if b"<testsuite" in head or b"<TestRun" in head:
+ found.append(full)
+ if len(found) > MAX_FILES:
+ problems.append("More than %d result files were found; the rest were not read." % MAX_FILES)
+ return found[:MAX_FILES], problems
+ else:
+ problems.append("%s is not a file or folder that can be read." % clean(p, 120))
+ return found, problems
+
+
+def module_label(path):
+ """The module folder a Maven or Gradle report belongs to (its parent of target/ or build/), or ''."""
+ parts = os.path.abspath(path).split(os.sep)
+ for i in range(len(parts) - 2, 1, -1):
+ if parts[i] in ("target", "build") and parts[i + 1] in ("surefire-reports", "failsafe-reports", "test-results"):
+ return os.sep.join(parts[:i]) or os.sep
+ return ""
+
+
+def read_results(paths):
+ """-> {tests {id: status}, skipReasons {id: bool}, classes, modules, suites, files, unreadable [str], problems [str], oldest, newest}."""
+ files, problems = result_files(paths)
+ out = {"tests": {}, "skipReasons": {}, "classes": {}, "modules": {}, "suites": {}, "files": 0, "unreadable": [], "problems": problems, "oldest": None, "newest": None}
+ for path in files:
+ label = clean(os.path.basename(path), 100)
+ try:
+ if os.path.getsize(path) > MAX_FILE:
+ raise ValueError("the file is larger than %d MB" % (MAX_FILE >> 20))
+ cases = parse_results(path)
+ except (OSError, ValueError) as err:
+ out["unreadable"].append("%s: %s" % (label, clean(err, 120)))
+ continue
+ root = module_label(path)
+ roots = sorted({root, os.path.realpath(root), root[len("/private"):] if root.startswith("/private/") else root} - {os.sep}, key=len, reverse=True) if root else []
+ seen, mtime = {}, os.path.getmtime(path)
+ out["files"] += 1
+ out["oldest"] = mtime if out["oldest"] is None else min(out["oldest"], mtime)
+ out["newest"] = mtime if out["newest"] is None else max(out["newest"], mtime)
+ for klass, name, status, has_reason, suite in cases:
+ for r in roots:
+ name = name.replace(r, "<MODULE>")
+ name = re.sub(r"(test)\d+(\.tmp)", r"\1<N>\2", name)
+ base = "%s#%s" % (klass, name) if klass else name
+ n = seen.get(base, 0)
+ seen[base] = n + 1
+ tid = base if n == 0 else "%s~%d" % (base, n)
+ out["tests"][tid] = status
+ out["skipReasons"][tid] = has_reason
+ for key, store in ((klass, out["classes"]), (root and os.path.basename(root), out["modules"]), (suite, out["suites"])):
+ if key:
+ store.setdefault(key, []).append(status)
+ for store in ("classes", "modules", "suites"):
+ out[store] = {k: tally(v) for k, v in out[store].items()}
+ return out
+
+
+# ---------------------------------------------------------------- raw runner logs (a saved run's own summary lines)
+LOG_CAP, LOG_LINE = 16 << 20, 2000
+NUM_WORD = re.compile(r"(\d+)\s+([A-Za-z]+)")
+
+
+def words(text):
+ """{'passed': 10, 'failed': 2} from '2 failed, 10 passed' style text."""
+ out = {}
+ for n, w in NUM_WORD.findall(text[:LOG_LINE]):
+ out[w.lower()] = out.get(w.lower(), 0) + int(n)
+ return out
+
+
+def _maven(m):
+ run, f, e, sk = (int(g) for g in m.groups())
+ return max(0, run - f - e - sk), f + e, sk
+
+
+def _pytest(m):
+ w = words(m.group(1))
+ return w.get("passed", 0) + w.get("xpassed", 0) + w.get("xfailed", 0), w.get("failed", 0) + w.get("error", 0) + w.get("errors", 0), w.get("skipped", 0)
+
+
+def _jest(m):
+ w = words(m.group(1))
+ return w.get("passed", 0), w.get("failed", 0), w.get("skipped", 0)
+
+
+def _php(m):
+ w = {k.lower(): int(v) for k, v in re.findall(r"([A-Za-z]+):\s*(\d+)", m.group(1)[:LOG_LINE])}
+ failed, skipped = w.get("failures", 0) + w.get("errors", 0), w.get("skipped", 0) + w.get("incomplete", 0)
+ return max(0, w.get("tests", 0) - failed - skipped), failed, skipped
+
+
+def _ctest(m):
+ failed, total = int(m.group(1)), int(m.group(2))
+ return max(0, total - failed), failed, 0
+
+
+def _gradle(m):
+ total, failed, skipped = int(m.group(1)), int(m.group(2) or 0), int(m.group(3) or 0)
+ return max(0, total - failed - skipped), failed, skipped
+
+
+# runner, a line pattern, and how to read (passed, failed, skipped) from a match. Every pattern is anchored and bounded.
+LOG_LINES = (
+ ("maven", re.compile(r"^(?:\[\w+\]\s*)?Tests run: (\d+), Failures: (\d+), Errors: (\d+), Skipped: (\d+)\s*$"), _maven),
+ ("gradle", re.compile(r"^(\d+) tests? completed(?:, (\d+) failed)?(?:, (\d+) skipped)?\s*$"), _gradle),
+ ("cargo", re.compile(r"^test result: (?:ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored;"), lambda m: (int(m.group(1)), int(m.group(2)), int(m.group(3)))),
+ ("pytest", re.compile(r"^(?:=+ )?((?:\d+ [a-z]+(?:, )?)+) in [\d.]+s(?: \([\d:]+\))?(?: =+)?\s*$"), _pytest),
+ ("pytest", re.compile(r"^(?:=+ )?no tests ran in [\d.]+s"), lambda m: (0, 0, 0)),
+ ("go test", re.compile(r"^\s*--- (PASS|FAIL|SKIP): \S"), lambda m: {"PASS": (1, 0, 0), "FAIL": (0, 1, 0), "SKIP": (0, 0, 1)}[m.group(1)]),
+ ("go test -json", re.compile(r'^\{.*"Action":"(pass|fail|skip)".*"Test":"[^"]+"'), lambda m: {"pass": (1, 0, 0), "fail": (0, 1, 0), "skip": (0, 0, 1)}[m.group(1)]),
+ ("dotnet test", re.compile(r"^(?:Passed|Failed)!\s+-\s+Failed:\s+(\d+),\s+Passed:\s+(\d+),\s+Skipped:\s+(\d+)"), lambda m: (int(m.group(2)), int(m.group(1)), int(m.group(3)))),
+ ("jest", re.compile(r"^\s*Tests:\s+(.*\b\d+ total)\s*$"), _jest),
+ ("vitest", re.compile(r"^\s*Tests\s{2,}(.*)\(\d+\)\s*$"), _jest),
+ ("ctest", re.compile(r"^\d+% tests passed, (\d+) tests? failed out of (\d+)"), _ctest),
+ ("phpunit", re.compile(r"^OK \((\d+) tests?, \d+ assertions?\)"), lambda m: (int(m.group(1)), 0, 0)),
+ ("phpunit", re.compile(r"^(Tests: \d+, Assertions: \d+.*?)\.?\s*$"), _php),
+)
+UNITTEST_RAN = re.compile(r"^Ran (\d+) tests? in [\d.]+s\s*$")
+UNITTEST_END = re.compile(r"^(?:OK|FAILED)(?: \(([^)]*)\))?\s*$")
+
+
+def parse_runner_log(text):
+ """(executed, failed, skipped, runners) from the runner's own summary lines, summed over the log; None when no line is recognised.
+ Recognised: Maven/Gradle, cargo, pytest, unittest, go test (-v or -json), dotnet test, jest, vitest, ctest, phpunit."""
+ passed = failed = skipped = 0
+ seen, ran = [], None
+ for line in text.split("\n")[:3000000]:
+ line = line.rstrip("\r")[:LOG_LINE]
+ if ran is not None and line.strip():
+ m = UNITTEST_END.match(line)
+ if m:
+ w = {k: int(v) for k, v in re.findall(r"(\w+)=(\d+)", (m.group(1) or "")[:LOG_LINE])}
+ f, sk = w.get("failures", 0) + w.get("errors", 0), w.get("skipped", 0)
+ passed, failed, skipped = passed + max(0, ran - f - sk), failed + f, skipped + sk
+ seen.append("unittest")
+ ran = None
+ m = UNITTEST_RAN.match(line)
+ if m:
+ ran = int(m.group(1))
+ continue
+ for name, rx, read in LOG_LINES:
+ m = rx.match(line)
+ if m:
+ p, f, sk = read(m)
+ passed, failed, skipped = passed + p, failed + f, skipped + sk
+ seen.append(name)
+ break
+ return (passed + failed, failed, skipped, sorted(set(seen))) if seen else None
+
+
+def read_logs(paths):
+ """Saved runner logs -> {"files", "executed", "failed", "skipped", "runners", "oldest", "newest", "unrecognised"}. A log without a summary line of a known runner is no evidence."""
+ out = {"files": 0, "executed": 0, "failed": 0, "skipped": 0, "runners": [], "oldest": None, "newest": None, "unrecognised": []}
+ for path in paths:
+ try:
+ st = os.lstat(path)
+ if not stat.S_ISREG(st.st_mode) or st.st_size > LOG_CAP:
+ out["unrecognised"].append("%s (a link, not a file, or larger than %d MB)" % (clean(os.path.basename(path), 80), LOG_CAP >> 20))
+ continue
+ with open(path, "rb") as fh:
+ got = parse_runner_log(fh.read(LOG_CAP).decode("utf-8", "replace"))
+ except OSError:
+ out["unrecognised"].append("%s (could not be read)" % clean(os.path.basename(path), 80))
+ continue
+ if got is None:
+ out["unrecognised"].append("%s (no summary line of a runner this script knows)" % clean(os.path.basename(path), 80))
+ continue
+ out["files"] += 1
+ out["executed"], out["failed"], out["skipped"] = out["executed"] + got[0], out["failed"] + got[1], out["skipped"] + got[2]
+ out["runners"] = sorted(set(out["runners"]) | set(got[3]))
+ out["oldest"] = st.st_mtime if out["oldest"] is None else min(out["oldest"], st.st_mtime)
+ out["newest"] = st.st_mtime if out["newest"] is None else max(out["newest"], st.st_mtime)
+ return out
+
+
+# ---------------------------------------------------------------- measured baselines: result files, raw logs, machine-readable results
+EVIDENCE_EXT = (".xml", ".trx", ".json", ".txt", ".log", ".out")
+PATH_TOKEN = re.compile(r"[A-Za-z0-9_.@+~-]+(?:/[A-Za-z0-9_.@+~-]+)*/?")
+
+
+RECORDED = re.compile(r"^[ \t>*-]*\**[ \t]*(?:recorded|machine-readable)\b[^:\n]{0,40}:", re.I)
+
+
+def evidence_paths(baseline_path, text):
+ """Where the old version's measured results are kept: the folder analysis/<system>/baseline/, and the paths on a line of BASELINE.md
+ that starts with "Recorded:" or "Machine-readable:" (a folder, or a file ending in .xml .trx .json .txt .log .out). Paths mentioned
+ anywhere else in the file are prose, not evidence. A path with a `..`, a link anywhere below the analysis folder, or a place outside
+ analysis/<system>/ is never used. -> the existing paths, at most 50."""
+ adir = os.path.dirname(os.path.abspath(baseline_path))
+ system, real_adir, found, seen = os.path.basename(adir), os.path.realpath(adir), [], set()
+
+ def add(comps):
+ cur = adir
+ for c in comps:
+ cur = os.path.join(cur, c)
+ if os.path.islink(cur):
+ return
+ if not os.path.lexists(cur):
+ return
+ real = os.path.realpath(cur)
+ try:
+ ok = os.path.commonpath([real_adir, real]) == real_adir
+ except ValueError:
+ ok = False
+ if ok and real not in seen and (os.path.isdir(cur) or os.path.isfile(cur)):
+ seen.add(real)
+ found.append(cur)
+
+ add(["baseline"])
+ for line in text.split("\n")[:20000]:
+ if not RECORDED.match(line[:4000]):
+ continue
+ for m in PATH_TOKEN.finditer(line[:4000]):
+ tok = m.group(0)
+ if not (tok.lower().endswith(EVIDENCE_EXT) or tok.endswith("/")):
+ continue
+ comps = [c for c in tok.split("/") if c]
+ if comps and comps[0] == "analysis":
+ comps = comps[2:] if len(comps) >= 3 and comps[1] == system else []
+ if comps and not any(c in (".", "..") for c in comps):
+ add(comps)
+ if len(found) >= 50:
+ return found
+ return found
+
+
+def parse_results_json(data):
+ """{test id: status} from a machine-readable per-test results document, or None. Recognised: {"test id": "PASS", ...}, {"tests": ...}
+ and [{"name": ..., "status": ...}, ...]. A document that only holds counts ({"PASS": 10, "FAIL": 1}) is not evidence of any test: anyone
+ can type one, so it is ignored."""
+ def status(v):
+ return STATUS.get(re.sub(r"[^a-z]", "", v.lower())) if isinstance(v, str) else None
+
+ obj = data["tests"] if isinstance(data, dict) and isinstance(data.get("tests"), (dict, list)) else data
+ tests = {}
+ if isinstance(obj, dict):
+ items = list(obj.items())[:MAX_CASES]
+ good = {str(k)[:400]: status(v) for k, v in items if status(v)}
+ if good and len(good) >= 0.9 * len(items):
+ tests = good
+ elif isinstance(obj, list):
+ for e in obj[:MAX_CASES]:
+ if isinstance(e, dict):
+ name = next((e[k] for k in ("test", "id", "name", "fullName", "fullname") if isinstance(e.get(k), str) and e[k]), None)
+ st = next((status(e[k]) for k in ("status", "result", "outcome") if status(e.get(k))), None)
+ if name and st:
+ tests.setdefault(name[:400], st)
+ return tests or None
+
+
+def read_baseline_evidence(paths):
+ """Read what evidence_paths found: JUnit/TRX XML, machine-readable per-test JSON, and raw runner logs with a known summary line.
+ Two sources that disagree are reported as conflicts, never resolved by taking the first.
+ -> {"tests": {id: status} or None, "counts": {...} or None (a log alone), "sources", "considered", "ignored", "unreadable", "conflicts", "measured"}."""
+ out = {"tests": None, "counts": None, "sources": [], "considered": [], "ignored": [], "unreadable": [], "conflicts": [], "measured": False}
+ files = []
+ for p in paths:
+ if os.path.islink(p):
+ continue
+ if os.path.isdir(p):
+ for base, dirs, names in os.walk(p, followlinks=False):
+ dirs[:] = [d for d in sorted(dirs) if d not in SKIP_DIRS and not os.path.islink(os.path.join(base, d))]
+ files += [os.path.join(base, n) for n in sorted(names) if n.lower().endswith(EVIDENCE_EXT) and not os.path.islink(os.path.join(base, n))]
+ if len(files) >= 2000:
+ break
+ elif p.lower().endswith(EVIDENCE_EXT):
+ files.append(p)
+ files = files[:2000]
+ out["considered"] = [clean(os.path.basename(f), 80) for f in files[:20]]
+ cat = lambda st: "BAD" if st in BAD else st # noqa: E731
+ tests, conflicts = {}, []
+
+ def merge(more, label):
+ for k, v in more.items():
+ if k not in tests:
+ tests[k] = v
+ elif cat(tests[k]) != cat(v):
+ conflicts.append("%s: %s in one source, %s in %s" % (clean(k, 100), tests[k], v, label))
+
+ xml = [f for f in files if f.lower().endswith((".xml", ".trx"))]
+ if xml:
+ got = read_results(xml)
+ out["unreadable"] += got["unreadable"][:5]
+ if got["files"]:
+ merge(got["tests"], "the result files")
+ out["sources"].append({"kind": "result files", "files": got["files"], "tests": len(got["tests"])})
+ for f in [f for f in files if f.lower().endswith(".json")]:
+ name = clean(os.path.basename(f), 80)
+ try:
+ t = parse_results_json(json.loads(read_capped(f, MAX_FILE).decode("utf-8-sig")))
+ except (InputError, ValueError, RecursionError) as err:
+ out["unreadable"].append("%s: %s" % (name, clean(err, 100)))
+ continue
+ if t:
+ merge(t, name)
+ out["sources"].append({"kind": "machine-readable results", "files": 1, "tests": len(t), "name": name})
+ else:
+ out["ignored"].append(name) # counts only, or no test entries: nothing that can be checked test by test
+ logs = read_logs([f for f in files if f.lower().endswith((".txt", ".log", ".out"))])
+ if logs["files"]:
+ out["sources"].append({"kind": "runner log", "files": logs["files"], "executed": logs["executed"], "runners": logs["runners"]})
+ if tests:
+ c = tally(tests.values())
+ if c["executed"] != logs["executed"] or c["fail"] + c["error"] != logs["failed"]:
+ conflicts.append("the runner log says %d executed and %d failed; the per-test results say %d and %d" % (logs["executed"], logs["failed"], c["executed"], c["fail"] + c["error"]))
+ out["tests"] = tests or None
+ out["counts"] = {"executed": logs["executed"], "failed": logs["failed"], "skipped": logs["skipped"]} if logs["files"] and not tests else None
+ out["conflicts"] = conflicts[:20]
+ out["measured"] = bool(tests or out["counts"])
+ return out
+
+
+def assess_baseline(base, measured):
+ """Is BASELINE.md measured or only typed? -> {"kind": "target-only" | "measured" | "typed" | "none", "sources", "disagree", "examples", ...}.
+ A typed table is measured only when a result file, a machine-readable results file or a raw log that this script parsed backs it, and
+ agrees with it (the same tests with the same outcomes, or the same executed and failed counts)."""
+ typed = bool(base["tests"] or base["modules"] or base["totals"])
+ info = {"kind": "typed" if typed else "none", "sources": [], "disagree": 0, "examples": [], "conflict": 0, "conflictExamples": [], "considered": list((measured or {}).get("considered", []))[:20],
+ "ignored": list((measured or {}).get("ignored", []))[:10], "unreadable": list((measured or {}).get("unreadable", []))[:5], "measuredTests": 0,
+ "measuredExecuted": None, "measuredFailed": None}
+ if base["targetOnly"]:
+ info["kind"] = "target-only"
+ return info
+ if not measured or not measured["measured"]:
+ return info
+ info["kind"], info["sources"] = "measured", list(measured["sources"])[:10]
+ info["conflict"] = len(measured.get("conflicts", []))
+ info["conflictExamples"] = [clean(c, 200) for c in measured.get("conflicts", [])[:3]]
+ cat = lambda st: "BAD" if st in BAD else st # noqa: E731
+ typed_counts, _ = baseline_counts(base)
+ if measured["tests"] is not None:
+ c = tally(measured["tests"].values())
+ info.update(measuredTests=len(measured["tests"]), measuredExecuted=c["executed"], measuredFailed=c["fail"] + c["error"])
+ if base["tests"]:
+ bad = [t for t, st in base["tests"].items() if cat(measured["tests"].get(t, "")) != cat(st)]
+ info["disagree"], info["examples"] = len(bad), [clean(t, 120) for t in bad[:5]]
+ elif typed_counts and (typed_counts["executed"] != c["executed"] or typed_counts["fail"] + typed_counts["error"] != c["fail"] + c["error"]):
+ info["disagree"], info["examples"] = 1, ["typed: %d executed, %d failed; measured: %d executed, %d failed" % (
+ typed_counts["executed"], typed_counts["fail"] + typed_counts["error"], c["executed"], c["fail"] + c["error"])]
+ else:
+ m = measured["counts"]
+ info.update(measuredExecuted=m["executed"], measuredFailed=m["failed"])
+ if typed_counts and (typed_counts["executed"] != m["executed"] or typed_counts["fail"] + typed_counts["error"] != m["failed"]):
+ info["disagree"], info["examples"] = 1, ["typed: %d executed, %d failed; measured: %d executed, %d failed" % (
+ typed_counts["executed"], typed_counts["fail"] + typed_counts["error"], m["executed"], m["failed"])]
+ totals = base["totals"] # a Totals table typed beside the rows must say what the evidence says
+ ex, fl = info["measuredExecuted"], info["measuredFailed"]
+ if totals and ex is not None and (("executed" in totals and totals["executed"] != ex) or (("fail" in totals or "error" in totals) and totals.get("fail", 0) + totals.get("error", 0) != fl)):
+ info["disagree"] += 1
+ info["examples"].append("the Totals table says %s executed and %s failed; measured: %d and %d" % (totals.get("executed", "?"), totals.get("fail", 0) + totals.get("error", 0), ex, fl))
+ return info
+
+
+# ---------------------------------------------------------------- the comparison
+def keep(items):
+ return items[:KEEP]
+
+
+def compare(base, fresh):
+ """-> the report dict (see the module docstring)."""
+ counts_b, source = baseline_counts(base)
+ counts_f = tally(fresh["tests"].values())
+ if base.get("measuredTests"):
+ source = "measured results"
+ rep = {"baseline": {"targetOnly": base["targetOnly"], "why": base["why"], "source": source, "counts": counts_b, "assessment": base.get("assessment")},
+ "fresh": {"files": fresh["files"], "counts": counts_f, "unreadable": fresh["unreadable"][:20], "unreadableCount": len(fresh["unreadable"]),
+ "oldest": fresh["oldest"], "newest": fresh["newest"]},
+ "regressions": [], "flaky": [], "newFailures": [], "stillFailing": [], "fixed": [], "newlySkipped": [], "missing": [], "renamed": 0,
+ "missingModules": [], "moduleDiffs": [], "executedDrop": None, "skippedGrowth": None, "problems": list(fresh["problems"])}
+ bt, ft, flaky = base["tests"], fresh["tests"], set(base.get("flaky", []))
+ if bt:
+ missing = [t for t in bt if t not in ft]
+ added = [t for t in ft if t not in bt]
+ for t, b in bt.items():
+ f = ft.get(t)
+ if f is None:
+ continue
+ if b == "PASS" and f in BAD:
+ (rep["flaky"] if t in flaky else rep["regressions"]).append({"id": t, "before": b, "now": f})
+ elif b in BAD and f in BAD:
+ rep["stillFailing"].append(t)
+ elif b in BAD and f == "PASS":
+ rep["fixed"].append(t)
+ elif b == "PASS" and f == "SKIP":
+ rep["newlySkipped"].append(t)
+ elif b == "SKIP" and f in BAD:
+ rep["newFailures"].append(t)
+ by_class = {}
+ for t in missing:
+ by_class.setdefault(t.split("#", 1)[0], [[], []])[0].append(t)
+ for t in added:
+ by_class.setdefault(t.split("#", 1)[0], [[], []])[1].append(t)
+ drifting = {k for k, (gone, new) in by_class.items() if gone and new}
+ grouped_b, grouped_f = {}, {}
+ for src, dst in ((bt, grouped_b), (ft, grouped_f)):
+ for t, v in src.items():
+ if t.split("#", 1)[0] in drifting:
+ dst.setdefault(t.split("#", 1)[0], []).append(v)
+ for klass, (gone, new) in by_class.items():
+ b_counts, f_counts = tally(grouped_b.get(klass, [])), tally(grouped_f.get(klass, []))
+ if gone and new and f_counts["executed"] >= b_counts["executed"] and f_counts["fail"] + f_counts["error"] <= b_counts["fail"] + b_counts["error"]:
+ rep["renamed"] += len(gone)
+ else:
+ rep["missing"] += gone
+ rep["newFailures"] += [t for t in new if ft[t] in BAD]
+ for name, b in base["modules"].items():
+ f = fresh["classes"].get(name) or fresh["modules"].get(name) or fresh["suites"].get(name)
+ if f is None:
+ rep["missingModules"].append(name)
+ continue
+ problems = []
+ if f["executed"] < b["pass"] + b["fail"] + b["error"]:
+ problems.append("%d test(s) executed, %d before" % (f["executed"], b["pass"] + b["fail"] + b["error"]))
+ if f["skip"] > b["skip"]:
+ problems.append("%d skipped, %d before" % (f["skip"], b["skip"]))
+ if not bt and f["fail"] + f["error"] > b["fail"] + b["error"]:
+ rep["regressions"].append({"id": name + " (all its tests)", "before": "%d failing" % (b["fail"] + b["error"]), "now": "%d failing" % (f["fail"] + f["error"])})
+ if problems:
+ rep["moduleDiffs"].append({"module": name, "problems": problems})
+ if counts_b:
+ if counts_f["executed"] < counts_b["executed"]:
+ rep["executedDrop"] = {"before": counts_b["executed"], "now": counts_f["executed"]}
+ if counts_f["skip"] > counts_b["skip"]:
+ rep["skippedGrowth"] = {"before": counts_b["skip"], "now": counts_f["skip"]}
+ rep["approved"] = []
+ for key, label in (("regressions", "passed before, fails now"), ("newFailures", "not in the baseline, fails now")):
+ kept = []
+ for item in rep[key]:
+ tid = item["id"] if isinstance(item, dict) else item
+ if tid in base.get("approved", {}):
+ rep["approved"].append({"id": tid, "reason": base["approved"][tid], "was": label})
+ else:
+ kept.append(item)
+ rep[key] = kept
+ for key in ("regressions", "flaky", "newFailures", "stillFailing", "fixed", "newlySkipped", "missing", "missingModules", "moduleDiffs", "approved"):
+ rep[key + "Count"], rep[key] = len(rep[key]), keep(rep[key])
+ rep["ok"] = counts_f["executed"] > 0 and not rep["regressionsCount"] and not rep["newFailuresCount"]
+ return rep
+
+
+def render(rep, limit=10):
+ b, f = rep["baseline"]["counts"], rep["fresh"]["counts"]
+ show = lambda c: "%d executed (%d pass, %d fail, %d error, %d skip)" % (c["executed"], c["pass"], c["fail"], c["error"], c["skip"]) # noqa: E731
+ lines = ["baseline (%s): %s" % (rep["baseline"]["source"], show(b) if b else "nothing readable"),
+ "this run (%d result file(s)): %s" % (rep["fresh"]["files"], show(f))]
+ if rep["baseline"]["targetOnly"]:
+ lines.append("BASELINE.md says target-only (%s): there is no old-version result to compare with." % (rep["baseline"]["why"] or "no reason given"))
+ a = rep["baseline"].get("assessment")
+ if a and a["kind"] == "measured":
+ lines.append("baseline evidence: measured (%s)" % "; ".join("%s%s" % (s["kind"], " %s" % s["name"] if s.get("name") else "") for s in a["sources"][:4]))
+ if a["conflict"]:
+ lines.append(" but the sources disagree with each other: %s" % "; ".join(a["conflictExamples"][:3]))
+ elif a["disagree"]:
+ lines.append(" but the typed table disagrees with it on %d test(s): %s" % (a["disagree"], "; ".join(a["examples"][:3])))
+ elif a and a["kind"] == "typed":
+ lines.append("baseline evidence: typed table only. No result file, per-test results file or raw log that this script can read was found in analysis/<system>/baseline/ or on a Recorded: or Machine-readable: line of BASELINE.md.%s" % (
+ " Ignored (counts only): " + ", ".join(a["ignored"][:3]) if a["ignored"] else ""))
+
+ def block(title, key, fmt=lambda x: clean(x, 160)):
+ if rep[key + "Count"]:
+ lines.append("%s: %d" % (title, rep[key + "Count"]))
+ lines.extend(" " + fmt(x) for x in rep[key][:limit])
+ if rep[key + "Count"] > limit:
+ lines.append(" ... and %d more (use --json)" % (rep[key + "Count"] - limit))
+ else:
+ lines.append("%s: 0" % title)
+ block("regressions (passed before, fail now)", "regressions", lambda x: "%s: %s -> %s" % (clean(x["id"], 140), x["before"], x["now"]))
+ block("differences a person approved in BASELINE.md (not counted)", "approved", lambda x: "%s: %s (%s)" % (clean(x["id"], 120), x["was"], clean(x["reason"], 100)))
+ block("flaky flips the baseline lists (not counted)", "flaky", lambda x: "%s: %s -> %s" % (clean(x["id"], 140), x["before"], x["now"]))
+ block("new failures (not in the baseline)", "newFailures")
+ block("still failing, as in the baseline", "stillFailing")
+ block("now passing (failed before)", "fixed")
+ block("skipped now that ran before", "newlySkipped")
+ block("baseline tests that did not run", "missing")
+ block("modules missing from this run", "missingModules")
+ block("modules with fewer tests or more skips", "moduleDiffs", lambda x: "%s: %s" % (clean(x["module"], 100), "; ".join(x["problems"])))
+ if rep["renamed"]:
+ lines.append("renamed: %d test name(s) changed but their class did not get worse" % rep["renamed"])
+ if rep["executedDrop"]:
+ lines.append("executed tests dropped from %(before)d to %(now)d" % rep["executedDrop"])
+ if rep["skippedGrowth"]:
+ lines.append("skipped tests grew from %(before)d to %(now)d" % rep["skippedGrowth"])
+ lines += ["%d result file(s) could not be read: %s" % (rep["fresh"]["unreadableCount"], "; ".join(rep["fresh"]["unreadable"][:3])) if rep["fresh"]["unreadableCount"] else "all result files were read"]
+ lines += rep["problems"]
+ lines.append("OK: nothing regressed and %d test(s) executed" % f["executed"] if rep["ok"] else "NOT OK: %s" % (
+ "no test executed" if not f["executed"] else "%d regression(s), %d new failure(s)" % (rep["regressionsCount"], rep["newFailuresCount"])))
+ return "\n".join(lines)
+
+
+def run(baseline_path, paths, measured="auto"):
+ """-> the report. Raises InputError for a baseline or results that cannot be used. `measured` is the old version's own evidence
+ (from read_baseline_evidence); by default it is looked for beside BASELINE.md (see evidence_paths). Measured per-test results,
+ when there are any, are what the fresh results are compared with; the typed table is only checked against them."""
+ if os.path.islink(baseline_path):
+ raise InputError("%s is a symbolic link" % baseline_path)
+ try:
+ with open(baseline_path, "rb") as fh:
+ text = fh.read(16 << 20).decode("utf-8", "replace")
+ except OSError as err:
+ raise InputError("cannot read %s: %s" % (baseline_path, err.__class__.__name__))
+ base = parse_baseline(text)
+ if not (base["targetOnly"] or base["modules"] or base["tests"] or base["totals"]):
+ raise InputError("%s has no per-test table, per-module table, totals or target-only line that can be read" % baseline_path)
+ if measured == "auto":
+ found = evidence_paths(baseline_path, text)
+ measured = read_baseline_evidence(found) if found else None
+ base["assessment"] = assess_baseline(base, measured)
+ if base["assessment"]["kind"] == "measured" and measured["tests"]:
+ base["tests"], base["measuredTests"] = measured["tests"], True
+ return compare(base, read_results(paths))
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser(description="Compare a fresh test run with BASELINE.md.")
+ ap.add_argument("baseline", help="analysis/<system>/BASELINE.md")
+ ap.add_argument("--junit", "--trx", action="append", default=[], metavar="PATH", help="a results file or a folder to search (repeat for several)")
+ ap.add_argument("--baseline-evidence", action="append", default=[], metavar="PATH",
+ help="a file or folder with the old version's own results (XML, JSON map, raw log); default: analysis/<system>/baseline/ and the paths BASELINE.md names")
+ ap.add_argument("--json", action="store_true", help="print the full report as JSON")
+ ap.add_argument("--max-list", type=int, default=10, help="how many entries to print per list (default 10)")
+ args = ap.parse_args(argv)
+ if not args.junit:
+ print("baseline_diff.py: give at least one --junit file or folder", file=sys.stderr)
+ return 2
+ try:
+ rep = run(args.baseline, args.junit, read_baseline_evidence(args.baseline_evidence) if args.baseline_evidence else "auto")
+ except InputError as err:
+ print("baseline_diff.py: %s" % err, file=sys.stderr)
+ return 2
+ print(json.dumps(rep, indent=1, ensure_ascii=True) if args.json else render(rep, max(1, args.max_list)))
+ return 0 if rep["ok"] else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())