summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPeter Lai <peterlai@anthropic.com>2026-11-14 07:16:42 -0800
committerGitHub <noreply@github.com>2026-11-14 07:16:42 -0800
commit1780c5e5a552fd4d1c9d6a7b0cab26b0c2bef064 (patch)
treef9c04d957809f6f1a314355821139f477263c43c
parent2c5958ecac7cee2b583967a2c4981d0f4a75c579 (diff)
Update docx, pptx, xlsx, and pdf skills (#2217)
-rw-r--r--skills/docx/SKILL.md4
-rw-r--r--skills/docx/scripts/office/helpers/__init__.py26
-rw-r--r--skills/docx/scripts/office/helpers/pptx_chart.py135
-rw-r--r--skills/docx/scripts/office/helpers/pptx_slide.py46
-rw-r--r--skills/docx/scripts/office/soffice.py2
-rwxr-xr-xskills/docx/scripts/office/validate.py41
-rw-r--r--skills/docx/scripts/office/validators/base.py20
-rw-r--r--skills/docx/scripts/office/validators/docx.py65
-rw-r--r--skills/docx/scripts/office/validators/pptx.py124
-rw-r--r--skills/docx/scripts/office/validators/redlining.py405
-rw-r--r--skills/pdf/FORMS.md (renamed from skills/pdf/forms.md)2
-rw-r--r--skills/pdf/REFERENCE.md (renamed from skills/pdf/reference.md)5
-rw-r--r--skills/pptx/SKILL.md55
-rwxr-xr-xskills/pptx/scripts/add_slide.py98
-rw-r--r--skills/pptx/scripts/apply_theme.js107
-rw-r--r--skills/pptx/scripts/office/helpers/__init__.py26
-rw-r--r--skills/pptx/scripts/office/helpers/pptx_chart.py135
-rw-r--r--skills/pptx/scripts/office/helpers/pptx_slide.py46
-rw-r--r--skills/pptx/scripts/office/soffice.py2
-rwxr-xr-xskills/pptx/scripts/office/validate.py41
-rw-r--r--skills/pptx/scripts/office/validators/base.py20
-rw-r--r--skills/pptx/scripts/office/validators/docx.py65
-rw-r--r--skills/pptx/scripts/office/validators/pptx.py124
-rw-r--r--skills/pptx/scripts/office/validators/redlining.py405
-rwxr-xr-xskills/pptx/scripts/slide_text.py132
-rw-r--r--skills/xlsx/SKILL.md33
-rw-r--r--skills/xlsx/scripts/office/helpers/__init__.py26
-rw-r--r--skills/xlsx/scripts/office/helpers/pptx_chart.py135
-rw-r--r--skills/xlsx/scripts/office/helpers/pptx_slide.py46
-rw-r--r--skills/xlsx/scripts/office/soffice.py2
-rwxr-xr-xskills/xlsx/scripts/office/validate.py41
-rw-r--r--skills/xlsx/scripts/office/validators/base.py20
-rw-r--r--skills/xlsx/scripts/office/validators/docx.py65
-rw-r--r--skills/xlsx/scripts/office/validators/pptx.py124
-rw-r--r--skills/xlsx/scripts/office/validators/redlining.py405
-rwxr-xr-xskills/xlsx/scripts/recalc.py867
36 files changed, 3273 insertions, 622 deletions
diff --git a/skills/docx/SKILL.md b/skills/docx/SKILL.md
index acc50ea..8620c46 100644
--- a/skills/docx/SKILL.md
+++ b/skills/docx/SKILL.md
@@ -28,6 +28,7 @@ A `.docx` is a ZIP archive of XML files. Choose your approach by task:
- **`ImageRun` requires `type:`** (`"png"`, `"jpg"`, …).
- **`PageBreak` must be inside a `Paragraph`.**
- **Never use `\n`** — use separate `Paragraph` elements.
+- **Headings:** style every level you use (`title`, `heading1`, `heading2`…) in `styles.default`: `heading1: { run: { font: "Arial", size: 32, bold: true }, paragraph: { spacing: { before: 240, after: 120 }, keepNext: true } }`. Every level needs its own font (even with a document default), `before` and `after` (even if 0) and `keepNext`: the built-in styles set none of them.
- **TOC:** headings must use built-in `HeadingLevel.*`; custom heading styles need `outlineLevel` set or they won't appear.
- **Don't use a table as a horizontal rule** — use a paragraph bottom border instead.
- **Dot-leader / right-aligned-on-same-line:** use `PositionalTab` (`alignment: PositionalTabAlignment.RIGHT`, `leader: PositionalTabLeader.DOT`) inside a `TextRun`, not literal `.` or space padding.
@@ -37,6 +38,7 @@ A `.docx` is a ZIP archive of XML files. Choose your approach by task:
After writing a `.docx`, render it and look at it:
```bash
+zip -d output.docx '*/' >/dev/null || true # drop the ZIP's folder entries: Word has refused a file that has them when opened through a cloud co-editing service
python scripts/office/soffice.py --headless --convert-to pdf output.docx
pdftoppm -jpeg -r 100 output.pdf page
ls page-*.jpg # then Read the images
@@ -53,7 +55,7 @@ unzip -q doc.docx -d unpacked/
find unpacked -type l -delete # strip symlink entries — docx from external parties is untrusted
python scripts/merge_runs.py unpacked/ # coalesce fragmented runs so text is findable
# edit unpacked/word/document.xml in place — do NOT reformat or pretty-print
-(cd unpacked && rm -f ../out.docx && zip -Xr ../out.docx .)
+(cd unpacked && rm -f ../out.docx && zip -XrD ../out.docx .)
python scripts/office/validate.py out.docx --original doc.docx # XSD checks; --auto-repair fixes common issues
# redlining? add --author "<the name you redlined under>" to check every edit is tracked
```
diff --git a/skills/docx/scripts/office/helpers/__init__.py b/skills/docx/scripts/office/helpers/__init__.py
index d3c5817..4902e2a 100644
--- a/skills/docx/scripts/office/helpers/__init__.py
+++ b/skills/docx/scripts/office/helpers/__init__.py
@@ -5,7 +5,7 @@ import stat
import tempfile
import urllib.parse
import zipfile
-from pathlib import Path
+from pathlib import Path, PureWindowsPath
OOXML_FAMILY = {
".docx": "docx",
@@ -21,16 +21,24 @@ _SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:")
SLIDE_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide"
+def one_line(text: str, limit: int) -> str:
+ return "".join(c if c.isprintable() else c.encode("unicode_escape").decode() for c in text)[:limit]
+
+
+def names_another_root(name: str) -> bool:
+ return bool(PureWindowsPath(name).anchor)
+
+
def opc_target(target: str, source_part: str, target_mode: str = "") -> str | None:
if not target:
return None
if target_mode.lower() == "external":
return None
+ target = urllib.parse.unquote(target)
+
if _SCHEME_RE.match(target):
return None
- target = urllib.parse.unquote(target)
-
if "\\" in target:
raise ValueError(f"relationship target is not a POSIX part name: {target!r}")
@@ -52,12 +60,15 @@ def opc_target(target: str, source_part: str, target_mode: str = "") -> str | No
if not parts:
raise ValueError(f"relationship target resolves to nothing: {target!r}")
+ if names_another_root("/".join(parts)):
+ raise ValueError(f"relationship target is not a part name: {target!r}")
return "/".join(parts)
def rels_source_part(rels_file: Path, unpacked_dir: Path) -> str:
owner_dir = rels_file.parent.parent.relative_to(unpacked_dir)
- return posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")]).lstrip("./")
+ part = posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")])
+ return part.removeprefix("./")
def part_text(data: bytes) -> str:
@@ -76,12 +87,19 @@ def safe_extract(zf: zipfile.ZipFile, dest: Path) -> None:
for m in zf.infolist():
if stat.S_ISLNK(m.external_attr >> 16):
raise ValueError(f"symlink archive entry not allowed: {m.filename!r}")
+ if names_another_root(m.filename):
+ raise ValueError(f"unsafe archive entry: {m.filename!r}")
target = (dest / m.filename).resolve()
if not target.is_relative_to(dest):
raise ValueError(f"unsafe archive entry: {m.filename!r}")
zf.extract(m, dest)
+def folder_entries(package: Path) -> list[str]:
+ with zipfile.ZipFile(package) as zf:
+ return [info.filename for info in zf.infolist() if info.is_dir()]
+
+
def rezip(src_dir: Path, out_path: Path) -> None:
files = sorted(p for p in src_dir.rglob("*") if p.is_file())
ct = src_dir / "[Content_Types].xml"
diff --git a/skills/docx/scripts/office/helpers/pptx_chart.py b/skills/docx/scripts/office/helpers/pptx_chart.py
index 209cb7c..0e86aed 100644
--- a/skills/docx/scripts/office/helpers/pptx_chart.py
+++ b/skills/docx/scripts/office/helpers/pptx_chart.py
@@ -1,15 +1,20 @@
-"""Find chart XML that PowerPoint refuses but the schema accepts.
+"""Find chart XML that PowerPoint refuses or repairs.
-Detection only: for either fault more than one repair is valid, and only the
-author knows which was meant.
+Detection only: more than one repair is usually valid, and only the author knows
+which was meant.
"""
from __future__ import annotations
+import html
import re
+from collections import Counter
from typing import Mapping
+from defusedxml import ElementTree
+from defusedxml.common import DefusedXmlException
+
from . import part_text
@@ -19,8 +24,12 @@ _GROUPING_RE = re.compile(r"""<c:grouping\b[^>]*?\bval=["'](\w+)["']""")
_DLBL_POS_RE = re.compile(r"""<c:dLblPos\b[^>]*?\bval=["'](\w+)["']""")
def _strip_ext_lst(text: str) -> str:
+ return _strip_outermost(text, "c:extLst")
+
+
+def _strip_outermost(text: str, name: str) -> str:
out, cursor = [], 0
- for lo, hi in _ext_lst_spans(text):
+ for lo, hi in _outermost_spans(text, name):
out.append(text[cursor:lo])
cursor = hi
out.append(text[cursor:])
@@ -139,18 +148,113 @@ def _check_chart_axis_references(part: str, xml: str) -> list[str]:
return problems
-def _ext_lst_spans(text: str) -> list[tuple[int, int]]:
+def _shown(value: str) -> str:
+ return '"' + repr(value[:20])[1:-1] + '"'
+
+
+DECIMAL_SETTINGS = (
+ "crossesAt", "max", "min", "majorUnit", "minorUnit", "logBase",
+ "forward", "backward", "intercept", "x", "y", "w", "h",
+ "splitPos", "custUnit", "pictureStackUnit",
+)
+_DECIMAL_SETTING_RE = re.compile(
+ rf"""<c:({"|".join(DECIMAL_SETTINGS)})\b[^<>]*?\bval=["']([^"'<>]*)["']"""
+)
+_DECIMAL_RE = re.compile(
+ r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?"
+ r"|[+-]?(inf|infinity|nan(\(\w*\))?|\u221e|1\.#(inf|qnan|snan|ind)\d*)",
+ re.IGNORECASE,
+)
+
+
+def _check_decimal_settings(part: str, xml: str) -> list[str]:
+ problems = []
+ for name, raw in sorted(set(_DECIMAL_SETTING_RE.findall(_strip_ext_lst(xml)))):
+ value = html.unescape(raw)
+ if _DECIMAL_RE.fullmatch(value.strip()):
+ continue
+ fix = "Fix: write a plain number, or remove the element"
+ if name == "crossesAt" and value.strip() == "autoZero":
+ fix = ("Fix: leave valAxisCrossesAt: 0 out of the pptxgenjs options, the axis "
+ "crosses at zero anyway. In the XML, write "
+ '<c:crosses val="autoZero"/> or a number')
+ problems.append(
+ f"{part}: <c:{name} val={_shown(value)}> is not a number, and PowerPoint "
+ f"will offer to repair the deck. {fix}"
+ )
+ return problems
+
+
+def _local(tag: str) -> str:
+ return tag.rsplit("}", 1)[-1]
+
+
+def _drawn_series(root) -> list:
+ found, to_visit = [], [root]
+ while to_visit:
+ for child in to_visit.pop():
+ name = _local(child.tag)
+ if name == "ser":
+ found.append(child)
+ elif name == "AlternateContent":
+ to_visit.extend(child[:1])
+ elif name != "extLst":
+ to_visit.append(child)
+ return found
+
+
+def _check_series_order(part: str, xml: str) -> list[str]:
+ try:
+ root = ElementTree.fromstring(xml.encode("utf-8", "surrogateescape"))
+ except (ElementTree.ParseError, DefusedXmlException, LookupError, ValueError):
+ return []
+ orders = Counter()
+ for series in _drawn_series(root):
+ own = [c.get("val", "") for c in series if _local(c.tag) == "order"]
+ if own and own[0].isascii() and own[0].isdigit():
+ orders[own[0]] += 1
+ repeated = sorted((o for o, n in orders.items() if n > 1), key=lambda o: (len(o), o))
+ if not repeated:
+ return []
+ return [
+ f"{part}: two or more series share the <c:order> value {', '.join(o[:12] for o in repeated)}, "
+ f"and PowerPoint will offer to repair the deck. Fix: number <c:order> "
+ f"(and <c:idx>) 0, 1, 2... across every series in the chart, a combo "
+ f"chart's line series carrying on from its bars"
+ ]
+
+
+PRESET_DASHES = frozenset({
+ "solid", "dot", "dash", "lgDash", "dashDot", "lgDashDot", "lgDashDotDot",
+ "sysDash", "sysDot", "sysDashDot", "sysDashDotDot",
+})
+_PRESET_DASH_RE = re.compile(r"""<a:prstDash\b[^<>]*?\bval=["']([^"'<>]*)["']""")
+
+
+def _check_preset_dashes(part: str, xml: str) -> list[str]:
+ values = {html.unescape(v) for v in _PRESET_DASH_RE.findall(_strip_ext_lst(xml))}
+ return [
+ f"{part}: <a:prstDash val={_shown(value)}> is not a dash style, and PowerPoint "
+ f"will offer to repair the deck. Use one of {', '.join(sorted(PRESET_DASHES))}. "
+ f"In pptxgenjs, lineDash takes one such word for the whole chart, not an array"
+ for value in sorted(values)
+ if value.strip() not in PRESET_DASHES
+ ]
+
+
+def _outermost_spans(text: str, name: str) -> list[tuple[int, int]]:
spans: list[tuple[int, int]] = []
depth = 0
start = 0
- for match in re.finditer(r"<(/?)c:extLst\b[^>]*?(/?)>", text):
+ for match in re.finditer(rf"<(/?){name}\b[^<>]*?(/?)>", text):
closing, self_closing = match.group(1), match.group(2)
if self_closing:
continue
if closing:
- depth -= 1
- if depth == 0:
- spans.append((start, match.end()))
+ if depth:
+ depth -= 1
+ if depth == 0:
+ spans.append((start, match.end()))
else:
if depth == 0:
start = match.start()
@@ -158,13 +262,22 @@ def _ext_lst_spans(text: str) -> list[tuple[int, int]]:
return spans
-CHART_CHECKS = (_check_stacked_label_positions, _check_chart_axis_references)
+CHART_CHECKS = (
+ _check_stacked_label_positions,
+ _check_chart_axis_references,
+ _check_decimal_settings,
+ _check_series_order,
+ _check_preset_dashes,
+)
+
+
+_COMMENT_OR_PI_RE = re.compile(r"<!--.*?(?:-->|\Z)|<\?(?!xml\s).*?(?:\?>|\Z)", re.DOTALL)
def find_chart_problems(files: Mapping[str, bytes]) -> list[str]:
problems: list[str] = []
for part in sorted(n for n in files if _CHART_PART_RE.fullmatch(n)):
- xml = part_text(files[part])
+ xml = _strip_outermost(_COMMENT_OR_PI_RE.sub("", part_text(files[part])), "mc:Fallback")
for check in CHART_CHECKS:
problems.extend(check(part, xml))
return problems
diff --git a/skills/docx/scripts/office/helpers/pptx_slide.py b/skills/docx/scripts/office/helpers/pptx_slide.py
index 22f9aee..55274b8 100644
--- a/skills/docx/scripts/office/helpers/pptx_slide.py
+++ b/skills/docx/scripts/office/helpers/pptx_slide.py
@@ -1,7 +1,8 @@
-"""Pick the slide-XML schema errors PowerPoint refuses the file over.
+"""Find the slide-XML defects known or predicted to make PowerPoint refuse or repair the file.
-A denylist over lxml's messages, so an unrecognised error class is a miss rather
-than a false alarm.
+Schema errors are picked by a denylist over lxml's messages, so an unrecognised
+error class is a miss rather than a false alarm. Animations that name a shape
+the slide no longer has are found by reading the part.
"""
@@ -43,6 +44,14 @@ FATAL_SLIDE_ERRORS: tuple[tuple[re.Pattern[str], str], ...] = (
re.compile(r"\}nvGrpSpPr': Missing child element"),
"a <p:nvGrpSpPr> with no children",
),
+ (
+ re.compile(r"\}(?:off|ext|chOff|chExt)', attribute '\w+': '-?(?:Infinity|NaN)'"),
+ "a position or size that is not a number (check the x, y, w, h you worked out)",
+ ),
+ (
+ re.compile(r"\}prstDash', attribute 'val'"),
+ "an unknown line dash style (for line.dashType try dash, sysDash or solid)",
+ ),
)
@@ -50,6 +59,37 @@ def is_schema_verdict(error: str) -> bool:
return error.startswith("Element ")
+_PML = "{http://schemas.openxmlformats.org/presentationml/2006/main}"
+_MC = "{http://schemas.openxmlformats.org/markup-compatibility/2006}"
+
+_NAMES_A_SHAPE = frozenset({f"{_PML}spTgt", f"{_PML}bldP", f"{_PML}bldGraphic"})
+
+
+def _shape_id(value: str | None) -> int | str | None:
+ text = (value or "").strip()
+ if text.isascii() and text.isdigit() and len(text) <= 18:
+ return int(text)
+ return text or None
+
+
+def stranded_animation_targets(root) -> list[str]:
+ timing = root.find(f"{_PML}timing")
+ if timing is None:
+ timing = root.find(f"{_MC}AlternateContent/{_MC}Choice/{_PML}timing")
+ if timing is None:
+ return []
+
+ on_slide = {_shape_id(e.get("id")) for e in root.iter("{*}cNvPr")}
+ named = {_shape_id(e.get("spid")) for e in timing.iter(*_NAMES_A_SHAPE)} - {None}
+ stranded = named - on_slide
+ return [
+ f"<p:timing> animates shape id {spid if isinstance(spid, int) else repr(spid[:16])}, which is gone, so PowerPoint will "
+ f"offer to repair the deck. Fix: delete the slide's whole <p:timing> element "
+ f"(the slide loses its animations), or put the shape back"
+ for spid in sorted(stranded, key=str)
+ ]
+
+
def fatal_slide_errors(errors: set[str]) -> list[str]:
out = []
for error in sorted(errors):
diff --git a/skills/docx/scripts/office/soffice.py b/skills/docx/scripts/office/soffice.py
index 0b4c99d..eec1687 100644
--- a/skills/docx/scripts/office/soffice.py
+++ b/skills/docx/scripts/office/soffice.py
@@ -64,7 +64,7 @@ def _ensure_shim() -> Path:
return _SHIM_SO
src = Path(tempfile.gettempdir()) / "lo_socket_shim.c"
- src.write_text(_SHIM_SOURCE)
+ src.write_text(_SHIM_SOURCE, encoding="utf-8")
subprocess.run(
["gcc", "-shared", "-fPIC", "-o", str(_SHIM_SO), str(src), "-ldl"],
check=True,
diff --git a/skills/docx/scripts/office/validate.py b/skills/docx/scripts/office/validate.py
index 29ca186..ff3bde0 100755
--- a/skills/docx/scripts/office/validate.py
+++ b/skills/docx/scripts/office/validate.py
@@ -14,6 +14,7 @@ Auto-repair fixes:
"""
import argparse
+import shlex
import sys
import tempfile
import zipfile
@@ -22,12 +23,21 @@ from pathlib import Path
import defusedxml.ElementTree as ET
from defusedxml.common import DefusedXmlException
-from helpers import OOXML_FAMILY, rezip, safe_extract
+from helpers import OOXML_FAMILY, folder_entries, rezip, safe_extract
from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator
WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+def _as_argument(path: Path) -> str:
+ name = str(path)
+ return shlex.quote("./" + name if name.startswith("-") else name)
+
+
+def _count(folders: list[str]) -> str:
+ return "1 folder entry" if len(folders) == 1 else f"{len(folders)} folder entries"
+
+
def _fail(message: str):
print(f"Error: {message}", file=sys.stderr)
sys.exit(2)
@@ -150,16 +160,43 @@ def main():
print(f"Error: Validation not supported for file type {family}")
sys.exit(1)
+ folders = folder_entries(packed_file) if packed_file is not None else []
+
+ unsaved_repairs = 0
if args.auto_repair:
total_repairs = sum(v.repair() for v in validators)
if total_repairs:
print(f"Auto-repaired {total_repairs} issue(s)")
- if packed_file is not None:
+ if packed_file is not None and (total_repairs or folders):
+ try:
rezip(unpacked_dir, packed_file)
+ except OSError as e:
+ unsaved_repairs = total_repairs
+ left = []
+ if total_repairs:
+ left.append(f"{total_repairs} repair(s) not saved")
+ if folders:
+ left.append(f"{_count(folders)} in its ZIP")
+ print(f"Could not write {packed_file} ({e.strerror or e}): it is as it was, "
+ f"with {' and '.join(left)}. Run --auto-repair on a copy in a folder "
+ "you can write to.")
+ else:
+ if folders:
+ print(f"Removed {_count(folders)} from the ZIP")
print(f"Wrote repaired file to {packed_file}")
+ elif folders:
+ print(
+ f"Note: {_count(folders)} in the ZIP. Such files have failed to open or co-edit "
+ f"through a cloud service. Remove them: zip -dq {_as_argument(packed_file)} '*/'"
+ )
success = all([v.validate() for v in validators])
+ if unsaved_repairs:
+ print(f"FAILED - {packed_file} was not rewritten and still has the {unsaved_repairs} "
+ "issue(s) repaired above; the checks ran on a repaired copy")
+ success = False
+
if temp_dir_ctx is not None:
temp_dir_ctx.cleanup()
diff --git a/skills/docx/scripts/office/validators/base.py b/skills/docx/scripts/office/validators/base.py
index 33fc97b..04d6169 100644
--- a/skills/docx/scripts/office/validators/base.py
+++ b/skills/docx/scripts/office/validators/base.py
@@ -2,6 +2,7 @@
Base validator with common validation logic for document files.
"""
+import os
import re
from pathlib import Path
@@ -10,7 +11,7 @@ from functools import lru_cache
import lxml.etree
-from helpers import safe_extract
+from helpers import names_another_root, safe_extract
@lru_cache(maxsize=None)
@@ -352,6 +353,9 @@ class BaseSchemaValidator:
if target and not target.startswith(
("http", "mailto:")
):
+ if names_another_root(target.lstrip("/")):
+ broken_refs.append((target, rel.sourceline))
+ continue
if target.startswith("/"):
target_path = self.unpacked_dir / target.lstrip("/")
elif rels_file.name == ".rels":
@@ -360,9 +364,18 @@ class BaseSchemaValidator:
base_dir = rels_dir.parent
target_path = base_dir / target
+ target_path = Path(os.path.normpath(target_path))
+ if not target_path.is_relative_to(self.unpacked_dir):
+ broken_refs.append((target, rel.sourceline))
+ continue
+
try:
target_path = target_path.resolve()
- if target_path.exists() and target_path.is_file():
+ if (
+ target_path.is_relative_to(self.unpacked_dir)
+ and target_path.exists()
+ and target_path.is_file()
+ ):
referenced_files.add(target_path)
all_referenced_files.add(target_path)
else:
@@ -783,8 +796,7 @@ class BaseSchemaValidator:
try:
schema = _load_schema(str(schema_path))
- with open(xml_file, "r") as f:
- xml_doc = lxml.etree.parse(f)
+ xml_doc = lxml.etree.parse(str(xml_file))
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)
xml_doc = self._preprocess_for_mc_ignorable(xml_doc)
diff --git a/skills/docx/scripts/office/validators/docx.py b/skills/docx/scripts/office/validators/docx.py
index b181499..2acdaf4 100644
--- a/skills/docx/scripts/office/validators/docx.py
+++ b/skills/docx/scripts/office/validators/docx.py
@@ -6,12 +6,13 @@ import random
import re
import tempfile
import zipfile
+from collections import Counter
from pathlib import Path
import defusedxml.minidom
import lxml.etree
-from helpers import safe_extract
+from helpers import one_line, safe_extract
from .base import BaseSchemaValidator
@@ -62,6 +63,9 @@ class DOCXSchemaValidator(BaseSchemaValidator):
if not self.validate_comment_markers():
all_valid = False
+ if not self.validate_style_ids():
+ all_valid = False
+
self.compare_paragraph_counts()
return all_valid
@@ -264,7 +268,9 @@ class DOCXSchemaValidator(BaseSchemaValidator):
for xml_file in self.xml_files:
try:
- for elem in lxml.etree.parse(str(xml_file)).iter():
+ tree = lxml.etree.parse(str(xml_file))
+ is_numbering = lxml.etree.QName(tree.getroot()).localname == "numbering"
+ for elem in tree.iter():
if val := elem.get(para_id_attr):
try:
if self._parse_id_value(val, base=16) >= 0x80000000:
@@ -278,7 +284,7 @@ class DOCXSchemaValidator(BaseSchemaValidator):
)
if val := elem.get(durable_id_attr):
- if xml_file.name == "numbering.xml":
+ if is_numbering:
try:
if self._parse_id_value(val, base=10) >= 0x7FFFFFFF:
errors.append(
@@ -288,7 +294,7 @@ class DOCXSchemaValidator(BaseSchemaValidator):
except ValueError:
errors.append(
f" {xml_file.name}:{elem.sourceline}: "
- f"durableId={val} must be decimal in numbering.xml"
+ f"durableId={val} must be decimal in a numbering part"
)
else:
try:
@@ -308,11 +314,58 @@ class DOCXSchemaValidator(BaseSchemaValidator):
if errors:
print(f"FAILED - {len(errors)} ID constraint violations:")
for e in errors:
- print(e)
+ print(one_line(e, 300))
elif self.verbose:
print("PASSED - All paraId/durableId values within constraints")
return not errors
+ def _repeated_style_ids(self, styles_root) -> set[str]:
+ attr = f"{{{self.WORD_2006_NAMESPACE}}}styleId"
+ ids = Counter(s.get(attr) for s in styles_root.iter(f"{{{self.WORD_2006_NAMESPACE}}}style"))
+ return {i for i, n in ids.items() if i is not None and n > 1}
+
+ def _original_part(self, xml_file) -> bytes | None:
+ if self.original_file is None:
+ return None
+ name = xml_file.relative_to(self.unpacked_dir).as_posix()
+ try:
+ with zipfile.ZipFile(self.original_file) as zf:
+ return zf.read(name)
+ except (KeyError, OSError, zipfile.BadZipFile):
+ return None
+
+ def validate_style_ids(self):
+ errors = []
+ styles_tag = f"{{{self.WORD_2006_NAMESPACE}}}styles"
+ if self.original_file is None:
+ return True
+
+ for xml_file in self.xml_files:
+ try:
+ root = lxml.etree.parse(str(xml_file)).getroot()
+ if root.tag != styles_tag:
+ continue
+ repeated = self._repeated_style_ids(root)
+ original = self._original_part(xml_file) if repeated else None
+ if original is not None:
+ repeated -= self._repeated_style_ids(lxml.etree.fromstring(original))
+ except lxml.etree.XMLSyntaxError:
+ continue
+ errors.extend(
+ f" {xml_file.relative_to(self.unpacked_dir)}: style id {style_id!r:.40} is now "
+ f"defined more than once, and Word uses only the last definition, without "
+ f"warning. Delete the earlier ones"
+ for style_id in sorted(repeated)
+ )
+
+ if errors:
+ print(f"FAILED - Found {len(errors)} style id(s) defined more than once:")
+ for error in errors:
+ print(one_line(error, 300))
+ elif self.verbose:
+ print("PASSED - The edit defined no style id a second time")
+ return not errors
+
def validate_comment_markers(self):
errors = []
@@ -415,7 +468,7 @@ class DOCXSchemaValidator(BaseSchemaValidator):
try:
content = xml_file.read_text(encoding="utf-8")
dom = defusedxml.minidom.parseString(content)
- is_numbering = xml_file.name == "numbering.xml"
+ is_numbering = dom.documentElement.localName == "numbering"
base = 10 if is_numbering else 16
pending = []
seen_in_file = set()
diff --git a/skills/docx/scripts/office/validators/pptx.py b/skills/docx/scripts/office/validators/pptx.py
index 318f0e6..bb06b82 100644
--- a/skills/docx/scripts/office/validators/pptx.py
+++ b/skills/docx/scripts/office/validators/pptx.py
@@ -63,6 +63,9 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if not self.validate_master_theme_uniqueness():
all_valid = False
+ if not self.validate_tags():
+ all_valid = False
+
if not self.validate_charts():
all_valid = False
@@ -87,6 +90,30 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if p.is_file()
}
+ @staticmethod
+ def _print_line(text: str, limit: int) -> None:
+ from helpers import one_line
+
+ print(one_line(text, limit))
+
+ def _listed_slides(self) -> set[str] | None:
+ import lxml.etree
+
+ from helpers import SLIDE_REL_TYPE
+
+ r_id = f"{{{self.OFFICE_RELATIONSHIPS_NAMESPACE}}}id"
+ try:
+ pres = lxml.etree.parse(str(self.unpacked_dir / "ppt" / "presentation.xml"))
+ rels = lxml.etree.parse(str(self.unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"))
+ listed = {e.get(r_id) for e in pres.iter(f"{{{self.PRESENTATIONML_NAMESPACE}}}sldId")}
+ return {
+ opc_target(rel.get("Target", ""), "ppt/presentation.xml", rel.get("TargetMode", ""))
+ for rel in rels.getroot()
+ if rel.get("Id") in listed and rel.get("Type") == SLIDE_REL_TYPE
+ }
+ except (OSError, ValueError, lxml.etree.XMLSyntaxError):
+ return None
+
def validate_master_theme_uniqueness(self):
from helpers.pptx_theme import _NOTES_MASTERS, live_shared_master_themes
@@ -94,7 +121,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if shared:
print(f"FAILED - Found {len(shared)} master(s) sharing a theme part:")
for message in shared:
- print(f" {message}")
+ self._print_line(f" {message}", 300)
if any(m.startswith(_NOTES_MASTERS) for m in shared):
print(" Fix: in ppt/presentation.xml, move <p:notesMasterIdLst> back to "
"directly after <p:sldIdLst>. PowerPoint reads that happily.")
@@ -106,6 +133,64 @@ class PPTXSchemaValidator(BaseSchemaValidator):
print("PASSED - No master shares a theme part in a way PowerPoint refuses")
return True
+ def validate_tags(self):
+ from collections import Counter
+
+ import lxml.etree
+
+ ns = {"p": self.PRESENTATIONML_NAMESPACE, "mc": self.MC_NAMESPACE}
+ r_id = f"{{{self.OFFICE_RELATIONSHIPS_NAMESPACE}}}id"
+ errors = []
+ owners: dict[str, list[str]] = {}
+ listed = self._listed_slides()
+
+ for slide in sorted(self.unpacked_dir.glob("ppt/slides/*.xml")):
+ relative = slide.relative_to(self.unpacked_dir).as_posix()
+ if listed is not None and relative not in listed:
+ continue
+ try:
+ root = lxml.etree.parse(str(slide)).getroot()
+ named = [t.get(r_id) for t in root.xpath(
+ ".//p:tags[not(ancestor::mc:Fallback) and not(ancestor::mc:Choice[preceding-sibling::mc:Choice])]",
+ namespaces=ns)]
+ rels_file = slide.parent / "_rels" / f"{slide.name}.rels"
+ targets = {
+ rel.get("Id"): opc_target(rel.get("Target", ""), relative, rel.get("TargetMode", ""))
+ for rel in lxml.etree.parse(str(rels_file)).getroot()
+ if rel.get("Type", "").endswith("/relationships/tags")
+ } if rels_file.is_file() else {}
+ except (lxml.etree.XMLSyntaxError, ValueError):
+ continue
+ for rid, count in sorted(Counter(r for r in named if r).items()):
+ if count > 1:
+ errors.append(
+ f" {relative[:80]}: {count} <p:tags> elements name the tags relationship "
+ f"{rid!r:.20}. Fix: a copied shape kept its <p:custDataLst>; delete that "
+ f"from the copy, or give it a tags part and relationship of its own"
+ )
+ if targets.get(rid):
+ owners.setdefault(targets[rid], []).append(relative)
+
+ for part, slides in sorted(owners.items()):
+ if len(slides) > 1:
+ named_from = sorted(set(slides))
+ more = f" and {len(named_from) - 3} more" if len(named_from) > 3 else ""
+ errors.append(
+ f" {part[:80]}: {len(slides)} tags relationships name it, from "
+ f"{', '.join(s[:60] for s in named_from[:3])}{more}. Fix: copy the part under a new name, add "
+ f"its Override to [Content_Types].xml and point every relationship but one at a copy of its own "
+ f"(add_slide.py does this when it duplicates a slide)"
+ )
+
+ if errors:
+ print(f"FAILED - Found {len(errors)} tags part(s) with more than one owner; PowerPoint offers to repair the deck:")
+ for error in errors:
+ self._print_line(error, 700)
+ return False
+ if self.verbose:
+ print("PASSED - Every tags part has one owner")
+ return True
+
def validate_charts(self):
from helpers.pptx_chart import find_chart_problems
@@ -113,7 +198,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if problems:
print(f"FAILED - Found {len(problems)} chart problem(s) PowerPoint rejects:")
for message in problems:
- print(f" {message}")
+ self._print_line(f" {message}", 600)
return False
if self.verbose:
@@ -151,14 +236,18 @@ class PPTXSchemaValidator(BaseSchemaValidator):
return found
def validate_slides(self):
+ import lxml.etree
+
from helpers.pptx_slide import (
SLIDE_PART_RE,
fatal_slide_errors,
is_schema_verdict,
+ stranded_animation_targets,
)
schema = self.schemas_dir / self.SCHEMA_MAPPINGS["ppt"]
inherited = self._original_slide_defects(schema)
+ listed = self._listed_slides()
problems: list[str] = []
broken: list[str] = []
@@ -166,6 +255,14 @@ class PPTXSchemaValidator(BaseSchemaValidator):
relative = xml_file.relative_to(self.unpacked_dir).as_posix()
if not SLIDE_PART_RE.fullmatch(relative):
continue
+ unlisted = relative.startswith("ppt/slides/") and listed is not None and relative not in listed
+ try:
+ root = lxml.etree.parse(str(xml_file)).getroot()
+ stranded = [] if unlisted else stranded_animation_targets(root)
+ except lxml.etree.XMLSyntaxError:
+ stranded = []
+ problems.extend(f"{relative}: {m}" for m in stranded)
+
ok, errors = self._validate_single_file_xsd(
xml_file.resolve(), self.unpacked_dir.resolve(), schema_path=schema
)
@@ -187,12 +284,12 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if broken:
print(f"FAILED - Could not check {len(broken)} slide part(s):")
for message in sorted(broken):
- print(f" {message[:240]}")
+ self._print_line(f" {message}", 240)
if problems:
print(f"FAILED - Found {len(problems)} slide problem(s) PowerPoint rejects:")
for message in sorted(problems):
- print(f" {message[:240]}")
+ self._print_line(f" {message}", 240)
if broken or problems:
return False
@@ -204,6 +301,17 @@ class PPTXSchemaValidator(BaseSchemaValidator):
def _get_schema_path(self, xml_file):
if xml_file.parent.name == "charts" and xml_file.name.startswith("chart"):
return None
+ if xml_file.parent.name == "ppt" and xml_file.suffix == ".xml":
+ import lxml.etree
+
+ try:
+ root = lxml.etree.parse(str(xml_file)).getroot()
+ except (lxml.etree.XMLSyntaxError, OSError):
+ root = None
+ if root is not None and (lxml.etree.QName(root).namespace or "").startswith(
+ "http://schemas.microsoft.com/office/"
+ ):
+ return None
return super()._get_schema_path(xml_file)
def _preprocess_for_schema(self, xml_doc, relative_path):
@@ -256,7 +364,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if errors:
print(f"FAILED - Found {len(errors)} UUID ID validation errors:")
for error in errors:
- print(error)
+ self._print_line(error, 300)
return False
else:
if self.verbose:
@@ -325,7 +433,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if errors:
print(f"FAILED - Found {len(errors)} slide layout ID validation errors:")
for error in errors:
- print(error)
+ self._print_line(error, 300)
print(
"Remove invalid references or add missing slide layouts to the relationships file."
)
@@ -366,7 +474,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if errors:
print("FAILED - Found slides with duplicate slideLayout references:")
for error in errors:
- print(error)
+ self._print_line(error, 300)
return False
else:
if self.verbose:
@@ -428,7 +536,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
f"FAILED - Found {len([e for e in errors if not e.startswith(' ')])} notes slide reference validation errors:"
)
for error in errors:
- print(error)
+ self._print_line(error, 300)
print("Each slide may optionally have its own slide file.")
return False
else:
diff --git a/skills/docx/scripts/office/validators/redlining.py b/skills/docx/scripts/office/validators/redlining.py
index 4185c51..4406912 100644
--- a/skills/docx/scripts/office/validators/redlining.py
+++ b/skills/docx/scripts/office/validators/redlining.py
@@ -1,19 +1,24 @@
"""
Validator for tracked changes in Word documents.
-Detects untracked edits in word/document.xml: text that differs from the
+Detects untracked edits in a Word document: text that differs from the
original without a <w:ins>/<w:del> wrapper recording it. The tracked changes
that are new relative to the original are undone, and the result is compared
against the original; whatever text still differs was edited without being
tracked.
-Only the document body is compared. Headers, footers, footnotes and endnotes
-are separate parts and are not checked.
+The body (word/document.xml) is compared, and so is each header, footer,
+footnotes and endnotes part, against the original's part of the same name.
+Other parts, comments among them, are not checked. A page-number field
+whose cached result is a plain number counts as its name in braces
+("{PAGE}"), not as that number, which any application that repaginates the
+document rewrites.
"""
-import subprocess
+import re
import tempfile
import zipfile
+from difflib import SequenceMatcher
from pathlib import Path
import defusedxml.ElementTree as ET
@@ -22,12 +27,35 @@ from defusedxml.common import DefusedXmlException
from helpers import rendered_text, safe_extract
+XML_ERRORS = (ET.ParseError, DefusedXmlException, LookupError, ValueError)
+
+
+class _WorkSpent(Exception):
+
+ pass
+
+
+class _Matcher(SequenceMatcher):
+
+ def __init__(self, a, b, allowance):
+ self.allowance = allowance
+ super().__init__(None, a, b, autojunk=False)
+
+ def find_longest_match(self, alo=0, ahi=None, blo=0, bhi=None):
+ items = self.a[alo:ahi]
+ self.allowance[0] -= len(items) + sum(len(self.b2j.get(x, ())) for x in items)
+ if self.allowance[0] < 0:
+ raise _WorkSpent
+ return super().find_longest_match(alo, ahi, blo, bhi)
+
+
class RedliningValidator:
def __init__(self, unpacked_dir, original_docx, verbose=False):
self.unpacked_dir = Path(unpacked_dir)
self.original_docx = Path(original_docx)
self.verbose = verbose
+ self._work_left = [self.WORK_ALLOWED]
self.namespaces = {
"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
}
@@ -58,39 +86,81 @@ class RedliningValidator:
)
return False
- try:
- modified_tree = ET.parse(modified_file)
- modified_root = modified_tree.getroot()
- original_tree = ET.parse(original_file)
- original_root = original_tree.getroot()
- except (ET.ParseError, DefusedXmlException) as e:
- print(f"FAILED - Error parsing XML files: {e}")
- return False
-
- new_changes = self._new_tracked_changes(original_root, modified_root)
- self._remove_tracked_changes(modified_root, new_changes)
-
- modified_text = self._extract_text_content(modified_root)
- original_text = self._extract_text_content(original_root)
-
- if modified_text != original_text:
- error_message = self._generate_detailed_diff(
- original_text, modified_text
- )
- print(error_message)
+ sides = (("original", temp_path), ("modified", self.unpacked_dir))
+ change_count, failures, differences = 0, [], []
+ for part in self._parts_to_compare(temp_path, self.unpacked_dir):
+ roots = []
+ for side, unpacked in sides:
+ try:
+ roots.append(self._parse_part(unpacked / part))
+ except XML_ERRORS as e:
+ failures.append(
+ f"FAILED - Error parsing {part} of the {side} document: {e}"
+ )
+ if len(roots) < len(sides):
+ continue
+ new_changes, original_text, modified_text = self._compare_part(*roots)
+ change_count += len(new_changes)
+ if modified_text != original_text:
+ differences.append((part, original_text, modified_text))
+
+ if differences:
+ failures.append(self._generate_detailed_diff(differences))
+ if failures:
+ print("\n".join(failures))
return False
if self.verbose:
print(
- f"PASSED - All {len(new_changes)} change(s) against the original "
+ f"PASSED - All {change_count} change(s) against the original "
"are properly tracked"
)
return True
+ OTHER_PARTS = (
+ "word/header*.xml",
+ "word/footer*.xml",
+ "word/footnotes.xml",
+ "word/endnotes.xml",
+ )
+
+ def _parts_to_compare(self, original_dir, modified_dir):
+ others = set()
+ for unpacked in (Path(original_dir), Path(modified_dir)):
+ for pattern in self.OTHER_PARTS:
+ others.update(
+ path.relative_to(unpacked).as_posix()
+ for path in unpacked.glob(pattern)
+ )
+ return ["word/document.xml", *sorted(others)]
+
+ def _compare_part(self, original_root, modified_root):
+ for root in (original_root, modified_root):
+ self._flatten_text_elements(root)
+ new_changes = self._new_tracked_changes(original_root, modified_root)
+ self._remove_tracked_changes(modified_root, new_changes)
+ return (
+ new_changes,
+ self._extract_text_content(original_root),
+ self._extract_text_content(modified_root),
+ )
+
+ def _parse_part(self, path):
+ if not path.is_file():
+ return ET.fromstring("<absent/>")
+ return ET.parse(path).getroot()
+
+ def _flatten_text_elements(self, root):
+ for elem in self._text_elements(root):
+ if len(elem):
+ elem.text = "".join(elem.itertext())
+ for child in list(elem):
+ elem.remove(child)
+
def _tracked_change_elements(self, root):
- ins_tag = f"{{{self.namespaces['w']}}}ins"
- del_tag = f"{{{self.namespaces['w']}}}del"
- return [elem for elem in root.iter() if elem.tag in (ins_tag, del_tag)]
+ w = self.namespaces["w"]
+ tags = (f"{{{w}}}ins", f"{{{w}}}del", f"{{{w}}}moveFrom", f"{{{w}}}moveTo")
+ return [elem for elem in root.iter() if elem.tag in tags]
def _rendered_text(self, elem):
preserve = elem.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve"
@@ -148,7 +218,8 @@ class RedliningValidator:
new.update(elems)
return new
- def _generate_detailed_diff(self, original_text, modified_text):
+ def _generate_detailed_diff(self, differences):
+ self._work_left = [self.WORK_ALLOWED]
error_parts = [
"FAILED - Document text doesn't match after removing the tracked changes",
"",
@@ -170,130 +241,238 @@ class RedliningValidator:
"",
]
- git_diff = self._get_git_word_diff(original_text, modified_text)
- if git_diff:
- error_parts.extend(["Differences:", "============", git_diff])
- else:
- error_parts.append("Unable to generate word diff (git not available)")
+ explain_page_fields = False
+ for part, original_text, modified_text in differences:
+ body = part == "word/document.xml"
+ heading = "Differences:" if body else f"Differences in {part}:"
+ if error_parts[-1]:
+ error_parts.append("")
+ word_diff = self._word_diff(original_text, modified_text)
+ error_parts.extend([heading, "=" * len(heading), word_diff])
+ if original_text.replace("\n", "") == modified_text.replace("\n", ""):
+ error_parts.append("Only paragraph breaks differ here.")
+ if re.search(r"\{(%s)\}" % "|".join(self.PAGE_FIELDS), word_diff):
+ explain_page_fields = True
+ if explain_page_fields:
+ error_parts.extend(["", self.PAGE_FIELD_NOTE])
+ if self._work_left[0] < 0:
+ error_parts.extend(["", self.WORK_SPENT_NOTE])
return "\n".join(error_parts)
- def _get_git_word_diff(self, original_text, modified_text):
- try:
- with tempfile.TemporaryDirectory() as temp_dir:
- temp_path = Path(temp_dir)
-
- original_file = temp_path / "original.txt"
- modified_file = temp_path / "modified.txt"
-
- original_file.write_text(original_text, encoding="utf-8")
- modified_file.write_text(modified_text, encoding="utf-8")
-
- result = subprocess.run(
- [
- "git",
- "diff",
- "--word-diff=plain",
- "--word-diff-regex=.",
- "-U0",
- "--no-index",
- str(original_file),
- str(modified_file),
- ],
- capture_output=True,
- text=True,
- )
+ def _word_diff(self, original_text, modified_text):
+ out = []
+ steps = self._steps(original_text.split("\n"), modified_text.split("\n"))
+ for tag, was, now in steps:
+ if tag == "replace" and len(was) == len(now):
+ out.extend(self._mark_changes(x, y) for x, y in zip(was, now) if x != y)
+ elif tag != "equal":
+ out.append(self._mark_changes("\n".join(was), "\n".join(now)))
+ return "\n".join(out)
+
+ def _mark_changes(self, old, new, by_character=False, trim=True):
+
+ def mark(text, opening, closing):
+ marked = re.sub(r"[^\n]+", lambda m: opening + m[0] + closing, text)
+ return marked if marked.strip("\n") or opening == "{+" else ""
+
+ pattern = "(?s)." if by_character else r"\S+\s*|\s+"
+ out = []
+ steps = self._steps(re.findall(pattern, old), re.findall(pattern, new), trim)
+ for tag, was, now in steps:
+ lone_word = len(was) == len(now) == 1
+ was, now = "".join(was), "".join(now)
+ if tag == "equal":
+ out.append(now)
+ elif tag == "replace" and not by_character:
+ long_word = lone_word and min(len(was), len(now)) > self.LONG_WORD
+ marked = self._mark_changes(was, now, by_character=True, trim=long_word)
+ out.append(marked)
+ else:
+ out.append(mark(was, "[-", "-]") + mark(now, "{+", "+}"))
+ return "".join(out)
- if result.stdout.strip():
- lines = result.stdout.split("\n")
- content_lines = []
- in_content = False
- for line in lines:
- if line.startswith("@@"):
- in_content = True
- continue
- if in_content and line.strip():
- content_lines.append(line)
-
- if content_lines:
- return "\n".join(content_lines)
-
- result = subprocess.run(
- [
- "git",
- "diff",
- "--word-diff=plain",
- "-U0",
- "--no-index",
- str(original_file),
- str(modified_file),
- ],
- capture_output=True,
- text=True,
- )
+ WORK_ALLOWED = 10_000_000
+ LONG_WORD = 200
- if result.stdout.strip():
- lines = result.stdout.split("\n")
- content_lines = []
- in_content = False
- for line in lines:
- if line.startswith("@@"):
- in_content = True
- continue
- if in_content and line.strip():
- content_lines.append(line)
- return "\n".join(content_lines)
+ def _steps(self, a, b, trim=True):
- except (subprocess.CalledProcessError, FileNotFoundError, Exception):
- pass
+ def shared_start(x, y):
+ return next(
+ (i for i, (p, q) in enumerate(zip(x, y)) if p != q), min(len(x), len(y))
+ )
- return None
+ head = shared_start(a, b) if trim else 0
+ tail = shared_start(a[head:][::-1], b[head:][::-1]) if trim else 0
+ a_mid, b_mid = a[head : len(a) - tail], b[head : len(b) - tail]
+ try:
+ if self._work_left[0] < len(a_mid):
+ self._work_left[0] = -1
+ raise _WorkSpent
+ numbers = {}
+ numbered = [
+ [numbers.setdefault(x, len(numbers)) for x in s] for s in (a_mid, b_mid)
+ ]
+ opcodes = _Matcher(*numbered, self._work_left).get_opcodes()
+ steps = [(tag, a_mid[i:j], b_mid[m:n]) for tag, i, j, m, n in opcodes]
+ except _WorkSpent:
+ steps = [("replace", a_mid, b_mid)]
+ return [
+ ("equal", a[:head], b[:head]),
+ *steps,
+ ("equal", a[len(a) - tail :], b[len(b) - tail :]),
+ ]
def _remove_tracked_changes(self, root, targets):
- ins_tag = f"{{{self.namespaces['w']}}}ins"
- del_tag = f"{{{self.namespaces['w']}}}del"
+ w = self.namespaces["w"]
+ ins_tag = f"{{{w}}}ins"
+ del_tag = f"{{{w}}}del"
+ move_from_tag = f"{{{w}}}moveFrom"
+ move_to_tag = f"{{{w}}}moveTo"
+
+ self._merge_split_paragraphs(root, targets)
for parent in root.iter():
to_remove = []
for child in parent:
- if child.tag == ins_tag and child in targets:
+ if child.tag in (ins_tag, move_to_tag) and child in targets:
to_remove.append(child)
for elem in to_remove:
parent.remove(elem)
- deltext_tag = f"{{{self.namespaces['w']}}}delText"
- t_tag = f"{{{self.namespaces['w']}}}t"
+ renamed = {
+ f"{{{w}}}delText": f"{{{w}}}t",
+ f"{{{w}}}delInstrText": f"{{{w}}}instrText",
+ }
for parent in root.iter():
to_process = []
for child in parent:
- if child.tag == del_tag and child in targets:
+ if child.tag in (del_tag, move_from_tag) and child in targets:
to_process.append((child, list(parent).index(child)))
for del_elem, del_index in reversed(to_process):
for elem in del_elem.iter():
- if elem.tag == deltext_tag:
- elem.tag = t_tag
+ elem.tag = renamed.get(elem.tag, elem.tag)
for child in reversed(list(del_elem)):
parent.insert(del_index, child)
parent.remove(del_elem)
+ range_tags = (
+ f"{{{w}}}moveFromRangeStart",
+ f"{{{w}}}moveFromRangeEnd",
+ f"{{{w}}}moveToRangeStart",
+ f"{{{w}}}moveToRangeEnd",
+ )
+ for parent in root.iter():
+ to_remove = [
+ c
+ for c in parent
+ if c.tag in range_tags and len(c) == 0 and not (c.text or "").strip()
+ ]
+ for elem in to_remove:
+ parent.remove(elem)
+
+ def _merge_split_paragraphs(self, root, targets):
+ w = self.namespaces["w"]
+ p_tag = f"{{{w}}}p"
+ ppr_tag = f"{{{w}}}pPr"
+ marker_path = f"{ppr_tag}/{{{w}}}rPr/{{{w}}}ins"
+
+ for parent in list(root.iter()):
+ for para in reversed([child for child in parent if child.tag == p_tag]):
+ if not any(m in targets for m in para.findall(marker_path)):
+ continue
+ siblings = list(parent)
+ index = siblings.index(para)
+ following = siblings[index + 1] if index + 1 < len(siblings) else None
+ if following is None or following.tag != p_tag:
+ continue
+ if any(self._text_elements(c) for c in para if c.tag == ppr_tag):
+ continue
+ insert_at = 1 if len(following) and following[0].tag == ppr_tag else 0
+ for child in reversed([c for c in para if c.tag != ppr_tag]):
+ following.insert(insert_at, child)
+ parent.remove(para)
+
def _extract_text_content(self, root):
p_tag = f"{{{self.namespaces['w']}}}p"
- t_tag = f"{{{self.namespaces['w']}}}t"
paragraphs = []
for p_elem in root.findall(f".//{p_tag}"):
- text_parts = []
- for t_elem in p_elem.findall(f".//{t_tag}"):
- text_parts.append(self._rendered_text(t_elem))
- paragraph_text = "".join(text_parts)
+ paragraph_text = self._paragraph_text(p_elem)
if paragraph_text:
paragraphs.append(paragraph_text)
return "\n".join(paragraphs)
+ PAGE_FIELDS = {"PAGE", "NUMPAGES", "SECTIONPAGES"}
+ PAGE_NUMBER = re.compile(r"-? ?[0-9]{1,6} ?-?|[ivxlcdm]{1,8}|[a-z]{1,2}|", re.I)
+ PAGE_FIELD_NOTE = (
+ "Note: {PAGE}, {NUMPAGES} or {SECTIONPAGES} above stands for a page-number\n"
+ "field. Word recomputes the number it displays, so the number stored in\n"
+ "the file is not compared, but adding or removing the field still counts."
+ )
+
+ WORK_SPENT_NOTE = (
+ "Note: the texts were too long or repetitive to compare closely all the\n"
+ "way through, so a passage shown whole, as removed and then added, may\n"
+ "differ in only a few characters. Compare such a passage with the\n"
+ "original directly to find the edit."
+ )
+
+ def _paragraph_text(self, p_elem):
+ w = self.namespaces["w"]
+ t_tag, instr_tag = f"{{{w}}}t", f"{{{w}}}instrText"
+ fld_char, fld_simple = f"{{{w}}}fldChar", f"{{{w}}}fldSimple"
+
+ def page_field(instruction, field):
+ words = instruction.split()
+ locked = field.get(f"{{{w}}}fldLock") in ("1", "true", "on")
+ if words and words[0].upper() in self.PAGE_FIELDS and not locked:
+ return words[0].upper()
+
+ def shown(name, result):
+ return "{%s}" % name if self.PAGE_NUMBER.fullmatch(result) else result
+
+ parts, fields = [], []
+ skipped = set()
+
+ def sink():
+ for *_, result in reversed(fields):
+ if result is not None:
+ return result
+ return parts
+
+ for elem in p_elem.iter():
+ if elem.tag == fld_simple:
+ name = page_field(elem.get(f"{{{w}}}instr", ""), elem)
+ if name:
+ texts = [self._rendered_text(t) for t in elem.iter(t_tag)]
+ sink().append(shown(name, "".join(texts).strip()))
+ skipped.update(elem.iter(t_tag))
+ elif elem.tag == fld_char:
+ kind = elem.get(f"{{{w}}}fldCharType")
+ if kind == "begin":
+ fields.append([elem, "", False, None])
+ elif kind == "separate" and fields and not fields[-1][2]:
+ fields[-1][2] = True
+ if page_field(fields[-1][1], fields[-1][0]):
+ fields[-1][3] = []
+ elif kind == "end" and fields:
+ begin, instruction, _, result = fields.pop()
+ name = page_field(instruction, begin)
+ if name:
+ sink().append(shown(name, "".join(result or []).strip()))
+ elif elem.tag == instr_tag and fields and not fields[-1][2]:
+ fields[-1][1] += elem.text or ""
+ elif elem.tag == t_tag and elem not in skipped:
+ sink().append(self._rendered_text(elem))
+ for *_, result in fields:
+ parts.extend(result or [])
+ return "".join(parts)
+
if __name__ == "__main__":
raise RuntimeError("This module should not be run directly.")
diff --git a/skills/pdf/forms.md b/skills/pdf/FORMS.md
index 6e7e1e0..1349385 100644
--- a/skills/pdf/forms.md
+++ b/skills/pdf/FORMS.md
@@ -1,7 +1,7 @@
**CRITICAL: You MUST complete these steps in order. Do not skip ahead to writing code.**
If you need to fill out a PDF form, first check to see if the PDF has fillable form fields. Run this script from this file's directory:
- `python scripts/check_fillable_fields <file.pdf>`, and depending on the result go to either the "Fillable fields" or "Non-fillable fields" and follow those instructions.
+ `python scripts/check_fillable_fields.py <file.pdf>`, and depending on the result go to either the "Fillable fields" or "Non-fillable fields" and follow those instructions.
# Fillable fields
If the PDF has fillable form fields:
diff --git a/skills/pdf/reference.md b/skills/pdf/REFERENCE.md
index 41400bf..7748e37 100644
--- a/skills/pdf/reference.md
+++ b/skills/pdf/REFERENCE.md
@@ -39,7 +39,8 @@ import pypdfium2 as pdfium
pdf = pdfium.PdfDocument("document.pdf")
for i, page in enumerate(pdf):
- text = page.get_text()
+ textpage = page.get_textpage()
+ text = textpage.get_text_range()
print(f"Page {i+1} text length: {len(text)} chars")
```
@@ -533,7 +534,7 @@ with open("cropped.pdf", "wb") as output:
- Process pages individually with pypdfium2
### 2. For Text Extraction
-- `pdftotext -bbox-layout` is fastest for plain text extraction
+- `pdftotext` is fastest for plain text extraction
- Use pdfplumber for structured data and tables
- Avoid `pypdf.extract_text()` for very large documents
diff --git a/skills/pptx/SKILL.md b/skills/pptx/SKILL.md
index a90df9c..785fea6 100644
--- a/skills/pptx/SKILL.md
+++ b/skills/pptx/SKILL.md
@@ -10,9 +10,9 @@ A `.pptx` is a ZIP archive of XML files. Choose your approach by task:
| Task | Approach |
|---|---|
-| **Create** a new deck | Write a `pptxgenjs` script — see gotchas below |
+| **Create** a new deck | Write a `pptxgenjs` script — see gotchas below — and build it as a [structured deck](#structured-decks) unless the user asks for a quick throwaway |
| **Edit** an existing deck, or build from a template | unzip → edit `ppt/slides/slideN.xml` → zip |
-| **Read** content | `markitdown deck.pptx` (one block per slide under `<!-- Slide number: N -->` markers); visual grid: `python scripts/thumbnail.py deck.pptx` |
+| **Read** content | `python scripts/slide_text.py deck.pptx` (one block per slide under `<!-- Slide number: N -->` markers); visual grid: `python scripts/thumbnail.py deck.pptx` |
## Scripts
@@ -20,9 +20,11 @@ Paths are relative to this skill's directory. Everything else is plain Python, `
| Script | What it does |
|---|---|
+| `scripts/slide_text.py deck.pptx` | Every slide's title, text, tables, chart data (scatter and bubble charts: series names only), picture alt text and speaker notes, as markdown. Reads `.potx` too |
| `scripts/thumbnail.py deck.pptx [prefix]` | Labeled grid of every slide, for picking template layouts. `.pptx` only. Pass `prefix` — it defaults to `thumbnails`, which overwrites the grids of any other deck done in the same directory |
| `scripts/add_slide.py unpacked/ slide2.xml [--after slideN.xml]` | Duplicate a slide (or a `slideLayoutN.xml`) with all the package bookkeeping. Also takes a `.pptx` directly with `-o out.pptx` |
| `scripts/clean.py unpacked/` | Delete slides, media, and rels no longer referenced. Run **after** `<p:sldIdLst>` is final |
+| `scripts/apply_theme.js` | Node module: `const { applyTheme } = require("<this skill's directory>/scripts/apply_theme.js")`. `await applyTheme("deck.pptx", THEME)` after `pres.writeFile()` writes `THEME`'s colors and name (escaped for XML) into the deck's theme, for [structured decks](#structured-decks). Also a CLI: `node scripts/apply_theme.js deck.pptx theme.json` |
| `scripts/office/validate.py deck.pptx [--original src.pptx]` | Schema, relationship, content-type, chart and slide checks; each failure names its fix. Pass `--original` for any template-derived deck — it baselines the schema checks against the template, so the template's own XSD errors don't read as yours |
| `scripts/office/soffice.py --headless --convert-to pdf deck.pptx` | LibreOffice wrapper — bare `soffice` hangs in this sandbox |
@@ -39,19 +41,34 @@ Paths are relative to this skill's directory. Everything else is plain Python, `
- **One `new pptxgen()` per output file** — never reuse an instance.
- **`rectRadius` only works on `ROUNDED_RECTANGLE`**, not `RECTANGLE`.
- **Gradient fills aren't supported** — use a gradient image as the background instead.
+- **Every `addText` call without a `placeholder` needs `isTextBox: true`** — without it the shape lacks `txBox="1"`, so screen readers announce the text as a "graphic" instead of a text box. No visual change.
- **Text boxes have built-in internal padding** — set `margin: 0` whenever text must align with a shape, line, or icon at the same x.
- **Speaker notes go in `slide.addNotes("...")`** (plain text, once per slide), never in a text box on the slide.
- **Keep charts native.** Use `addChart()` for everything PowerPoint can chart (pass an array of `{type, data, options}` for combos). For PowerPoint-native features the library doesn't expose (trendlines, error bars), compute the extra series yourself or post-process the generated OOXML — do not fall back to a rendered image. Only chart types PowerPoint has no native form for (Sankey, network, chord) go in as images.
- **Default charts render bare** — no title, no data labels, dated palette. Set `showTitle` + `title`, `showValue: true` + `dataLabelPosition`, `chartColors: [...]` from your palette, and quiet the frame (`catAxisLabelColor`/`valAxisLabelColor`, `valGridLine: { color, size }`, `catGridLine: { style: "none" }`, `showLegend: false` for a single series).
- **On a stacked bar or column chart, `dataLabelPosition` must be `ctr`, `inEnd`, or `inBase`.** `outEnd` **corrupts the file**.
- **A combo series using `secondaryValAxis`/`secondaryCatAxis` needs both `valAxes` and `catAxes` on the chart options, two entries each.** Without them pptxgenjs writes axis *ids* it never declares, and PowerPoint **discards that chart** and reports the file as corrupt. Supplying only `valAxes` is not enough.
-- **After `writeFile()`, run `python scripts/office/validate.py deck.pptx`.** It reports the two chart faults above and the slide-XML defects PowerPoint refuses, and names the fix for each. Fix them in your generator, not by hand-editing the packed XML.
+- **After `writeFile()`, run `python scripts/office/validate.py deck.pptx`.** It reports many of the chart and slide-XML faults PowerPoint refuses or repairs, the two above among them, and names the fix for each. Fix them in your generator, not by hand-editing the packed XML.
- **Never reorder the children of `<p:presentation>`.** pptxgenjs writes `<p:notesMasterIdLst>` right after `<p:sldIdLst>` and points both masters at one theme part. PowerPoint reads that happily — move the element and the same deck becomes unopenable.
- **Icons:** render `react-icons` to SVG (`ReactDOMServer.renderToStaticMarkup`), rasterize with `sharp` at ≥256px, and insert via `addImage({ data: "image/png;base64," + buf.toString("base64") })` — the `image/png;base64,` prefix is required (`react-icons`, `react`, `react-dom`, and `sharp` are preinstalled — `npm install react-icons react react-dom sharp` only if a require fails).
+- **`defineSlideMaster()` creates a slide *layout*, not a master** — one `slideLayoutN.xml` per call, under a single master you cannot change. Call them layouts when talking to the user.
+- **`pres.company` and a `defineSlideMaster()` `title` are written into the XML unescaped** — an `&` in either (`"Harrow & Sons"`) leaves `docProps/app.xml` or the layout unparseable, and `validate.py` fails. Write `&amp;` there, and in the matching `masterName`. `title`, `subject`, `author`, section titles, `objectName`, notes and slide text are escaped for you.
+
+### Structured decks
+
+A deck built on a theme, layouts, placeholders and sections can be restyled in one place and edited in PowerPoint without fighting it. Build every new deck this way unless the user asks for a quick throwaway.
+
+1. **Theme first:** `const THEME = { name, headFontFace, bodyFontFace, colors: { dk1, lt1, dk2, lt2, accent1 … accent6, hlink, folHlink } }`, colors in hex. `pres.theme` takes the two fonts and nothing else. Name no font after that — titles get the heading font, other text the body font — except on chart text, which is hard-coded Arial: give its `*FontFace` options `"+mn-lt"` (theme body font; `"+mj-lt"` is the heading font).
+2. **`const C = pres.SchemeColor` for every color** — `C.text1`/`C.text2` are `dk1`/`dk2`, `C.background1`/`C.background2` are `lt1`/`lt2`. These take hex only, read from `THEME`: `valGridLine.color`, `catGridLine.color`, `shadow.color`, and on a one-series bar chart `invertedColors` or more than one `chartColors`. A scheme color there is written as `<a:srgbClr val="bg2"/>` without complaint, and being hex, these will not follow a later change of theme.
+3. **Sections:** `pres.addSection({ title })` just before that section's first slide, then `sectionTitle` on every `addSlide`. A slide without one — or, when sections are all declared up front, a table's auto-paged continuation slides — lands in a stray "Default-N" section.
+4. **One layout per slide frame** (`defineSlideMaster()`): title, section divider, title only, chart + commentary, table; dark and light variants if the design alternates. Text regions are named placeholders carrying position, size, weight and `color` — chart, image and table placeholders need a `color` too, or prompt text and arrays of runs come out hard-coded `000000`. Footer text, logo and `slideNumber` go on the layout, never on a slide.
+5. **Fill by name:** `addSlide({ masterName, sectionTitle })`, then `addText("…", { placeholder: "title" })` with no position, size or color — the placeholder's own options win, and anything it doesn't set (italic, `align`, `fontFace`) still applies; style one run with `[{ text, options }]`. A misspelt `placeholder` or `masterName` fails silently (a stray text box at 0,0; the blank layout). Only `title`, `body` and `chart` placeholders fill by name alone: `addImage({ placeholder })` takes the placeholder's x and y but not its size (a 1" × 1" picture) — pass `w`/`h` too. `addTable({ placeholder })` takes its x and y only — always pass `w`; when it auto-pages it ignores the placeholder and runs off the slide, so also pass `x`, `y` and `autoPageSlideStartY` (same as `y`, or continuation pages are laid out from the top margin), give the layout a `margin` whose bottom clears the footer, and fill the continuation slides' titles through `slide.newAutoPagedSlides`.
+6. **A layout is the frame, not the "layout" Design Ideas means.** Cards, icons, stat callouts and images are still composed slide by slide on top of it, in scheme colors, each with an `objectName` — every Design Idea applies, and most content slides sit on the title-only layout.
+7. **Write, then apply the theme:** `await pres.writeFile({ fileName: "deck.pptx" })`, then `await applyTheme("deck.pptx", THEME)`. pptxgenjs cannot write theme colors: until this runs, scheme colors resolve to Office's stock palette. It throws, changing nothing, on a scheme color in a hex-only option — fix what it names and rebuild.
## Editing existing decks and templates
-Pick layouts first: `python scripts/thumbnail.py template.pptx template-thumbs` writes a labeled grid of every slide and prints the file(s) it created — `template-thumbs.jpg`, split into `template-thumbs-N.jpg` past 12 slides. **Always pass that second argument, named after the deck.** It defaults to `thumbnails`, so two decks thumbnailed in one directory silently overwrite each other's grids — the first deck's are simply gone (template analysis only — visual QA needs the full-resolution renders from [Converting to Images](#converting-to-images); it only accepts `.pptx`, so copy a `.potx` to a `.pptx` name first). Use it with `markitdown` to map each content section onto a template slide, and vary the layouts — don't put every section on the same title-and-bullets slide.
+Pick layouts first: `python scripts/thumbnail.py template.pptx template-thumbs` writes a labeled grid of every slide and prints the file(s) it created — `template-thumbs.jpg`, split into `template-thumbs-N.jpg` past 12 slides. **Always pass that second argument, named after the deck.** It defaults to `thumbnails`, so two decks thumbnailed in one directory silently overwrite each other's grids — the first deck's are simply gone (template analysis only — visual QA needs the full-resolution renders from [Converting to Images](#converting-to-images); it only accepts `.pptx`, so copy a `.potx` to a `.pptx` name first). Use it with `slide_text.py` to map each content section onto a template slide, and vary the layouts — don't put every section on the same title-and-bullets slide.
```bash
python3 -c "import sys,zipfile; zipfile.ZipFile(sys.argv[1]).extractall('unpacked')" deck.pptx
@@ -59,7 +76,7 @@ python scripts/add_slide.py unpacked/ slide2.xml --after slide2.xml # duplicat
# reorder / delete slides = edit <p:sldIdLst> in ppt/presentation.xml
python scripts/clean.py unpacked/ # after deletions: removes orphaned slides, media, rels
# edit slide content in ppt/slides/slideN.xml
-(cd unpacked && rm -f ../out.pptx && zip -Xr ../out.pptx .) # zip from INSIDE the dir; rm first or deleted parts survive
+(cd unpacked && rm -f ../out.pptx && zip -XrD ../out.pptx .) # zip from INSIDE the dir; rm first or deleted parts survive
python scripts/office/validate.py out.pptx --original deck.pptx
```
@@ -77,6 +94,15 @@ When filling in a template:
- Let bullets inherit from the layout; only add `<a:buChar>`, `<a:buAutoNum>` (numbered), or `<a:buNone>` to override — never a literal `•` in the text.
- Text with leading or trailing spaces needs `xml:space="preserve"` on its `<a:t>`.
+## Deck content and flow
+
+- **One message per slide.** A deck goes theme by theme, message by message.
+- **A common structure is the pyramid:** the main message up front, an executive summary that previews the sections, the sections themselves, then a conclusion, usually with next steps that act on the findings.
+- **Match the structure to the size of the deck.** A short deck might not need an introduction and a conclusion; a long deck might need dividers between topics.
+- **Build toward what the deck is for:** a decision, consensus, a celebration, resolving a disagreement, or opening a discussion.
+- **Build for how it will be used.** A live deck gets narration and possibly discussion; a pre-read has to carry the whole narrative itself. An executive audience in a live meeting needs few words and highly visual slides. Detailed, word-heavy slides are sometimes right for subject-matter experts or for a deck read offline. Analytical reports lean on charts and tables.
+- **Size a live deck to the talk:** about 1.5 to 2 minutes per slide, with time left for questions if there will be any.
+
## Design Ideas
**Don't create boring slides.** Plain bullets on a white background won't impress anyone. Consider ideas from this list for each slide.
@@ -150,6 +176,9 @@ Choose colors that match your topic — don't default to generic blue. Use these
### Avoid (Common Mistakes)
- **Don't repeat the same layout** — vary columns, cards, and callouts across slides
+- **Don't end a slide title with a period**
+- **Don't let titles drift between slides of the same type** — content slides with the title at the top put it at the same position, font, size, and alignment on every slide; a different slide type (a section divider with the title mid-slide) can place it differently
+- **Don't change how chart labels look from chart to chart** — axis and data labels use the same color, font, and size on every chart in the deck; vary one only where a chart needs it (a second axis, a highlighted series, a label sitting on a dark bar)
- **Don't center body text** — left-align paragraphs and lists; center only titles
- **Don't skimp on size contrast** — titles need 36pt+ to stand out from 14-16pt body
- **Don't default to blue** — pick colors that reflect the specific topic
@@ -161,16 +190,16 @@ Choose colors that match your topic — don't default to generic blue. Use these
- **NEVER use accent lines under titles** — these are a hallmark of AI-generated slides; use whitespace or background color instead
- **NEVER add decorative color bars or accent stripes** — this includes: header/footer bars spanning the slide width, vertical sidebar stripes down one edge of the slide, thin accent stripes along one edge of a card or content block, and "single-side borders" on rectangles. These read as AI-generated filler. If you want to set a card apart, use a subtle background tint, a drop shadow, or an icon — not an edge stripe.
- **Don't default to cream/beige backgrounds** — when no background is specified, use white (`FFFFFF`) or the user's brand palette; avoid warm-neutral defaults like `F5F5DC`, `FAF0E6`, `FAEBD7`, `FFF8E1`
-- **Don't ship text that overflows its shape** — if text doesn't fit, reduce font size, split across slides, or enlarge the container; never leave content cut off or spilling past bounds
+- **Don't ship text that overflows its shape** — if text doesn't fit, reduce font size, split across slides, or enlarge the container; never leave content cut off or spilling past bounds. **Don't shrink below the sizes in the Typography table** (body text 14pt, captions 10pt) in a deck you create — once you reach that floor, cut words, enlarge the container, or split the slide instead; when editing an existing deck or template, match its sizes
## QA (Required)
Your first render usually has a few real issues — overlaps, overflow, misalignment. Find and fix those, re-render only the slides you changed, and stop.
-### Content QA
+### Content QA (required)
```bash
-markitdown output.pptx
+python scripts/slide_text.py output.pptx
```
Check for missing content, typos, wrong order.
@@ -178,7 +207,7 @@ Check for missing content, typos, wrong order.
**When using templates, check for leftover placeholder text:**
```bash
-markitdown output.pptx | grep -iE "\bx{3,}\b|lorem|ipsum|\bTODO|\[insert|this.*(page|slide).*layout"
+python scripts/slide_text.py output.pptx | grep -iE "\bx{3,}\b|lorem|ipsum|\bTODO|\[insert|this.*(page|slide).*layout"
```
If grep returns results, fix them before declaring success.
@@ -201,7 +230,9 @@ pptxgenjs emits chart XML PowerPoint refuses to open, and every other tool
accepts: python-pptx opens those decks, LibreOffice renders them, the XSD
passes them. Every failure names its fix. Fix it in the generator and rebuild.
-### Visual QA
+**A pass is not proof of a valid file:** `validate.py` does not catch everything PowerPoint repairs (a chart element repeated past its limit passes); when you delete an animated shape from a template slide, delete the slide's `<p:timing>` element too (its animations go with it), or PowerPoint offers to repair the deck.
+
+### Visual QA (required)
Convert the slides to images (see [Converting to Images](#converting-to-images)) and inspect every one. After staring at the generating code you tend to see what you expect rather than what rendered, so look at the images fresh (a subagent works well for this if you have one). User-visible defects to look for:
@@ -216,6 +247,8 @@ Convert the slides to images (see [Converting to Images](#converting-to-images))
- Template decoration mispositioned after text replacement — e.g., a title underline positioned for one line, but the replaced title wrapped to two
- Low-contrast icons (e.g., dark icons on dark backgrounds without a contrasting circle)
- Text boxes too narrow causing excessive wrapping
+- Titles at different heights, or chart labels that look different from chart to chart, across slides of the same type — compare slides with each other, not only each slide with itself
+- A word or number broken across two lines — a word split mid-word, or a figure separated from its unit ("26" on one line, "M" on the next)
- Leftover placeholder content
## Converting to Images
@@ -235,4 +268,4 @@ ls -1 "$PWD"/slide-*.jpg
## Dependencies
-`pptxgenjs` (npm, preinstalled — install only if `require('pptxgenjs')` fails) · `markitdown[pptx]`, `Pillow`, `defusedxml`, `lxml` (pip — text dump, thumbnail, clean, validate) · LibreOffice (`soffice`, auto-configured for sandboxed environments via `scripts/office/soffice.py`) · `pdftoppm` (Poppler)
+`pptxgenjs` (npm, preinstalled — install only if `require('pptxgenjs')` fails) · `python-pptx`, `Pillow`, `defusedxml`, `lxml` (pip — slide_text, thumbnail, clean, validate) · LibreOffice (`soffice`, auto-configured for sandboxed environments via `scripts/office/soffice.py`) · `pdftoppm` (Poppler)
diff --git a/skills/pptx/scripts/add_slide.py b/skills/pptx/scripts/add_slide.py
index f013ea9..9b418bc 100755
--- a/skills/pptx/scripts/add_slide.py
+++ b/skills/pptx/scripts/add_slide.py
@@ -7,6 +7,7 @@ Does all of the package bookkeeping, so the deck stays valid:
- adds a slide relationship with a fresh rId to presentation.xml.rels
- inserts <p:sldId id="..." r:id="..."/> with a fresh id into
<p:sldIdLst> — at the end, or after --after SLIDE
+ - counts the new slide in docProps/app.xml
Works on an unpacked directory (during an editing session) or directly on a
.pptx/.potx file (extracted to a temp dir, then rezipped atomically; the
@@ -25,6 +26,7 @@ A duplicated slide still holds the source's content: edit ppt/slides/slideN.xml
"""
import argparse
+import posixpath
import re
import shutil
import sys
@@ -32,8 +34,12 @@ from typing import NoReturn
import tempfile
import zipfile
from pathlib import Path
+from xml.parsers.expat import ExpatError
-from office.helpers import rezip, safe_extract
+import defusedxml.minidom
+from defusedxml.common import DefusedXmlException
+
+from office.helpers import opc_target, rezip, safe_extract
MINIMAL_SLIDE_XML = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<p:sld xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main">
@@ -62,6 +68,8 @@ MINIMAL_SLIDE_XML = '''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
SHARED_PART_TYPES = ("chart", "diagramData", "oleObject", "package")
NOTES_SLIDE_TYPE_RE = re.compile(r"""Type=["'][^"']*/relationships/notesSlide["']""")
+TAGS_TYPE_RE = re.compile(r"""Type=["'][^"']*/relationships/tags["']""")
+TAGS_CONTENT_TYPE = "application/vnd.openxmlformats-officedocument.presentationml.tags+xml"
RELATIONSHIP_RE = re.compile(r"<Relationship\b[^>]*?(?:/>|>.*?</Relationship\s*>)", re.DOTALL)
SLIDE_ID_MIN = 256
@@ -136,6 +144,11 @@ def duplicate_slide(unpacked_dir: Path, source: str, after: str | None = None) -
lambda m: "" if NOTES_SLIDE_TYPE_RE.search(m.group(0)) else m.group(0),
rels_content,
)
+ rels_content = RELATIONSHIP_RE.sub(
+ lambda m: _with_own_tags_part(unpacked_dir, m.group(0), f"ppt/slides/{dest}")
+ if TAGS_TYPE_RE.search(m.group(0)) else m.group(0),
+ rels_content,
+ )
dest_rels.write_text(rels_content, encoding="utf-8")
shared_parts = sorted({
t for t in re.findall(r'Type="[^"]*/relationships/(\w+)"', rels_content)
@@ -151,6 +164,32 @@ def duplicate_slide(unpacked_dir: Path, source: str, after: str | None = None) -
return dest
+def _with_own_tags_part(unpacked_dir: Path, relationship: str, slide_part: str) -> str:
+ target = re.search(r"""Target=["']([^"']*)["']""", relationship)
+ try:
+ source = opc_target(target.group(1), slide_part) if target else None
+ except ValueError:
+ source = None
+ if source is None or not (unpacked_dir / source).is_file():
+ return relationship
+
+ tags_dir = (unpacked_dir / source).parent
+ content_types_path = unpacked_dir / "[Content_Types].xml"
+ content_types = content_types_path.read_text(encoding="utf-8")
+ names = [f.name for f in tags_dir.iterdir()] + re.findall(r"""PartName=["'][^"']*/([^/"']+)["']""", content_types)
+ taken = [int(m.group(1)) for name in names if (m := re.fullmatch(r"tag(\d{1,9})\.xml", name, re.I))]
+ copy = tags_dir / f"tag{max(taken, default=0) + 1}.xml"
+ shutil.copy2(unpacked_dir / source, copy)
+
+ part = copy.relative_to(unpacked_dir).as_posix()
+ override = f'<Override PartName="/{part}" ContentType="{TAGS_CONTENT_TYPE}"/>'
+ content_types_path.write_text(
+ content_types.replace("</Types>", f" {override}\n</Types>"), encoding="utf-8"
+ )
+ relative = posixpath.relpath(part, posixpath.dirname(slide_part))
+ return relationship.replace(target.group(0), f'Target="{relative}"')
+
+
def _precheck_registration(unpacked_dir: Path, after: str | None, dest: str) -> str | None:
pres_path = unpacked_dir / "ppt" / "presentation.xml"
if not pres_path.exists():
@@ -194,6 +233,8 @@ def _register_slide(unpacked_dir: Path, dest: str, source_desc: str, after_rid:
slide_id = _get_next_slide_id(unpacked_dir)
pos, total = _insert_into_sld_id_lst(unpacked_dir, slide_id, rid, after_rid)
+ _count_slide_in_app_properties(unpacked_dir, pos)
+
print(f"Created ppt/slides/{dest} from {source_desc}")
print(
f'Inserted <p:sldId id="{slide_id}" r:id="{rid}"/> into <p:sldIdLst> '
@@ -201,6 +242,61 @@ def _register_slide(unpacked_dir: Path, dest: str, source_desc: str, after_rid:
)
+def _count_slide_in_app_properties(unpacked_dir: Path, position: int) -> None:
+ path = unpacked_dir / "docProps" / "app.xml"
+ if not path.is_file():
+ return
+ try:
+ dom = defusedxml.minidom.parse(str(path))
+ slides = _only_child(dom.documentElement, "Slides")
+ was = int(slides.firstChild.data)
+ if not _add_slide_title(dom, was, position):
+ return
+ except (AttributeError, LookupError, ValueError, TypeError, OSError, ExpatError, DefusedXmlException):
+ return
+ slides.firstChild.data = str(was + 1)
+ path.write_bytes(dom.toxml(encoding="UTF-8", standalone=dom.standalone))
+
+
+def _add_slide_title(dom, was: int, position: int) -> bool:
+ heading_pairs = _only_child(dom.documentElement, "HeadingPairs")
+ titles_of_parts = _only_child(dom.documentElement, "TitlesOfParts")
+ if heading_pairs is None and titles_of_parts is None:
+ return True
+ headings, titles = _only_child(heading_pairs, "vector"), _only_child(titles_of_parts, "vector")
+ pairs = [v.getElementsByTagName("*")[0] for v in _elements(headings)]
+ names = [n.firstChild.data if n.firstChild else "" for n in pairs[0::2]]
+ counts = pairs[1::2]
+ sizes = [int(c.firstChild.data) for c in counts]
+ entries = _elements(titles)
+ consistent = (
+ int(headings.getAttribute("size")) == 2 * len(counts) == len(_elements(headings))
+ and int(titles.getAttribute("size")) == len(entries) == sum(sizes)
+ )
+ titled = [i for i, name in enumerate(names) if name == "Slide Titles"]
+ as_long = [i for i in titled or range(len(sizes)) if sizes[i] == was > 0]
+ if not consistent or len(as_long) != 1:
+ return False
+ (group,) = as_long
+
+ title = entries[0].cloneNode(False)
+ title.appendChild(dom.createTextNode(f"Slide {was + 1}"))
+ at = sum(sizes[:group]) + min(position - 1, was)
+ titles.insertBefore(title, entries[at] if at < len(entries) else None)
+ titles.setAttribute("size", str(len(entries) + 1))
+ counts[group].firstChild.data = str(was + 1)
+ return True
+
+
+def _elements(node) -> list:
+ return [c for c in node.childNodes if c.nodeType == c.ELEMENT_NODE]
+
+
+def _only_child(node, local_name: str):
+ found = [c for c in _elements(node) if c.localName == local_name] if node else []
+ return found[0] if len(found) == 1 else None
+
+
def _add_to_content_types(unpacked_dir: Path, dest: str) -> None:
content_types_path = unpacked_dir / "[Content_Types].xml"
content_types = content_types_path.read_text(encoding="utf-8")
diff --git a/skills/pptx/scripts/apply_theme.js b/skills/pptx/scripts/apply_theme.js
new file mode 100644
index 0000000..ea6e45a
--- /dev/null
+++ b/skills/pptx/scripts/apply_theme.js
@@ -0,0 +1,107 @@
+// Give a deck pptxgenjs wrote its own theme colors and theme name. pptxgenjs sets the theme's
+// two fonts (pres.theme) but hard-codes Office's palette, so scheme colors resolve to Office
+// blue and orange until the deck's colors are written into ppt/theme/theme1.xml.
+//
+// const { applyTheme } = require("<this skill's directory>/scripts/apply_theme.js");
+// await pres.writeFile({ fileName: "deck.pptx" });
+// await applyTheme("deck.pptx", {
+// name: "Northwind",
+// colors: { dk1, lt1, dk2, lt2, accent1, accent2, accent3, accent4, accent5, accent6, hlink, folHlink },
+// }); // extra keys (headFontFace, ...) are ignored, and the twelve colors may sit beside name instead
+//
+// node scripts/apply_theme.js deck.pptx theme.json [out.pptx] # the same, from a shell
+//
+// Colors are six hex digits, no "#". The deck is rewritten in place unless a third argument
+// names another file. It throws, changing nothing, if a color anywhere in the deck is not six
+// hex digits, which is what a scheme color passed to a hex-only option turns into.
+//
+// As the step that rewrites the deck after pres.writeFile(), it also handles two faults pptxgenjs writes
+// that make PowerPoint offer to repair the deck: it mends <c:crossesAt val="autoZero"/>, and it
+// throws on a line dash that is not a dash style.
+const fs = require("fs");
+
+// jszip comes with pptxgenjs but is not always installed where a bare require("jszip") finds
+// it, so it is loaded from beside pptxgenjs. Both are found the way any require() in this file
+// would find them: from this file's directory upward, then NODE_PATH and Node's global folders.
+// No search starts at the working directory or the calling script's directory.
+function loadJSZip() {
+ try {
+ return require(require.resolve("jszip", { paths: [require.resolve("pptxgenjs")] }));
+ } catch {
+ throw new Error(`pptxgenjs and the jszip that comes with it cannot be found from ${__filename}. If pptxgenjs is installed beside your own script, run node with NODE_PATH set to that node_modules folder. Nothing was changed.`);
+ }
+}
+
+const SLOTS = ["dk1", "lt1", "dk2", "lt2", "accent1", "accent2", "accent3", "accent4", "accent5", "accent6", "hlink", "folHlink"];
+// A name or value from the deck, for an error a model reads: quoted, cut, and with nothing unprintable left raw.
+const shown = (s) => JSON.stringify(String(s).slice(0, 80)).replace(/[\p{C}\u2028\u2029]/gu, (c) => "\\u{" + c.codePointAt(0).toString(16) + "}");
+const DASHES = ["solid", "dot", "dash", "lgDash", "dashDot", "lgDashDot", "lgDashDotDot", "sysDash", "sysDot", "sysDashDot", "sysDashDotDot"];
+
+function themeXml(xml, theme) {
+ if (!theme || !theme.name) throw new Error("theme.name is required. Nothing was changed.");
+ // The name goes into XML attributes: drop control characters, escape & < > and the double quote.
+ const name = String(theme.name).replace(/[\x00-\x08\x0B\x0C\x0E-\x1F]/g, "").replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+ const colors = theme.colors || theme; // { name, colors: {...} }, or flat { name, dk1, ... }
+ for (const k of SLOTS) {
+ if (!/^[0-9A-Fa-f]{6}$/.test(String(colors[k]))) throw new Error(`theme.colors.${k} is ${JSON.stringify(colors[k])}; it must be six hex digits with no "#" (slots: ${SLOTS.join(" ")}). Nothing was changed.`);
+ }
+ const scheme = `<a:clrScheme name="${name}">` + SLOTS.map((k) => `<a:${k}><a:srgbClr val="${String(colors[k]).toUpperCase()}"/></a:${k}>`).join("") + "</a:clrScheme>";
+ // Function replacements, so a "$" in the name is not read as a replacement pattern.
+ // The theme, its color scheme and its font scheme each carry a name; none should still say Office.
+ const out = xml
+ .replace(/<a:clrScheme\b[\s\S]*?<\/a:clrScheme>/, () => scheme)
+ .replace(/(<a:(?:theme|fontScheme)\b[^>]*?\bname=")[^"]*"/g, (_, head) => `${head}${name}"`);
+ if (!out.includes(scheme)) throw new Error("ppt/theme/theme1.xml has no <a:clrScheme> to replace; this script expects the theme part pptxgenjs writes. Nothing was changed.");
+ return out;
+}
+
+async function applyTheme(deckPath, theme, outPath = deckPath) {
+ const zip = await loadJSZip().loadAsync(fs.readFileSync(deckPath));
+ const part = "ppt/theme/theme1.xml"; // the one theme part pptxgenjs writes; the slide master and the notes master share it
+ if (!zip.file(part)) throw new Error(`${deckPath} has no ${part}; this script is for decks pptxgenjs wrote. Nothing was changed.`);
+ zip.file(part, themeXml(await zip.file(part).async("string"), theme));
+
+ // A scheme color given to a hex-only option is written as <a:srgbClr val="bg2"/>, which is
+ // not a color. pptxgenjs does not warn; stop here instead.
+ for (const name of Object.keys(zip.files)) {
+ if (!name.endsWith(".xml")) continue;
+ const xml = await zip.file(name).async("string");
+ const bad = xml.match(/<a:srgbClr val="((?![0-9A-Fa-f]{6}")[^"]*)"/);
+ if (bad) throw new Error(`${shown(name)} has an <a:srgbClr> whose val is ${shown(bad[1])}, not six hex digits. Usually a scheme color was given to an option that takes hex only (valGridLine.color, catGridLine.color, shadow.color, or chartColors / invertedColors on a one-series bar chart). Pass the six-digit hex from your theme object there and rebuild. Nothing was changed.`);
+
+ // pptxgenjs writes a dash option through unchecked, and an array as "solid,dash".
+ const dash = [...xml.matchAll(/<a:prstDash val="([^"]*)"/g)].find((m) => !DASHES.includes(m[1].trim()));
+ if (dash) throw new Error(`${shown(name)} has an <a:prstDash> whose val is ${shown(dash[1])}, which is not a dash style, and PowerPoint would offer to repair the deck. Give line.dashType, a chart's lineDash (one word for the whole chart, not an array), catAxisLineStyle or valAxisLineStyle one of: ${DASHES.join(" ")}. Then rebuild. Nothing was changed.`);
+
+ // pptxgenjs writes valAxisCrossesAt: 0 as <c:crossesAt val="autoZero"/>. <c:crossesAt> takes
+ // a number; the word belongs in <c:crosses>, where it means the same: cross at zero.
+ if (xml.includes('<c:crossesAt val="autoZero"/>')) zip.file(name, xml.replaceAll('<c:crossesAt val="autoZero"/>', '<c:crosses val="autoZero"/>'));
+ }
+ // A ZIP entry for a folder ("ppt/slides/") is not a part, and pptxgenjs writes one per folder.
+ // A cloud co-editing service reported issues editing decks that carry them. They are dropped
+ // here for that reason, not to save space. (zip.remove() would take the folder's files too.)
+ for (const name of Object.keys(zip.files)) if (zip.files[name].dir) delete zip.files[name];
+
+ fs.writeFileSync(outPath, await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }));
+}
+
+module.exports = { applyTheme };
+
+if (require.main === module) {
+ const [deck, themeFile, out] = process.argv.slice(2);
+ if (!deck || !themeFile) {
+ console.error("usage: node apply_theme.js deck.pptx theme.json [out.pptx]");
+ process.exit(2);
+ }
+ let theme;
+ try {
+ theme = JSON.parse(fs.readFileSync(themeFile, "utf8"));
+ } catch (e) {
+ console.error(`Error: cannot read ${themeFile} as JSON (${e.message}). Nothing was changed.`);
+ process.exit(1);
+ }
+ applyTheme(deck, theme, out || deck).then(
+ () => console.log(`Theme written to ${out || deck}`),
+ (e) => { console.error(`Error: ${e.message}`); process.exit(1); }
+ );
+}
diff --git a/skills/pptx/scripts/office/helpers/__init__.py b/skills/pptx/scripts/office/helpers/__init__.py
index d3c5817..4902e2a 100644
--- a/skills/pptx/scripts/office/helpers/__init__.py
+++ b/skills/pptx/scripts/office/helpers/__init__.py
@@ -5,7 +5,7 @@ import stat
import tempfile
import urllib.parse
import zipfile
-from pathlib import Path
+from pathlib import Path, PureWindowsPath
OOXML_FAMILY = {
".docx": "docx",
@@ -21,16 +21,24 @@ _SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:")
SLIDE_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide"
+def one_line(text: str, limit: int) -> str:
+ return "".join(c if c.isprintable() else c.encode("unicode_escape").decode() for c in text)[:limit]
+
+
+def names_another_root(name: str) -> bool:
+ return bool(PureWindowsPath(name).anchor)
+
+
def opc_target(target: str, source_part: str, target_mode: str = "") -> str | None:
if not target:
return None
if target_mode.lower() == "external":
return None
+ target = urllib.parse.unquote(target)
+
if _SCHEME_RE.match(target):
return None
- target = urllib.parse.unquote(target)
-
if "\\" in target:
raise ValueError(f"relationship target is not a POSIX part name: {target!r}")
@@ -52,12 +60,15 @@ def opc_target(target: str, source_part: str, target_mode: str = "") -> str | No
if not parts:
raise ValueError(f"relationship target resolves to nothing: {target!r}")
+ if names_another_root("/".join(parts)):
+ raise ValueError(f"relationship target is not a part name: {target!r}")
return "/".join(parts)
def rels_source_part(rels_file: Path, unpacked_dir: Path) -> str:
owner_dir = rels_file.parent.parent.relative_to(unpacked_dir)
- return posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")]).lstrip("./")
+ part = posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")])
+ return part.removeprefix("./")
def part_text(data: bytes) -> str:
@@ -76,12 +87,19 @@ def safe_extract(zf: zipfile.ZipFile, dest: Path) -> None:
for m in zf.infolist():
if stat.S_ISLNK(m.external_attr >> 16):
raise ValueError(f"symlink archive entry not allowed: {m.filename!r}")
+ if names_another_root(m.filename):
+ raise ValueError(f"unsafe archive entry: {m.filename!r}")
target = (dest / m.filename).resolve()
if not target.is_relative_to(dest):
raise ValueError(f"unsafe archive entry: {m.filename!r}")
zf.extract(m, dest)
+def folder_entries(package: Path) -> list[str]:
+ with zipfile.ZipFile(package) as zf:
+ return [info.filename for info in zf.infolist() if info.is_dir()]
+
+
def rezip(src_dir: Path, out_path: Path) -> None:
files = sorted(p for p in src_dir.rglob("*") if p.is_file())
ct = src_dir / "[Content_Types].xml"
diff --git a/skills/pptx/scripts/office/helpers/pptx_chart.py b/skills/pptx/scripts/office/helpers/pptx_chart.py
index 209cb7c..0e86aed 100644
--- a/skills/pptx/scripts/office/helpers/pptx_chart.py
+++ b/skills/pptx/scripts/office/helpers/pptx_chart.py
@@ -1,15 +1,20 @@
-"""Find chart XML that PowerPoint refuses but the schema accepts.
+"""Find chart XML that PowerPoint refuses or repairs.
-Detection only: for either fault more than one repair is valid, and only the
-author knows which was meant.
+Detection only: more than one repair is usually valid, and only the author knows
+which was meant.
"""
from __future__ import annotations
+import html
import re
+from collections import Counter
from typing import Mapping
+from defusedxml import ElementTree
+from defusedxml.common import DefusedXmlException
+
from . import part_text
@@ -19,8 +24,12 @@ _GROUPING_RE = re.compile(r"""<c:grouping\b[^>]*?\bval=["'](\w+)["']""")
_DLBL_POS_RE = re.compile(r"""<c:dLblPos\b[^>]*?\bval=["'](\w+)["']""")
def _strip_ext_lst(text: str) -> str:
+ return _strip_outermost(text, "c:extLst")
+
+
+def _strip_outermost(text: str, name: str) -> str:
out, cursor = [], 0
- for lo, hi in _ext_lst_spans(text):
+ for lo, hi in _outermost_spans(text, name):
out.append(text[cursor:lo])
cursor = hi
out.append(text[cursor:])
@@ -139,18 +148,113 @@ def _check_chart_axis_references(part: str, xml: str) -> list[str]:
return problems
-def _ext_lst_spans(text: str) -> list[tuple[int, int]]:
+def _shown(value: str) -> str:
+ return '"' + repr(value[:20])[1:-1] + '"'
+
+
+DECIMAL_SETTINGS = (
+ "crossesAt", "max", "min", "majorUnit", "minorUnit", "logBase",
+ "forward", "backward", "intercept", "x", "y", "w", "h",
+ "splitPos", "custUnit", "pictureStackUnit",
+)
+_DECIMAL_SETTING_RE = re.compile(
+ rf"""<c:({"|".join(DECIMAL_SETTINGS)})\b[^<>]*?\bval=["']([^"'<>]*)["']"""
+)
+_DECIMAL_RE = re.compile(
+ r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?"
+ r"|[+-]?(inf|infinity|nan(\(\w*\))?|\u221e|1\.#(inf|qnan|snan|ind)\d*)",
+ re.IGNORECASE,
+)
+
+
+def _check_decimal_settings(part: str, xml: str) -> list[str]:
+ problems = []
+ for name, raw in sorted(set(_DECIMAL_SETTING_RE.findall(_strip_ext_lst(xml)))):
+ value = html.unescape(raw)
+ if _DECIMAL_RE.fullmatch(value.strip()):
+ continue
+ fix = "Fix: write a plain number, or remove the element"
+ if name == "crossesAt" and value.strip() == "autoZero":
+ fix = ("Fix: leave valAxisCrossesAt: 0 out of the pptxgenjs options, the axis "
+ "crosses at zero anyway. In the XML, write "
+ '<c:crosses val="autoZero"/> or a number')
+ problems.append(
+ f"{part}: <c:{name} val={_shown(value)}> is not a number, and PowerPoint "
+ f"will offer to repair the deck. {fix}"
+ )
+ return problems
+
+
+def _local(tag: str) -> str:
+ return tag.rsplit("}", 1)[-1]
+
+
+def _drawn_series(root) -> list:
+ found, to_visit = [], [root]
+ while to_visit:
+ for child in to_visit.pop():
+ name = _local(child.tag)
+ if name == "ser":
+ found.append(child)
+ elif name == "AlternateContent":
+ to_visit.extend(child[:1])
+ elif name != "extLst":
+ to_visit.append(child)
+ return found
+
+
+def _check_series_order(part: str, xml: str) -> list[str]:
+ try:
+ root = ElementTree.fromstring(xml.encode("utf-8", "surrogateescape"))
+ except (ElementTree.ParseError, DefusedXmlException, LookupError, ValueError):
+ return []
+ orders = Counter()
+ for series in _drawn_series(root):
+ own = [c.get("val", "") for c in series if _local(c.tag) == "order"]
+ if own and own[0].isascii() and own[0].isdigit():
+ orders[own[0]] += 1
+ repeated = sorted((o for o, n in orders.items() if n > 1), key=lambda o: (len(o), o))
+ if not repeated:
+ return []
+ return [
+ f"{part}: two or more series share the <c:order> value {', '.join(o[:12] for o in repeated)}, "
+ f"and PowerPoint will offer to repair the deck. Fix: number <c:order> "
+ f"(and <c:idx>) 0, 1, 2... across every series in the chart, a combo "
+ f"chart's line series carrying on from its bars"
+ ]
+
+
+PRESET_DASHES = frozenset({
+ "solid", "dot", "dash", "lgDash", "dashDot", "lgDashDot", "lgDashDotDot",
+ "sysDash", "sysDot", "sysDashDot", "sysDashDotDot",
+})
+_PRESET_DASH_RE = re.compile(r"""<a:prstDash\b[^<>]*?\bval=["']([^"'<>]*)["']""")
+
+
+def _check_preset_dashes(part: str, xml: str) -> list[str]:
+ values = {html.unescape(v) for v in _PRESET_DASH_RE.findall(_strip_ext_lst(xml))}
+ return [
+ f"{part}: <a:prstDash val={_shown(value)}> is not a dash style, and PowerPoint "
+ f"will offer to repair the deck. Use one of {', '.join(sorted(PRESET_DASHES))}. "
+ f"In pptxgenjs, lineDash takes one such word for the whole chart, not an array"
+ for value in sorted(values)
+ if value.strip() not in PRESET_DASHES
+ ]
+
+
+def _outermost_spans(text: str, name: str) -> list[tuple[int, int]]:
spans: list[tuple[int, int]] = []
depth = 0
start = 0
- for match in re.finditer(r"<(/?)c:extLst\b[^>]*?(/?)>", text):
+ for match in re.finditer(rf"<(/?){name}\b[^<>]*?(/?)>", text):
closing, self_closing = match.group(1), match.group(2)
if self_closing:
continue
if closing:
- depth -= 1
- if depth == 0:
- spans.append((start, match.end()))
+ if depth:
+ depth -= 1
+ if depth == 0:
+ spans.append((start, match.end()))
else:
if depth == 0:
start = match.start()
@@ -158,13 +262,22 @@ def _ext_lst_spans(text: str) -> list[tuple[int, int]]:
return spans
-CHART_CHECKS = (_check_stacked_label_positions, _check_chart_axis_references)
+CHART_CHECKS = (
+ _check_stacked_label_positions,
+ _check_chart_axis_references,
+ _check_decimal_settings,
+ _check_series_order,
+ _check_preset_dashes,
+)
+
+
+_COMMENT_OR_PI_RE = re.compile(r"<!--.*?(?:-->|\Z)|<\?(?!xml\s).*?(?:\?>|\Z)", re.DOTALL)
def find_chart_problems(files: Mapping[str, bytes]) -> list[str]:
problems: list[str] = []
for part in sorted(n for n in files if _CHART_PART_RE.fullmatch(n)):
- xml = part_text(files[part])
+ xml = _strip_outermost(_COMMENT_OR_PI_RE.sub("", part_text(files[part])), "mc:Fallback")
for check in CHART_CHECKS:
problems.extend(check(part, xml))
return problems
diff --git a/skills/pptx/scripts/office/helpers/pptx_slide.py b/skills/pptx/scripts/office/helpers/pptx_slide.py
index 22f9aee..55274b8 100644
--- a/skills/pptx/scripts/office/helpers/pptx_slide.py
+++ b/skills/pptx/scripts/office/helpers/pptx_slide.py
@@ -1,7 +1,8 @@
-"""Pick the slide-XML schema errors PowerPoint refuses the file over.
+"""Find the slide-XML defects known or predicted to make PowerPoint refuse or repair the file.
-A denylist over lxml's messages, so an unrecognised error class is a miss rather
-than a false alarm.
+Schema errors are picked by a denylist over lxml's messages, so an unrecognised
+error class is a miss rather than a false alarm. Animations that name a shape
+the slide no longer has are found by reading the part.
"""
@@ -43,6 +44,14 @@ FATAL_SLIDE_ERRORS: tuple[tuple[re.Pattern[str], str], ...] = (
re.compile(r"\}nvGrpSpPr': Missing child element"),
"a <p:nvGrpSpPr> with no children",
),
+ (
+ re.compile(r"\}(?:off|ext|chOff|chExt)', attribute '\w+': '-?(?:Infinity|NaN)'"),
+ "a position or size that is not a number (check the x, y, w, h you worked out)",
+ ),
+ (
+ re.compile(r"\}prstDash', attribute 'val'"),
+ "an unknown line dash style (for line.dashType try dash, sysDash or solid)",
+ ),
)
@@ -50,6 +59,37 @@ def is_schema_verdict(error: str) -> bool:
return error.startswith("Element ")
+_PML = "{http://schemas.openxmlformats.org/presentationml/2006/main}"
+_MC = "{http://schemas.openxmlformats.org/markup-compatibility/2006}"
+
+_NAMES_A_SHAPE = frozenset({f"{_PML}spTgt", f"{_PML}bldP", f"{_PML}bldGraphic"})
+
+
+def _shape_id(value: str | None) -> int | str | None:
+ text = (value or "").strip()
+ if text.isascii() and text.isdigit() and len(text) <= 18:
+ return int(text)
+ return text or None
+
+
+def stranded_animation_targets(root) -> list[str]:
+ timing = root.find(f"{_PML}timing")
+ if timing is None:
+ timing = root.find(f"{_MC}AlternateContent/{_MC}Choice/{_PML}timing")
+ if timing is None:
+ return []
+
+ on_slide = {_shape_id(e.get("id")) for e in root.iter("{*}cNvPr")}
+ named = {_shape_id(e.get("spid")) for e in timing.iter(*_NAMES_A_SHAPE)} - {None}
+ stranded = named - on_slide
+ return [
+ f"<p:timing> animates shape id {spid if isinstance(spid, int) else repr(spid[:16])}, which is gone, so PowerPoint will "
+ f"offer to repair the deck. Fix: delete the slide's whole <p:timing> element "
+ f"(the slide loses its animations), or put the shape back"
+ for spid in sorted(stranded, key=str)
+ ]
+
+
def fatal_slide_errors(errors: set[str]) -> list[str]:
out = []
for error in sorted(errors):
diff --git a/skills/pptx/scripts/office/soffice.py b/skills/pptx/scripts/office/soffice.py
index 0b4c99d..eec1687 100644
--- a/skills/pptx/scripts/office/soffice.py
+++ b/skills/pptx/scripts/office/soffice.py
@@ -64,7 +64,7 @@ def _ensure_shim() -> Path:
return _SHIM_SO
src = Path(tempfile.gettempdir()) / "lo_socket_shim.c"
- src.write_text(_SHIM_SOURCE)
+ src.write_text(_SHIM_SOURCE, encoding="utf-8")
subprocess.run(
["gcc", "-shared", "-fPIC", "-o", str(_SHIM_SO), str(src), "-ldl"],
check=True,
diff --git a/skills/pptx/scripts/office/validate.py b/skills/pptx/scripts/office/validate.py
index 29ca186..ff3bde0 100755
--- a/skills/pptx/scripts/office/validate.py
+++ b/skills/pptx/scripts/office/validate.py
@@ -14,6 +14,7 @@ Auto-repair fixes:
"""
import argparse
+import shlex
import sys
import tempfile
import zipfile
@@ -22,12 +23,21 @@ from pathlib import Path
import defusedxml.ElementTree as ET
from defusedxml.common import DefusedXmlException
-from helpers import OOXML_FAMILY, rezip, safe_extract
+from helpers import OOXML_FAMILY, folder_entries, rezip, safe_extract
from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator
WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+def _as_argument(path: Path) -> str:
+ name = str(path)
+ return shlex.quote("./" + name if name.startswith("-") else name)
+
+
+def _count(folders: list[str]) -> str:
+ return "1 folder entry" if len(folders) == 1 else f"{len(folders)} folder entries"
+
+
def _fail(message: str):
print(f"Error: {message}", file=sys.stderr)
sys.exit(2)
@@ -150,16 +160,43 @@ def main():
print(f"Error: Validation not supported for file type {family}")
sys.exit(1)
+ folders = folder_entries(packed_file) if packed_file is not None else []
+
+ unsaved_repairs = 0
if args.auto_repair:
total_repairs = sum(v.repair() for v in validators)
if total_repairs:
print(f"Auto-repaired {total_repairs} issue(s)")
- if packed_file is not None:
+ if packed_file is not None and (total_repairs or folders):
+ try:
rezip(unpacked_dir, packed_file)
+ except OSError as e:
+ unsaved_repairs = total_repairs
+ left = []
+ if total_repairs:
+ left.append(f"{total_repairs} repair(s) not saved")
+ if folders:
+ left.append(f"{_count(folders)} in its ZIP")
+ print(f"Could not write {packed_file} ({e.strerror or e}): it is as it was, "
+ f"with {' and '.join(left)}. Run --auto-repair on a copy in a folder "
+ "you can write to.")
+ else:
+ if folders:
+ print(f"Removed {_count(folders)} from the ZIP")
print(f"Wrote repaired file to {packed_file}")
+ elif folders:
+ print(
+ f"Note: {_count(folders)} in the ZIP. Such files have failed to open or co-edit "
+ f"through a cloud service. Remove them: zip -dq {_as_argument(packed_file)} '*/'"
+ )
success = all([v.validate() for v in validators])
+ if unsaved_repairs:
+ print(f"FAILED - {packed_file} was not rewritten and still has the {unsaved_repairs} "
+ "issue(s) repaired above; the checks ran on a repaired copy")
+ success = False
+
if temp_dir_ctx is not None:
temp_dir_ctx.cleanup()
diff --git a/skills/pptx/scripts/office/validators/base.py b/skills/pptx/scripts/office/validators/base.py
index 33fc97b..04d6169 100644
--- a/skills/pptx/scripts/office/validators/base.py
+++ b/skills/pptx/scripts/office/validators/base.py
@@ -2,6 +2,7 @@
Base validator with common validation logic for document files.
"""
+import os
import re
from pathlib import Path
@@ -10,7 +11,7 @@ from functools import lru_cache
import lxml.etree
-from helpers import safe_extract
+from helpers import names_another_root, safe_extract
@lru_cache(maxsize=None)
@@ -352,6 +353,9 @@ class BaseSchemaValidator:
if target and not target.startswith(
("http", "mailto:")
):
+ if names_another_root(target.lstrip("/")):
+ broken_refs.append((target, rel.sourceline))
+ continue
if target.startswith("/"):
target_path = self.unpacked_dir / target.lstrip("/")
elif rels_file.name == ".rels":
@@ -360,9 +364,18 @@ class BaseSchemaValidator:
base_dir = rels_dir.parent
target_path = base_dir / target
+ target_path = Path(os.path.normpath(target_path))
+ if not target_path.is_relative_to(self.unpacked_dir):
+ broken_refs.append((target, rel.sourceline))
+ continue
+
try:
target_path = target_path.resolve()
- if target_path.exists() and target_path.is_file():
+ if (
+ target_path.is_relative_to(self.unpacked_dir)
+ and target_path.exists()
+ and target_path.is_file()
+ ):
referenced_files.add(target_path)
all_referenced_files.add(target_path)
else:
@@ -783,8 +796,7 @@ class BaseSchemaValidator:
try:
schema = _load_schema(str(schema_path))
- with open(xml_file, "r") as f:
- xml_doc = lxml.etree.parse(f)
+ xml_doc = lxml.etree.parse(str(xml_file))
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)
xml_doc = self._preprocess_for_mc_ignorable(xml_doc)
diff --git a/skills/pptx/scripts/office/validators/docx.py b/skills/pptx/scripts/office/validators/docx.py
index b181499..2acdaf4 100644
--- a/skills/pptx/scripts/office/validators/docx.py
+++ b/skills/pptx/scripts/office/validators/docx.py
@@ -6,12 +6,13 @@ import random
import re
import tempfile
import zipfile
+from collections import Counter
from pathlib import Path
import defusedxml.minidom
import lxml.etree
-from helpers import safe_extract
+from helpers import one_line, safe_extract
from .base import BaseSchemaValidator
@@ -62,6 +63,9 @@ class DOCXSchemaValidator(BaseSchemaValidator):
if not self.validate_comment_markers():
all_valid = False
+ if not self.validate_style_ids():
+ all_valid = False
+
self.compare_paragraph_counts()
return all_valid
@@ -264,7 +268,9 @@ class DOCXSchemaValidator(BaseSchemaValidator):
for xml_file in self.xml_files:
try:
- for elem in lxml.etree.parse(str(xml_file)).iter():
+ tree = lxml.etree.parse(str(xml_file))
+ is_numbering = lxml.etree.QName(tree.getroot()).localname == "numbering"
+ for elem in tree.iter():
if val := elem.get(para_id_attr):
try:
if self._parse_id_value(val, base=16) >= 0x80000000:
@@ -278,7 +284,7 @@ class DOCXSchemaValidator(BaseSchemaValidator):
)
if val := elem.get(durable_id_attr):
- if xml_file.name == "numbering.xml":
+ if is_numbering:
try:
if self._parse_id_value(val, base=10) >= 0x7FFFFFFF:
errors.append(
@@ -288,7 +294,7 @@ class DOCXSchemaValidator(BaseSchemaValidator):
except ValueError:
errors.append(
f" {xml_file.name}:{elem.sourceline}: "
- f"durableId={val} must be decimal in numbering.xml"
+ f"durableId={val} must be decimal in a numbering part"
)
else:
try:
@@ -308,11 +314,58 @@ class DOCXSchemaValidator(BaseSchemaValidator):
if errors:
print(f"FAILED - {len(errors)} ID constraint violations:")
for e in errors:
- print(e)
+ print(one_line(e, 300))
elif self.verbose:
print("PASSED - All paraId/durableId values within constraints")
return not errors
+ def _repeated_style_ids(self, styles_root) -> set[str]:
+ attr = f"{{{self.WORD_2006_NAMESPACE}}}styleId"
+ ids = Counter(s.get(attr) for s in styles_root.iter(f"{{{self.WORD_2006_NAMESPACE}}}style"))
+ return {i for i, n in ids.items() if i is not None and n > 1}
+
+ def _original_part(self, xml_file) -> bytes | None:
+ if self.original_file is None:
+ return None
+ name = xml_file.relative_to(self.unpacked_dir).as_posix()
+ try:
+ with zipfile.ZipFile(self.original_file) as zf:
+ return zf.read(name)
+ except (KeyError, OSError, zipfile.BadZipFile):
+ return None
+
+ def validate_style_ids(self):
+ errors = []
+ styles_tag = f"{{{self.WORD_2006_NAMESPACE}}}styles"
+ if self.original_file is None:
+ return True
+
+ for xml_file in self.xml_files:
+ try:
+ root = lxml.etree.parse(str(xml_file)).getroot()
+ if root.tag != styles_tag:
+ continue
+ repeated = self._repeated_style_ids(root)
+ original = self._original_part(xml_file) if repeated else None
+ if original is not None:
+ repeated -= self._repeated_style_ids(lxml.etree.fromstring(original))
+ except lxml.etree.XMLSyntaxError:
+ continue
+ errors.extend(
+ f" {xml_file.relative_to(self.unpacked_dir)}: style id {style_id!r:.40} is now "
+ f"defined more than once, and Word uses only the last definition, without "
+ f"warning. Delete the earlier ones"
+ for style_id in sorted(repeated)
+ )
+
+ if errors:
+ print(f"FAILED - Found {len(errors)} style id(s) defined more than once:")
+ for error in errors:
+ print(one_line(error, 300))
+ elif self.verbose:
+ print("PASSED - The edit defined no style id a second time")
+ return not errors
+
def validate_comment_markers(self):
errors = []
@@ -415,7 +468,7 @@ class DOCXSchemaValidator(BaseSchemaValidator):
try:
content = xml_file.read_text(encoding="utf-8")
dom = defusedxml.minidom.parseString(content)
- is_numbering = xml_file.name == "numbering.xml"
+ is_numbering = dom.documentElement.localName == "numbering"
base = 10 if is_numbering else 16
pending = []
seen_in_file = set()
diff --git a/skills/pptx/scripts/office/validators/pptx.py b/skills/pptx/scripts/office/validators/pptx.py
index 318f0e6..bb06b82 100644
--- a/skills/pptx/scripts/office/validators/pptx.py
+++ b/skills/pptx/scripts/office/validators/pptx.py
@@ -63,6 +63,9 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if not self.validate_master_theme_uniqueness():
all_valid = False
+ if not self.validate_tags():
+ all_valid = False
+
if not self.validate_charts():
all_valid = False
@@ -87,6 +90,30 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if p.is_file()
}
+ @staticmethod
+ def _print_line(text: str, limit: int) -> None:
+ from helpers import one_line
+
+ print(one_line(text, limit))
+
+ def _listed_slides(self) -> set[str] | None:
+ import lxml.etree
+
+ from helpers import SLIDE_REL_TYPE
+
+ r_id = f"{{{self.OFFICE_RELATIONSHIPS_NAMESPACE}}}id"
+ try:
+ pres = lxml.etree.parse(str(self.unpacked_dir / "ppt" / "presentation.xml"))
+ rels = lxml.etree.parse(str(self.unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"))
+ listed = {e.get(r_id) for e in pres.iter(f"{{{self.PRESENTATIONML_NAMESPACE}}}sldId")}
+ return {
+ opc_target(rel.get("Target", ""), "ppt/presentation.xml", rel.get("TargetMode", ""))
+ for rel in rels.getroot()
+ if rel.get("Id") in listed and rel.get("Type") == SLIDE_REL_TYPE
+ }
+ except (OSError, ValueError, lxml.etree.XMLSyntaxError):
+ return None
+
def validate_master_theme_uniqueness(self):
from helpers.pptx_theme import _NOTES_MASTERS, live_shared_master_themes
@@ -94,7 +121,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if shared:
print(f"FAILED - Found {len(shared)} master(s) sharing a theme part:")
for message in shared:
- print(f" {message}")
+ self._print_line(f" {message}", 300)
if any(m.startswith(_NOTES_MASTERS) for m in shared):
print(" Fix: in ppt/presentation.xml, move <p:notesMasterIdLst> back to "
"directly after <p:sldIdLst>. PowerPoint reads that happily.")
@@ -106,6 +133,64 @@ class PPTXSchemaValidator(BaseSchemaValidator):
print("PASSED - No master shares a theme part in a way PowerPoint refuses")
return True
+ def validate_tags(self):
+ from collections import Counter
+
+ import lxml.etree
+
+ ns = {"p": self.PRESENTATIONML_NAMESPACE, "mc": self.MC_NAMESPACE}
+ r_id = f"{{{self.OFFICE_RELATIONSHIPS_NAMESPACE}}}id"
+ errors = []
+ owners: dict[str, list[str]] = {}
+ listed = self._listed_slides()
+
+ for slide in sorted(self.unpacked_dir.glob("ppt/slides/*.xml")):
+ relative = slide.relative_to(self.unpacked_dir).as_posix()
+ if listed is not None and relative not in listed:
+ continue
+ try:
+ root = lxml.etree.parse(str(slide)).getroot()
+ named = [t.get(r_id) for t in root.xpath(
+ ".//p:tags[not(ancestor::mc:Fallback) and not(ancestor::mc:Choice[preceding-sibling::mc:Choice])]",
+ namespaces=ns)]
+ rels_file = slide.parent / "_rels" / f"{slide.name}.rels"
+ targets = {
+ rel.get("Id"): opc_target(rel.get("Target", ""), relative, rel.get("TargetMode", ""))
+ for rel in lxml.etree.parse(str(rels_file)).getroot()
+ if rel.get("Type", "").endswith("/relationships/tags")
+ } if rels_file.is_file() else {}
+ except (lxml.etree.XMLSyntaxError, ValueError):
+ continue
+ for rid, count in sorted(Counter(r for r in named if r).items()):
+ if count > 1:
+ errors.append(
+ f" {relative[:80]}: {count} <p:tags> elements name the tags relationship "
+ f"{rid!r:.20}. Fix: a copied shape kept its <p:custDataLst>; delete that "
+ f"from the copy, or give it a tags part and relationship of its own"
+ )
+ if targets.get(rid):
+ owners.setdefault(targets[rid], []).append(relative)
+
+ for part, slides in sorted(owners.items()):
+ if len(slides) > 1:
+ named_from = sorted(set(slides))
+ more = f" and {len(named_from) - 3} more" if len(named_from) > 3 else ""
+ errors.append(
+ f" {part[:80]}: {len(slides)} tags relationships name it, from "
+ f"{', '.join(s[:60] for s in named_from[:3])}{more}. Fix: copy the part under a new name, add "
+ f"its Override to [Content_Types].xml and point every relationship but one at a copy of its own "
+ f"(add_slide.py does this when it duplicates a slide)"
+ )
+
+ if errors:
+ print(f"FAILED - Found {len(errors)} tags part(s) with more than one owner; PowerPoint offers to repair the deck:")
+ for error in errors:
+ self._print_line(error, 700)
+ return False
+ if self.verbose:
+ print("PASSED - Every tags part has one owner")
+ return True
+
def validate_charts(self):
from helpers.pptx_chart import find_chart_problems
@@ -113,7 +198,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if problems:
print(f"FAILED - Found {len(problems)} chart problem(s) PowerPoint rejects:")
for message in problems:
- print(f" {message}")
+ self._print_line(f" {message}", 600)
return False
if self.verbose:
@@ -151,14 +236,18 @@ class PPTXSchemaValidator(BaseSchemaValidator):
return found
def validate_slides(self):
+ import lxml.etree
+
from helpers.pptx_slide import (
SLIDE_PART_RE,
fatal_slide_errors,
is_schema_verdict,
+ stranded_animation_targets,
)
schema = self.schemas_dir / self.SCHEMA_MAPPINGS["ppt"]
inherited = self._original_slide_defects(schema)
+ listed = self._listed_slides()
problems: list[str] = []
broken: list[str] = []
@@ -166,6 +255,14 @@ class PPTXSchemaValidator(BaseSchemaValidator):
relative = xml_file.relative_to(self.unpacked_dir).as_posix()
if not SLIDE_PART_RE.fullmatch(relative):
continue
+ unlisted = relative.startswith("ppt/slides/") and listed is not None and relative not in listed
+ try:
+ root = lxml.etree.parse(str(xml_file)).getroot()
+ stranded = [] if unlisted else stranded_animation_targets(root)
+ except lxml.etree.XMLSyntaxError:
+ stranded = []
+ problems.extend(f"{relative}: {m}" for m in stranded)
+
ok, errors = self._validate_single_file_xsd(
xml_file.resolve(), self.unpacked_dir.resolve(), schema_path=schema
)
@@ -187,12 +284,12 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if broken:
print(f"FAILED - Could not check {len(broken)} slide part(s):")
for message in sorted(broken):
- print(f" {message[:240]}")
+ self._print_line(f" {message}", 240)
if problems:
print(f"FAILED - Found {len(problems)} slide problem(s) PowerPoint rejects:")
for message in sorted(problems):
- print(f" {message[:240]}")
+ self._print_line(f" {message}", 240)
if broken or problems:
return False
@@ -204,6 +301,17 @@ class PPTXSchemaValidator(BaseSchemaValidator):
def _get_schema_path(self, xml_file):
if xml_file.parent.name == "charts" and xml_file.name.startswith("chart"):
return None
+ if xml_file.parent.name == "ppt" and xml_file.suffix == ".xml":
+ import lxml.etree
+
+ try:
+ root = lxml.etree.parse(str(xml_file)).getroot()
+ except (lxml.etree.XMLSyntaxError, OSError):
+ root = None
+ if root is not None and (lxml.etree.QName(root).namespace or "").startswith(
+ "http://schemas.microsoft.com/office/"
+ ):
+ return None
return super()._get_schema_path(xml_file)
def _preprocess_for_schema(self, xml_doc, relative_path):
@@ -256,7 +364,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if errors:
print(f"FAILED - Found {len(errors)} UUID ID validation errors:")
for error in errors:
- print(error)
+ self._print_line(error, 300)
return False
else:
if self.verbose:
@@ -325,7 +433,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if errors:
print(f"FAILED - Found {len(errors)} slide layout ID validation errors:")
for error in errors:
- print(error)
+ self._print_line(error, 300)
print(
"Remove invalid references or add missing slide layouts to the relationships file."
)
@@ -366,7 +474,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if errors:
print("FAILED - Found slides with duplicate slideLayout references:")
for error in errors:
- print(error)
+ self._print_line(error, 300)
return False
else:
if self.verbose:
@@ -428,7 +536,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
f"FAILED - Found {len([e for e in errors if not e.startswith(' ')])} notes slide reference validation errors:"
)
for error in errors:
- print(error)
+ self._print_line(error, 300)
print("Each slide may optionally have its own slide file.")
return False
else:
diff --git a/skills/pptx/scripts/office/validators/redlining.py b/skills/pptx/scripts/office/validators/redlining.py
index 4185c51..4406912 100644
--- a/skills/pptx/scripts/office/validators/redlining.py
+++ b/skills/pptx/scripts/office/validators/redlining.py
@@ -1,19 +1,24 @@
"""
Validator for tracked changes in Word documents.
-Detects untracked edits in word/document.xml: text that differs from the
+Detects untracked edits in a Word document: text that differs from the
original without a <w:ins>/<w:del> wrapper recording it. The tracked changes
that are new relative to the original are undone, and the result is compared
against the original; whatever text still differs was edited without being
tracked.
-Only the document body is compared. Headers, footers, footnotes and endnotes
-are separate parts and are not checked.
+The body (word/document.xml) is compared, and so is each header, footer,
+footnotes and endnotes part, against the original's part of the same name.
+Other parts, comments among them, are not checked. A page-number field
+whose cached result is a plain number counts as its name in braces
+("{PAGE}"), not as that number, which any application that repaginates the
+document rewrites.
"""
-import subprocess
+import re
import tempfile
import zipfile
+from difflib import SequenceMatcher
from pathlib import Path
import defusedxml.ElementTree as ET
@@ -22,12 +27,35 @@ from defusedxml.common import DefusedXmlException
from helpers import rendered_text, safe_extract
+XML_ERRORS = (ET.ParseError, DefusedXmlException, LookupError, ValueError)
+
+
+class _WorkSpent(Exception):
+
+ pass
+
+
+class _Matcher(SequenceMatcher):
+
+ def __init__(self, a, b, allowance):
+ self.allowance = allowance
+ super().__init__(None, a, b, autojunk=False)
+
+ def find_longest_match(self, alo=0, ahi=None, blo=0, bhi=None):
+ items = self.a[alo:ahi]
+ self.allowance[0] -= len(items) + sum(len(self.b2j.get(x, ())) for x in items)
+ if self.allowance[0] < 0:
+ raise _WorkSpent
+ return super().find_longest_match(alo, ahi, blo, bhi)
+
+
class RedliningValidator:
def __init__(self, unpacked_dir, original_docx, verbose=False):
self.unpacked_dir = Path(unpacked_dir)
self.original_docx = Path(original_docx)
self.verbose = verbose
+ self._work_left = [self.WORK_ALLOWED]
self.namespaces = {
"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
}
@@ -58,39 +86,81 @@ class RedliningValidator:
)
return False
- try:
- modified_tree = ET.parse(modified_file)
- modified_root = modified_tree.getroot()
- original_tree = ET.parse(original_file)
- original_root = original_tree.getroot()
- except (ET.ParseError, DefusedXmlException) as e:
- print(f"FAILED - Error parsing XML files: {e}")
- return False
-
- new_changes = self._new_tracked_changes(original_root, modified_root)
- self._remove_tracked_changes(modified_root, new_changes)
-
- modified_text = self._extract_text_content(modified_root)
- original_text = self._extract_text_content(original_root)
-
- if modified_text != original_text:
- error_message = self._generate_detailed_diff(
- original_text, modified_text
- )
- print(error_message)
+ sides = (("original", temp_path), ("modified", self.unpacked_dir))
+ change_count, failures, differences = 0, [], []
+ for part in self._parts_to_compare(temp_path, self.unpacked_dir):
+ roots = []
+ for side, unpacked in sides:
+ try:
+ roots.append(self._parse_part(unpacked / part))
+ except XML_ERRORS as e:
+ failures.append(
+ f"FAILED - Error parsing {part} of the {side} document: {e}"
+ )
+ if len(roots) < len(sides):
+ continue
+ new_changes, original_text, modified_text = self._compare_part(*roots)
+ change_count += len(new_changes)
+ if modified_text != original_text:
+ differences.append((part, original_text, modified_text))
+
+ if differences:
+ failures.append(self._generate_detailed_diff(differences))
+ if failures:
+ print("\n".join(failures))
return False
if self.verbose:
print(
- f"PASSED - All {len(new_changes)} change(s) against the original "
+ f"PASSED - All {change_count} change(s) against the original "
"are properly tracked"
)
return True
+ OTHER_PARTS = (
+ "word/header*.xml",
+ "word/footer*.xml",
+ "word/footnotes.xml",
+ "word/endnotes.xml",
+ )
+
+ def _parts_to_compare(self, original_dir, modified_dir):
+ others = set()
+ for unpacked in (Path(original_dir), Path(modified_dir)):
+ for pattern in self.OTHER_PARTS:
+ others.update(
+ path.relative_to(unpacked).as_posix()
+ for path in unpacked.glob(pattern)
+ )
+ return ["word/document.xml", *sorted(others)]
+
+ def _compare_part(self, original_root, modified_root):
+ for root in (original_root, modified_root):
+ self._flatten_text_elements(root)
+ new_changes = self._new_tracked_changes(original_root, modified_root)
+ self._remove_tracked_changes(modified_root, new_changes)
+ return (
+ new_changes,
+ self._extract_text_content(original_root),
+ self._extract_text_content(modified_root),
+ )
+
+ def _parse_part(self, path):
+ if not path.is_file():
+ return ET.fromstring("<absent/>")
+ return ET.parse(path).getroot()
+
+ def _flatten_text_elements(self, root):
+ for elem in self._text_elements(root):
+ if len(elem):
+ elem.text = "".join(elem.itertext())
+ for child in list(elem):
+ elem.remove(child)
+
def _tracked_change_elements(self, root):
- ins_tag = f"{{{self.namespaces['w']}}}ins"
- del_tag = f"{{{self.namespaces['w']}}}del"
- return [elem for elem in root.iter() if elem.tag in (ins_tag, del_tag)]
+ w = self.namespaces["w"]
+ tags = (f"{{{w}}}ins", f"{{{w}}}del", f"{{{w}}}moveFrom", f"{{{w}}}moveTo")
+ return [elem for elem in root.iter() if elem.tag in tags]
def _rendered_text(self, elem):
preserve = elem.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve"
@@ -148,7 +218,8 @@ class RedliningValidator:
new.update(elems)
return new
- def _generate_detailed_diff(self, original_text, modified_text):
+ def _generate_detailed_diff(self, differences):
+ self._work_left = [self.WORK_ALLOWED]
error_parts = [
"FAILED - Document text doesn't match after removing the tracked changes",
"",
@@ -170,130 +241,238 @@ class RedliningValidator:
"",
]
- git_diff = self._get_git_word_diff(original_text, modified_text)
- if git_diff:
- error_parts.extend(["Differences:", "============", git_diff])
- else:
- error_parts.append("Unable to generate word diff (git not available)")
+ explain_page_fields = False
+ for part, original_text, modified_text in differences:
+ body = part == "word/document.xml"
+ heading = "Differences:" if body else f"Differences in {part}:"
+ if error_parts[-1]:
+ error_parts.append("")
+ word_diff = self._word_diff(original_text, modified_text)
+ error_parts.extend([heading, "=" * len(heading), word_diff])
+ if original_text.replace("\n", "") == modified_text.replace("\n", ""):
+ error_parts.append("Only paragraph breaks differ here.")
+ if re.search(r"\{(%s)\}" % "|".join(self.PAGE_FIELDS), word_diff):
+ explain_page_fields = True
+ if explain_page_fields:
+ error_parts.extend(["", self.PAGE_FIELD_NOTE])
+ if self._work_left[0] < 0:
+ error_parts.extend(["", self.WORK_SPENT_NOTE])
return "\n".join(error_parts)
- def _get_git_word_diff(self, original_text, modified_text):
- try:
- with tempfile.TemporaryDirectory() as temp_dir:
- temp_path = Path(temp_dir)
-
- original_file = temp_path / "original.txt"
- modified_file = temp_path / "modified.txt"
-
- original_file.write_text(original_text, encoding="utf-8")
- modified_file.write_text(modified_text, encoding="utf-8")
-
- result = subprocess.run(
- [
- "git",
- "diff",
- "--word-diff=plain",
- "--word-diff-regex=.",
- "-U0",
- "--no-index",
- str(original_file),
- str(modified_file),
- ],
- capture_output=True,
- text=True,
- )
+ def _word_diff(self, original_text, modified_text):
+ out = []
+ steps = self._steps(original_text.split("\n"), modified_text.split("\n"))
+ for tag, was, now in steps:
+ if tag == "replace" and len(was) == len(now):
+ out.extend(self._mark_changes(x, y) for x, y in zip(was, now) if x != y)
+ elif tag != "equal":
+ out.append(self._mark_changes("\n".join(was), "\n".join(now)))
+ return "\n".join(out)
+
+ def _mark_changes(self, old, new, by_character=False, trim=True):
+
+ def mark(text, opening, closing):
+ marked = re.sub(r"[^\n]+", lambda m: opening + m[0] + closing, text)
+ return marked if marked.strip("\n") or opening == "{+" else ""
+
+ pattern = "(?s)." if by_character else r"\S+\s*|\s+"
+ out = []
+ steps = self._steps(re.findall(pattern, old), re.findall(pattern, new), trim)
+ for tag, was, now in steps:
+ lone_word = len(was) == len(now) == 1
+ was, now = "".join(was), "".join(now)
+ if tag == "equal":
+ out.append(now)
+ elif tag == "replace" and not by_character:
+ long_word = lone_word and min(len(was), len(now)) > self.LONG_WORD
+ marked = self._mark_changes(was, now, by_character=True, trim=long_word)
+ out.append(marked)
+ else:
+ out.append(mark(was, "[-", "-]") + mark(now, "{+", "+}"))
+ return "".join(out)
- if result.stdout.strip():
- lines = result.stdout.split("\n")
- content_lines = []
- in_content = False
- for line in lines:
- if line.startswith("@@"):
- in_content = True
- continue
- if in_content and line.strip():
- content_lines.append(line)
-
- if content_lines:
- return "\n".join(content_lines)
-
- result = subprocess.run(
- [
- "git",
- "diff",
- "--word-diff=plain",
- "-U0",
- "--no-index",
- str(original_file),
- str(modified_file),
- ],
- capture_output=True,
- text=True,
- )
+ WORK_ALLOWED = 10_000_000
+ LONG_WORD = 200
- if result.stdout.strip():
- lines = result.stdout.split("\n")
- content_lines = []
- in_content = False
- for line in lines:
- if line.startswith("@@"):
- in_content = True
- continue
- if in_content and line.strip():
- content_lines.append(line)
- return "\n".join(content_lines)
+ def _steps(self, a, b, trim=True):
- except (subprocess.CalledProcessError, FileNotFoundError, Exception):
- pass
+ def shared_start(x, y):
+ return next(
+ (i for i, (p, q) in enumerate(zip(x, y)) if p != q), min(len(x), len(y))
+ )
- return None
+ head = shared_start(a, b) if trim else 0
+ tail = shared_start(a[head:][::-1], b[head:][::-1]) if trim else 0
+ a_mid, b_mid = a[head : len(a) - tail], b[head : len(b) - tail]
+ try:
+ if self._work_left[0] < len(a_mid):
+ self._work_left[0] = -1
+ raise _WorkSpent
+ numbers = {}
+ numbered = [
+ [numbers.setdefault(x, len(numbers)) for x in s] for s in (a_mid, b_mid)
+ ]
+ opcodes = _Matcher(*numbered, self._work_left).get_opcodes()
+ steps = [(tag, a_mid[i:j], b_mid[m:n]) for tag, i, j, m, n in opcodes]
+ except _WorkSpent:
+ steps = [("replace", a_mid, b_mid)]
+ return [
+ ("equal", a[:head], b[:head]),
+ *steps,
+ ("equal", a[len(a) - tail :], b[len(b) - tail :]),
+ ]
def _remove_tracked_changes(self, root, targets):
- ins_tag = f"{{{self.namespaces['w']}}}ins"
- del_tag = f"{{{self.namespaces['w']}}}del"
+ w = self.namespaces["w"]
+ ins_tag = f"{{{w}}}ins"
+ del_tag = f"{{{w}}}del"
+ move_from_tag = f"{{{w}}}moveFrom"
+ move_to_tag = f"{{{w}}}moveTo"
+
+ self._merge_split_paragraphs(root, targets)
for parent in root.iter():
to_remove = []
for child in parent:
- if child.tag == ins_tag and child in targets:
+ if child.tag in (ins_tag, move_to_tag) and child in targets:
to_remove.append(child)
for elem in to_remove:
parent.remove(elem)
- deltext_tag = f"{{{self.namespaces['w']}}}delText"
- t_tag = f"{{{self.namespaces['w']}}}t"
+ renamed = {
+ f"{{{w}}}delText": f"{{{w}}}t",
+ f"{{{w}}}delInstrText": f"{{{w}}}instrText",
+ }
for parent in root.iter():
to_process = []
for child in parent:
- if child.tag == del_tag and child in targets:
+ if child.tag in (del_tag, move_from_tag) and child in targets:
to_process.append((child, list(parent).index(child)))
for del_elem, del_index in reversed(to_process):
for elem in del_elem.iter():
- if elem.tag == deltext_tag:
- elem.tag = t_tag
+ elem.tag = renamed.get(elem.tag, elem.tag)
for child in reversed(list(del_elem)):
parent.insert(del_index, child)
parent.remove(del_elem)
+ range_tags = (
+ f"{{{w}}}moveFromRangeStart",
+ f"{{{w}}}moveFromRangeEnd",
+ f"{{{w}}}moveToRangeStart",
+ f"{{{w}}}moveToRangeEnd",
+ )
+ for parent in root.iter():
+ to_remove = [
+ c
+ for c in parent
+ if c.tag in range_tags and len(c) == 0 and not (c.text or "").strip()
+ ]
+ for elem in to_remove:
+ parent.remove(elem)
+
+ def _merge_split_paragraphs(self, root, targets):
+ w = self.namespaces["w"]
+ p_tag = f"{{{w}}}p"
+ ppr_tag = f"{{{w}}}pPr"
+ marker_path = f"{ppr_tag}/{{{w}}}rPr/{{{w}}}ins"
+
+ for parent in list(root.iter()):
+ for para in reversed([child for child in parent if child.tag == p_tag]):
+ if not any(m in targets for m in para.findall(marker_path)):
+ continue
+ siblings = list(parent)
+ index = siblings.index(para)
+ following = siblings[index + 1] if index + 1 < len(siblings) else None
+ if following is None or following.tag != p_tag:
+ continue
+ if any(self._text_elements(c) for c in para if c.tag == ppr_tag):
+ continue
+ insert_at = 1 if len(following) and following[0].tag == ppr_tag else 0
+ for child in reversed([c for c in para if c.tag != ppr_tag]):
+ following.insert(insert_at, child)
+ parent.remove(para)
+
def _extract_text_content(self, root):
p_tag = f"{{{self.namespaces['w']}}}p"
- t_tag = f"{{{self.namespaces['w']}}}t"
paragraphs = []
for p_elem in root.findall(f".//{p_tag}"):
- text_parts = []
- for t_elem in p_elem.findall(f".//{t_tag}"):
- text_parts.append(self._rendered_text(t_elem))
- paragraph_text = "".join(text_parts)
+ paragraph_text = self._paragraph_text(p_elem)
if paragraph_text:
paragraphs.append(paragraph_text)
return "\n".join(paragraphs)
+ PAGE_FIELDS = {"PAGE", "NUMPAGES", "SECTIONPAGES"}
+ PAGE_NUMBER = re.compile(r"-? ?[0-9]{1,6} ?-?|[ivxlcdm]{1,8}|[a-z]{1,2}|", re.I)
+ PAGE_FIELD_NOTE = (
+ "Note: {PAGE}, {NUMPAGES} or {SECTIONPAGES} above stands for a page-number\n"
+ "field. Word recomputes the number it displays, so the number stored in\n"
+ "the file is not compared, but adding or removing the field still counts."
+ )
+
+ WORK_SPENT_NOTE = (
+ "Note: the texts were too long or repetitive to compare closely all the\n"
+ "way through, so a passage shown whole, as removed and then added, may\n"
+ "differ in only a few characters. Compare such a passage with the\n"
+ "original directly to find the edit."
+ )
+
+ def _paragraph_text(self, p_elem):
+ w = self.namespaces["w"]
+ t_tag, instr_tag = f"{{{w}}}t", f"{{{w}}}instrText"
+ fld_char, fld_simple = f"{{{w}}}fldChar", f"{{{w}}}fldSimple"
+
+ def page_field(instruction, field):
+ words = instruction.split()
+ locked = field.get(f"{{{w}}}fldLock") in ("1", "true", "on")
+ if words and words[0].upper() in self.PAGE_FIELDS and not locked:
+ return words[0].upper()
+
+ def shown(name, result):
+ return "{%s}" % name if self.PAGE_NUMBER.fullmatch(result) else result
+
+ parts, fields = [], []
+ skipped = set()
+
+ def sink():
+ for *_, result in reversed(fields):
+ if result is not None:
+ return result
+ return parts
+
+ for elem in p_elem.iter():
+ if elem.tag == fld_simple:
+ name = page_field(elem.get(f"{{{w}}}instr", ""), elem)
+ if name:
+ texts = [self._rendered_text(t) for t in elem.iter(t_tag)]
+ sink().append(shown(name, "".join(texts).strip()))
+ skipped.update(elem.iter(t_tag))
+ elif elem.tag == fld_char:
+ kind = elem.get(f"{{{w}}}fldCharType")
+ if kind == "begin":
+ fields.append([elem, "", False, None])
+ elif kind == "separate" and fields and not fields[-1][2]:
+ fields[-1][2] = True
+ if page_field(fields[-1][1], fields[-1][0]):
+ fields[-1][3] = []
+ elif kind == "end" and fields:
+ begin, instruction, _, result = fields.pop()
+ name = page_field(instruction, begin)
+ if name:
+ sink().append(shown(name, "".join(result or []).strip()))
+ elif elem.tag == instr_tag and fields and not fields[-1][2]:
+ fields[-1][1] += elem.text or ""
+ elif elem.tag == t_tag and elem not in skipped:
+ sink().append(self._rendered_text(elem))
+ for *_, result in fields:
+ parts.extend(result or [])
+ return "".join(parts)
+
if __name__ == "__main__":
raise RuntimeError("This module should not be run directly.")
diff --git a/skills/pptx/scripts/slide_text.py b/skills/pptx/scripts/slide_text.py
new file mode 100755
index 0000000..c583efa
--- /dev/null
+++ b/skills/pptx/scripts/slide_text.py
@@ -0,0 +1,132 @@
+"""Print each slide of a .pptx, .pptm, .potx or .ppsx as markdown under `<!-- Slide number: N -->`.
+
+Title, text, tables, chart data, picture alt text and speaker notes; shapes top to bottom.
+
+Usage: python slide_text.py deck.pptx
+"""
+
+import os
+import re
+import sys
+import zipfile
+
+from pptx.package import Package
+
+
+def _position(shape):
+ try:
+ return (shape.top or 0, shape.left or 0)
+ except Exception:
+ return (0, 0)
+
+
+def _one_line(text) -> str:
+ return " ".join(str(text).split())
+
+
+def _table(rows) -> list[str]:
+ lines = ["| " + " | ".join(_one_line(c).replace("|", "\\|") for c in r) + " |" for r in rows]
+ if lines:
+ lines.insert(1, "|" + " --- |" * len(rows[0]))
+ return lines
+
+
+def _chart(chart) -> list[str]:
+ heading = "### Chart"
+ try:
+ if chart.has_title and chart.chart_title.has_text_frame:
+ heading += ": " + _one_line(chart.chart_title.text_frame.text)
+ categories = [c.label for c in chart.plots[0].categories]
+ series = [(s.name, list(s.values)) for s in chart.series]
+ except Exception as e:
+ return [
+ heading,
+ f"[chart data not readable ({type(e).__name__}): render the slide to see it]",
+ ]
+ rows = [["Category", *(name for name, _ in series)]]
+ for i, category in enumerate(categories):
+ rows.append([category, *(v[i] if i < len(v) else "" for _, v in series)])
+ return [heading, "", *_table(rows)]
+
+
+def _shape(shape) -> list[str]:
+ tag = shape._element.tag
+ if tag.endswith("}grpSp"):
+ return _shapes(shape.shapes)
+ if tag.endswith("}pic"):
+ alt = _one_line(shape._element._nvXxPr.cNvPr.get("descr", ""))
+ return [f"![{alt}]({shape.name})"]
+ if getattr(shape, "has_table", False):
+ return ["", *_table([[c.text for c in r.cells] for r in shape.table.rows]), ""]
+ if getattr(shape, "has_chart", False):
+ return ["", *_chart(shape.chart), ""]
+ if shape.has_text_frame and shape.text_frame.text.strip():
+ text = shape.text_frame.text.replace("\v", "\n")
+ is_title = shape.is_placeholder and shape.placeholder_format.idx == 0
+ return ["# " + text.lstrip() if is_title else text]
+ return []
+
+
+def _members(shapes) -> list:
+ try:
+ return list(shapes)
+ except Exception:
+ pass
+ members = []
+ for i in range(len(shapes)):
+ try:
+ members.append(shapes[i])
+ except Exception as e:
+ members.append(e)
+ return members
+
+
+def _shapes(shapes) -> list[str]:
+ lines = []
+ for shape in sorted(_members(shapes), key=_position):
+ try:
+ if isinstance(shape, Exception):
+ raise shape
+ lines += _shape(shape)
+ except Exception as e:
+ lines.append(f"[shape not readable ({type(e).__name__}): render the slide to see it]")
+ return lines
+
+
+def slide_text(slide, number: int) -> str:
+ lines = [f"<!-- Slide number: {number} -->", *_shapes(slide.shapes)]
+ try:
+ notes = slide.notes_slide.notes_text_frame if slide.has_notes_slide else None
+ if notes is not None and notes.text.strip():
+ lines += ["", "### Notes:", notes.text.replace("\v", "\n")]
+ except Exception as e:
+ lines += ["", f"[notes not readable ({type(e).__name__})]"]
+ return re.sub(r"\n{3,}", "\n\n", "\n".join(lines).strip())
+
+
+def main() -> None:
+ if len(sys.argv) != 2:
+ sys.exit("Usage: python slide_text.py deck.pptx")
+ path = sys.argv[1]
+ if not os.path.isfile(path):
+ sys.exit(f"Error: {path} {'is not a file' if os.path.exists(path) else 'does not exist'}")
+ if not zipfile.is_zipfile(path):
+ sys.exit(
+ f"Error: {path} is corrupt, password-protected, or an old .ppt (not a zip archive)"
+ )
+ try:
+ deck = getattr(Package.open(path).main_document_part, "presentation", None)
+ except Exception as e:
+ sys.exit(f"Error: cannot read {path}: {e}")
+ if deck is None:
+ sys.exit(f"Error: {path} is not a .pptx, .pptm, .potx or .ppsx file")
+ sys.stdout.reconfigure(errors="replace")
+ try:
+ blocks = [slide_text(s, n) for n, s in enumerate(deck.slides, 1)]
+ print("\n\n".join(blocks) or "[no slides; layouts and masters are not printed]", flush=True)
+ except BrokenPipeError:
+ os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
+
+
+if __name__ == "__main__":
+ main()
diff --git a/skills/xlsx/SKILL.md b/skills/xlsx/SKILL.md
index cc34720..419ecd3 100644
--- a/skills/xlsx/SKILL.md
+++ b/skills/xlsx/SKILL.md
@@ -20,7 +20,7 @@ license: Complete terms in LICENSE.txt
## Requirements for every output
- **Professional font** (Arial, Times New Roman) throughout, unless the user says otherwise.
-- **Zero formula errors.** Never ship while `recalc.py` reports `errors_found`. If you think an error predates you, prove it: load the *original* with `data_only=True` and look at that cell. An error you introduced looks exactly like one you inherited.
+- **Zero formula errors.** Never ship while `recalc.py` reports `errors_found`; the one exception is under `not_computed` below. If you think an error predates you, prove it: load the *original* with `data_only=True` and look at that cell. An error you introduced looks exactly like one you inherited.
- **Use formulas, never hardcoded results.** Write `sheet['B10'] = '=SUM(B2:B9)'`, not the Python-computed total. The sheet must recalculate when its inputs change.
- **Follow the user's spec literally.** Exact tab names, exact column headers, and the formula they spelled out. A redesign that computes something else fails, however elegant.
- **Document every assumption and hardcoded number** where the reader will see it — a cell comment, or an adjacent cell at a table's end. Cite a real source when one exists (`Source: Company 10-K, FY2024, Page 45, Revenue Note, [SEC EDGAR URL]`); when the number came from the user, say so plainly.
@@ -37,7 +37,7 @@ formula cell reads back as `None` to anything reading cached values — `pandas`
python scripts/recalc.py output.xlsx [timeout_seconds] # default 30
```
-LibreOffice computes every formula, the file is **rewritten in place**, and you get JSON:
+LibreOffice computes the formulas **on a copy**, only their values go into your file, and you get JSON:
`status` (`success` | `errors_found`), `total_formulas`, `total_errors`, and an
`error_summary` naming up to 100 cells per error type (`locations_truncated` says how many it
withheld — trust `total_errors`, not the length of the list). Fix what it names and run it
@@ -49,24 +49,23 @@ workbook.
range or a reference to the wrong row yields a clean, error-free file with wrong numbers.
Write 2–3 formulas first and check they pull the values you expect, before building out a grid.
-**A workbook that links to another file loses those links** if you re-save it with openpyxl and
-then recalculate. Such a formula reads `='[1]Returns Analysis'!$B$2` — the `[1]` is an index
-into the workbook's external-reference list, naming a *separate file on disk*, not a sheet.
-That file is rarely present here, so the cell's cached value is the only thing holding its
-data. openpyxl strips that value on save; LibreOffice then has to resolve the reference for
-real, fails, writes `#NAME?`, and deletes every link. `recalc.py` refuses to run in that state
-— copy those cells' values out of the original before you save over them (`--force` overrides,
-and accepts the loss).
+**`not_computed` lists formulas LibreOffice cannot evaluate, even on `success`**: a function it lacks
+(`_xlfn.XLOOKUP`, `_xlfn.LET`…) or a link to another workbook (`='[1]Returns Analysis'!$B$2`). They
+keep what they held: after an openpyxl save, nothing until Excel opens it. For their numbers sooner,
+read the original with `data_only=True`; keep the formulas. A sound formula depending on a listed one,
+directly or not, reports `#NAME?` too: leave it and tell the user. A cell that tests one
+(`IFERROR(B1,0)`) gets LibreOffice's fallback, and a kept value may be stale: say so. After editing
+sheet XML, delete each changed formula's `<v>`: a mistake that keeps its value is listed, not reported.
## Choosing formulas that survive verification
-LibreOffice implements fewer functions than Excel, and one it cannot evaluate becomes a
-literal `#NAME?` baked into the file you deliver.
+LibreOffice implements fewer functions than Excel, and a formula it cannot evaluate has no value
+until Excel opens the file.
- **Prefer Excel-2007-era functions** — `SUMIFS`, `INDEX`, `MATCH`, `IFERROR`, `SUMPRODUCT` — which need no prefix.
- **Six post-2007 functions work, but only with an `_xlfn.` prefix**, because openpyxl writes your formula into the XML verbatim and Excel stores post-2007 names prefixed (its UI hides the prefix): `_xlfn.TEXTJOIN`, `_xlfn.CONCAT`, `_xlfn.IFS`, `_xlfn.SWITCH`, `_xlfn.MAXIFS`, `_xlfn.MINIFS`. Written bare, each yields `#NAME?`.
-- **Never use `XLOOKUP`, `XMATCH`, `SORT`, `FILTER`, `UNIQUE`, or `SEQUENCE`.** The runtime's LibreOffice cannot evaluate them under *any* prefix. Newer builds do evaluate them, but they are spilling array functions and an openpyxl-written file has no spill metadata, so only the top-left cell of the range gets a value — and `recalc.py` reports `total_errors: 0` on the truncated result. Use `INDEX`/`MATCH` for lookups, and sort, filter, and de-duplicate in Python before writing the cells.
-- A formula LibreOffice could not parse is written back **lowercased** — a quick tell beside a `#NAME?`.
+- **Never use `XLOOKUP`, `XMATCH`, `SORT`, `FILTER`, `UNIQUE`, or `SEQUENCE`.** The runtime's LibreOffice cannot evaluate them, and one that can fills only the top-left cell of their result, since an openpyxl-written file has no spill metadata. Use `INDEX`/`MATCH` for lookups, and sort, filter, and de-duplicate in Python before writing the cells.
+- A missing `)` is reported as `unbalanced brackets`. Too few arguments (`=ROUND(A1)`) is not: LibreOffice returns a value, and Excel may offer to repair the file.
## openpyxl gotchas
@@ -76,6 +75,12 @@ literal `#NAME?` baked into the file you deliver.
- **Merged cells: write the top-left anchor only.** Every other cell in the range is a `MergedCell` whose `.value` is read-only.
- **`.xlsm` loses its macros unless you pass `keep_vba=True`** to `load_workbook`.
- **A sheet name containing a space must be quoted** in a cross-sheet reference: `='Assumptions Inputs'!$B$5`. Unquoted, it evaluates to `#VALUE!`.
+- **Saving an upload drops its text boxes, shapes and slicers.** First search `xl/drawings/` for `<xdr:sp ` (with the space) and look for an `xl/slicers/` part; if found, tell the user or edit the sheet XML directly.
+- **Nothing mends a malformed workbook**: an unclosed quote in a number format (`'0.00 "USD'`), a defined name starting with `=`, or duplicate or empty table headers are saved as written, and Excel may offer to repair the file.
+- **Partly bold or italic cell text is flattened** unless `load_workbook(rich_text=True)`; such a `.value` is then a `CellRichText`, not a `str`.
+- **Every new formula needs its own `number_format`** (date, %, currency): nothing infers it, so a date shows as `45337` and 30% as `0.3`.
+- **On every new chart set `chart.roundedCorners = False`, `.overlay = False` on its legend and on each title you gave it (`chart.legend`, `chart.title`, `chart.x_axis.title`…), and, if it has axes (not pie or doughnut), `chart.x_axis.delete = False` and `chart.y_axis.delete = False`**, or Excel may draw the title or legend over the plot, round the frame, or draw no axes.
+- **With a `DateAxis`, set `chart.y_axis.crossAx = 500`**, or Excel may offer to repair the file.
## Financial models
diff --git a/skills/xlsx/scripts/office/helpers/__init__.py b/skills/xlsx/scripts/office/helpers/__init__.py
index d3c5817..4902e2a 100644
--- a/skills/xlsx/scripts/office/helpers/__init__.py
+++ b/skills/xlsx/scripts/office/helpers/__init__.py
@@ -5,7 +5,7 @@ import stat
import tempfile
import urllib.parse
import zipfile
-from pathlib import Path
+from pathlib import Path, PureWindowsPath
OOXML_FAMILY = {
".docx": "docx",
@@ -21,16 +21,24 @@ _SCHEME_RE = re.compile(r"^[A-Za-z][A-Za-z0-9+.\-]*:")
SLIDE_REL_TYPE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/slide"
+def one_line(text: str, limit: int) -> str:
+ return "".join(c if c.isprintable() else c.encode("unicode_escape").decode() for c in text)[:limit]
+
+
+def names_another_root(name: str) -> bool:
+ return bool(PureWindowsPath(name).anchor)
+
+
def opc_target(target: str, source_part: str, target_mode: str = "") -> str | None:
if not target:
return None
if target_mode.lower() == "external":
return None
+ target = urllib.parse.unquote(target)
+
if _SCHEME_RE.match(target):
return None
- target = urllib.parse.unquote(target)
-
if "\\" in target:
raise ValueError(f"relationship target is not a POSIX part name: {target!r}")
@@ -52,12 +60,15 @@ def opc_target(target: str, source_part: str, target_mode: str = "") -> str | No
if not parts:
raise ValueError(f"relationship target resolves to nothing: {target!r}")
+ if names_another_root("/".join(parts)):
+ raise ValueError(f"relationship target is not a part name: {target!r}")
return "/".join(parts)
def rels_source_part(rels_file: Path, unpacked_dir: Path) -> str:
owner_dir = rels_file.parent.parent.relative_to(unpacked_dir)
- return posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")]).lstrip("./")
+ part = posixpath.join(owner_dir.as_posix(), rels_file.name[: -len(".rels")])
+ return part.removeprefix("./")
def part_text(data: bytes) -> str:
@@ -76,12 +87,19 @@ def safe_extract(zf: zipfile.ZipFile, dest: Path) -> None:
for m in zf.infolist():
if stat.S_ISLNK(m.external_attr >> 16):
raise ValueError(f"symlink archive entry not allowed: {m.filename!r}")
+ if names_another_root(m.filename):
+ raise ValueError(f"unsafe archive entry: {m.filename!r}")
target = (dest / m.filename).resolve()
if not target.is_relative_to(dest):
raise ValueError(f"unsafe archive entry: {m.filename!r}")
zf.extract(m, dest)
+def folder_entries(package: Path) -> list[str]:
+ with zipfile.ZipFile(package) as zf:
+ return [info.filename for info in zf.infolist() if info.is_dir()]
+
+
def rezip(src_dir: Path, out_path: Path) -> None:
files = sorted(p for p in src_dir.rglob("*") if p.is_file())
ct = src_dir / "[Content_Types].xml"
diff --git a/skills/xlsx/scripts/office/helpers/pptx_chart.py b/skills/xlsx/scripts/office/helpers/pptx_chart.py
index 209cb7c..0e86aed 100644
--- a/skills/xlsx/scripts/office/helpers/pptx_chart.py
+++ b/skills/xlsx/scripts/office/helpers/pptx_chart.py
@@ -1,15 +1,20 @@
-"""Find chart XML that PowerPoint refuses but the schema accepts.
+"""Find chart XML that PowerPoint refuses or repairs.
-Detection only: for either fault more than one repair is valid, and only the
-author knows which was meant.
+Detection only: more than one repair is usually valid, and only the author knows
+which was meant.
"""
from __future__ import annotations
+import html
import re
+from collections import Counter
from typing import Mapping
+from defusedxml import ElementTree
+from defusedxml.common import DefusedXmlException
+
from . import part_text
@@ -19,8 +24,12 @@ _GROUPING_RE = re.compile(r"""<c:grouping\b[^>]*?\bval=["'](\w+)["']""")
_DLBL_POS_RE = re.compile(r"""<c:dLblPos\b[^>]*?\bval=["'](\w+)["']""")
def _strip_ext_lst(text: str) -> str:
+ return _strip_outermost(text, "c:extLst")
+
+
+def _strip_outermost(text: str, name: str) -> str:
out, cursor = [], 0
- for lo, hi in _ext_lst_spans(text):
+ for lo, hi in _outermost_spans(text, name):
out.append(text[cursor:lo])
cursor = hi
out.append(text[cursor:])
@@ -139,18 +148,113 @@ def _check_chart_axis_references(part: str, xml: str) -> list[str]:
return problems
-def _ext_lst_spans(text: str) -> list[tuple[int, int]]:
+def _shown(value: str) -> str:
+ return '"' + repr(value[:20])[1:-1] + '"'
+
+
+DECIMAL_SETTINGS = (
+ "crossesAt", "max", "min", "majorUnit", "minorUnit", "logBase",
+ "forward", "backward", "intercept", "x", "y", "w", "h",
+ "splitPos", "custUnit", "pictureStackUnit",
+)
+_DECIMAL_SETTING_RE = re.compile(
+ rf"""<c:({"|".join(DECIMAL_SETTINGS)})\b[^<>]*?\bval=["']([^"'<>]*)["']"""
+)
+_DECIMAL_RE = re.compile(
+ r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?"
+ r"|[+-]?(inf|infinity|nan(\(\w*\))?|\u221e|1\.#(inf|qnan|snan|ind)\d*)",
+ re.IGNORECASE,
+)
+
+
+def _check_decimal_settings(part: str, xml: str) -> list[str]:
+ problems = []
+ for name, raw in sorted(set(_DECIMAL_SETTING_RE.findall(_strip_ext_lst(xml)))):
+ value = html.unescape(raw)
+ if _DECIMAL_RE.fullmatch(value.strip()):
+ continue
+ fix = "Fix: write a plain number, or remove the element"
+ if name == "crossesAt" and value.strip() == "autoZero":
+ fix = ("Fix: leave valAxisCrossesAt: 0 out of the pptxgenjs options, the axis "
+ "crosses at zero anyway. In the XML, write "
+ '<c:crosses val="autoZero"/> or a number')
+ problems.append(
+ f"{part}: <c:{name} val={_shown(value)}> is not a number, and PowerPoint "
+ f"will offer to repair the deck. {fix}"
+ )
+ return problems
+
+
+def _local(tag: str) -> str:
+ return tag.rsplit("}", 1)[-1]
+
+
+def _drawn_series(root) -> list:
+ found, to_visit = [], [root]
+ while to_visit:
+ for child in to_visit.pop():
+ name = _local(child.tag)
+ if name == "ser":
+ found.append(child)
+ elif name == "AlternateContent":
+ to_visit.extend(child[:1])
+ elif name != "extLst":
+ to_visit.append(child)
+ return found
+
+
+def _check_series_order(part: str, xml: str) -> list[str]:
+ try:
+ root = ElementTree.fromstring(xml.encode("utf-8", "surrogateescape"))
+ except (ElementTree.ParseError, DefusedXmlException, LookupError, ValueError):
+ return []
+ orders = Counter()
+ for series in _drawn_series(root):
+ own = [c.get("val", "") for c in series if _local(c.tag) == "order"]
+ if own and own[0].isascii() and own[0].isdigit():
+ orders[own[0]] += 1
+ repeated = sorted((o for o, n in orders.items() if n > 1), key=lambda o: (len(o), o))
+ if not repeated:
+ return []
+ return [
+ f"{part}: two or more series share the <c:order> value {', '.join(o[:12] for o in repeated)}, "
+ f"and PowerPoint will offer to repair the deck. Fix: number <c:order> "
+ f"(and <c:idx>) 0, 1, 2... across every series in the chart, a combo "
+ f"chart's line series carrying on from its bars"
+ ]
+
+
+PRESET_DASHES = frozenset({
+ "solid", "dot", "dash", "lgDash", "dashDot", "lgDashDot", "lgDashDotDot",
+ "sysDash", "sysDot", "sysDashDot", "sysDashDotDot",
+})
+_PRESET_DASH_RE = re.compile(r"""<a:prstDash\b[^<>]*?\bval=["']([^"'<>]*)["']""")
+
+
+def _check_preset_dashes(part: str, xml: str) -> list[str]:
+ values = {html.unescape(v) for v in _PRESET_DASH_RE.findall(_strip_ext_lst(xml))}
+ return [
+ f"{part}: <a:prstDash val={_shown(value)}> is not a dash style, and PowerPoint "
+ f"will offer to repair the deck. Use one of {', '.join(sorted(PRESET_DASHES))}. "
+ f"In pptxgenjs, lineDash takes one such word for the whole chart, not an array"
+ for value in sorted(values)
+ if value.strip() not in PRESET_DASHES
+ ]
+
+
+def _outermost_spans(text: str, name: str) -> list[tuple[int, int]]:
spans: list[tuple[int, int]] = []
depth = 0
start = 0
- for match in re.finditer(r"<(/?)c:extLst\b[^>]*?(/?)>", text):
+ for match in re.finditer(rf"<(/?){name}\b[^<>]*?(/?)>", text):
closing, self_closing = match.group(1), match.group(2)
if self_closing:
continue
if closing:
- depth -= 1
- if depth == 0:
- spans.append((start, match.end()))
+ if depth:
+ depth -= 1
+ if depth == 0:
+ spans.append((start, match.end()))
else:
if depth == 0:
start = match.start()
@@ -158,13 +262,22 @@ def _ext_lst_spans(text: str) -> list[tuple[int, int]]:
return spans
-CHART_CHECKS = (_check_stacked_label_positions, _check_chart_axis_references)
+CHART_CHECKS = (
+ _check_stacked_label_positions,
+ _check_chart_axis_references,
+ _check_decimal_settings,
+ _check_series_order,
+ _check_preset_dashes,
+)
+
+
+_COMMENT_OR_PI_RE = re.compile(r"<!--.*?(?:-->|\Z)|<\?(?!xml\s).*?(?:\?>|\Z)", re.DOTALL)
def find_chart_problems(files: Mapping[str, bytes]) -> list[str]:
problems: list[str] = []
for part in sorted(n for n in files if _CHART_PART_RE.fullmatch(n)):
- xml = part_text(files[part])
+ xml = _strip_outermost(_COMMENT_OR_PI_RE.sub("", part_text(files[part])), "mc:Fallback")
for check in CHART_CHECKS:
problems.extend(check(part, xml))
return problems
diff --git a/skills/xlsx/scripts/office/helpers/pptx_slide.py b/skills/xlsx/scripts/office/helpers/pptx_slide.py
index 22f9aee..55274b8 100644
--- a/skills/xlsx/scripts/office/helpers/pptx_slide.py
+++ b/skills/xlsx/scripts/office/helpers/pptx_slide.py
@@ -1,7 +1,8 @@
-"""Pick the slide-XML schema errors PowerPoint refuses the file over.
+"""Find the slide-XML defects known or predicted to make PowerPoint refuse or repair the file.
-A denylist over lxml's messages, so an unrecognised error class is a miss rather
-than a false alarm.
+Schema errors are picked by a denylist over lxml's messages, so an unrecognised
+error class is a miss rather than a false alarm. Animations that name a shape
+the slide no longer has are found by reading the part.
"""
@@ -43,6 +44,14 @@ FATAL_SLIDE_ERRORS: tuple[tuple[re.Pattern[str], str], ...] = (
re.compile(r"\}nvGrpSpPr': Missing child element"),
"a <p:nvGrpSpPr> with no children",
),
+ (
+ re.compile(r"\}(?:off|ext|chOff|chExt)', attribute '\w+': '-?(?:Infinity|NaN)'"),
+ "a position or size that is not a number (check the x, y, w, h you worked out)",
+ ),
+ (
+ re.compile(r"\}prstDash', attribute 'val'"),
+ "an unknown line dash style (for line.dashType try dash, sysDash or solid)",
+ ),
)
@@ -50,6 +59,37 @@ def is_schema_verdict(error: str) -> bool:
return error.startswith("Element ")
+_PML = "{http://schemas.openxmlformats.org/presentationml/2006/main}"
+_MC = "{http://schemas.openxmlformats.org/markup-compatibility/2006}"
+
+_NAMES_A_SHAPE = frozenset({f"{_PML}spTgt", f"{_PML}bldP", f"{_PML}bldGraphic"})
+
+
+def _shape_id(value: str | None) -> int | str | None:
+ text = (value or "").strip()
+ if text.isascii() and text.isdigit() and len(text) <= 18:
+ return int(text)
+ return text or None
+
+
+def stranded_animation_targets(root) -> list[str]:
+ timing = root.find(f"{_PML}timing")
+ if timing is None:
+ timing = root.find(f"{_MC}AlternateContent/{_MC}Choice/{_PML}timing")
+ if timing is None:
+ return []
+
+ on_slide = {_shape_id(e.get("id")) for e in root.iter("{*}cNvPr")}
+ named = {_shape_id(e.get("spid")) for e in timing.iter(*_NAMES_A_SHAPE)} - {None}
+ stranded = named - on_slide
+ return [
+ f"<p:timing> animates shape id {spid if isinstance(spid, int) else repr(spid[:16])}, which is gone, so PowerPoint will "
+ f"offer to repair the deck. Fix: delete the slide's whole <p:timing> element "
+ f"(the slide loses its animations), or put the shape back"
+ for spid in sorted(stranded, key=str)
+ ]
+
+
def fatal_slide_errors(errors: set[str]) -> list[str]:
out = []
for error in sorted(errors):
diff --git a/skills/xlsx/scripts/office/soffice.py b/skills/xlsx/scripts/office/soffice.py
index 0b4c99d..eec1687 100644
--- a/skills/xlsx/scripts/office/soffice.py
+++ b/skills/xlsx/scripts/office/soffice.py
@@ -64,7 +64,7 @@ def _ensure_shim() -> Path:
return _SHIM_SO
src = Path(tempfile.gettempdir()) / "lo_socket_shim.c"
- src.write_text(_SHIM_SOURCE)
+ src.write_text(_SHIM_SOURCE, encoding="utf-8")
subprocess.run(
["gcc", "-shared", "-fPIC", "-o", str(_SHIM_SO), str(src), "-ldl"],
check=True,
diff --git a/skills/xlsx/scripts/office/validate.py b/skills/xlsx/scripts/office/validate.py
index 29ca186..ff3bde0 100755
--- a/skills/xlsx/scripts/office/validate.py
+++ b/skills/xlsx/scripts/office/validate.py
@@ -14,6 +14,7 @@ Auto-repair fixes:
"""
import argparse
+import shlex
import sys
import tempfile
import zipfile
@@ -22,12 +23,21 @@ from pathlib import Path
import defusedxml.ElementTree as ET
from defusedxml.common import DefusedXmlException
-from helpers import OOXML_FAMILY, rezip, safe_extract
+from helpers import OOXML_FAMILY, folder_entries, rezip, safe_extract
from validators import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator
WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+def _as_argument(path: Path) -> str:
+ name = str(path)
+ return shlex.quote("./" + name if name.startswith("-") else name)
+
+
+def _count(folders: list[str]) -> str:
+ return "1 folder entry" if len(folders) == 1 else f"{len(folders)} folder entries"
+
+
def _fail(message: str):
print(f"Error: {message}", file=sys.stderr)
sys.exit(2)
@@ -150,16 +160,43 @@ def main():
print(f"Error: Validation not supported for file type {family}")
sys.exit(1)
+ folders = folder_entries(packed_file) if packed_file is not None else []
+
+ unsaved_repairs = 0
if args.auto_repair:
total_repairs = sum(v.repair() for v in validators)
if total_repairs:
print(f"Auto-repaired {total_repairs} issue(s)")
- if packed_file is not None:
+ if packed_file is not None and (total_repairs or folders):
+ try:
rezip(unpacked_dir, packed_file)
+ except OSError as e:
+ unsaved_repairs = total_repairs
+ left = []
+ if total_repairs:
+ left.append(f"{total_repairs} repair(s) not saved")
+ if folders:
+ left.append(f"{_count(folders)} in its ZIP")
+ print(f"Could not write {packed_file} ({e.strerror or e}): it is as it was, "
+ f"with {' and '.join(left)}. Run --auto-repair on a copy in a folder "
+ "you can write to.")
+ else:
+ if folders:
+ print(f"Removed {_count(folders)} from the ZIP")
print(f"Wrote repaired file to {packed_file}")
+ elif folders:
+ print(
+ f"Note: {_count(folders)} in the ZIP. Such files have failed to open or co-edit "
+ f"through a cloud service. Remove them: zip -dq {_as_argument(packed_file)} '*/'"
+ )
success = all([v.validate() for v in validators])
+ if unsaved_repairs:
+ print(f"FAILED - {packed_file} was not rewritten and still has the {unsaved_repairs} "
+ "issue(s) repaired above; the checks ran on a repaired copy")
+ success = False
+
if temp_dir_ctx is not None:
temp_dir_ctx.cleanup()
diff --git a/skills/xlsx/scripts/office/validators/base.py b/skills/xlsx/scripts/office/validators/base.py
index 33fc97b..04d6169 100644
--- a/skills/xlsx/scripts/office/validators/base.py
+++ b/skills/xlsx/scripts/office/validators/base.py
@@ -2,6 +2,7 @@
Base validator with common validation logic for document files.
"""
+import os
import re
from pathlib import Path
@@ -10,7 +11,7 @@ from functools import lru_cache
import lxml.etree
-from helpers import safe_extract
+from helpers import names_another_root, safe_extract
@lru_cache(maxsize=None)
@@ -352,6 +353,9 @@ class BaseSchemaValidator:
if target and not target.startswith(
("http", "mailto:")
):
+ if names_another_root(target.lstrip("/")):
+ broken_refs.append((target, rel.sourceline))
+ continue
if target.startswith("/"):
target_path = self.unpacked_dir / target.lstrip("/")
elif rels_file.name == ".rels":
@@ -360,9 +364,18 @@ class BaseSchemaValidator:
base_dir = rels_dir.parent
target_path = base_dir / target
+ target_path = Path(os.path.normpath(target_path))
+ if not target_path.is_relative_to(self.unpacked_dir):
+ broken_refs.append((target, rel.sourceline))
+ continue
+
try:
target_path = target_path.resolve()
- if target_path.exists() and target_path.is_file():
+ if (
+ target_path.is_relative_to(self.unpacked_dir)
+ and target_path.exists()
+ and target_path.is_file()
+ ):
referenced_files.add(target_path)
all_referenced_files.add(target_path)
else:
@@ -783,8 +796,7 @@ class BaseSchemaValidator:
try:
schema = _load_schema(str(schema_path))
- with open(xml_file, "r") as f:
- xml_doc = lxml.etree.parse(f)
+ xml_doc = lxml.etree.parse(str(xml_file))
xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc)
xml_doc = self._preprocess_for_mc_ignorable(xml_doc)
diff --git a/skills/xlsx/scripts/office/validators/docx.py b/skills/xlsx/scripts/office/validators/docx.py
index b181499..2acdaf4 100644
--- a/skills/xlsx/scripts/office/validators/docx.py
+++ b/skills/xlsx/scripts/office/validators/docx.py
@@ -6,12 +6,13 @@ import random
import re
import tempfile
import zipfile
+from collections import Counter
from pathlib import Path
import defusedxml.minidom
import lxml.etree
-from helpers import safe_extract
+from helpers import one_line, safe_extract
from .base import BaseSchemaValidator
@@ -62,6 +63,9 @@ class DOCXSchemaValidator(BaseSchemaValidator):
if not self.validate_comment_markers():
all_valid = False
+ if not self.validate_style_ids():
+ all_valid = False
+
self.compare_paragraph_counts()
return all_valid
@@ -264,7 +268,9 @@ class DOCXSchemaValidator(BaseSchemaValidator):
for xml_file in self.xml_files:
try:
- for elem in lxml.etree.parse(str(xml_file)).iter():
+ tree = lxml.etree.parse(str(xml_file))
+ is_numbering = lxml.etree.QName(tree.getroot()).localname == "numbering"
+ for elem in tree.iter():
if val := elem.get(para_id_attr):
try:
if self._parse_id_value(val, base=16) >= 0x80000000:
@@ -278,7 +284,7 @@ class DOCXSchemaValidator(BaseSchemaValidator):
)
if val := elem.get(durable_id_attr):
- if xml_file.name == "numbering.xml":
+ if is_numbering:
try:
if self._parse_id_value(val, base=10) >= 0x7FFFFFFF:
errors.append(
@@ -288,7 +294,7 @@ class DOCXSchemaValidator(BaseSchemaValidator):
except ValueError:
errors.append(
f" {xml_file.name}:{elem.sourceline}: "
- f"durableId={val} must be decimal in numbering.xml"
+ f"durableId={val} must be decimal in a numbering part"
)
else:
try:
@@ -308,11 +314,58 @@ class DOCXSchemaValidator(BaseSchemaValidator):
if errors:
print(f"FAILED - {len(errors)} ID constraint violations:")
for e in errors:
- print(e)
+ print(one_line(e, 300))
elif self.verbose:
print("PASSED - All paraId/durableId values within constraints")
return not errors
+ def _repeated_style_ids(self, styles_root) -> set[str]:
+ attr = f"{{{self.WORD_2006_NAMESPACE}}}styleId"
+ ids = Counter(s.get(attr) for s in styles_root.iter(f"{{{self.WORD_2006_NAMESPACE}}}style"))
+ return {i for i, n in ids.items() if i is not None and n > 1}
+
+ def _original_part(self, xml_file) -> bytes | None:
+ if self.original_file is None:
+ return None
+ name = xml_file.relative_to(self.unpacked_dir).as_posix()
+ try:
+ with zipfile.ZipFile(self.original_file) as zf:
+ return zf.read(name)
+ except (KeyError, OSError, zipfile.BadZipFile):
+ return None
+
+ def validate_style_ids(self):
+ errors = []
+ styles_tag = f"{{{self.WORD_2006_NAMESPACE}}}styles"
+ if self.original_file is None:
+ return True
+
+ for xml_file in self.xml_files:
+ try:
+ root = lxml.etree.parse(str(xml_file)).getroot()
+ if root.tag != styles_tag:
+ continue
+ repeated = self._repeated_style_ids(root)
+ original = self._original_part(xml_file) if repeated else None
+ if original is not None:
+ repeated -= self._repeated_style_ids(lxml.etree.fromstring(original))
+ except lxml.etree.XMLSyntaxError:
+ continue
+ errors.extend(
+ f" {xml_file.relative_to(self.unpacked_dir)}: style id {style_id!r:.40} is now "
+ f"defined more than once, and Word uses only the last definition, without "
+ f"warning. Delete the earlier ones"
+ for style_id in sorted(repeated)
+ )
+
+ if errors:
+ print(f"FAILED - Found {len(errors)} style id(s) defined more than once:")
+ for error in errors:
+ print(one_line(error, 300))
+ elif self.verbose:
+ print("PASSED - The edit defined no style id a second time")
+ return not errors
+
def validate_comment_markers(self):
errors = []
@@ -415,7 +468,7 @@ class DOCXSchemaValidator(BaseSchemaValidator):
try:
content = xml_file.read_text(encoding="utf-8")
dom = defusedxml.minidom.parseString(content)
- is_numbering = xml_file.name == "numbering.xml"
+ is_numbering = dom.documentElement.localName == "numbering"
base = 10 if is_numbering else 16
pending = []
seen_in_file = set()
diff --git a/skills/xlsx/scripts/office/validators/pptx.py b/skills/xlsx/scripts/office/validators/pptx.py
index 318f0e6..bb06b82 100644
--- a/skills/xlsx/scripts/office/validators/pptx.py
+++ b/skills/xlsx/scripts/office/validators/pptx.py
@@ -63,6 +63,9 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if not self.validate_master_theme_uniqueness():
all_valid = False
+ if not self.validate_tags():
+ all_valid = False
+
if not self.validate_charts():
all_valid = False
@@ -87,6 +90,30 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if p.is_file()
}
+ @staticmethod
+ def _print_line(text: str, limit: int) -> None:
+ from helpers import one_line
+
+ print(one_line(text, limit))
+
+ def _listed_slides(self) -> set[str] | None:
+ import lxml.etree
+
+ from helpers import SLIDE_REL_TYPE
+
+ r_id = f"{{{self.OFFICE_RELATIONSHIPS_NAMESPACE}}}id"
+ try:
+ pres = lxml.etree.parse(str(self.unpacked_dir / "ppt" / "presentation.xml"))
+ rels = lxml.etree.parse(str(self.unpacked_dir / "ppt" / "_rels" / "presentation.xml.rels"))
+ listed = {e.get(r_id) for e in pres.iter(f"{{{self.PRESENTATIONML_NAMESPACE}}}sldId")}
+ return {
+ opc_target(rel.get("Target", ""), "ppt/presentation.xml", rel.get("TargetMode", ""))
+ for rel in rels.getroot()
+ if rel.get("Id") in listed and rel.get("Type") == SLIDE_REL_TYPE
+ }
+ except (OSError, ValueError, lxml.etree.XMLSyntaxError):
+ return None
+
def validate_master_theme_uniqueness(self):
from helpers.pptx_theme import _NOTES_MASTERS, live_shared_master_themes
@@ -94,7 +121,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if shared:
print(f"FAILED - Found {len(shared)} master(s) sharing a theme part:")
for message in shared:
- print(f" {message}")
+ self._print_line(f" {message}", 300)
if any(m.startswith(_NOTES_MASTERS) for m in shared):
print(" Fix: in ppt/presentation.xml, move <p:notesMasterIdLst> back to "
"directly after <p:sldIdLst>. PowerPoint reads that happily.")
@@ -106,6 +133,64 @@ class PPTXSchemaValidator(BaseSchemaValidator):
print("PASSED - No master shares a theme part in a way PowerPoint refuses")
return True
+ def validate_tags(self):
+ from collections import Counter
+
+ import lxml.etree
+
+ ns = {"p": self.PRESENTATIONML_NAMESPACE, "mc": self.MC_NAMESPACE}
+ r_id = f"{{{self.OFFICE_RELATIONSHIPS_NAMESPACE}}}id"
+ errors = []
+ owners: dict[str, list[str]] = {}
+ listed = self._listed_slides()
+
+ for slide in sorted(self.unpacked_dir.glob("ppt/slides/*.xml")):
+ relative = slide.relative_to(self.unpacked_dir).as_posix()
+ if listed is not None and relative not in listed:
+ continue
+ try:
+ root = lxml.etree.parse(str(slide)).getroot()
+ named = [t.get(r_id) for t in root.xpath(
+ ".//p:tags[not(ancestor::mc:Fallback) and not(ancestor::mc:Choice[preceding-sibling::mc:Choice])]",
+ namespaces=ns)]
+ rels_file = slide.parent / "_rels" / f"{slide.name}.rels"
+ targets = {
+ rel.get("Id"): opc_target(rel.get("Target", ""), relative, rel.get("TargetMode", ""))
+ for rel in lxml.etree.parse(str(rels_file)).getroot()
+ if rel.get("Type", "").endswith("/relationships/tags")
+ } if rels_file.is_file() else {}
+ except (lxml.etree.XMLSyntaxError, ValueError):
+ continue
+ for rid, count in sorted(Counter(r for r in named if r).items()):
+ if count > 1:
+ errors.append(
+ f" {relative[:80]}: {count} <p:tags> elements name the tags relationship "
+ f"{rid!r:.20}. Fix: a copied shape kept its <p:custDataLst>; delete that "
+ f"from the copy, or give it a tags part and relationship of its own"
+ )
+ if targets.get(rid):
+ owners.setdefault(targets[rid], []).append(relative)
+
+ for part, slides in sorted(owners.items()):
+ if len(slides) > 1:
+ named_from = sorted(set(slides))
+ more = f" and {len(named_from) - 3} more" if len(named_from) > 3 else ""
+ errors.append(
+ f" {part[:80]}: {len(slides)} tags relationships name it, from "
+ f"{', '.join(s[:60] for s in named_from[:3])}{more}. Fix: copy the part under a new name, add "
+ f"its Override to [Content_Types].xml and point every relationship but one at a copy of its own "
+ f"(add_slide.py does this when it duplicates a slide)"
+ )
+
+ if errors:
+ print(f"FAILED - Found {len(errors)} tags part(s) with more than one owner; PowerPoint offers to repair the deck:")
+ for error in errors:
+ self._print_line(error, 700)
+ return False
+ if self.verbose:
+ print("PASSED - Every tags part has one owner")
+ return True
+
def validate_charts(self):
from helpers.pptx_chart import find_chart_problems
@@ -113,7 +198,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if problems:
print(f"FAILED - Found {len(problems)} chart problem(s) PowerPoint rejects:")
for message in problems:
- print(f" {message}")
+ self._print_line(f" {message}", 600)
return False
if self.verbose:
@@ -151,14 +236,18 @@ class PPTXSchemaValidator(BaseSchemaValidator):
return found
def validate_slides(self):
+ import lxml.etree
+
from helpers.pptx_slide import (
SLIDE_PART_RE,
fatal_slide_errors,
is_schema_verdict,
+ stranded_animation_targets,
)
schema = self.schemas_dir / self.SCHEMA_MAPPINGS["ppt"]
inherited = self._original_slide_defects(schema)
+ listed = self._listed_slides()
problems: list[str] = []
broken: list[str] = []
@@ -166,6 +255,14 @@ class PPTXSchemaValidator(BaseSchemaValidator):
relative = xml_file.relative_to(self.unpacked_dir).as_posix()
if not SLIDE_PART_RE.fullmatch(relative):
continue
+ unlisted = relative.startswith("ppt/slides/") and listed is not None and relative not in listed
+ try:
+ root = lxml.etree.parse(str(xml_file)).getroot()
+ stranded = [] if unlisted else stranded_animation_targets(root)
+ except lxml.etree.XMLSyntaxError:
+ stranded = []
+ problems.extend(f"{relative}: {m}" for m in stranded)
+
ok, errors = self._validate_single_file_xsd(
xml_file.resolve(), self.unpacked_dir.resolve(), schema_path=schema
)
@@ -187,12 +284,12 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if broken:
print(f"FAILED - Could not check {len(broken)} slide part(s):")
for message in sorted(broken):
- print(f" {message[:240]}")
+ self._print_line(f" {message}", 240)
if problems:
print(f"FAILED - Found {len(problems)} slide problem(s) PowerPoint rejects:")
for message in sorted(problems):
- print(f" {message[:240]}")
+ self._print_line(f" {message}", 240)
if broken or problems:
return False
@@ -204,6 +301,17 @@ class PPTXSchemaValidator(BaseSchemaValidator):
def _get_schema_path(self, xml_file):
if xml_file.parent.name == "charts" and xml_file.name.startswith("chart"):
return None
+ if xml_file.parent.name == "ppt" and xml_file.suffix == ".xml":
+ import lxml.etree
+
+ try:
+ root = lxml.etree.parse(str(xml_file)).getroot()
+ except (lxml.etree.XMLSyntaxError, OSError):
+ root = None
+ if root is not None and (lxml.etree.QName(root).namespace or "").startswith(
+ "http://schemas.microsoft.com/office/"
+ ):
+ return None
return super()._get_schema_path(xml_file)
def _preprocess_for_schema(self, xml_doc, relative_path):
@@ -256,7 +364,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if errors:
print(f"FAILED - Found {len(errors)} UUID ID validation errors:")
for error in errors:
- print(error)
+ self._print_line(error, 300)
return False
else:
if self.verbose:
@@ -325,7 +433,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if errors:
print(f"FAILED - Found {len(errors)} slide layout ID validation errors:")
for error in errors:
- print(error)
+ self._print_line(error, 300)
print(
"Remove invalid references or add missing slide layouts to the relationships file."
)
@@ -366,7 +474,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
if errors:
print("FAILED - Found slides with duplicate slideLayout references:")
for error in errors:
- print(error)
+ self._print_line(error, 300)
return False
else:
if self.verbose:
@@ -428,7 +536,7 @@ class PPTXSchemaValidator(BaseSchemaValidator):
f"FAILED - Found {len([e for e in errors if not e.startswith(' ')])} notes slide reference validation errors:"
)
for error in errors:
- print(error)
+ self._print_line(error, 300)
print("Each slide may optionally have its own slide file.")
return False
else:
diff --git a/skills/xlsx/scripts/office/validators/redlining.py b/skills/xlsx/scripts/office/validators/redlining.py
index 4185c51..4406912 100644
--- a/skills/xlsx/scripts/office/validators/redlining.py
+++ b/skills/xlsx/scripts/office/validators/redlining.py
@@ -1,19 +1,24 @@
"""
Validator for tracked changes in Word documents.
-Detects untracked edits in word/document.xml: text that differs from the
+Detects untracked edits in a Word document: text that differs from the
original without a <w:ins>/<w:del> wrapper recording it. The tracked changes
that are new relative to the original are undone, and the result is compared
against the original; whatever text still differs was edited without being
tracked.
-Only the document body is compared. Headers, footers, footnotes and endnotes
-are separate parts and are not checked.
+The body (word/document.xml) is compared, and so is each header, footer,
+footnotes and endnotes part, against the original's part of the same name.
+Other parts, comments among them, are not checked. A page-number field
+whose cached result is a plain number counts as its name in braces
+("{PAGE}"), not as that number, which any application that repaginates the
+document rewrites.
"""
-import subprocess
+import re
import tempfile
import zipfile
+from difflib import SequenceMatcher
from pathlib import Path
import defusedxml.ElementTree as ET
@@ -22,12 +27,35 @@ from defusedxml.common import DefusedXmlException
from helpers import rendered_text, safe_extract
+XML_ERRORS = (ET.ParseError, DefusedXmlException, LookupError, ValueError)
+
+
+class _WorkSpent(Exception):
+
+ pass
+
+
+class _Matcher(SequenceMatcher):
+
+ def __init__(self, a, b, allowance):
+ self.allowance = allowance
+ super().__init__(None, a, b, autojunk=False)
+
+ def find_longest_match(self, alo=0, ahi=None, blo=0, bhi=None):
+ items = self.a[alo:ahi]
+ self.allowance[0] -= len(items) + sum(len(self.b2j.get(x, ())) for x in items)
+ if self.allowance[0] < 0:
+ raise _WorkSpent
+ return super().find_longest_match(alo, ahi, blo, bhi)
+
+
class RedliningValidator:
def __init__(self, unpacked_dir, original_docx, verbose=False):
self.unpacked_dir = Path(unpacked_dir)
self.original_docx = Path(original_docx)
self.verbose = verbose
+ self._work_left = [self.WORK_ALLOWED]
self.namespaces = {
"w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
}
@@ -58,39 +86,81 @@ class RedliningValidator:
)
return False
- try:
- modified_tree = ET.parse(modified_file)
- modified_root = modified_tree.getroot()
- original_tree = ET.parse(original_file)
- original_root = original_tree.getroot()
- except (ET.ParseError, DefusedXmlException) as e:
- print(f"FAILED - Error parsing XML files: {e}")
- return False
-
- new_changes = self._new_tracked_changes(original_root, modified_root)
- self._remove_tracked_changes(modified_root, new_changes)
-
- modified_text = self._extract_text_content(modified_root)
- original_text = self._extract_text_content(original_root)
-
- if modified_text != original_text:
- error_message = self._generate_detailed_diff(
- original_text, modified_text
- )
- print(error_message)
+ sides = (("original", temp_path), ("modified", self.unpacked_dir))
+ change_count, failures, differences = 0, [], []
+ for part in self._parts_to_compare(temp_path, self.unpacked_dir):
+ roots = []
+ for side, unpacked in sides:
+ try:
+ roots.append(self._parse_part(unpacked / part))
+ except XML_ERRORS as e:
+ failures.append(
+ f"FAILED - Error parsing {part} of the {side} document: {e}"
+ )
+ if len(roots) < len(sides):
+ continue
+ new_changes, original_text, modified_text = self._compare_part(*roots)
+ change_count += len(new_changes)
+ if modified_text != original_text:
+ differences.append((part, original_text, modified_text))
+
+ if differences:
+ failures.append(self._generate_detailed_diff(differences))
+ if failures:
+ print("\n".join(failures))
return False
if self.verbose:
print(
- f"PASSED - All {len(new_changes)} change(s) against the original "
+ f"PASSED - All {change_count} change(s) against the original "
"are properly tracked"
)
return True
+ OTHER_PARTS = (
+ "word/header*.xml",
+ "word/footer*.xml",
+ "word/footnotes.xml",
+ "word/endnotes.xml",
+ )
+
+ def _parts_to_compare(self, original_dir, modified_dir):
+ others = set()
+ for unpacked in (Path(original_dir), Path(modified_dir)):
+ for pattern in self.OTHER_PARTS:
+ others.update(
+ path.relative_to(unpacked).as_posix()
+ for path in unpacked.glob(pattern)
+ )
+ return ["word/document.xml", *sorted(others)]
+
+ def _compare_part(self, original_root, modified_root):
+ for root in (original_root, modified_root):
+ self._flatten_text_elements(root)
+ new_changes = self._new_tracked_changes(original_root, modified_root)
+ self._remove_tracked_changes(modified_root, new_changes)
+ return (
+ new_changes,
+ self._extract_text_content(original_root),
+ self._extract_text_content(modified_root),
+ )
+
+ def _parse_part(self, path):
+ if not path.is_file():
+ return ET.fromstring("<absent/>")
+ return ET.parse(path).getroot()
+
+ def _flatten_text_elements(self, root):
+ for elem in self._text_elements(root):
+ if len(elem):
+ elem.text = "".join(elem.itertext())
+ for child in list(elem):
+ elem.remove(child)
+
def _tracked_change_elements(self, root):
- ins_tag = f"{{{self.namespaces['w']}}}ins"
- del_tag = f"{{{self.namespaces['w']}}}del"
- return [elem for elem in root.iter() if elem.tag in (ins_tag, del_tag)]
+ w = self.namespaces["w"]
+ tags = (f"{{{w}}}ins", f"{{{w}}}del", f"{{{w}}}moveFrom", f"{{{w}}}moveTo")
+ return [elem for elem in root.iter() if elem.tag in tags]
def _rendered_text(self, elem):
preserve = elem.get("{http://www.w3.org/XML/1998/namespace}space") == "preserve"
@@ -148,7 +218,8 @@ class RedliningValidator:
new.update(elems)
return new
- def _generate_detailed_diff(self, original_text, modified_text):
+ def _generate_detailed_diff(self, differences):
+ self._work_left = [self.WORK_ALLOWED]
error_parts = [
"FAILED - Document text doesn't match after removing the tracked changes",
"",
@@ -170,130 +241,238 @@ class RedliningValidator:
"",
]
- git_diff = self._get_git_word_diff(original_text, modified_text)
- if git_diff:
- error_parts.extend(["Differences:", "============", git_diff])
- else:
- error_parts.append("Unable to generate word diff (git not available)")
+ explain_page_fields = False
+ for part, original_text, modified_text in differences:
+ body = part == "word/document.xml"
+ heading = "Differences:" if body else f"Differences in {part}:"
+ if error_parts[-1]:
+ error_parts.append("")
+ word_diff = self._word_diff(original_text, modified_text)
+ error_parts.extend([heading, "=" * len(heading), word_diff])
+ if original_text.replace("\n", "") == modified_text.replace("\n", ""):
+ error_parts.append("Only paragraph breaks differ here.")
+ if re.search(r"\{(%s)\}" % "|".join(self.PAGE_FIELDS), word_diff):
+ explain_page_fields = True
+ if explain_page_fields:
+ error_parts.extend(["", self.PAGE_FIELD_NOTE])
+ if self._work_left[0] < 0:
+ error_parts.extend(["", self.WORK_SPENT_NOTE])
return "\n".join(error_parts)
- def _get_git_word_diff(self, original_text, modified_text):
- try:
- with tempfile.TemporaryDirectory() as temp_dir:
- temp_path = Path(temp_dir)
-
- original_file = temp_path / "original.txt"
- modified_file = temp_path / "modified.txt"
-
- original_file.write_text(original_text, encoding="utf-8")
- modified_file.write_text(modified_text, encoding="utf-8")
-
- result = subprocess.run(
- [
- "git",
- "diff",
- "--word-diff=plain",
- "--word-diff-regex=.",
- "-U0",
- "--no-index",
- str(original_file),
- str(modified_file),
- ],
- capture_output=True,
- text=True,
- )
+ def _word_diff(self, original_text, modified_text):
+ out = []
+ steps = self._steps(original_text.split("\n"), modified_text.split("\n"))
+ for tag, was, now in steps:
+ if tag == "replace" and len(was) == len(now):
+ out.extend(self._mark_changes(x, y) for x, y in zip(was, now) if x != y)
+ elif tag != "equal":
+ out.append(self._mark_changes("\n".join(was), "\n".join(now)))
+ return "\n".join(out)
+
+ def _mark_changes(self, old, new, by_character=False, trim=True):
+
+ def mark(text, opening, closing):
+ marked = re.sub(r"[^\n]+", lambda m: opening + m[0] + closing, text)
+ return marked if marked.strip("\n") or opening == "{+" else ""
+
+ pattern = "(?s)." if by_character else r"\S+\s*|\s+"
+ out = []
+ steps = self._steps(re.findall(pattern, old), re.findall(pattern, new), trim)
+ for tag, was, now in steps:
+ lone_word = len(was) == len(now) == 1
+ was, now = "".join(was), "".join(now)
+ if tag == "equal":
+ out.append(now)
+ elif tag == "replace" and not by_character:
+ long_word = lone_word and min(len(was), len(now)) > self.LONG_WORD
+ marked = self._mark_changes(was, now, by_character=True, trim=long_word)
+ out.append(marked)
+ else:
+ out.append(mark(was, "[-", "-]") + mark(now, "{+", "+}"))
+ return "".join(out)
- if result.stdout.strip():
- lines = result.stdout.split("\n")
- content_lines = []
- in_content = False
- for line in lines:
- if line.startswith("@@"):
- in_content = True
- continue
- if in_content and line.strip():
- content_lines.append(line)
-
- if content_lines:
- return "\n".join(content_lines)
-
- result = subprocess.run(
- [
- "git",
- "diff",
- "--word-diff=plain",
- "-U0",
- "--no-index",
- str(original_file),
- str(modified_file),
- ],
- capture_output=True,
- text=True,
- )
+ WORK_ALLOWED = 10_000_000
+ LONG_WORD = 200
- if result.stdout.strip():
- lines = result.stdout.split("\n")
- content_lines = []
- in_content = False
- for line in lines:
- if line.startswith("@@"):
- in_content = True
- continue
- if in_content and line.strip():
- content_lines.append(line)
- return "\n".join(content_lines)
+ def _steps(self, a, b, trim=True):
- except (subprocess.CalledProcessError, FileNotFoundError, Exception):
- pass
+ def shared_start(x, y):
+ return next(
+ (i for i, (p, q) in enumerate(zip(x, y)) if p != q), min(len(x), len(y))
+ )
- return None
+ head = shared_start(a, b) if trim else 0
+ tail = shared_start(a[head:][::-1], b[head:][::-1]) if trim else 0
+ a_mid, b_mid = a[head : len(a) - tail], b[head : len(b) - tail]
+ try:
+ if self._work_left[0] < len(a_mid):
+ self._work_left[0] = -1
+ raise _WorkSpent
+ numbers = {}
+ numbered = [
+ [numbers.setdefault(x, len(numbers)) for x in s] for s in (a_mid, b_mid)
+ ]
+ opcodes = _Matcher(*numbered, self._work_left).get_opcodes()
+ steps = [(tag, a_mid[i:j], b_mid[m:n]) for tag, i, j, m, n in opcodes]
+ except _WorkSpent:
+ steps = [("replace", a_mid, b_mid)]
+ return [
+ ("equal", a[:head], b[:head]),
+ *steps,
+ ("equal", a[len(a) - tail :], b[len(b) - tail :]),
+ ]
def _remove_tracked_changes(self, root, targets):
- ins_tag = f"{{{self.namespaces['w']}}}ins"
- del_tag = f"{{{self.namespaces['w']}}}del"
+ w = self.namespaces["w"]
+ ins_tag = f"{{{w}}}ins"
+ del_tag = f"{{{w}}}del"
+ move_from_tag = f"{{{w}}}moveFrom"
+ move_to_tag = f"{{{w}}}moveTo"
+
+ self._merge_split_paragraphs(root, targets)
for parent in root.iter():
to_remove = []
for child in parent:
- if child.tag == ins_tag and child in targets:
+ if child.tag in (ins_tag, move_to_tag) and child in targets:
to_remove.append(child)
for elem in to_remove:
parent.remove(elem)
- deltext_tag = f"{{{self.namespaces['w']}}}delText"
- t_tag = f"{{{self.namespaces['w']}}}t"
+ renamed = {
+ f"{{{w}}}delText": f"{{{w}}}t",
+ f"{{{w}}}delInstrText": f"{{{w}}}instrText",
+ }
for parent in root.iter():
to_process = []
for child in parent:
- if child.tag == del_tag and child in targets:
+ if child.tag in (del_tag, move_from_tag) and child in targets:
to_process.append((child, list(parent).index(child)))
for del_elem, del_index in reversed(to_process):
for elem in del_elem.iter():
- if elem.tag == deltext_tag:
- elem.tag = t_tag
+ elem.tag = renamed.get(elem.tag, elem.tag)
for child in reversed(list(del_elem)):
parent.insert(del_index, child)
parent.remove(del_elem)
+ range_tags = (
+ f"{{{w}}}moveFromRangeStart",
+ f"{{{w}}}moveFromRangeEnd",
+ f"{{{w}}}moveToRangeStart",
+ f"{{{w}}}moveToRangeEnd",
+ )
+ for parent in root.iter():
+ to_remove = [
+ c
+ for c in parent
+ if c.tag in range_tags and len(c) == 0 and not (c.text or "").strip()
+ ]
+ for elem in to_remove:
+ parent.remove(elem)
+
+ def _merge_split_paragraphs(self, root, targets):
+ w = self.namespaces["w"]
+ p_tag = f"{{{w}}}p"
+ ppr_tag = f"{{{w}}}pPr"
+ marker_path = f"{ppr_tag}/{{{w}}}rPr/{{{w}}}ins"
+
+ for parent in list(root.iter()):
+ for para in reversed([child for child in parent if child.tag == p_tag]):
+ if not any(m in targets for m in para.findall(marker_path)):
+ continue
+ siblings = list(parent)
+ index = siblings.index(para)
+ following = siblings[index + 1] if index + 1 < len(siblings) else None
+ if following is None or following.tag != p_tag:
+ continue
+ if any(self._text_elements(c) for c in para if c.tag == ppr_tag):
+ continue
+ insert_at = 1 if len(following) and following[0].tag == ppr_tag else 0
+ for child in reversed([c for c in para if c.tag != ppr_tag]):
+ following.insert(insert_at, child)
+ parent.remove(para)
+
def _extract_text_content(self, root):
p_tag = f"{{{self.namespaces['w']}}}p"
- t_tag = f"{{{self.namespaces['w']}}}t"
paragraphs = []
for p_elem in root.findall(f".//{p_tag}"):
- text_parts = []
- for t_elem in p_elem.findall(f".//{t_tag}"):
- text_parts.append(self._rendered_text(t_elem))
- paragraph_text = "".join(text_parts)
+ paragraph_text = self._paragraph_text(p_elem)
if paragraph_text:
paragraphs.append(paragraph_text)
return "\n".join(paragraphs)
+ PAGE_FIELDS = {"PAGE", "NUMPAGES", "SECTIONPAGES"}
+ PAGE_NUMBER = re.compile(r"-? ?[0-9]{1,6} ?-?|[ivxlcdm]{1,8}|[a-z]{1,2}|", re.I)
+ PAGE_FIELD_NOTE = (
+ "Note: {PAGE}, {NUMPAGES} or {SECTIONPAGES} above stands for a page-number\n"
+ "field. Word recomputes the number it displays, so the number stored in\n"
+ "the file is not compared, but adding or removing the field still counts."
+ )
+
+ WORK_SPENT_NOTE = (
+ "Note: the texts were too long or repetitive to compare closely all the\n"
+ "way through, so a passage shown whole, as removed and then added, may\n"
+ "differ in only a few characters. Compare such a passage with the\n"
+ "original directly to find the edit."
+ )
+
+ def _paragraph_text(self, p_elem):
+ w = self.namespaces["w"]
+ t_tag, instr_tag = f"{{{w}}}t", f"{{{w}}}instrText"
+ fld_char, fld_simple = f"{{{w}}}fldChar", f"{{{w}}}fldSimple"
+
+ def page_field(instruction, field):
+ words = instruction.split()
+ locked = field.get(f"{{{w}}}fldLock") in ("1", "true", "on")
+ if words and words[0].upper() in self.PAGE_FIELDS and not locked:
+ return words[0].upper()
+
+ def shown(name, result):
+ return "{%s}" % name if self.PAGE_NUMBER.fullmatch(result) else result
+
+ parts, fields = [], []
+ skipped = set()
+
+ def sink():
+ for *_, result in reversed(fields):
+ if result is not None:
+ return result
+ return parts
+
+ for elem in p_elem.iter():
+ if elem.tag == fld_simple:
+ name = page_field(elem.get(f"{{{w}}}instr", ""), elem)
+ if name:
+ texts = [self._rendered_text(t) for t in elem.iter(t_tag)]
+ sink().append(shown(name, "".join(texts).strip()))
+ skipped.update(elem.iter(t_tag))
+ elif elem.tag == fld_char:
+ kind = elem.get(f"{{{w}}}fldCharType")
+ if kind == "begin":
+ fields.append([elem, "", False, None])
+ elif kind == "separate" and fields and not fields[-1][2]:
+ fields[-1][2] = True
+ if page_field(fields[-1][1], fields[-1][0]):
+ fields[-1][3] = []
+ elif kind == "end" and fields:
+ begin, instruction, _, result = fields.pop()
+ name = page_field(instruction, begin)
+ if name:
+ sink().append(shown(name, "".join(result or []).strip()))
+ elif elem.tag == instr_tag and fields and not fields[-1][2]:
+ fields[-1][1] += elem.text or ""
+ elif elem.tag == t_tag and elem not in skipped:
+ sink().append(self._rendered_text(elem))
+ for *_, result in fields:
+ parts.extend(result or [])
+ return "".join(parts)
+
if __name__ == "__main__":
raise RuntimeError("This module should not be run directly.")
diff --git a/skills/xlsx/scripts/recalc.py b/skills/xlsx/scripts/recalc.py
index 6232be2..9f1d1b9 100755
--- a/skills/xlsx/scripts/recalc.py
+++ b/skills/xlsx/scripts/recalc.py
@@ -1,32 +1,38 @@
"""
Excel Formula Recalculation Script
Recalculates all formulas in an Excel file using LibreOffice
+
+LibreOffice recalculates a copy of the workbook. Only the values it computed are then written
+into the workbook itself, so nothing else in the file is rewritten.
"""
import contextlib
+import copy
+import html
+import io
import json
+import math
import os
import platform
+import posixpath
import re
import shutil
+import signal
import subprocess
import sys
import tempfile
import time
import zipfile
from pathlib import Path
+from xml.parsers import expat
from office.soffice import get_soffice_env, run_soffice
-from openpyxl import load_workbook
-
MACRO_FILENAME = "Module1.xba"
SOFFICE_MISSING = "soffice not found on PATH; LibreOffice is required to recalculate"
MAX_LOCATIONS = 100
-EXTERNAL_REF_RE = re.compile(r"""(?<![\w"\[])'?\[\d+\][^!"\[\]]*'?!""")
-
RECALCULATE_MACRO = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE script:module PUBLIC "-//OpenOffice.org//DTD OfficeDocument 1.0//EN" "module.dtd">
<script:module xmlns:script="http://openoffice.org/2000/script" script:name="Module1" script:language="StarBasic">
@@ -78,90 +84,92 @@ def setup_libreoffice_macro(profile_dir: Path, timeout=30):
return url, None
-def external_links_at_risk(filename):
- try:
- with zipfile.ZipFile(filename) as archive:
- names = archive.namelist()
- except (zipfile.BadZipFile, OSError):
- return []
- if not any(n.startswith("xl/externalLinks/") for n in names):
+def _existing_cells(ws):
+ if not hasattr(ws, "iter_rows"):
return []
+ return [ws._cells[key] for key in sorted(ws._cells)]
- with contextlib.ExitStack() as stack:
- formulas = load_workbook(filename, data_only=False)
- stack.callback(formulas.close)
- values = load_workbook(filename, data_only=True)
- stack.callback(values.close)
-
- external_names = [
- name
- for name, dn in formulas.defined_names.items()
- if isinstance(getattr(dn, "value", None), str) and EXTERNAL_REF_RE.search(dn.value)
- ]
- name_re = (
- re.compile(r"\b(" + "|".join(re.escape(n) for n in external_names) + r")\b")
- if external_names
- else None
- )
- at_risk = []
- for sheet in formulas.sheetnames:
- ws = formulas[sheet]
- if not hasattr(ws, "iter_rows"):
+def _is_workbook(path):
+ try:
+ with zipfile.ZipFile(path) as archive:
+ return _workbook_part(archive) is not None
+ except Exception:
+ return False
+
+
+def _part_that_is_not_xml(path):
+ with zipfile.ZipFile(path) as archive:
+ for item in archive.infolist():
+ if not item.filename.endswith((".xml", ".rels")) or item.file_size > 50_000_000:
continue
- cached = values[sheet]
- for row in ws.iter_rows():
- for cell in row:
- v = cell.value
- if not (isinstance(v, str) and v.startswith("=")):
- continue
- reaches_out = EXTERNAL_REF_RE.search(v) or (name_re and name_re.search(v))
- if reaches_out and cached[cell.coordinate].value is None:
- at_risk.append(f"{sheet}!{cell.coordinate}")
- return at_risk
+ data = archive.read(item)
+ plain = data.replace(b"\x00", b"") if b"\x00" in data[:64] else data
+ if b"<!" in plain:
+ continue
+ try:
+ expat.ParserCreate().Parse(data, True)
+ except expat.ExpatError as e:
+ return f"{item.filename} is not well-formed XML: {e}"
+ return None
+
+
+def _has_formulas(path):
+ try:
+ with zipfile.ZipFile(path) as archive:
+ for _, part_name in _worksheets(archive, any_namespace=True):
+ raw = archive.read(part_name)
+ plain = raw.replace(b"\x00", b"") if b"\x00" in raw[:64] else raw
+ if re.search(rb"<(?:\w+:)?f[\s/>]", plain):
+ return True
+ if b"<!" not in plain:
+ expat.ParserCreate().Parse(raw, True)
+ return False
+ except Exception:
+ return True
def recalc(filename, timeout=30, force=False):
if not Path(filename).exists():
return {"error": f"File {filename} does not exist"}
- abs_path = str(Path(filename).absolute())
+ abs_path = os.path.realpath(filename)
- if not os.access(abs_path, os.W_OK):
- return {"error": f"{filename} is not writable; recalculation rewrites the file in place"}
+ if not _is_workbook(abs_path):
+ return {"error": f"{filename} is not an .xlsx or .xlsm workbook; recalc.py cannot read it"}
+
+ if not _has_formulas(abs_path):
+ report = _report(filename)
+ if "error" not in report:
+ return report
+ broken = _part_that_is_not_xml(abs_path)
+ if broken:
+ return {"error": f"{broken}; {filename} is unchanged"}
+ return {"status": "success", "total_errors": 0, "error_summary": {}, "total_formulas": 0}
+
+ if not (os.access(abs_path, os.W_OK) and os.access(os.path.dirname(abs_path), os.W_OK)):
+ return {
+ "error": f"{filename} or its folder is not writable; recalc.py needs to write to both"
+ }
try:
get_soffice_env()
except Exception as e:
return {"error": f"Could not prepare the LibreOffice environment: {e}"}
- if not force:
+ with tempfile.TemporaryDirectory(prefix="recalc-lo-", ignore_cleanup_errors=True) as work:
+ copy = os.path.join(work, "copy", os.path.basename(abs_path))
+ profile_dir = Path(work) / "profile"
try:
- at_risk = external_links_at_risk(filename)
- except Exception as e:
- return {"error": f"Could not inspect {filename} for external links: {e}"}
- if at_risk:
- shown = at_risk[:MAX_LOCATIONS]
- return {
- "error": (
- "Refusing to recalculate: this workbook links to another workbook, and "
- f"{len(at_risk)} linked cell(s) have lost their cached value (openpyxl strips "
- "these on save). Recalculating would resolve them to #NAME? and delete the "
- "external links for good. Copy those cells' values from the original file "
- "before saving, or pass --force to accept the loss. Charts and conditional "
- "formats can hold external references too, so this list may not be exhaustive."
- ),
- "external_link_cells": shown,
- "external_link_cells_truncated": max(0, len(at_risk) - len(shown)),
- }
-
- with tempfile.TemporaryDirectory(
- prefix="recalc-lo-profile-", ignore_cleanup_errors=True
- ) as profile_dir:
- return _recalc_with_profile(filename, abs_path, timeout, Path(profile_dir))
-
-
-def _recalc_with_profile(filename, abs_path, timeout, profile_dir: Path):
+ os.mkdir(os.path.dirname(copy))
+ shutil.copyfile(abs_path, copy)
+ profile_dir.mkdir()
+ except OSError as e:
+ return {"error": f"Could not copy {filename} for LibreOffice to work on: {e}"}
+ return _recalc_with_profile(filename, abs_path, copy, timeout, profile_dir)
+
+
+def _recalc_with_profile(filename, abs_path, copy, timeout, profile_dir: Path):
started = time.monotonic()
profile_url, err = setup_libreoffice_macro(profile_dir, timeout=timeout)
if err:
@@ -169,7 +177,7 @@ def _recalc_with_profile(filename, abs_path, timeout, profile_dir: Path):
timeout = max(5, int(timeout - (time.monotonic() - started)))
- before = _stamp(abs_path)
+ before = _stamp(copy)
cmd = [
"soffice",
@@ -177,7 +185,7 @@ def _recalc_with_profile(filename, abs_path, timeout, profile_dir: Path):
"--norestore",
f"-env:UserInstallation={profile_url}",
"vnd.sun.star.script:Standard.Module1.RecalculateAndSave?language=Basic&location=application",
- abs_path,
+ copy,
]
if platform.system() == "Linux" and shutil.which("timeout"):
@@ -203,101 +211,688 @@ def _recalc_with_profile(filename, abs_path, timeout, profile_dir: Path):
detail = (result.stderr or "").strip() or f"soffice exited {result.returncode}"
return {"error": f"LibreOffice failed to recalculate: {detail}"}
- if _stamp(abs_path) == before:
+ try:
+ rewritten = _stamp(copy) != before
+ except OSError:
+ rewritten = False
+ if not rewritten:
return {
"error": (
- "LibreOffice exited cleanly but never rewrote the file, so nothing was "
- "recalculated. Check that no other LibreOffice instance is running, then retry."
+ "LibreOffice exited cleanly but never rewrote its copy of the file, so nothing was "
+ "recalculated. Run it again; if the same happens, LibreOffice is not running the "
+ "recalculation macro on this machine."
)
}
try:
- wb = load_workbook(filename, data_only=True)
-
- excel_errors = [
- "#VALUE!",
- "#DIV/0!",
- "#REF!",
- "#NAME?",
- "#NULL!",
- "#NUM!",
- "#N/A",
- ]
- error_details = {err: [] for err in excel_errors}
- total_errors = 0
-
- for sheet_name in wb.sheetnames:
- ws = wb[sheet_name]
- if not hasattr(ws, "iter_rows"):
- continue
- for row in ws.iter_rows():
- for cell in row:
- if cell.value is not None and isinstance(cell.value, str):
- for err in excel_errors:
- if err in cell.value:
- location = f"{sheet_name}!{cell.coordinate}"
- error_details[err].append(location)
- total_errors += 1
- break
-
- result = {
- "status": "success" if total_errors == 0 else "errors_found",
- "total_errors": total_errors,
- "error_summary": {},
- }
+ found = _write_values(abs_path, copy)
+ except Exception as e:
+ why = f"which is unchanged: {str(e) or type(e).__name__}"
+ return {"error": f"Could not write the new values into {filename}, {why}"}
- for err_type, locations in error_details.items():
- if locations:
- entry = {"count": len(locations), "locations": locations[:MAX_LOCATIONS]}
- if len(locations) > MAX_LOCATIONS:
- entry["locations_truncated"] = len(locations) - MAX_LOCATIONS
- result["error_summary"][err_type] = entry
+ return _report(filename, found)
- wb.close()
- wb_formulas = load_workbook(filename, data_only=False)
- formula_count = 0
- for sheet_name in wb_formulas.sheetnames:
- ws = wb_formulas[sheet_name]
- if not hasattr(ws, "iter_rows"):
- continue
- for row in ws.iter_rows():
- for cell in row:
- if (
- cell.value
- and isinstance(cell.value, str)
- and cell.value.startswith("=")
- ):
- formula_count += 1
- wb_formulas.close()
+EXCEL_ERRORS = ["#VALUE!", "#DIV/0!", "#REF!", "#NAME?", "#NULL!", "#NUM!", "#N/A"]
+
+_MAIN_NS = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
+_REL_NS = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
+_CELL_REF_RE = re.compile(r"([A-Z]{1,3})([0-9]+)")
+MAX_MERGE_CHECKS = 1_000_000
+_MERGE_RE = re.compile(rb'<(?:\w+:)?mergeCell\b[^>]*\bref="([^"]*)"')
+
+
+def _error_in(text):
+ if "#" in text:
+ for err in EXCEL_ERRORS:
+ if err in text:
+ return err
+ return None
+
+
+def _rich_text(node):
+ runs = node.findall(f"{_MAIN_NS}t") + node.findall(f"{_MAIN_NS}r/{_MAIN_NS}t")
+ return "".join(t.text or "" for t in runs)
+
+
+def _row_major(coordinate):
+ address = _CELL_REF_RE.fullmatch(coordinate)
+ if address is None:
+ raise ValueError(f"{coordinate[:20]!r} is not the address of a cell")
+ column, row = address.groups()
+ number = 0
+ for letter in column:
+ number = number * 26 + ord(letter) - ord("A") + 1
+ return int(row), number
+
+
+def _merged_ranges(sheet_xml):
+ ranges = []
+ for ref in _MERGE_RE.findall(sheet_xml):
+ first, _, last = ref.decode().partition(":")
+ ranges.append((*_row_major(first), *_row_major(last)))
+ return ranges
+
- result["total_formulas"] = formula_count
+def _hidden_by_a_merge(merged, row, column):
+ return any(
+ top <= row <= bottom and left <= column <= right and (row, column) != (top, left)
+ for top, left, bottom, right in merged
+ )
- return result
- except Exception as e:
+def _utf8_or_refuse(data, what):
+ declared = re.search(rb"""<\?xml[^>]*\sencoding\s*=\s*["']([^"']+)""", data[:200])
+ utf8 = declared is None or declared[1].lower().replace(b"-", b"") == b"utf8"
+ if data[:2] in (b"\xff\xfe", b"\xfe\xff") or b"\x00" in data[:64] or not utf8:
+ raise ValueError(f"{what} is not in UTF-8, which is all this script can read")
+
+
+def _xml(data, what="an XML part"):
+ try:
+ from defusedxml.ElementTree import fromstring
+ except ImportError:
+ _utf8_or_refuse(data, what)
+ if b"<!DOCTYPE" in data or b"<!ENTITY" in data:
+ raise ValueError("XML with a DTD, and defusedxml is not installed") from None
+ from xml.etree.ElementTree import fromstring
+ return fromstring(data)
+
+
+def _workbook_part(archive):
+ if "_rels/.rels" not in archive.namelist():
+ return None
+ for rel in _xml(archive.read("_rels/.rels"), "_rels/.rels"):
+ if rel.get("Type", "").endswith("/officeDocument"):
+ name = posixpath.normpath(rel.get("Target", "")).lstrip("/")
+ return name if name in archive.namelist() else None
+ return None
+
+
+def _workbook_xml(archive):
+ name = _workbook_part(archive)
+ if name is None:
+ raise ValueError("the package names no workbook part")
+ if archive.getinfo(name).file_size > 50_000_000:
+ raise ValueError(f"{name} is too large to be a workbook's")
+ return _xml(archive.read(name), name)
+
+
+def _worksheets(archive, any_namespace=False):
+ workbook = _workbook_xml(archive)
+ if not any_namespace and workbook.tag != f"{_MAIN_NS}workbook":
+ namespace = workbook.tag.partition("}")[0].lstrip("{")
+ if "/ooxml/" in namespace:
+ raise ValueError("it is in Strict Open XML format, which this script cannot edit")
+ raise ValueError(f"its XML is in a namespace this script does not read ({namespace})")
+ folder, file_name = posixpath.split(_workbook_part(archive))
+ rels_name = posixpath.join(folder, "_rels", f"{file_name}.rels")
+ rels = {rel.get("Id"): rel for rel in _xml(archive.read(rels_name), rels_name)}
+ found = []
+ for sheet in workbook.iter():
+ if not sheet.tag.endswith("}sheet"):
+ continue
+ rel_id = next((value for key, value in sheet.attrib.items() if key.endswith("}id")), None)
+ rel = rels.get(rel_id)
+ if rel is None or not rel.get("Type", "").endswith("/worksheet"):
+ continue
+ target = rel.get("Target")
+ if target.startswith("/"):
+ part_name = target.lstrip("/")
+ else:
+ part_name = posixpath.normpath(posixpath.join(folder, target))
+ found.append((sheet.get("name"), part_name))
+ return found
+
+
+def _sheet_names(archive):
+ return [s.get("name") for s in _workbook_xml(archive).iter() if s.tag.endswith("}sheet")]
+
+
+def _scan_xml(filename):
+ from defusedxml.ElementTree import iterparse
+
+ error_details = {err: [] for err in EXCEL_ERRORS}
+ formula_count = 0
+ merge_checks = 0
+
+ with zipfile.ZipFile(filename) as archive:
+ shared_errors, shared_equals = {}, set()
+ if "xl/sharedStrings.xml" in archive.namelist():
+ with archive.open("xl/sharedStrings.xml") as part:
+ index = 0
+ for _, node in iterparse(part):
+ if node.tag != f"{_MAIN_NS}si":
+ continue
+ text = _rich_text(node).replace("x005F_", "")
+ err = _error_in(text)
+ if err:
+ shared_errors[index] = err
+ if text.startswith("="):
+ shared_equals.add(index)
+ index += 1
+ node.clear()
+
+ for sheet_name, part_name in _worksheets(archive):
+ found = []
+ xml = archive.read(part_name)
+ merged = _merged_ranges(xml)
+ for _, node in iterparse(io.BytesIO(xml)):
+ if node.tag == f"{_MAIN_NS}row":
+ node.clear()
+ if node.tag != f"{_MAIN_NS}c":
+ continue
+ if node.get("r") is None:
+ raise ValueError("a cell without its address")
+ formula = node.find(f"{_MAIN_NS}f")
+ is_formula = formula is not None and formula.get("t") != "dataTable"
+ err = None
+ cell_type = node.get("t", "n")
+ if cell_type == "s":
+ index = int(node.findtext(f"{_MAIN_NS}v"))
+ err = shared_errors.get(index)
+ is_formula = is_formula or (formula is None and index in shared_equals)
+ elif cell_type not in ("n", "b", "d"):
+ inline = node.find(f"{_MAIN_NS}is")
+ text = node.findtext(f"{_MAIN_NS}v") if inline is None else _rich_text(inline)
+ err = _error_in(text or "")
+ is_formula = is_formula or (formula is None and (text or "").startswith("="))
+ if not is_formula and not err:
+ continue
+ row, column = _row_major(node.get("r"))
+ merge_checks += len(merged)
+ if merge_checks > MAX_MERGE_CHECKS:
+ raise ValueError("too many merged ranges for this many cells")
+ if _hidden_by_a_merge(merged, row, column):
+ continue
+ formula_count += is_formula
+ if err:
+ found.append(((row, column), err, node.get("r")))
+ for _, err, coordinate in sorted(found):
+ error_details[err].append(f"{sheet_name}!{coordinate}")
+
+ return error_details, formula_count
+
+
+def _scan_with_openpyxl(filename):
+ from openpyxl import load_workbook
+ from openpyxl.worksheet.formula import ArrayFormula
+
+ error_details = {err: [] for err in EXCEL_ERRORS}
+ wb = load_workbook(filename, data_only=True)
+ for sheet_name in wb.sheetnames:
+ ws = wb[sheet_name]
+ for cell in _existing_cells(ws):
+ if isinstance(cell.value, str):
+ err = _error_in(cell.value)
+ if err:
+ error_details[err].append(f"{sheet_name}!{cell.coordinate}")
+ wb.close()
+
+ wb_formulas = load_workbook(filename, data_only=False)
+ formula_count = 0
+ for sheet_name in wb_formulas.sheetnames:
+ ws = wb_formulas[sheet_name]
+ for cell in _existing_cells(ws):
+ v = cell.value
+ if isinstance(v, ArrayFormula):
+ v = v.text
+ if isinstance(v, str) and v.startswith("="):
+ formula_count += 1
+ wb_formulas.close()
+
+ return error_details, formula_count
+
+
+def _sheet_names_too_long(filename):
+ try:
+ with zipfile.ZipFile(filename) as archive:
+ return [name for name in _sheet_names(archive) if len(name or "") > 31]
+ except Exception:
+ return []
+
+
+def _report(filename, found=None):
+ try:
+ try:
+ error_details, formula_count = _scan_xml(filename)
+ except Exception:
+ error_details, formula_count = _scan_with_openpyxl(filename)
+ except Exception as e:
return {"error": str(e)}
+ found = found or {}
+ kept = found.get("not_computed", [])
+ apart = set(kept) | set(found.get("#NAME?", ()))
+ in_file = [where for where in error_details["#NAME?"] if where not in apart]
+ error_details["#NAME?"] = in_file + found.get("#NAME?", [])
+ for error in EXCEL_ERRORS:
+ if error != "#NAME?":
+ error_details[error] += found.get(error, [])
+ error_details[UNBALANCED] = found.get(UNBALANCED, [])
+ error_details[TOO_LONG] = _sheet_names_too_long(filename)
+
+ total_errors = sum(len(locations) for locations in error_details.values())
+ result = {
+ "status": "success" if total_errors == 0 else "errors_found",
+ "total_errors": total_errors,
+ "error_summary": {},
+ }
+
+ for err_type, locations in error_details.items():
+ if locations:
+ entry = {"count": len(locations), "locations": locations[:MAX_LOCATIONS]}
+ if len(locations) > MAX_LOCATIONS:
+ entry["locations_truncated"] = len(locations) - MAX_LOCATIONS
+ result["error_summary"][err_type] = entry
+
+ result["total_formulas"] = formula_count
+ if kept:
+ result["not_computed"] = {"count": len(kept), "locations": kept[:MAX_LOCATIONS]}
+ if len(kept) > MAX_LOCATIONS:
+ result["not_computed"]["locations_truncated"] = len(kept) - MAX_LOCATIONS
+ return result
+
+
+
+MAX_ARRAY_CELLS = 1_000_000
+
+_ATTRS = rb"""((?:[^>"']|"[^"]*"|'[^']*')*?)"""
+_ROW_OR_CELL_RE = re.compile(
+ rb"<((?:\w+:)?)(?:row(?=[\s/>])" + _ATTRS + rb"/?>"
+ rb"|c(?=[\s/>])" + _ATTRS + rb"(?:/>|>(.*?)</\1c\s*>))",
+ re.S,
+)
+_F_RE = re.compile(rb"<(?:\w+:)?f(?=[\s/>])" + _ATTRS + rb"(?:/>|>(.*?)</(?:\w+:)?f\s*>)", re.S)
+_V_RE = re.compile(rb"<(?:\w+:)?v(?=[\s/>])[^>]*?(?:/>|>(.*?)</(?:\w+:)?v\s*>)", re.S)
+_T_OR_VM_RE = re.compile(rb"""\s(?:t|vm)\s*=\s*(?:"[^"]*"|'[^']*')""")
+_EXT_LST_RE = re.compile(rb"<(?:\w+:)?extLst\b.*</(?:\w+:)?extLst\s*>", re.S)
+_NAME_ERROR = (b"e", b"#NAME?")
+_STRING_RE = re.compile(r'"[^"]*"')
+_QUOTED_RE = re.compile(r""""[^"]*"|'[^']*'|\[[^\[\]]*\]""")
+_CALL_RE = re.compile(r"(?<![\w.])([A-Za-z_][\w.]*)\(")
+_DELIMS = r"\s+\-*/^&=<>(),;:!%{}#@~|"
+_WORD_RE = re.compile(rf"(?<![^{_DELIMS}])[^{_DELIMS}]+(?![^{_DELIMS}]|[(!])")
+_EXTERNAL_REF_RE = re.compile(r"""(?<![\w"\[])'?\[[1-9]\d*\][^!"\[\]]*'?!""")
+_PREFIX_RE = re.compile(r"^(?:_xlfn\.|_xlws\.)+", re.I)
+UNBALANCED = "unbalanced brackets"
+TOO_LONG = "sheet name over 31 characters"
+_STORED_BARE = frozenset(
+ "CUBEVALUE CUBEMEMBER CUBESET CUBESETCOUNT CUBERANKEDMEMBER CUBEKPIMEMBER CUBEMEMBERPROPERTY "
+ "RTD CALL DBCS PHONETIC REGISTER.ID SQL.REQUEST USDOLLAR BAHTTEXT ECMA.CEILING".split()
+)
+LIBREOFFICE_LACKS = _STORED_BARE | frozenset(
+ "XLOOKUP XMATCH SORT SORTBY FILTER UNIQUE SEQUENCE RANDARRAY LET LAMBDA MAP REDUCE SCAN "
+ "MAKEARRAY BYROW BYCOL ISOMITTED TEXTBEFORE TEXTAFTER TEXTSPLIT VSTACK HSTACK TOROW TOCOL "
+ "WRAPROWS WRAPCOLS TAKE DROP CHOOSEROWS CHOOSECOLS EXPAND VALUETOTEXT ARRAYTOTEXT REGEXTEST "
+ "REGEXEXTRACT REGEXREPLACE PERCENTOF GROUPBY PIVOTBY TRIMRANGE IMAGE STOCKHISTORY FIELDVALUE "
+ "PY COPILOT TRANSLATE DETECTLANGUAGE ANCHORARRAY SINGLE BINOM.DIST.RANGE IMCOSH IMCOT IMCSC "
+ "IMCSCH IMSEC IMSECH IMSINH IMTAN QUERYSTRING".split()
+)
+
+
+def _attr_re(name):
+ return re.compile(rb"\s" + name + rb"""\s*=\s*(?:"([^"]*)"|'([^']*)')""")
+
+
+_R_RE, _T_RE, _REF_RE, _SI_RE = _attr_re(b"r"), _attr_re(b"t"), _attr_re(b"ref"), _attr_re(b"si")
+
+
+def _attr(pattern, attrs):
+ m = pattern.search(attrs)
+ if m is None:
+ return None
+ return m[1] if m[1] is not None else m[2]
+
+
+def _sheet_data_span(xml):
+ if b"<!" in xml:
+ return None
+ parser = expat.ParserCreate(namespace_separator="}")
+ depth, found = [0], []
+
+ def start(name, _attributes):
+ depth[0] += 1
+ if depth[0] == 2 and name == _MAIN_NS[1:] + "sheetData" and not found:
+ found.append(parser.CurrentByteIndex)
+
+ def end(name):
+ if depth[0] == 2 and name == _MAIN_NS[1:] + "sheetData" and len(found) == 1:
+ found.append(parser.CurrentByteIndex)
+ depth[0] -= 1
+
+ parser.StartElementHandler, parser.EndElementHandler = start, end
+ parser.Parse(xml, True)
+ return tuple(found) if len(found) == 2 else None
+
+
+def _cells(xml):
+ span = _sheet_data_span(xml)
+ row = column = 0
+ for m in _ROW_OR_CELL_RE.finditer(xml, *span) if span else ():
+ if m[3] is None:
+ number = _attr(_R_RE, m[2])
+ row, column = int(number) if number else row + 1, 0
+ continue
+ ref = _attr(_R_RE, m[3])
+ if ref:
+ row, column = _row_major(ref.decode())
+ else:
+ column += 1
+ yield row, column, m
+
+
+def _sheet_xml(archive, part_name):
+ xml = archive.read(part_name)
+ _utf8_or_refuse(xml, part_name)
+ body = xml[3:] if xml.startswith(b"\xef\xbb\xbf") else xml
+ if b"<!" in xml or body.count(b"<?") > bool(re.match(rb"<\?xml\s", body)):
+ raise ValueError(
+ f"{part_name} holds a comment, CDATA, a DTD or a processing instruction, which this "
+ "script cannot edit. Load and save the workbook with openpyxl first, or leave it as is"
+ )
+ try:
+ expat.ParserCreate().Parse(xml, True)
+ except expat.ExpatError as e:
+ raise ValueError(f"{part_name} is not well-formed XML: {e}") from None
+ return xml
+
+
+def _array_cells(xml, budget):
+ cells = {}
+ for f in _F_RE.finditer(xml):
+ if _attr(_T_RE, f[1]) not in (b"array", b"dataTable"):
+ continue
+ first, _, last = (_attr(_REF_RE, f[1]) or b"").decode().partition(":")
+ if not last:
+ continue
+ (top, left), (bottom, right) = _row_major(first), _row_major(last)
+ if bottom < top or right < left:
+ continue
+ if len(cells) + (bottom - top + 1) * (right - left + 1) > budget:
+ continue
+ cells.update(
+ ((r, c), (top, left)) for r in range(top, bottom + 1) for c in range(left, right + 1)
+ )
+ return cells
+
+
+def _computed(copy_path, array_cells, names):
+ computed, lacked = {}, {}
+ with zipfile.ZipFile(copy_path) as archive:
+ strings = None
+ its_names = _sheet_names(archive)
+ by_name = {ours.casefold(): ours for ours in names}
+ ours = {its: by_name[its.casefold()] for its in its_names if its.casefold() in by_name}
+ if len(its_names) == len(names):
+ for its, in_that_place in zip(its_names, names):
+ if its not in ours and in_that_place not in ours.values():
+ ours[its] = in_that_place
+ for name, part_name in _worksheets(archive):
+ name = ours.get(name, name)
+ cells, lacks = computed.setdefault(name, {}), lacked.setdefault(name, set())
+ wanted = array_cells.get(name, ())
+ for row, column, m in _cells(archive.read(part_name)):
+ content = m[4] or b""
+ if (row, column) not in wanted and not _F_RE.search(content):
+ continue
+ v = _V_RE.search(content)
+ kind, text = _attr(_T_RE, m[3]) or b"n", v and v[1]
+ if text is not None and kind == b"s":
+ if strings is None:
+ shared = _xml(archive.read("xl/sharedStrings.xml"))
+ strings = [_rich_text(si) for si in shared.iter(f"{_MAIN_NS}si")]
+ kind, text = b"str", html.escape(strings[int(text)], quote=False).encode()
+ f = _F_RE.search(content)
+ its_formula = (f and f[2]) or b""
+ if re.search(rb"[a-z]", its_formula):
+ called = [_PREFIX_RE.sub("", n) for n in _called(_text(its_formula))]
+ if any(n != n.upper() and n.upper() in LIBREOFFICE_LACKS for n in called):
+ lacks.add((row, column))
+ if _calls_its_own(_text(its_formula)):
+ kind, text = _NAME_ERROR
+ if text is not None and (kind == b"str" or (text and kind in (b"n", b"b", b"e"))):
+ cells[row, column] = (kind, text)
+ return computed, lacked
+
+
+def _same_value(old_kind, old, kind, text):
+ if old is None:
+ return False
+ if (old_kind, old) == (kind, text):
+ return True
+ if {old_kind, kind} <= {b"n", b"b"}:
+ try:
+ return math.isclose(float(old), float(text), rel_tol=1e-14, abs_tol=1e-12)
+ except ValueError:
+ return False
+ return old_kind == kind and html.unescape(old.decode()) == html.unescape(text.decode())
+
+
+def _cell_with_value(m, f, kind, text):
+ p = m[1] or b""
+ attrs = _T_OR_VM_RE.sub(b"", m[3])
+ if kind != b"n":
+ attrs += b' t="' + kind + b'"'
+ kept = b"" if f is None else m[4][: f.end()]
+ ext_lst = _EXT_LST_RE.search(m[4] or b"")
+ rest = ext_lst[0] if ext_lst else b""
+ return b"<%sc%s>%s<%sv>%s</%sv>%s</%sc>" % (p, attrs, kept, p, text, p, rest, p)
+
+
+def _coordinate(row, column):
+ letters = ""
+ while column:
+ column, index = divmod(column - 1, 26)
+ letters = chr(ord("A") + index) + letters
+ return f"{letters}{row}"
+
+
+def _text(formula):
+ text = formula.decode()
+ for entity, character in (("&lt;", "<"), ("&gt;", ">"), ("&quot;", '"'), ("&apos;", "'")):
+ text = text.replace(entity, character)
+ return text.replace("&amp;", "&")
+
+
+def _called(formula):
+ names = _CALL_RE.findall(_QUOTED_RE.sub("", formula))
+ return [name for name in names if not name.lower().startswith("_xlpm.")]
+
+
+def _lacking(name):
+ bare = _PREFIX_RE.sub("", name)
+ if bare.upper() not in LIBREOFFICE_LACKS:
+ return None
+ return bare != name or bare.upper() in _STORED_BARE
+
+
+def _calls_its_own(formula):
+ names = [_PREFIX_RE.sub("", name) for name in _called(formula)]
+ return any(name != name.upper() and name != name.lower() for name in names)
+
+
+def _unbalanced(formula):
+ bare = _QUOTED_RE.sub("", formula)
+ return bare.count("(") != bare.count(")")
+
+
+def _reads_another_workbook(formula, reaching):
+ plain = _STRING_RE.sub("", formula)
+ if _EXTERNAL_REF_RE.search(plain):
+ return True
+ words = _WORD_RE.findall(_QUOTED_RE.sub("", plain)) if reaching else ()
+ return any(word.lower() in reaching for word in words)
+
+
+def _names_reaching_another_workbook(archive):
+ reaching, used_by = set(), {}
+ for name in _workbook_xml(archive).iter(f"{_MAIN_NS}definedName"):
+ key, plain = name.get("name", "").lower(), _STRING_RE.sub("", name.text or "")
+ if _EXTERNAL_REF_RE.search(plain):
+ reaching.add(key)
+ for word in _WORD_RE.findall(_QUOTED_RE.sub("", plain)):
+ used_by.setdefault(word.lower(), set()).add(key)
+ todo = list(reaching)
+ while todo:
+ for user in used_by.get(todo.pop(), ()):
+ if user not in reaching:
+ reaching.add(user)
+ todo.append(user)
+ return reaching
+
+
+def _verdict(formula, reaching=frozenset()):
+ lacking = [_lacking(name) for name in _called(formula)]
+ if False in lacking:
+ return "mistake"
+ if _reads_another_workbook(formula, reaching):
+ return "link"
+ return "lacks" if True in lacking else None
+
+
+def _splice(xml, computed, lacked, array_cells, sheet_name, reaching, found):
+ out, done = [], 0
+ filling, filled, shared = set(), set(), {}
+ for row, column, m in _cells(xml):
+ content = m[4] or b""
+ f = _F_RE.search(content)
+ if f is None and array_cells.get((row, column)) not in filling:
+ continue
+ v = _V_RE.search(content, f.end() if f else 0)
+ old = v and (v[1] or b"")
+ kind = _attr(_T_RE, m[3]) or b"n"
+ new = computed.get((row, column))
+ verdict = None
+ if f is None:
+ filled.add((row, column))
+ else:
+ where = f"{sheet_name}!{_coordinate(row, column)}"
+ text, si = _text(f[2] or b""), _attr(_SI_RE, f[1])
+ verdict = _verdict(text, reaching) if text else shared.get(si)
+ if text and si:
+ shared[si] = verdict
+ if verdict == "lacks" and (row, column) not in lacked:
+ verdict = None
+ if _unbalanced(text):
+ found[UNBALANCED].append(where)
+ if verdict or new is None or new == _NAME_ERROR:
+ if f is not None:
+ has_value = old or (old is not None and kind == b"str")
+ held = has_value and (kind, old) != _NAME_ERROR
+ no_value_held = not verdict and new == _NAME_ERROR and not held
+ mistake = verdict == "mistake" or no_value_held
+ found["#NAME?" if mistake else "not_computed"].append(where)
+ continue
+ if f is not None:
+ filling.add((row, column))
+ if _same_value(kind, old, *new):
+ continue
+ out += [xml[done : m.start()], _cell_with_value(m, f, *new)]
+ done = m.end()
+ for cell, first in array_cells.items():
+ new = computed.get(cell)
+ held_by_the_sheet = cell == first or cell in filled
+ if first in filling and not held_by_the_sheet and new and new[0] == b"e":
+ error = new[1].decode()
+ if error != "#NAME?" and error in EXCEL_ERRORS:
+ found.setdefault(error, []).append(f"{sheet_name}!{_coordinate(*cell)}")
+ if not out:
+ return xml
+ out.append(xml[done:])
+ return b"".join(out)
+
+
+def _replace_parts(path, parts):
+ handle, new_path = tempfile.mkstemp(
+ dir=os.path.dirname(path), prefix=".recalc-", suffix=Path(path).suffix
+ )
+ try:
+ with (
+ os.fdopen(handle, "wb") as out,
+ zipfile.ZipFile(path) as old,
+ zipfile.ZipFile(out, "w") as new,
+ ):
+ new.comment = old.comment
+ for item in old.infolist():
+ if item.filename in parts:
+ new.writestr(item, parts[item.filename])
+ continue
+ with old.open(item) as source, new.open(copy.copy(item), "w") as target:
+ shutil.copyfileobj(source, target)
+ status = os.stat(path)
+ with contextlib.suppress(OSError, AttributeError):
+ os.chown(new_path, status.st_uid, status.st_gid)
+ shutil.copystat(path, new_path)
+ os.utime(new_path)
+ os.replace(new_path, path)
+ except BaseException:
+ with contextlib.suppress(OSError):
+ os.unlink(new_path)
+ raise
+
+
+def _write_values(path, copy_path):
+ with zipfile.ZipFile(path) as archive:
+ sheets = _worksheets(archive)
+ reaching = _names_reaching_another_workbook(archive)
+ array_cells = {}
+ for name, part_name in sheets:
+ budget = MAX_ARRAY_CELLS - sum(map(len, array_cells.values()))
+ array_cells[name] = _array_cells(_sheet_xml(archive, part_name), budget)
+ computed, lacked = _computed(copy_path, array_cells, _sheet_names(archive))
+ parts, found = {}, {"#NAME?": [], UNBALANCED: [], "not_computed": []}
+ for name, part_name in sheets:
+ xml = archive.read(part_name)
+ if name not in computed and _F_RE.search(xml):
+ raise ValueError(
+ f"LibreOffice's copy has no sheet called {name!r}: it renames or drops some "
+ "sheets, such as one whose name is over 31 characters, so this sheet's "
+ "formulas cannot be recalculated"
+ )
+ cells, lacks = computed.get(name, {}), lacked.get(name, ())
+ new = _splice(xml, cells, lacks, array_cells[name], name, reaching, found)
+ if new is not xml:
+ try:
+ expat.ParserCreate().Parse(new, True)
+ except expat.ExpatError as e:
+ raise ValueError(f"editing {part_name} as text broke its XML ({e})") from None
+ parts[part_name] = new
+ if parts:
+ _replace_parts(path, parts)
+ return found
+
def main():
+ signal.signal(signal.SIGTERM, lambda *_: sys.exit(143))
args = [a for a in sys.argv[1:] if a != "--force"]
force = "--force" in sys.argv[1:]
if not args:
print("Usage: python recalc.py <excel_file> [timeout_seconds] [--force]")
print("\nRecalculates all formulas in an Excel file using LibreOffice")
+ print("LibreOffice works on a copy; only the values it computes are written into the file.")
print("\nReturns JSON with error details:")
print(" - status: 'success' or 'errors_found'")
print(" - total_errors: Total number of Excel errors found")
print(" - total_formulas: Number of formulas in the file")
print(" - error_summary: Breakdown by error type with locations")
- print(" - #VALUE!, #DIV/0!, #REF!, #NAME?, #NULL!, #NUM!, #N/A")
+ print(" - #VALUE!, #DIV/0!, #REF!, #NAME?, #NULL!, #NUM!, #N/A, unbalanced brackets")
+ print(" - not_computed: only if LibreOffice could not compute some formulas: a function")
+ print(" it lacks, or a link to another workbook. Those cells are left as they were, and")
+ print(" Excel computes them when it opens a file openpyxl saved. A sound formula that")
+ print(" depends on one of them reports #NAME? too")
print("\nOn any failure the JSON has an 'error' key and no 'status'.")
- print("--force recalculates even when it would destroy external links.")
+ print("--force is accepted and ignored.")
sys.exit(1)
filename = args[0]
- timeout = int(args[1]) if len(args) > 1 else 30
+ try:
+ timeout = int(args[1]) if len(args) > 1 else 30
+ except ValueError:
+ result = {"error": f"timeout must be an integer number of seconds, got {args[1]!r}"}
+ print(json.dumps(result, indent=2))
+ sys.exit(1)
result = recalc(filename, timeout, force=force)
print(json.dumps(result, indent=2))