summaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
authorJesse Vincent <jesse@fsck.com>2026-09-18 18:38:59 +0000
committerJesse Vincent <jesse@fsck.com>2026-09-18 18:38:59 +0000
commit8412aaf0385cbfd850675093f01b16c88052f268 (patch)
tree88ae48d06c4d104d65e35d345e65f7c210189564 /tests
parentb36e0829c6d0140e93cfef2ca599b1b07d4a7797 (diff)
Release v6.4.0: diagnosing-superpowers and movie skills, Native execution, OpenCode 2.0, Muse, and Qwen Code supportrelease/v6.4.0
Everything on dev since the v6.3.0 merge-back (777ceb5), as one commit. Full notes: RELEASE-NOTES.md, v6.4.0 section. Claude-Session: https://claude.ai/code/session_01PRUnVUm4g4EcP9eNjAiT2B
Diffstat (limited to 'tests')
-rwxr-xr-xtests/claude-code/run-skill-tests.sh1
-rwxr-xr-xtests/claude-code/test-executing-plans-scripts.sh139
-rwxr-xr-xtests/claude-code/test-sdd-workspace.sh161
-rwxr-xr-xtests/codex-plugin-sync/test-sync-to-codex-plugin.sh6
-rwxr-xr-xtests/diagnosing-superpowers/test-skill-structure.sh151
-rwxr-xr-xtests/opencode/run-tests.sh4
-rw-r--r--tests/opencode/test-bootstrap-caching.mjs122
-rw-r--r--tests/opencode/test-session-bootstrap.mjs224
-rwxr-xr-xtests/opencode/test-session-bootstrap.sh4
-rw-r--r--tests/opencode/test-skill-registration.mjs205
-rwxr-xr-xtests/opencode/test-skill-registration.sh22
-rw-r--r--tests/proving-it-works-with-a-movie/README.md29
-rw-r--r--tests/proving-it-works-with-a-movie/fixtures.py299
-rw-r--r--tests/proving-it-works-with-a-movie/fixtures/terminal_app.py41
-rw-r--r--tests/proving-it-works-with-a-movie/run-tests.py64
-rw-r--r--tests/proving-it-works-with-a-movie/test_assembly.py409
-rw-r--r--tests/proving-it-works-with-a-movie/test_browser.py73
-rw-r--r--tests/proving-it-works-with-a-movie/test_browser_contract.py112
-rw-r--r--tests/proving-it-works-with-a-movie/test_checker.py194
-rw-r--r--tests/proving-it-works-with-a-movie/test_narration.py351
-rw-r--r--tests/proving-it-works-with-a-movie/test_narration_contract.py546
-rw-r--r--tests/proving-it-works-with-a-movie/test_paths.py104
-rw-r--r--tests/proving-it-works-with-a-movie/test_recorder_contract.py538
-rw-r--r--tests/proving-it-works-with-a-movie/test_subtitle_contract.py290
-rw-r--r--tests/proving-it-works-with-a-movie/test_subtitles.py162
-rw-r--r--tests/proving-it-works-with-a-movie/test_terminal.py316
26 files changed, 4567 insertions, 0 deletions
diff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.sh
index 83217cd..97bce6d 100755
--- a/tests/claude-code/run-skill-tests.sh
+++ b/tests/claude-code/run-skill-tests.sh
@@ -76,6 +76,7 @@ done
tests=(
"test-worktree-path-policy.sh"
"test-sdd-workspace.sh"
+ "test-executing-plans-scripts.sh"
"test-subagent-driven-development.sh"
)
diff --git a/tests/claude-code/test-executing-plans-scripts.sh b/tests/claude-code/test-executing-plans-scripts.sh
new file mode 100755
index 0000000..994733b
--- /dev/null
+++ b/tests/claude-code/test-executing-plans-scripts.sh
@@ -0,0 +1,139 @@
+#!/usr/bin/env bash
+# Tests for executing-plans' bookkeeping helpers: scripts/task-start extracts
+# the brief and records BASE in one call; scripts/task-done runs the task's
+# test command, records the result in the ledger, and refuses to record a
+# failing task.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+EP_SCRIPTS="$REPO_ROOT/skills/executing-plans/scripts"
+
+FAILURES=0
+TEST_ROOT=""
+
+pass() { echo " [PASS] $1"; }
+fail() {
+ echo " [FAIL] $1"
+ FAILURES=$((FAILURES + 1))
+}
+
+cleanup() {
+ if [[ -n "$TEST_ROOT" && -d "$TEST_ROOT" ]]; then
+ rm -rf "$TEST_ROOT"
+ fi
+}
+
+main() {
+ echo "=== Test: executing-plans scripts ==="
+
+ TEST_ROOT="$(mktemp -d)"
+ trap cleanup EXIT
+
+ git init -q -b main "$TEST_ROOT/repo"
+ local repo
+ repo="$(cd "$TEST_ROOT/repo" && git rev-parse --show-toplevel)"
+ local git_id=(-c user.email=t@example.com -c user.name=t -c commit.gpgsign=false)
+
+ cat > "$repo/plan.md" <<'PLAN'
+# Plan
+
+## Task 1: First thing
+
+Do the first thing.
+
+## Task 2: Second thing
+
+Do the second thing.
+PLAN
+ ( cd "$repo" && git add plan.md && git "${git_id[@]}" commit -qm fixture )
+ local base
+ base="$(cd "$repo" && git rev-parse HEAD)"
+
+ # --- task-start: argument validation ---
+ local rc=0
+ (cd "$repo" && "$EP_SCRIPTS/task-start" plan.md >/dev/null 2>&1) || rc=$?
+ if [[ "$rc" -eq 2 ]]; then
+ pass "task-start without a task number errors with exit 2"
+ else
+ fail "task-start without a task number errors with exit 2 (got $rc)"
+ fi
+
+ # --- task-start: brief path + BASE in one call ---
+ local out
+ out="$(cd "$repo" && "$EP_SCRIPTS/task-start" plan.md 1)"
+ if [[ "$out" == *"brief: $repo/.superpowers/sdd/plan/task-1-brief.md"* ]]; then
+ pass "task-start prints the brief path under the plan's workspace"
+ else
+ fail "task-start prints the brief path under the plan's workspace"
+ echo " got: $out"
+ fi
+ if [[ "$out" == *"base: $base"* ]]; then
+ pass "task-start prints BASE as the current HEAD"
+ else
+ fail "task-start prints BASE as the current HEAD"
+ echo " got: $out"
+ fi
+ if [[ -s "$repo/.superpowers/sdd/plan/task-1-brief.md" ]]; then
+ pass "task-start writes the brief file"
+ else
+ fail "task-start writes the brief file"
+ fi
+
+ # --- task-done: records a passing task ---
+ ( cd "$repo" && echo x > work.txt && git add work.txt && git "${git_id[@]}" commit -qm "task 1" )
+ local head
+ head="$(cd "$repo" && git rev-parse HEAD)"
+ out="$(cd "$repo" && "$EP_SCRIPTS/task-done" plan.md 1 "$base" -- sh -c 'echo "Ran 3 tests"; echo OK')"
+ rc=$?
+ local ledger="$repo/.superpowers/sdd/plan/progress.md"
+ local expected="Task 1: complete (commits ${base:0:7}..${head:0:7}, tests: sh -c 'echo \"Ran 3 tests\"; echo OK' → OK)"
+ if [[ -f "$ledger" ]] && grep -qF "$expected" "$ledger"; then
+ pass "task-done appends the completion line with commit range and test result"
+ else
+ fail "task-done appends the completion line with commit range and test result"
+ echo " expected: $expected"
+ echo " ledger:"; sed 's/^/ /' "$ledger" 2>/dev/null || echo " (missing)"
+ fi
+ if [[ "$out" == *"OK"* ]]; then
+ pass "task-done prints the tail of the test output"
+ else
+ fail "task-done prints the tail of the test output"
+ echo " got: $out"
+ fi
+ if [[ -s "$repo/.superpowers/sdd/plan/task-1-tests.log" ]]; then
+ pass "task-done keeps the full test output in the workspace"
+ else
+ fail "task-done keeps the full test output in the workspace"
+ fi
+
+ # --- task-done: refuses to record a failing task ---
+ rc=0
+ out="$(cd "$repo" && "$EP_SCRIPTS/task-done" plan.md 2 "$head" -- sh -c 'echo "FAILED (errors=1)"; exit 1' 2>&1)" || rc=$?
+ if [[ "$rc" -ne 0 ]]; then
+ pass "task-done exits non-zero when the test command fails"
+ else
+ fail "task-done exits non-zero when the test command fails"
+ fi
+ if ! grep -q "Task 2: complete" "$ledger"; then
+ pass "task-done does not record a failing task as complete"
+ else
+ fail "task-done does not record a failing task as complete"
+ fi
+ if [[ "$out" == *"FAILED"* ]]; then
+ pass "task-done shows the failing output"
+ else
+ fail "task-done shows the failing output"
+ echo " got: $out"
+ fi
+
+ echo
+ if [[ "$FAILURES" -eq 0 ]]; then
+ echo "PASS"
+ else
+ echo "FAIL ($FAILURES)"
+ exit 1
+ fi
+}
+
+main "$@"
diff --git a/tests/claude-code/test-sdd-workspace.sh b/tests/claude-code/test-sdd-workspace.sh
index 8417230..2e8c227 100755
--- a/tests/claude-code/test-sdd-workspace.sh
+++ b/tests/claude-code/test-sdd-workspace.sh
@@ -165,6 +165,30 @@ PLAN
echo " got: $rp_explicit"
fi
+ # --- range guards: BASE must be an ancestor of HEAD, range must be non-empty ---
+ local divergent
+ divergent="$(cd "$repo" && git "${git_id[@]}" commit-tree 'HEAD~1^{tree}' -p 'HEAD~1' -m divergent)"
+ rc=0
+ local guard_err
+ guard_err="$(cd "$repo" && "$SDD_SCRIPTS/review-package" plan-a.md "$divergent" HEAD 2>&1 >/dev/null)" || rc=$?
+ if [[ "$rc" -eq 3 && "$guard_err" == *"not a descendant"* ]]; then
+ pass "review-package rejects a BASE that is not an ancestor of HEAD with exit 3"
+ else
+ fail "review-package rejects a BASE that is not an ancestor of HEAD with exit 3"
+ echo " exit: $rc"
+ echo " stderr: $guard_err"
+ fi
+
+ rc=0
+ guard_err="$(cd "$repo" && "$SDD_SCRIPTS/review-package" plan-a.md HEAD HEAD 2>&1 >/dev/null)" || rc=$?
+ if [[ "$rc" -eq 3 && "$guard_err" == *"empty commit range"* ]]; then
+ pass "review-package rejects an empty BASE..HEAD range with exit 3"
+ else
+ fail "review-package rejects an empty BASE..HEAD range with exit 3"
+ echo " exit: $rc"
+ echo " stderr: $guard_err"
+ fi
+
# --- Worktree isolation: a linked worktree resolves its own workspace ---
local wt="$TEST_ROOT/wt"
( cd "$repo" && git worktree add -q "$wt" -b wt-feature )
@@ -189,6 +213,143 @@ PLAN
echo " status: $wt_status"
fi
+ # --- helpers survive a mode-stripping extractor dropping exec bits (#2040) ---
+ local stripped="$TEST_ROOT/stripped-scripts"
+ mkdir -p "$stripped"
+ cp "$SDD_SCRIPTS/sdd-workspace" "$SDD_SCRIPTS/task-brief" "$SDD_SCRIPTS/review-package" "$stripped/"
+ chmod -x "$stripped"/*
+ local noexec_out noexec_rc=0
+ noexec_out="$(cd "$repo" && bash "$stripped/task-brief" plan-b.md 1 2>&1)" || noexec_rc=$?
+ if [[ "$noexec_rc" -eq 0 && -f "$repo/.superpowers/sdd/plan-b/task-1-brief.md" ]]; then
+ pass "task-brief works with no exec bit on sdd-workspace"
+ else
+ fail "task-brief works with no exec bit on sdd-workspace"
+ echo " rc: $noexec_rc"
+ echo " output: $noexec_out"
+ fi
+
+ # --- Ownership markers: two plans with the same basename (#2045) ---
+ mkdir -p "$repo/docs/alpha" "$repo/docs/beta"
+ cat > "$repo/docs/alpha/plan.md" <<'PLAN'
+# Alpha Plan
+
+## Task 1: Alpha work
+
+Alpha-only requirement text.
+PLAN
+ cat > "$repo/docs/beta/plan.md" <<'PLAN'
+# Beta Plan
+
+## Task 1: Beta work
+
+Beta-only requirement text.
+PLAN
+
+ local dir_alpha dir_beta
+ dir_alpha="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" docs/alpha/plan.md)"
+ dir_beta="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" docs/beta/plan.md)"
+ if [[ "$dir_alpha" != "$dir_beta" ]]; then
+ pass "same-basename plans resolve to distinct workspaces"
+ else
+ fail "same-basename plans resolve to distinct workspaces"
+ echo " alpha: $dir_alpha"
+ echo " beta: $dir_beta"
+ fi
+
+ ( cd "$repo" && "$SDD_SCRIPTS/task-brief" docs/alpha/plan.md 1 >/dev/null )
+ ( cd "$repo" && "$SDD_SCRIPTS/task-brief" docs/beta/plan.md 1 >/dev/null )
+ if grep -q "Alpha-only requirement text." "$dir_alpha/task-1-brief.md" 2>/dev/null \
+ && grep -q "Beta-only requirement text." "$dir_beta/task-1-brief.md" 2>/dev/null; then
+ pass "same-basename plans keep both task briefs intact"
+ else
+ fail "same-basename plans keep both task briefs intact"
+ echo " alpha brief: $(cat "$dir_alpha/task-1-brief.md" 2>/dev/null)"
+ echo " beta brief: $(cat "$dir_beta/task-1-brief.md" 2>/dev/null)"
+ fi
+
+ # --- Legacy adoption: pre-existing workspace without a marker ---
+ printf '# Foo\n\n## Task 1: Foo\n\nFoo.\n' > "$repo/foo.md"
+ mkdir -p "$repo/.superpowers/sdd/foo"
+ printf 'ledger\n' > "$repo/.superpowers/sdd/foo/progress.md"
+ local dir_foo
+ dir_foo="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" foo.md)"
+ if [[ "$dir_foo" == "$repo/.superpowers/sdd/foo" \
+ && -f "$dir_foo/progress.md" \
+ && "$(cat "$dir_foo/plan-path" 2>/dev/null)" == "foo.md" ]]; then
+ pass "legacy markerless workspace is adopted in place and marked"
+ else
+ fail "legacy markerless workspace is adopted in place and marked"
+ echo " dir: $dir_foo"
+ echo " marker: $(cat "$dir_foo/plan-path" 2>/dev/null)"
+ fi
+
+ # --- Ownership conflict: marker names a different plan ---
+ printf '# Bar\n\n## Task 1: Bar\n\nBar.\n' > "$repo/bar.md"
+ mkdir -p "$repo/.superpowers/sdd/bar"
+ printf 'somewhere-else/bar.md\n' > "$repo/.superpowers/sdd/bar/plan-path"
+ printf 'other ledger\n' > "$repo/.superpowers/sdd/bar/progress.md"
+ local dir_bar
+ dir_bar="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" bar.md)"
+ if [[ "$dir_bar" == "$repo/.superpowers/sdd/bar-repo" \
+ && "$(cat "$dir_bar/plan-path" 2>/dev/null)" == "bar.md" ]]; then
+ pass "owned workspace disambiguates with parent-dir suffix"
+ else
+ fail "owned workspace disambiguates with parent-dir suffix"
+ echo " got: $dir_bar"
+ fi
+ if [[ "$(cat "$repo/.superpowers/sdd/bar/plan-path")" == "somewhere-else/bar.md" \
+ && "$(cat "$repo/.superpowers/sdd/bar/progress.md")" == "other ledger" ]]; then
+ pass "conflicting plan leaves the original workspace untouched"
+ else
+ fail "conflicting plan leaves the original workspace untouched"
+ fi
+
+ # --- Counter fallback: parent-suffixed workspace is owned too ---
+ printf '# Baz\n\n## Task 1: Baz\n\nBaz.\n' > "$repo/baz.md"
+ mkdir -p "$repo/.superpowers/sdd/baz" "$repo/.superpowers/sdd/baz-repo"
+ printf 'one/baz.md\n' > "$repo/.superpowers/sdd/baz/plan-path"
+ printf 'two/baz.md\n' > "$repo/.superpowers/sdd/baz-repo/plan-path"
+ local dir_baz
+ dir_baz="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" baz.md)"
+ if [[ "$dir_baz" == "$repo/.superpowers/sdd/baz-repo-2" \
+ && "$(cat "$dir_baz/plan-path" 2>/dev/null)" == "baz.md" ]]; then
+ pass "double conflict falls back to a counter suffix"
+ else
+ fail "double conflict falls back to a counter suffix"
+ echo " got: $dir_baz"
+ fi
+
+ # --- Same plan spelled differently resolves to one workspace ---
+ local dir_rel dir_abs dir_dotdot
+ dir_rel="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" docs/alpha/plan.md)"
+ dir_abs="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" "$repo/docs/alpha/plan.md")"
+ dir_dotdot="$(cd "$repo/docs/beta" && "$SDD_SCRIPTS/sdd-workspace" ../alpha/plan.md)"
+ if [[ "$dir_rel" == "$dir_abs" && "$dir_rel" == "$dir_dotdot" \
+ && "$(cat "$dir_rel/plan-path" 2>/dev/null)" == "docs/alpha/plan.md" ]]; then
+ pass "relative, absolute, and ../ spellings share one workspace and marker"
+ else
+ fail "relative, absolute, and ../ spellings share one workspace and marker"
+ echo " rel: $dir_rel"
+ echo " abs: $dir_abs"
+ echo " dotdot: $dir_dotdot"
+ echo " marker: $(cat "$dir_rel/plan-path" 2>/dev/null)"
+ fi
+
+ # --- Out-of-repo plans keep working, marker holds the absolute path ---
+ mkdir -p "$TEST_ROOT/outside"
+ printf '# Remote\n\n## Task 1: Remote\n\nRemote.\n' > "$TEST_ROOT/outside/remote-plan.md"
+ local outside_abs dir_out
+ outside_abs="$(cd "$TEST_ROOT/outside" && pwd -P)/remote-plan.md"
+ dir_out="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" "$TEST_ROOT/outside/remote-plan.md")"
+ if [[ "$dir_out" == "$repo/.superpowers/sdd/remote-plan" \
+ && "$(cat "$dir_out/plan-path" 2>/dev/null)" == "$outside_abs" ]]; then
+ pass "out-of-repo plan gets a basename slug and an absolute-path marker"
+ else
+ fail "out-of-repo plan gets a basename slug and an absolute-path marker"
+ echo " dir: $dir_out"
+ echo " marker: $(cat "$dir_out/plan-path" 2>/dev/null)"
+ fi
+
echo ""
if [[ "$FAILURES" -ne 0 ]]; then
echo "FAILED: $FAILURES assertion(s)."
diff --git a/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh b/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh
index 01de394..265aea1 100755
--- a/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh
+++ b/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh
@@ -196,6 +196,10 @@ write_upstream_fixture() {
}
EOF
+ cat > "$repo/index.js" <<'EOF'
+export { default } from "./.opencode/plugins/superpowers.js";
+EOF
+
cat > "$repo/.gitignore" <<'EOF'
.private-journal/
EOF
@@ -303,6 +307,7 @@ EOF
hooks/run-hook.cmd \
hooks/session-start \
hooks/session-start-codex \
+ index.js \
package.json \
scripts/sync-to-codex-plugin.sh \
skills/example/SKILL.md
@@ -664,6 +669,7 @@ main() {
assert_not_contains "$preview_section" "evals/" "Preview excludes eval harness"
assert_not_contains "$preview_section" ".gitmodules" "Preview excludes repo submodule metadata"
assert_not_contains "$preview_section" ".pre-commit-config.yaml" "Preview excludes repo pre-commit config"
+ assert_not_contains "$preview_section" "index.js" "Preview excludes OpenCode root entrypoint"
assert_not_contains "$preview_output" "Overlay file (.codex-plugin/plugin.json) will be regenerated" "Preview omits overlay regeneration note"
assert_not_contains "$preview_output" "Assets (superpowers-small.svg, app-icon.png) will be seeded from" "Preview omits assets seeding note"
assert_contains "$preview_section" "skills/example/SKILL.md" "Preview reflects dirty tracked destination file"
diff --git a/tests/diagnosing-superpowers/test-skill-structure.sh b/tests/diagnosing-superpowers/test-skill-structure.sh
new file mode 100755
index 0000000..9c31591
--- /dev/null
+++ b/tests/diagnosing-superpowers/test-skill-structure.sh
@@ -0,0 +1,151 @@
+#!/usr/bin/env bash
+# Structural checks for skills/diagnosing-superpowers. Behavior is tested by
+# scenario evals kept by the maintainer; this script only checks the things a
+# shell can check: frontmatter, referenced files exist, no local paths or
+# names leaked into shipped files, SKILL.md word budget.
+set -u
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+SKILL_DIR="$REPO_ROOT/skills/diagnosing-superpowers"
+SKILL_MD="$SKILL_DIR/SKILL.md"
+WORD_BUDGET=1000
+
+PASSES=0
+FAILURES=0
+
+pass() { echo " [PASS] $1"; PASSES=$((PASSES + 1)); }
+fail() { echo " [FAIL] $1"; FAILURES=$((FAILURES + 1)); }
+
+echo "diagnosing-superpowers structure"
+
+# --- SKILL.md frontmatter -------------------------------------------------
+if [ -f "$SKILL_MD" ]; then
+ pass "SKILL.md exists"
+ frontmatter="$(awk 'NR==1 && $0!="---"{exit} NR>1 && $0=="---"{exit} NR>1{print}' "$SKILL_MD")"
+ if printf '%s\n' "$frontmatter" | grep -q '^name: diagnosing-superpowers$'; then
+ pass "frontmatter name is diagnosing-superpowers"
+ else
+ fail "frontmatter name is diagnosing-superpowers"
+ fi
+ description="$(printf '%s\n' "$frontmatter" | awk '/^description:/{sub(/^description:[ ]*/,""); print; found=1; next} found && /^[ ]/{print} found && !/^[ ]/{exit}' | tr '\n' ' ')"
+ if printf '%s' "$description" | grep -q '^Use when'; then
+ pass "description starts with 'Use when'"
+ else
+ fail "description starts with 'Use when' (got: ${description:0:60})"
+ fi
+ if [ "${#description}" -le 1024 ]; then
+ pass "description under 1024 characters"
+ else
+ fail "description under 1024 characters (${#description})"
+ fi
+ for banned in "dispatch" "then" "step"; do
+ if printf '%s' "$description" | grep -qiw "$banned"; then
+ fail "description contains workflow word '$banned'"
+ else
+ pass "description avoids workflow word '$banned'"
+ fi
+ done
+
+ # --- word budget --------------------------------------------------------
+ body_words="$(awk 'BEGIN{fm=0} NR==1 && $0=="---"{fm=1; next} fm==1 && $0=="---"{fm=2; next} fm==2{print}' "$SKILL_MD" | wc -w | tr -d ' ')"
+ if [ "$body_words" -le "$WORD_BUDGET" ]; then
+ pass "SKILL.md body within $WORD_BUDGET words ($body_words)"
+ else
+ fail "SKILL.md body within $WORD_BUDGET words ($body_words)"
+ fi
+
+ # --- required sections --------------------------------------------------
+ for heading in "## Hard rules" "## Red Flags"; do
+ if grep -q "^$heading" "$SKILL_MD"; then
+ pass "SKILL.md has section '$heading'"
+ else
+ fail "SKILL.md has section '$heading'"
+ fi
+ done
+
+ # --- every referenced skill file exists --------------------------------
+ while IFS= read -r ref; do
+ if [ -f "$SKILL_DIR/$ref" ]; then
+ pass "referenced file exists: $ref"
+ else
+ fail "referenced file exists: $ref"
+ fi
+ done < <(grep -o '\(references\|prompts\|templates\)/[A-Za-z0-9._-]*\.md' "$SKILL_MD" | sort -u)
+else
+ fail "SKILL.md exists"
+fi
+
+# --- expected files -------------------------------------------------------
+expected_files=(
+ references/redaction-policy.md
+ references/session-discovery.md
+ references/context-safety.md
+ references/github-issues.md
+ prompts/analyst-common.md
+ prompts/skill-timeline.md
+ prompts/plan-adherence.md
+ prompts/repeated-work.md
+ prompts/stumbles.md
+ prompts/quality-evidence.md
+ prompts/request-conflicts.md
+ prompts/cost-and-time.md
+ prompts/scrub.md
+ prompts/scrub-audit.md
+ prompts/similar-session.md
+ templates/case.md
+ templates/report.md
+ templates/bundle-README.md
+ templates/issue.md
+)
+for rel in "${expected_files[@]}"; do
+ if [ -f "$SKILL_DIR/$rel" ]; then
+ pass "expected file present: $rel"
+ else
+ fail "expected file present: $rel"
+ fi
+done
+
+# --- removed harness recipes stay removed --------------------------------
+removed_files=(
+ references/claude-code-sessions.md
+ references/codex-sessions.md
+ references/other-harnesses.md
+)
+for rel in "${removed_files[@]}"; do
+ if [ ! -e "$SKILL_DIR/$rel" ]; then
+ pass "removed reference absent: $rel"
+ else
+ fail "removed reference absent: $rel"
+ fi
+done
+
+removed_reference_hits="$(grep -rn -E 'references/(claude-code-sessions|codex-sessions|other-harnesses)\.md' "$SKILL_DIR" --include='*.md' 2>/dev/null || true)"
+if [ -z "$removed_reference_hits" ]; then
+ pass "active skill prose has no references to removed harness recipes"
+else
+ fail "active skill prose has no references to removed harness recipes"
+ printf '%s\n' "$removed_reference_hits" | head -10 | sed 's/^/ /'
+fi
+
+# --- no local paths or names in shipped files ----------------------------
+leaks="$(grep -rn -E '/Users/|/home/|jesse' "$SKILL_DIR" "$SCRIPT_DIR" --exclude=test-skill-structure.sh 2>/dev/null || true)"
+if [ -z "$leaks" ]; then
+ pass "no machine-specific paths or names in shipped files (skills + tests)"
+else
+ fail "no machine-specific paths or names in shipped files (skills + tests)"
+ printf '%s\n' "$leaks" | head -10 | sed 's/^/ /'
+fi
+
+# --- "the user" never appears in skill prose -----------------------------
+user_hits="$(grep -rn -i 'the user' "$SKILL_DIR" --include='*.md' 2>/dev/null || true)"
+if [ -z "$user_hits" ]; then
+ pass "skill files say 'your human partner', not 'the user'"
+else
+ fail "skill files say 'your human partner', not 'the user'"
+ printf '%s\n' "$user_hits" | head -10 | sed 's/^/ /'
+fi
+
+echo
+echo "Passed: $PASSES Failed: $FAILURES"
+[ "$FAILURES" -eq 0 ]
diff --git a/tests/opencode/run-tests.sh b/tests/opencode/run-tests.sh
index d9b100e..43e1a3a 100755
--- a/tests/opencode/run-tests.sh
+++ b/tests/opencode/run-tests.sh
@@ -45,6 +45,8 @@ while [[ $# -gt 0 ]]; do
echo "Tests:"
echo " test-plugin-loading.sh Verify plugin installation and structure"
echo " test-bootstrap-caching.sh Verify bootstrap content caching"
+ echo " test-session-bootstrap.sh Verify session classification and lookup recovery"
+ echo " test-skill-registration.sh Verify V2 skill registration contract (2.0.4 path field)"
echo " test-tools.sh Test use_skill and find_skills tools (integration)"
echo " test-priority.sh Test skill priority resolution (integration)"
exit 0
@@ -61,6 +63,8 @@ done
tests=(
"test-plugin-loading.sh"
"test-bootstrap-caching.sh"
+ "test-session-bootstrap.sh"
+ "test-skill-registration.sh"
)
# Integration tests (require OpenCode)
diff --git a/tests/opencode/test-bootstrap-caching.mjs b/tests/opencode/test-bootstrap-caching.mjs
index 32149ae..386b7b1 100644
--- a/tests/opencode/test-bootstrap-caching.mjs
+++ b/tests/opencode/test-bootstrap-caching.mjs
@@ -32,6 +32,10 @@ const mod = await import(pathToFileURL(pluginPath).href);
const plugin = await mod.SuperpowersPlugin({ client: {}, directory: '.' });
const transform = plugin['experimental.chat.messages.transform'];
+// Mapping constants are flavor-specific (#opencode-v2): V1 keeps the 1.18.x
+// tool names, V2 teaches the renamed tools. Assert both directly.
+const mappingFailures = assertMappingConstants(mod);
+
const firstOutput = makeOutput(`${scenario} bootstrap first step`);
await transform({}, firstOutput);
const afterFirst = { existsCount, readCount };
@@ -40,6 +44,11 @@ const secondOutput = makeOutput(`${scenario} bootstrap second step`);
await transform({}, secondOutput);
const afterSecond = { existsCount, readCount };
+// Exercise the V2 path (setup() + ctx.session.hook("context")) with a mock
+// ctx so the V2_MAPPING wiring is verified, not just the constant. Run after
+// the V1 count snapshots: setup() reads SKILL.md files during registration.
+const v2Result = await runV2ContextHook(mod);
+
const result = {
scenario,
firstBootstrapParts: countBootstrapParts(firstOutput),
@@ -52,12 +61,24 @@ const result = {
secondReadCount: afterSecond.readCount,
firstExistsCount: afterFirst.existsCount,
secondExistsCount: afterSecond.existsCount,
+ v2BootstrapParts: v2Result.bootstrapParts,
+ mapsV2SubagentTool: v2Result.text.includes('`subagent` with `agent: "general"`'),
+ mapsV2SessionIDContinuation: v2Result.text.includes('`sessionID` to continue a previous subagent'),
+ mapsV2NoTodoTool: v2Result.text.includes('no todo tool'),
+ mapsV2MutationToPatch: v2Result.text.includes('`patch` with `patchText`'),
+ mapsV2Shell: v2Result.text.includes('`shell`'),
+ staleV1ToolsInV2: v2Result.text.includes('`apply_patch`') || v2Result.text.includes('`todowrite`') || v2Result.text.includes('`subagent_type`'),
};
const failures = scenario === 'present'
? assertPresentBootstrap(result)
: assertMissingBootstrap(result);
+if (scenario === 'present') {
+ failures.push(...assertV2Bootstrap(result));
+}
+failures.push(...mappingFailures);
+
if (failures.length > 0) {
console.error(JSON.stringify(result, null, 2));
for (const failure of failures) {
@@ -144,3 +165,104 @@ function assertMissingBootstrap(result) {
}
return failures;
}
+
+function assertMappingConstants(mod) {
+ const failures = [];
+ if (typeof mod.V1_MAPPING !== 'string' || typeof mod.V2_MAPPING !== 'string') {
+ failures.push('expected plugin to export V1_MAPPING and V2_MAPPING string constants');
+ return failures;
+ }
+ for (const needle of ['`todowrite`', '`task` with `subagent_type: "general"`', '`apply_patch`', '`bash`']) {
+ if (!mod.V1_MAPPING.includes(needle)) {
+ failures.push(`expected V1_MAPPING to keep the 1.18.x tool name ${needle}`);
+ }
+ }
+ for (const needle of [
+ '`subagent` with `agent: "general"`',
+ '`sessionID` to continue a previous subagent',
+ 'no todo tool',
+ '`write`',
+ '`edit`',
+ '`patch` with `patchText`',
+ '`shell`',
+ '`read`',
+ '`grep`, `glob`',
+ '`webfetch`',
+ '`websearch`',
+ ]) {
+ if (!mod.V2_MAPPING.includes(needle)) {
+ failures.push(`expected V2_MAPPING to teach the V2 tool ${needle}`);
+ }
+ }
+ for (const stale of ['`todowrite`', '`task` with', '`apply_patch`', '`bash`']) {
+ if (mod.V2_MAPPING.includes(stale)) {
+ failures.push(`expected V2_MAPPING not to teach the V1-only tool name ${stale}`);
+ }
+ }
+ return failures;
+}
+
+// Drive setup() with a mock V2 ctx and fire the captured "context" hook on a
+// top-level (parentID-less) session. Returns the injected-part count and the
+// injected bootstrap text ('' when nothing was injected).
+async function runV2ContextHook(mod) {
+ let contextHook = null;
+ const ctx = {
+ skill: {
+ transform: async (fn) => {
+ fn({ add: () => {} });
+ },
+ },
+ session: {
+ hook: async (name, cb) => {
+ if (name === 'context') contextHook = cb;
+ },
+ get: async ({ sessionID }) => ({ id: sessionID }), // top-level: no parentID
+ },
+ };
+ try {
+ await mod.default.setup(ctx);
+ } catch (err) {
+ console.error('[test] V2 setup() threw:', err);
+ return { bootstrapParts: 0, text: '' };
+ }
+ if (typeof contextHook !== 'function') {
+ return { bootstrapParts: 0, text: '' };
+ }
+ const event = {
+ sessionID: 'sess-v2-top',
+ messages: [{ role: 'user', content: [{ type: 'text', text: 'v2 bootstrap step' }] }],
+ };
+ await contextHook(event);
+ const parts = event.messages[0].content.filter(
+ (part) => part.type === 'text' && part.text.includes('EXTREMELY_IMPORTANT')
+ );
+ return { bootstrapParts: parts.length, text: parts[0]?.text || '' };
+}
+
+function assertV2Bootstrap(result) {
+ const failures = [];
+ if (result.v2BootstrapParts !== 1) {
+ failures.push(`expected V2 context hook to inject one bootstrap part, got ${result.v2BootstrapParts}`);
+ return failures;
+ }
+ if (!result.mapsV2SubagentTool) {
+ failures.push('expected V2 bootstrap to map general-purpose subagents to subagent with agent');
+ }
+ if (!result.mapsV2SessionIDContinuation) {
+ failures.push('expected V2 bootstrap to teach sessionID continuation for subagents');
+ }
+ if (!result.mapsV2NoTodoTool) {
+ failures.push('expected V2 bootstrap to state that V2 has no todo tool');
+ }
+ if (!result.mapsV2MutationToPatch) {
+ failures.push('expected V2 bootstrap to map file mutation to patch with patchText');
+ }
+ if (!result.mapsV2Shell) {
+ failures.push('expected V2 bootstrap to map shell commands to the shell tool');
+ }
+ if (result.staleV1ToolsInV2) {
+ failures.push('expected V2 bootstrap not to teach V1-only tool names (apply_patch/todowrite/subagent_type)');
+ }
+ return failures;
+}
diff --git a/tests/opencode/test-session-bootstrap.mjs b/tests/opencode/test-session-bootstrap.mjs
new file mode 100644
index 0000000..31658d5
--- /dev/null
+++ b/tests/opencode/test-session-bootstrap.mjs
@@ -0,0 +1,224 @@
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import { pathToFileURL } from 'node:url';
+
+const [, , inputPath] = process.argv;
+assert.ok(inputPath, 'pass the plugin module path');
+const pluginURL = pathToFileURL(fs.realpathSync(inputPath));
+const marker = '<EXTREMELY_IMPORTANT>\nYou have superpowers.';
+let generation = 0;
+
+function reply(flavor, session) {
+ return flavor === 'v1' ? { data: session } : session;
+}
+
+function makeEvent(flavor, sessionID) {
+ const text = { type: 'text', text: 'Execute the assigned task' };
+ return {
+ sessionID,
+ messages: [flavor === 'v1'
+ ? { info: { role: 'user', sessionID }, parts: [text] }
+ : { role: 'user', content: [text] }],
+ };
+}
+
+function bootstrapCount(event) {
+ return event.messages.flatMap((message) => message.parts ?? message.content ?? []).filter(
+ (part) => part.type === 'text' && part.text.startsWith(marker)
+ ).length;
+}
+
+async function makeHarness(flavor, fetchSession) {
+ const mod = await import(`${pluginURL.href}?session-test=${++generation}`);
+ const lookups = [];
+ const registered = [];
+ const get = async (id) => {
+ lookups.push(id);
+ return fetchSession(id, lookups.length);
+ };
+ let invoke;
+ if (flavor === 'v1') {
+ const hooks = await mod.SuperpowersPlugin({
+ client: { session: { get: ({ path: { id } }) => get(id) } },
+ directory: '.',
+ });
+ invoke = (event) => hooks['experimental.chat.messages.transform']({}, event);
+ } else {
+ await mod.default.setup({
+ skill: { transform: async (transform) => transform({ add: (skill) => registered.push(skill) }) },
+ session: {
+ get: ({ sessionID }) => get(sessionID),
+ hook: async (name, callback) => { if (name === 'context') invoke = callback; },
+ },
+ });
+ }
+ assert.equal(typeof invoke, 'function');
+ return { invoke, lookups, registered };
+}
+
+for (const flavor of ['v1', 'v2']) {
+ for (const [kind, extra, expected] of [
+ ['root', {}, 1],
+ ['child', { parentID: 'parent' }, 0],
+ ['fork', { fork: { sessionID: 'origin' } }, 1],
+ ]) {
+ const id = `${flavor}-${kind}`;
+ const h = await makeHarness(flavor, () => reply(flavor, { id, ...extra }));
+ const event = makeEvent(flavor, id);
+ await h.invoke(event);
+ assert.equal(bootstrapCount(event), expected, `${id}: first request`);
+ await h.invoke(event);
+ assert.equal(bootstrapCount(event), expected, `${id}: repeated event`);
+ const fresh = makeEvent(flavor, id);
+ await h.invoke(fresh);
+ assert.equal(bootstrapCount(fresh), expected, `${id}: fresh request`);
+ assert.deepEqual(h.lookups, [id], `${id}: cache successful classification`);
+ if (flavor === 'v2' && kind === 'child') {
+ assert.ok(h.registered.some((skill) => skill.id === 'brainstorming'));
+ }
+ }
+
+ const failures = [
+ ['throws', () => { throw new Error('temporary lookup failure'); }],
+ ['missing', () => undefined],
+ ['null', () => null],
+ ['empty', () => reply(flavor, {})],
+ ['wrong-id', () => reply(flavor, { id: 'different-session' })],
+ ['invalid-parent', (id) => reply(flavor, { id, parentID: 42 })],
+ ];
+ if (flavor === 'v1') {
+ failures.push(['resolved-http-error', () => ({
+ data: undefined,
+ error: { name: 'UnknownError', data: { message: 'temporary 503' } },
+ response: { ok: false, status: 503 },
+ })]);
+ }
+ for (const [kind, firstResult] of failures) {
+ const id = `${flavor}-${kind}`;
+ const h = await makeHarness(flavor, (sessionID, call) => call === 1
+ ? firstResult(sessionID)
+ : reply(flavor, { id: sessionID, parentID: 'parent' }));
+ const counts = [];
+ for (let step = 0; step < 2; step++) {
+ const event = makeEvent(flavor, id);
+ await h.invoke(event);
+ counts.push(bootstrapCount(event));
+ }
+ assert.deepEqual(counts, [1, 0], `${id}: recover on the next request`);
+ assert.deepEqual(h.lookups, [id, id], `${id}: never cache the failure`);
+ }
+
+ const isolated = await makeHarness(flavor, (id) => reply(flavor,
+ id === 'child-session' ? { id, parentID: 'parent' } : { id }));
+ for (const [id, expected] of [['root-session', 1], ['child-session', 0], ['root-session', 1], ['child-session', 0]]) {
+ const event = makeEvent(flavor, id);
+ await isolated.invoke(event);
+ assert.equal(bootstrapCount(event), expected);
+ }
+ assert.deepEqual(isolated.lookups, ['root-session', 'child-session']);
+
+ const bounded = await makeHarness(flavor, (id) => reply(flavor, { id, parentID: 'parent' }));
+ for (let index = 0; index <= 512; index++) {
+ const event = makeEvent(flavor, `eviction-${index}`);
+ await bounded.invoke(event);
+ assert.equal(bootstrapCount(event), 0);
+ }
+ const evicted = makeEvent(flavor, 'eviction-0');
+ await bounded.invoke(evicted);
+ assert.equal(bootstrapCount(evicted), 0);
+ assert.equal(bounded.lookups.filter((id) => id === 'eviction-0').length, 2);
+
+ const restarted = await makeHarness(flavor, (id) => reply(flavor, { id, parentID: 'parent' }));
+ const afterRestart = makeEvent(flavor, 'eviction-0');
+ await restarted.invoke(afterRestart);
+ assert.equal(bootstrapCount(afterRestart), 0);
+ assert.deepEqual(restarted.lookups, ['eviction-0']);
+
+ const unknown = await makeHarness(flavor, () => { throw new Error('must not look up a missing ID'); });
+ const noID = makeEvent(flavor, undefined);
+ await unknown.invoke(noID);
+ assert.equal(bootstrapCount(noID), 1);
+ assert.deepEqual(unknown.lookups, []);
+}
+
+function compactedEvent(sessionID) {
+ return {
+ sessionID,
+ system: [],
+ messages: [{
+ role: 'assistant',
+ content: [{ type: 'compaction', provider: 'fixture', encrypted: 'opaque-checkpoint' }],
+ }],
+ };
+}
+
+const compactedRoot = await makeHarness('v2', (id) => ({ id }));
+const rootEvent = compactedEvent('compacted-root');
+const checkpoint = structuredClone(rootEvent.messages[0]);
+await compactedRoot.invoke(rootEvent);
+assert.equal(bootstrapCount(rootEvent), 1);
+assert.deepEqual(rootEvent.messages[0], checkpoint);
+assert.equal(rootEvent.messages.length, 2);
+assert.equal(rootEvent.messages[1].role, 'user');
+assert.deepEqual(rootEvent.system, []);
+await compactedRoot.invoke(rootEvent);
+assert.equal(bootstrapCount(rootEvent), 1);
+assert.equal(rootEvent.messages.length, 2);
+const freshRootEvent = compactedEvent('compacted-root');
+await compactedRoot.invoke(freshRootEvent);
+assert.equal(bootstrapCount(freshRootEvent), 1);
+assert.deepEqual(compactedRoot.lookups, ['compacted-root']);
+
+const compactedChild = await makeHarness('v2', (id) => ({ id, parentID: 'parent' }));
+const childEvent = compactedEvent('compacted-child');
+const originalChild = structuredClone(childEvent);
+await compactedChild.invoke(childEvent);
+assert.equal(bootstrapCount(childEvent), 0);
+assert.deepEqual(childEvent, originalChild);
+assert.deepEqual(compactedChild.lookups, ['compacted-child']);
+
+const retryChild = await makeHarness('v2', (id, call) => {
+ if (call === 1) throw new Error('temporary lookup failure');
+ return { id, parentID: 'parent' };
+});
+const unknownChild = compactedEvent('retry-compacted-child');
+await retryChild.invoke(unknownChild);
+assert.equal(bootstrapCount(unknownChild), 1);
+const recoveredChild = compactedEvent('retry-compacted-child');
+await retryChild.invoke(recoveredChild);
+assert.equal(bootstrapCount(recoveredChild), 0);
+assert.equal(recoveredChild.messages.length, 1);
+assert.equal(retryChild.lookups.length, 2);
+
+const newPromptAfterCheckpoint = compactedEvent('new-prompt-after-checkpoint-root');
+newPromptAfterCheckpoint.messages.push({ role: 'user', content: [{ type: 'text', text: 'Continue' }] });
+await compactedRoot.invoke(newPromptAfterCheckpoint);
+assert.equal(bootstrapCount(newPromptAfterCheckpoint), 1);
+assert.equal(newPromptAfterCheckpoint.messages.length, 2);
+assert.equal(newPromptAfterCheckpoint.messages[1].content.length, 2);
+
+const retainedUser = compactedEvent('retained-user-root');
+retainedUser.messages.unshift({ role: 'user', content: [{ type: 'text', text: 'Keep going' }] });
+const retainedCheckpoint = structuredClone(retainedUser.messages[1]);
+await compactedRoot.invoke(retainedUser);
+assert.equal(bootstrapCount(retainedUser), 1);
+assert.equal(retainedUser.messages.length, 2);
+assert.equal(retainedUser.messages[0].content.length, 2);
+assert.ok(retainedUser.messages[0].content[0].text.startsWith(marker));
+assert.equal(retainedUser.messages[0].content[1].text, 'Keep going');
+assert.deepEqual(retainedUser.messages[1], retainedCheckpoint);
+await compactedRoot.invoke(retainedUser);
+assert.equal(bootstrapCount(retainedUser), 1);
+assert.equal(retainedUser.messages.length, 2);
+
+const retainedUserChild = compactedEvent('retained-user-child');
+retainedUserChild.messages.unshift({ role: 'user', content: [{ type: 'text', text: 'Keep going' }] });
+const originalRetainedUserChild = structuredClone(retainedUserChild);
+await compactedChild.invoke(retainedUserChild);
+assert.equal(bootstrapCount(retainedUserChild), 0);
+assert.deepEqual(retainedUserChild, originalRetainedUserChild);
+const empty = { sessionID: 'empty', messages: [] };
+await compactedRoot.invoke(empty);
+assert.deepEqual(empty.messages, []);
+
+console.log('Session classification, recovery and cache lifetime passed');
diff --git a/tests/opencode/test-session-bootstrap.sh b/tests/opencode/test-session-bootstrap.sh
new file mode 100755
index 0000000..accd85b
--- /dev/null
+++ b/tests/opencode/test-session-bootstrap.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+node "$SCRIPT_DIR/test-session-bootstrap.mjs" "$SCRIPT_DIR/../../.opencode/plugins/superpowers.js"
diff --git a/tests/opencode/test-skill-registration.mjs b/tests/opencode/test-skill-registration.mjs
new file mode 100644
index 0000000..a31fe4a
--- /dev/null
+++ b/tests/opencode/test-skill-registration.mjs
@@ -0,0 +1,205 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { pathToFileURL } from 'url';
+
+// Verifies the V2 skill registration payload matches OpenCode 2.0.4's
+// Skill.Info contract (packages/schema/src/skill.ts):
+// { id, name, description?, autoinvoke?, path, content }
+// Upstream commit 199aabe9e2 (first released in v2.0.4) renamed the required
+// file field `location` -> `path`. A wrong field name makes draft.add()
+// throw inside the host's transform rebuild, which asynchronously disables
+// the whole plugin ("Plugin disabled after skill.transform failed") and
+// takes the bootstrap hook down with it — see PR #2106 review by 80avin.
+
+const [, , inputPath] = process.argv;
+
+if (!inputPath) {
+ console.error('Usage: node test-skill-registration.mjs PLUGIN_PATH');
+ process.exit(2);
+}
+
+const pluginPath = fs.realpathSync(inputPath);
+const skillsDir = path.resolve(path.dirname(pluginPath), '../../skills');
+const mod = await import(pathToFileURL(pluginPath).href);
+
+const failures = [];
+
+// --- Run 1: passive capture of every draft.add payload -------------------
+const added = [];
+await mod.default.setup(makeCtx({ add: (skill) => added.push(skill) }));
+
+const expectedIds = fs.existsSync(skillsDir)
+ ? fs.readdirSync(skillsDir, { withFileTypes: true })
+ .filter((e) => e.isDirectory() && !e.name.startsWith('.'))
+ .filter((e) => fs.existsSync(path.join(skillsDir, e.name, 'SKILL.md')))
+ .map((e) => e.name)
+ .sort()
+ : [];
+
+if (added.length === 0) {
+ failures.push('expected setup() to register at least one skill via draft.add()');
+}
+if (JSON.stringify(added.map((s) => s.id).sort()) !== JSON.stringify(expectedIds)) {
+ failures.push(`expected draft.add() ids to match skills dir contents, got ${JSON.stringify(added.map((s) => s.id))}`);
+}
+
+for (const skill of added) {
+ if (typeof skill.path !== 'string' || !path.isAbsolute(skill.path)) {
+ failures.push(`skill "${skill.id}": expected required absolute Skill.Info field "path", got ${JSON.stringify(skill.path)}`);
+ } else if (skill.path !== path.join(skillsDir, skill.id, 'SKILL.md')) {
+ failures.push(`skill "${skill.id}": expected path ${path.join(skillsDir, skill.id, 'SKILL.md')}, got ${skill.path}`);
+ } else if (!fs.existsSync(skill.path)) {
+ failures.push(`skill "${skill.id}": path does not exist on disk: ${skill.path}`);
+ }
+ // Stale 2.0.3-era fields must not leak into the payload: the host strips
+ // unknown keys, but keeping them would silently mask a future regression
+ // to a schema that no longer accepts `path`.
+ if ('location' in skill) {
+ failures.push(`skill "${skill.id}": payload still carries the pre-2.0.4 field "location"`);
+ }
+ if ('slash' in skill) {
+ failures.push(`skill "${skill.id}": payload carries "slash", removed from Skill.Info in 2.0.4`);
+ }
+ if (typeof skill.id !== 'string' || skill.id.length === 0) failures.push(`skill payload missing non-empty "id"`);
+ if (typeof skill.name !== 'string' || skill.name.length === 0) failures.push(`skill "${skill.id}" missing non-empty "name"`);
+ if (typeof skill.content !== 'string' || !skill.content.trim()) failures.push(`skill "${skill.id}" missing non-empty "content"`);
+ if ('description' in skill && typeof skill.description !== 'string') {
+ failures.push(`skill "${skill.id}": "description" must be a string when present`);
+ } else if ('description' in skill && /["']$/.test(skill.description)) {
+ failures.push(`skill "${skill.id}": description ends with a dangling quote: ${JSON.stringify(skill.description)}`);
+ }
+ if (typeof skill.content === 'string' && skill.content.startsWith('---')) {
+ failures.push(`skill "${skill.id}": content still starts with the frontmatter delimiter`);
+ }
+}
+
+// --- Run 2: hostile draft.add must not abort the remaining registrations --
+// The real host swallows a throw escaping the transform callback and then
+// hard-disables the plugin asynchronously. Locally we can only observe the
+// synchronous half of that contract: when draft.add() rejects one skill, the
+// plugin must keep registering the rest instead of aborting the loop.
+const hostileId = added.length > 1 ? added[Math.floor(added.length / 2)].id : null;
+const survived = [];
+let setupThrew = null;
+let survivingContextHook;
+try {
+ await mod.default.setup(makeCtx({
+ add: (skill) => {
+ if (skill.id === hostileId) throw new Error('Simulated Skill.Info decode failure');
+ survived.push(skill.id);
+ },
+ onHook: (name, callback) => {
+ if (name === 'context') survivingContextHook = callback;
+ },
+ }));
+} catch (err) {
+ setupThrew = err;
+}
+if (setupThrew) {
+ failures.push(`expected setup() to contain draft.add() failures, but it threw: ${setupThrew.message}`);
+} else if (hostileId) {
+ const expectedSurvivors = added.map((s) => s.id).filter((id) => id !== hostileId);
+ if (JSON.stringify(survived.sort()) !== JSON.stringify(expectedSurvivors.sort())) {
+ failures.push(`expected all non-rejected skills to still register when one draft.add() throws, got ${JSON.stringify(survived)}`);
+ }
+}
+if (typeof survivingContextHook !== 'function') {
+ failures.push('expected bootstrap hook to survive a rejected skill');
+} else {
+ const event = {
+ sessionID: 'registration-survival-root',
+ messages: [{ role: 'user', content: [{ type: 'text', text: 'Continue' }] }],
+ };
+ await survivingContextHook(event);
+ const count = event.messages.flatMap((message) => message.content).filter(
+ (part) => part.type === 'text' && part.text.startsWith('<EXTREMELY_IMPORTANT>\nYou have superpowers.')
+ ).length;
+ if (count !== 1) failures.push(`expected surviving bootstrap once, got ${count}`);
+}
+
+// --- Run 3: quoted and multi-line frontmatter values ---------------------
+// The description is what the host shows in its skill list. A quoted value
+// that wraps onto indented continuation lines must register as one unquoted
+// line, so exercise each layout against a synthetic install: a copy of the
+// plugin next to fixture skills, laid out like a real package root.
+const frontmatterFixtures = {
+ 'multi-line-double': {
+ frontmatter: 'description: "Use when foo happens\n and bar continues\n and baz ends"',
+ expected: 'Use when foo happens and bar continues and baz ends',
+ },
+ 'multi-line-single': {
+ frontmatter: "description: 'Use when foo happens\n and bar continues\n and baz ends'",
+ expected: 'Use when foo happens and bar continues and baz ends',
+ },
+ 'single-line-quoted': {
+ frontmatter: 'description: "Plain quoted"',
+ expected: 'Plain quoted',
+ },
+ 'block-scalar': {
+ frontmatter: 'description: >\n Folded line one\n line two',
+ expected: 'Folded line one line two',
+ },
+};
+const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'superpowers-frontmatter-'));
+try {
+ const fixturePlugin = path.join(fixtureRoot, '.opencode', 'plugins', 'superpowers.js');
+ fs.mkdirSync(path.dirname(fixturePlugin), { recursive: true });
+ fs.copyFileSync(pluginPath, fixturePlugin);
+ for (const [id, { frontmatter }] of Object.entries(frontmatterFixtures)) {
+ const skillDir = path.join(fixtureRoot, 'skills', id);
+ fs.mkdirSync(skillDir, { recursive: true });
+ fs.writeFileSync(path.join(skillDir, 'SKILL.md'), `---\nname: ${id}\n${frontmatter}\n---\n# Title\n\nBody.\n`);
+ }
+ const fixtureMod = await import(pathToFileURL(fixturePlugin).href);
+ const fixtureAdded = [];
+ await fixtureMod.default.setup(makeCtx({ add: (skill) => fixtureAdded.push(skill) }));
+ for (const [id, { expected }] of Object.entries(frontmatterFixtures)) {
+ const skill = fixtureAdded.find((s) => s.id === id);
+ if (!skill) {
+ failures.push(`fixture "${id}": expected setup() to register it`);
+ continue;
+ }
+ if (skill.description !== expected) {
+ failures.push(`fixture "${id}": expected description ${JSON.stringify(expected)}, got ${JSON.stringify(skill.description)}`);
+ }
+ if (skill.content.startsWith('---')) {
+ failures.push(`fixture "${id}": content still starts with the frontmatter delimiter`);
+ }
+ }
+} finally {
+ fs.rmSync(fixtureRoot, { recursive: true, force: true });
+}
+
+const result = {
+ registered: added.length,
+ ids: added.map((s) => s.id),
+ allPathsValid: added.every((s) => s.path === path.join(skillsDir, s.id, 'SKILL.md') && fs.existsSync(s.path)),
+ staleLocationField: added.some((s) => 'location' in s),
+ hostileRejectedId: hostileId,
+ survivedHostileAdd: JSON.stringify(survived.sort()) === JSON.stringify(added.map((s) => s.id).filter((id) => id !== hostileId).sort()),
+};
+
+if (failures.length > 0) {
+ console.error(JSON.stringify(result, null, 2));
+ for (const failure of failures) {
+ console.error(`FAIL: ${failure}`);
+ }
+ process.exit(1);
+}
+
+console.log(JSON.stringify(result, null, 2));
+
+function makeCtx({ add, onHook = () => {} }) {
+ return {
+ skill: {
+ transform: async (fn) => {
+ await fn({ list: () => [], get: () => undefined, add, update: () => {}, remove: () => {} });
+ },
+ },
+ session: {
+ hook: async (name, callback) => onHook(name, callback),
+ get: async ({ sessionID }) => ({ id: sessionID }), // top-level: no parentID
+ },
+ };
+}
diff --git a/tests/opencode/test-skill-registration.sh b/tests/opencode/test-skill-registration.sh
new file mode 100755
index 0000000..ac882a4
--- /dev/null
+++ b/tests/opencode/test-skill-registration.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Test: V2 Skill Registration Contract (#2106 review)
+# Verifies setup() registers skills matching OpenCode 2.0.4's Skill.Info
+# schema (path field, no stale location/slash) and contains per-skill
+# draft.add() failures instead of aborting registration.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+
+echo "=== Test: V2 Skill Registration Contract ==="
+
+source "$SCRIPT_DIR/setup.sh"
+trap cleanup_test_env EXIT
+
+node "$SCRIPT_DIR/test-skill-registration.mjs" "$SUPERPOWERS_PLUGIN_FILE"
+node "$SCRIPT_DIR/test-skill-registration.mjs" "$OPENCODE_CONFIG_DIR/plugins/superpowers.js"
+
+echo " [PASS] Skill payloads match the 2.0.4 Skill.Info contract"
+echo " [PASS] A rejected draft.add() skips one skill without aborting the rest"
+echo " [PASS] Quoted and multi-line frontmatter values register unquoted"
+echo ""
+echo "=== All skill registration tests passed ==="
diff --git a/tests/proving-it-works-with-a-movie/README.md b/tests/proving-it-works-with-a-movie/README.md
new file mode 100644
index 0000000..2a0eb2d
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/README.md
@@ -0,0 +1,29 @@
+# Portable movie regressions
+
+Run a suite from the repository root:
+
+```sh
+uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite assembly
+uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite checker
+uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite contracts
+uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite narration
+uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite all
+```
+
+By default, unavailable external capabilities are reported as skips. Add
+`--require-capabilities` when the selected environment is required to provide
+them; any skip then makes the run fail.
+
+The assembly sine wave is only a synthetic timing fixture. The narration drift
+inputs exercise text comparison only. Neither is speech/ASR acceptance.
+
+The `contracts` suite is the safe entrypoint for mocked process/media boundaries
+and text fixtures. It does not run the existing media/session suites, inspect a
+finished movie, or replace live acceptance and the required human viewing gate.
+
+The `terminal` suite starts a real ttyd session and skips where ttyd or a
+Chrome-family browser is missing. It films `bash` by default on Unix and
+`powershell51` on Windows; set `MOVIE_TEST_SHELL` to `powershell51`,
+`powershell7`, `gitbash`, or `bash`, and `MOVIE_TEST_SHELL_EXE`,
+`MOVIE_TEST_TTYD`, or `MOVIE_TEST_BROWSER` when those executables are not on
+PATH.
diff --git a/tests/proving-it-works-with-a-movie/fixtures.py b/tests/proving-it-works-with-a-movie/fixtures.py
new file mode 100644
index 0000000..eac965b
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/fixtures.py
@@ -0,0 +1,299 @@
+"""Portable fixtures for the imported movie regression suites."""
+
+import json
+import importlib.util
+import importlib.machinery
+import shutil
+import subprocess
+import types
+import sys
+from pathlib import Path
+
+
+TIMEOUT_SECONDS = 900
+
+
+def missing_executables(*names: str) -> list[str]:
+ """Return executable names that cannot be resolved on PATH."""
+ return [name for name in names if shutil.which(name) is None]
+
+
+def output_text(result: subprocess.CompletedProcess[bytes]) -> str:
+ """Decode a captured command's combined output as UTF-8 evidence."""
+ return (result.stdout + result.stderr).decode("utf-8", errors="replace")
+
+
+def run_tool(
+ name: str,
+ args: list[str],
+ *,
+ cwd: Path,
+ env: dict[str, str] | None = None,
+) -> subprocess.CompletedProcess[bytes]:
+ """Invoke one extensionless movie tool through its PEP 723 environment."""
+ uv = shutil.which("uv")
+ if uv is None:
+ raise RuntimeError("uv is required for movie tool tests")
+ script = (
+ Path(__file__).resolve().parents[2]
+ / "skills/proving-it-works-with-a-movie/scripts"
+ / name
+ )
+ return subprocess.run(
+ [uv, "run", "--script", str(script), *args],
+ cwd=cwd,
+ env=env,
+ capture_output=True,
+ timeout=TIMEOUT_SECONDS,
+ )
+
+
+def load_script(name: str) -> types.ModuleType:
+ """Load an extensionless movie tool as a test module."""
+ script = (
+ Path(__file__).resolve().parents[2]
+ / "skills/proving-it-works-with-a-movie/scripts"
+ / name
+ )
+ if not script.exists():
+ script = script.with_suffix(".py")
+ sys.path.insert(0, str(script.parent))
+ loader = importlib.machinery.SourceFileLoader(f"movie_tool_{name}", str(script))
+ spec = importlib.util.spec_from_loader(loader.name, loader)
+ if spec is None or spec.loader is None:
+ raise ImportError(f"cannot load movie tool {name!r}")
+ module = importlib.util.module_from_spec(spec)
+ try:
+ spec.loader.exec_module(module)
+ finally:
+ sys.path.pop(0)
+ return module
+
+
+def duration(path: Path) -> float:
+ """Measure a media file's container duration with ffprobe."""
+ ffprobe = shutil.which("ffprobe")
+ if ffprobe is None:
+ raise RuntimeError("ffprobe is required for movie tool tests")
+ result = subprocess.run(
+ [
+ ffprobe,
+ "-v",
+ "error",
+ "-show_entries",
+ "format=duration",
+ "-of",
+ "csv=p=0",
+ str(path),
+ ],
+ capture_output=True,
+ timeout=TIMEOUT_SECONDS,
+ )
+ if result.returncode != 0:
+ raise RuntimeError(f"ffprobe failed for {path}: {output_text(result)}")
+ return float(result.stdout.decode("utf-8").strip())
+
+
+def _run_ffmpeg(args: list[str], *, cwd: Path) -> None:
+ ffmpeg = shutil.which("ffmpeg")
+ if ffmpeg is None:
+ raise RuntimeError("ffmpeg is required for movie tool tests")
+ result = subprocess.run(
+ [ffmpeg, "-nostdin", "-y", "-v", "error", *args],
+ cwd=cwd,
+ capture_output=True,
+ timeout=TIMEOUT_SECONDS,
+ )
+ if result.returncode != 0:
+ raise RuntimeError(f"ffmpeg fixture generation failed: {output_text(result)}")
+
+
+def assembly_fixture(work: Path) -> Path:
+ """Write the imported image/frames/synthetic-timing-audio assembly case."""
+ shots = work / "shots"
+ shots.mkdir(parents=True)
+ for index in range(1, 5):
+ _run_ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ f"color=c=0x{index}0{index}0{index}0:size=320x180:d=0.1",
+ "-frames:v",
+ "1",
+ str(shots / f"s0{index}.png"),
+ ],
+ cwd=work,
+ )
+
+ narration = work / "narration"
+ narration.mkdir()
+ wav = narration / "body.wav"
+ _run_ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=300:duration=6",
+ str(wav),
+ ],
+ cwd=work,
+ )
+ manifest = [
+ {
+ "id": "body",
+ "text": "one two three four five six seven eight nine ten",
+ "wav": "body.wav",
+ "duration": duration(wav),
+ }
+ ]
+ (narration / "manifest.json").write_text(
+ json.dumps(manifest), encoding="utf-8"
+ )
+
+ scenes = work / "scenes.yaml"
+ scenes.write_text(
+ """resolution: { width: 640, height: 360 }
+fps: 30
+scenes:
+ - id: opener
+ kind: image
+ src: shots/s01.png
+ duration: 2
+ - id: body
+ kind: frames
+ src: shots
+ rate: 1.0
+ narration: one two three four five six seven eight nine ten
+""",
+ encoding="utf-8",
+ )
+ return scenes
+
+
+def checker_fixture(work: Path) -> dict[str, Path]:
+ """Create the four imported checker movies and their subtitle sidecars."""
+ front_loaded = work / "front-loaded.mp4"
+ _run_ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc2=size=320x240:rate=10:d=2",
+ "-f",
+ "lavfi",
+ "-i",
+ "color=c=navy:size=320x240:rate=10:d=20",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=300:duration=22",
+ "-filter_complex",
+ "[0:v][1:v]concat=n=2:v=1:a=0[v]",
+ "-map",
+ "[v]",
+ "-map",
+ "2:a",
+ "-c:v",
+ "libx264",
+ "-pix_fmt",
+ "yuv420p",
+ "-c:a",
+ "aac",
+ "-shortest",
+ str(front_loaded),
+ ],
+ cwd=work,
+ )
+
+ paced = work / "paced.mp4"
+ _run_ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc2=size=320x240:rate=10:d=22",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=300:duration=22",
+ "-c:v",
+ "libx264",
+ "-pix_fmt",
+ "yuv420p",
+ "-c:a",
+ "aac",
+ "-shortest",
+ str(paced),
+ ],
+ cwd=work,
+ )
+
+ still = work / "still.mp4"
+ _run_ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ "color=c=navy:size=320x240:rate=10:d=12",
+ "-f",
+ "lavfi",
+ "-i",
+ "sine=frequency=300:duration=12",
+ "-c:v",
+ "libx264",
+ "-pix_fmt",
+ "yuv420p",
+ "-c:a",
+ "aac",
+ "-shortest",
+ str(still),
+ ],
+ cwd=work,
+ )
+
+ silent = work / "silent.mp4"
+ _run_ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ "testsrc2=size=320x240:rate=10:d=12",
+ "-c:v",
+ "libx264",
+ "-pix_fmt",
+ "yuv420p",
+ str(silent),
+ ],
+ cwd=work,
+ )
+
+ subtitles = """1
+00:00:00,000 --> 00:00:07,000
+A narrated movie needs subtitles:
+plenty of people watch muted.
+
+2
+00:00:07,000 --> 00:00:14,000
+The checker treats their absence
+as a defect, not a nicety.
+
+3
+00:00:14,000 --> 00:00:21,500
+And it notices when they stop
+before the narration does.
+"""
+ (work / "paced.srt").write_text(subtitles, encoding="utf-8")
+ (work / "short.srt").write_text(
+ "\n".join(subtitles.splitlines()[:8]) + "\n", encoding="utf-8"
+ )
+ short = work / "short.mp4"
+ shutil.copyfile(paced, short)
+
+ return {
+ "front-loaded": front_loaded,
+ "paced": paced,
+ "still": still,
+ "silent": silent,
+ "short": short,
+ }
diff --git a/tests/proving-it-works-with-a-movie/fixtures/terminal_app.py b/tests/proving-it-works-with-a-movie/fixtures/terminal_app.py
new file mode 100644
index 0000000..0b197d9
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/fixtures/terminal_app.py
@@ -0,0 +1,41 @@
+"""TUI fixture: three timed colour states, then wait for `q`.
+`tree DIR` instead leaves a three-deep process tree running for cleanup tests."""
+import json, os, subprocess, sys, time
+from pathlib import Path
+
+
+def getch():
+ if os.name == "nt":
+ import msvcrt
+ return msvcrt.getwch()
+ import termios, tty
+ fd = sys.stdin.fileno()
+ old = termios.tcgetattr(fd)
+ try:
+ tty.setcbreak(fd)
+ return sys.stdin.read(1)
+ finally:
+ termios.tcsetattr(fd, termios.TCSADRAIN, old)
+
+
+if len(sys.argv) > 1 and sys.argv[1] == "tree":
+ directory = Path(sys.argv[2])
+ directory.mkdir(exist_ok=True)
+ level = int(sys.argv[3]) if len(sys.argv) > 3 else 0
+ if level < 2:
+ subprocess.Popen([sys.executable, __file__, "tree", str(directory), str(level + 1)])
+ (directory / f"{level}.json").write_text(json.dumps(dict(pid=os.getpid(), level=level)))
+ while True:
+ time.sleep(1)
+
+print("WRAPPING OUTPUT " + ("0123456789 wrap proof " * 60), flush=True)
+time.sleep(1.4)
+states = []
+for label, color in [("STATE ONE RED", 41), ("STATE TWO GREEN", 42), ("STATE THREE BLUE", 44)]:
+ print("\x1b[2J\x1b[H" + f"\x1b[{color}m" + (label + " " * 60 + "\n") * 12 + "\x1b[0m", end="", flush=True)
+ states.append(dict(label=label, time=time.monotonic()))
+ Path("states.json").write_text(json.dumps(states))
+ time.sleep(1.5)
+while getch() != "q":
+ pass
+print("\nAUTOMATIC GATE COMPLETE", flush=True)
diff --git a/tests/proving-it-works-with-a-movie/run-tests.py b/tests/proving-it-works-with-a-movie/run-tests.py
new file mode 100644
index 0000000..441d39f
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/run-tests.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env -S uv run --script
+# /// script
+# requires-python = ">=3.10"
+# dependencies = ["pyyaml", "pillow", "websocket-client==1.9.0"]
+# ///
+"""Run the proving-it-works-with-a-movie regression suites portably."""
+
+import argparse
+import sys
+import unittest
+from pathlib import Path
+
+
+IMPLEMENTED_SUITES = {
+ "assembly": "test_assembly.py",
+ "browser": "test_browser.py",
+ "checker": "test_checker.py",
+ "contracts": "test_*contract*.py",
+ "narration": "test_narration.py",
+ "paths": "test_paths.py",
+ "subtitles": "test_subtitles.py",
+ "terminal": "test_terminal.py",
+}
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--suite",
+ required=True,
+ choices=[*IMPLEMENTED_SUITES, "all"],
+ )
+ parser.add_argument("--require-capabilities", action="store_true")
+ return parser.parse_args()
+
+
+def main() -> int:
+ args = parse_args()
+ patterns = (
+ list(IMPLEMENTED_SUITES.values())
+ if args.suite == "all"
+ else [IMPLEMENTED_SUITES[args.suite]]
+ )
+ test_directory = Path(__file__).resolve().parent
+ loader = unittest.TestLoader()
+ suite = unittest.TestSuite(
+ loader.discover(
+ str(test_directory),
+ pattern=pattern,
+ top_level_dir=str(test_directory),
+ )
+ for pattern in patterns
+ )
+ result = unittest.TextTestRunner(verbosity=2).run(suite)
+ if args.require_capabilities and result.skipped:
+ print(
+ f"required capabilities unavailable: {len(result.skipped)} "
+ "selected test(s) skipped",
+ file=sys.stderr,
+ )
+ return 1
+ return 0 if result.wasSuccessful() else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/proving-it-works-with-a-movie/test_assembly.py b/tests/proving-it-works-with-a-movie/test_assembly.py
new file mode 100644
index 0000000..dd771a8
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_assembly.py
@@ -0,0 +1,409 @@
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import fixtures
+from PIL import Image
+
+
+def run_ffmpeg(args: list[str], *, cwd: Path) -> subprocess.CompletedProcess[bytes]:
+ ffmpeg = shutil.which("ffmpeg")
+ if ffmpeg is None:
+ raise RuntimeError("ffmpeg is required for assembly fixtures")
+ result = subprocess.run(
+ [ffmpeg, "-nostdin", "-y", "-v", "error", *args],
+ cwd=cwd,
+ capture_output=True,
+ timeout=fixtures.TIMEOUT_SECONDS,
+ )
+ if result.returncode != 0:
+ raise RuntimeError(
+ f"ffmpeg fixture generation failed: {fixtures.output_text(result)}"
+ )
+ return result
+
+
+def make_tone(path: Path, duration: float, frequency: int, *, cwd: Path) -> None:
+ run_ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ f"sine=frequency={frequency}:sample_rate=8000:duration={duration}",
+ "-c:a",
+ "pcm_s16le",
+ str(path),
+ ],
+ cwd=cwd,
+ )
+
+
+def make_movie(path: Path, duration: float, frequency: int, *, cwd: Path) -> None:
+ run_ffmpeg(
+ [
+ "-f",
+ "lavfi",
+ "-i",
+ f"color=c=green:size=160x90:rate=10:duration={duration}",
+ "-f",
+ "lavfi",
+ "-i",
+ f"sine=frequency={frequency}:sample_rate=8000:duration={duration}",
+ "-c:v",
+ "libx264",
+ "-pix_fmt",
+ "yuv420p",
+ "-c:a",
+ "aac",
+ "-shortest",
+ str(path),
+ ],
+ cwd=cwd,
+ )
+
+
+def decoded_pixel(path: Path, timestamp: float, *, cwd: Path) -> tuple[int, int, int]:
+ result = run_ffmpeg(
+ [
+ "-ss",
+ str(timestamp),
+ "-i",
+ str(path),
+ "-frames:v",
+ "1",
+ "-vf",
+ "scale=1:1",
+ "-f",
+ "rawvideo",
+ "-pix_fmt",
+ "rgb24",
+ "-",
+ ],
+ cwd=cwd,
+ )
+ if len(result.stdout) < 3:
+ raise AssertionError(f"no decoded pixel from {path}")
+ return tuple(result.stdout[:3])
+
+
+def decoded_frequency(path: Path, *, cwd: Path) -> float:
+ sample_rate = 8000
+ result = run_ffmpeg(
+ [
+ "-ss",
+ "0.1",
+ "-t",
+ "0.5",
+ "-i",
+ str(path),
+ "-map",
+ "0:a:0",
+ "-f",
+ "s16le",
+ "-acodec",
+ "pcm_s16le",
+ "-ac",
+ "1",
+ "-ar",
+ str(sample_rate),
+ "-",
+ ],
+ cwd=cwd,
+ )
+ samples = [
+ int.from_bytes(result.stdout[index:index + 2], "little", signed=True)
+ for index in range(0, len(result.stdout) - 1, 2)
+ ]
+ nonzero = [sample for sample in samples if sample]
+ crossings = sum(
+ (left < 0 <= right) or (left > 0 >= right)
+ for left, right in zip(nonzero, nonzero[1:])
+ )
+ seconds = len(samples) / sample_rate
+ return crossings / (2 * seconds)
+
+
+def available_browser() -> str | None:
+ return fixtures.load_script("browser_tools").find_browser(os.environ.get("MOVIE_BROWSER"))
+
+
+class AssemblyRegression(unittest.TestCase):
+ def test_narration_padding_and_offsets(self):
+ missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe")
+ if missing:
+ self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}")
+
+ with tempfile.TemporaryDirectory() as directory:
+ work = Path(directory)
+ scenes = fixtures.assembly_fixture(work)
+ result = fixtures.run_tool(
+ "assemble", [str(scenes), str(work / "out.mp4")], cwd=work
+ )
+ self.assertEqual(result.returncode, 0, fixtures.output_text(result))
+ self.assertAlmostEqual(
+ fixtures.duration(work / "out.mp4"), 8, delta=0.4
+ )
+ offsets = json.loads(
+ (work / "segments/offsets.json").read_text(encoding="utf-8")
+ )
+ self.assertAlmostEqual(offsets["body"], 2, delta=0.3)
+
+ subtitles = fixtures.run_tool(
+ "make-subtitles",
+ [
+ str(work / "narration/manifest.json"),
+ str(work / "out.srt"),
+ "--offsets-json",
+ str(work / "segments/offsets.json"),
+ ],
+ cwd=work,
+ )
+ self.assertEqual(
+ subtitles.returncode, 0, fixtures.output_text(subtitles)
+ )
+ srt = (work / "out.srt").read_text(encoding="utf-8")
+ timing_line = next(line for line in srt.splitlines() if "-->" in line)
+ start = timing_line.partition("-->")[0].strip()
+ hours, minutes, seconds_millis = start.split(":")
+ seconds, millis = seconds_millis.split(",")
+ first_cue_start = (
+ int(hours) * 3600
+ + int(minutes) * 60
+ + int(seconds)
+ + int(millis) / 1000
+ )
+ self.assertAlmostEqual(
+ first_cue_start,
+ offsets["body"],
+ delta=0.001,
+ )
+
+ def test_image_frames_and_movie_paths_timing_order_and_cleanup(self):
+ missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe")
+ if missing:
+ self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}")
+
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ launch = root / "launch"
+ launch.mkdir()
+ project = root / "movie O'Brien λ & [take]"
+ assets = project / "assets"
+ frames = assets / "frames"
+ subtitles = assets / "subtitles" / "nested"
+ narration = project / "narration"
+ frames.mkdir(parents=True)
+ subtitles.mkdir(parents=True)
+ narration.mkdir()
+
+ still = assets / "still image.png"
+ Image.new("RGB", (160, 90), (230, 230, 230)).save(still)
+ Image.new("RGB", (160, 90), (255, 0, 0)).save(frames / "a.png")
+ Image.new("RGB", (160, 90), (0, 0, 255)).save(frames / "b.png")
+ source_pngs = {
+ path: path.read_bytes() for path in [still, *sorted(frames.glob("*.png"))]
+ }
+
+ source_movie = assets / "source movie.mp4"
+ make_movie(source_movie, 0.8, 440, cwd=launch)
+ make_tone(narration / "image.wav", 0.8, 660, cwd=launch)
+ make_tone(narration / "frames.wav", 1.4, 770, cwd=launch)
+ make_tone(narration / "movie.wav", 1.6, 880, cwd=launch)
+ (narration / "manifest.json").write_text(json.dumps([
+ {"id": "image", "text": "Image narration", "wav": "image.wav", "duration": 0.8},
+ {"id": "frames", "text": "Frames narration", "wav": "frames.wav", "duration": 1.4},
+ ]), encoding="utf-8")
+
+ nested_srt = subtitles / "captions.srt"
+ nested_srt.write_bytes(
+ b"1\r\n00:00:00,000 --> 00:00:00,500\r\nPortable paths\r\n"
+ )
+ scenes = project / "scenes.yaml"
+ yaml_text = f"""resolution: {{ width: 160, height: 90 }}
+fps: 10
+scenes:
+ - id: image
+ kind: image
+ src: assets/still image.png
+ duration: 0.3
+ narration: Image narration
+ - id: frames
+ kind: frames
+ src: {frames.resolve().as_posix()}
+ rate: 2.0
+ narration: Frames narration
+ - id: movie
+ kind: movie
+ src: assets/source movie.mp4
+"""
+ scenes.write_bytes(yaml_text.replace("\n", "\r\n").encode("utf-8"))
+
+ work = root / "generated outside launch"
+ output = project / "assembled output.mp4"
+ result = fixtures.run_tool(
+ "assemble",
+ [
+ os.path.relpath(scenes, launch),
+ os.path.relpath(output, launch),
+ "--narration",
+ str(narration.resolve()),
+ "--work",
+ str(work.resolve()),
+ ],
+ cwd=launch,
+ )
+ self.assertEqual(result.returncode, 0, fixtures.output_text(result))
+
+ image_duration = fixtures.duration(work / "image.mp4")
+ frames_duration = fixtures.duration(work / "frames.mp4")
+ movie_duration = fixtures.duration(work / "movie.mp4")
+ self.assertAlmostEqual(
+ image_duration,
+ max(fixtures.duration(narration / "image.wav"), 0.3),
+ delta=0.25,
+ )
+ self.assertAlmostEqual(
+ frames_duration,
+ max(fixtures.duration(narration / "frames.wav"), 1.0),
+ delta=0.25,
+ )
+ self.assertAlmostEqual(
+ movie_duration,
+ fixtures.duration(source_movie),
+ delta=0.25,
+ )
+ self.assertLess(movie_duration, fixtures.duration(narration / "movie.wav") - 0.4)
+ self.assertAlmostEqual(
+ decoded_frequency(work / "movie.mp4", cwd=launch),
+ 440,
+ delta=15,
+ )
+
+ offsets = json.loads(
+ (work / "offsets.json").read_text(encoding="utf-8")
+ )
+ self.assertAlmostEqual(offsets["image"], 0.0, delta=0.001)
+ self.assertAlmostEqual(offsets["frames"], image_duration, delta=0.001)
+ self.assertNotIn("movie", offsets)
+
+ first = decoded_pixel(work / "frames.mp4", 0.2, cwd=launch)
+ second = decoded_pixel(work / "frames.mp4", 0.7, cwd=launch)
+ self.assertGreater(first[0], first[2] + 100)
+ self.assertGreater(second[2], second[0] + 100)
+ self.assertEqual(
+ {path: path.read_bytes() for path in source_pngs},
+ source_pngs,
+ )
+ self.assertEqual(list(work.glob("frames-frames-*")), [])
+
+ subtitled = project / "subtitled output.mp4"
+ subtitle_result = fixtures.run_tool(
+ "burn-subtitles",
+ [
+ str(output.resolve()),
+ os.path.relpath(nested_srt, launch),
+ str(subtitled.resolve()),
+ "--soft",
+ ],
+ cwd=launch,
+ )
+ self.assertEqual(
+ subtitle_result.returncode,
+ 0,
+ fixtures.output_text(subtitle_result),
+ )
+ self.assertTrue(subtitled.is_file())
+
+ def test_card_uses_longer_narration_duration(self):
+ missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe")
+ if missing:
+ self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}")
+ browser = available_browser()
+ if browser is None:
+ self.skipTest("required browser unavailable for card assembly")
+
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ narration = root / "narration"
+ narration.mkdir()
+ make_tone(narration / "card.wav", 0.8, 550, cwd=root)
+ (narration / "manifest.json").write_text(json.dumps([
+ {"id": "card", "text": "Card narration", "wav": "card.wav", "duration": 0.8}
+ ]), encoding="utf-8")
+ scenes = root / "scenes.yaml"
+ scenes.write_bytes(
+ b"\xef\xbb\xbfresolution: { width: 160, height: 90 }\r\n"
+ b"fps: 10\r\n"
+ b"scenes:\r\n"
+ b" - id: card\r\n"
+ b" kind: card\r\n"
+ b" title: Portable\r\n"
+ b" subtitle: paths\r\n"
+ b" duration: 0.3\r\n"
+ b" narration: Card narration\r\n"
+ )
+ work = root / "work"
+ result = fixtures.run_tool(
+ "assemble",
+ [
+ str(scenes),
+ str(root / "out.mp4"),
+ "--work",
+ str(work),
+ "--browser",
+ browser,
+ ],
+ cwd=root,
+ )
+ self.assertEqual(result.returncode, 0, fixtures.output_text(result))
+ self.assertAlmostEqual(
+ fixtures.duration(work / "card.mp4"),
+ max(fixtures.duration(narration / "card.wav"), 0.3),
+ delta=0.25,
+ )
+ offsets = json.loads(
+ (work / "offsets.json").read_text(encoding="utf-8")
+ )
+ self.assertAlmostEqual(offsets["card"], 0.0, delta=0.001)
+
+ def test_frame_sources_survive_failed_assembly_cleanup(self):
+ missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe")
+ if missing:
+ self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}")
+
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ source = root / "movie O'Brien λ & [take]" / "frames"
+ source.mkdir(parents=True)
+ bad_frame = source / "a.png"
+ bad_frame.write_bytes(b"not a PNG")
+ scenes = root / "scenes.yaml"
+ scenes.write_text(
+ f"""resolution: {{ width: 160, height: 90 }}
+fps: 10
+scenes:
+ - id: broken
+ kind: frames
+ src: {source.resolve().as_posix()}
+ rate: 1.0
+""",
+ encoding="utf-8",
+ )
+ work = root / "work"
+ result = fixtures.run_tool(
+ "assemble",
+ [str(scenes), str(root / "out.mp4"), "--work", str(work)],
+ cwd=root,
+ )
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(bad_frame.read_bytes(), b"not a PNG")
+ self.assertEqual(list(work.glob("frames-broken-*")), [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/proving-it-works-with-a-movie/test_browser.py b/tests/proving-it-works-with-a-movie/test_browser.py
new file mode 100644
index 0000000..e14e37b
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_browser.py
@@ -0,0 +1,73 @@
+import os
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from PIL import Image
+import fixtures
+
+
+class BrowserToolsRegression(unittest.TestCase):
+ def test_explicit_unusable_browser_fails_authoritatively(self):
+ module = fixtures.load_script("browser_tools")
+ with self.assertRaises(FileNotFoundError):
+ module.find_browser("this-browser-does-not-exist")
+ if os.name != "nt":
+ with tempfile.NamedTemporaryFile() as file:
+ with self.assertRaises(FileNotFoundError):
+ module.find_browser(file.name)
+
+ def test_windows_chrome_edge_discovery(self):
+ module = fixtures.load_script("browser_tools")
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ for relative in ("Google/Chrome/Application/chrome.exe", "Microsoft/Edge/Application/msedge.exe"):
+ browser = root / relative
+ browser.parent.mkdir(parents=True)
+ browser.touch()
+ with patch.object(module.sys, "platform", "win32"), patch.dict(os.environ, {"LOCALAPPDATA": str(root)}, clear=True), patch.object(module.shutil, "which", return_value=None):
+ self.assertEqual(module.find_browser(None), str(browser.resolve()))
+ browser.unlink()
+ with patch.object(module.sys, "platform", "win32"), patch.dict(os.environ, {}, clear=True), patch.object(module.shutil, "which", side_effect=lambda name: "C:/Edge/msedge.exe" if name == "msedge.exe" else None):
+ self.assertEqual(module.find_browser(None), "C:/Edge/msedge.exe")
+
+ def browser(self, module):
+ browser = module.find_browser(os.environ.get("MOVIE_BROWSER"))
+ if not browser:
+ self.skipTest("Chrome/Edge is required")
+ return browser
+
+ def test_render_card_special_path_is_a_real_png(self):
+ module = fixtures.load_script("browser_tools")
+ browser = self.browser(module)
+ with tempfile.TemporaryDirectory() as directory:
+ work = Path(directory) / "special path O'Brien λ # %"
+ work.mkdir()
+ html, png = work / "card page.html", work / "card.png"
+ html.write_text("<meta charset='utf-8'><style>body{margin:0;background:rgb(255,0,0)}</style><p>λ</p>", encoding="utf-8")
+ module.render_card(html, png, browser=browser, width=640, height=360)
+ with Image.open(png) as image:
+ self.assertEqual(image.size, (640, 360))
+ self.assertEqual(image.convert("RGB").getpixel((500, 200)), (255, 0, 0))
+
+ def test_render_card_timeout_preserves_unrelated_process(self):
+ module = fixtures.load_script("browser_tools")
+ browser = self.browser(module)
+ sentinel = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"])
+ try:
+ with tempfile.TemporaryDirectory() as directory:
+ work = Path(directory)
+ html = work / "card.html"
+ html.write_text("<p>timeout</p>", encoding="utf-8")
+ start = time.monotonic()
+ with self.assertRaises(TimeoutError):
+ module.render_card(html, work / "card.png", browser=browser, width=640, height=360, timeout=0)
+ self.assertLess(time.monotonic() - start, 10)
+ self.assertIsNone(sentinel.poll())
+ finally:
+ sentinel.terminate()
+ sentinel.wait(timeout=10)
diff --git a/tests/proving-it-works-with-a-movie/test_browser_contract.py b/tests/proving-it-works-with-a-movie/test_browser_contract.py
new file mode 100644
index 0000000..64eb6bc
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_browser_contract.py
@@ -0,0 +1,112 @@
+"""Browser cleanup decisions with fake processes and a completed-output token."""
+import contextlib
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import fixtures
+
+
+class CompletedOutput:
+ """Stand in for the completed-output observation without creating an image."""
+ def startswith(self, prefix):
+ return True
+
+ def endswith(self, suffix):
+ return True
+
+
+@contextlib.contextmanager
+def rendering(*, exited=False, taskkill_status=0, wait_timeout=False, locked_profile=False):
+ module = fixtures.load_script("browser_tools")
+ with tempfile.TemporaryDirectory() as temp, contextlib.ExitStack() as stack:
+ html, png = Path(temp).resolve() / "card.html", Path(temp).resolve() / "card.png"
+ html.write_text("<p>card</p>")
+ process = SimpleNamespace(pid=1100, returncode=0 if exited else None)
+ process.poll = lambda: process.returncode
+ calls, profiles = [], []
+ original_is_file, original_unlink = Path.is_file, os.unlink
+
+ def popen(argv, **kwargs):
+ profiles.append(Path(kwargs["cwd"]))
+ return process
+
+ def taskkill(argv, **kwargs):
+ calls.append(argv)
+ if taskkill_status == 0 and not wait_timeout:
+ process.returncode = -9
+ return subprocess.CompletedProcess(argv, taskkill_status, b"", b"termination failed")
+
+ def wait(timeout):
+ if process.returncode is None:
+ raise subprocess.TimeoutExpired("fake browser", timeout)
+ return process.returncode
+
+ def unlink(path, *args, **kwargs):
+ if locked_profile and Path(path).name == "browser.log":
+ raise PermissionError("locked browser profile")
+ return original_unlink(path, *args, **kwargs)
+
+ process.wait = wait
+ for obj, name, value in (
+ (module.sys, "platform", "win32"),
+ (module.subprocess, "Popen", popen),
+ (module.subprocess, "run", taskkill),
+ (Path, "is_file", lambda path: True if path == png else original_is_file(path)),
+ (Path, "read_bytes", lambda path: CompletedOutput()),
+ (os, "unlink", unlink),
+ ):
+ stack.enter_context(patch.object(obj, name, value))
+ try:
+ yield SimpleNamespace(render=lambda: module.render_card(html, png, browser="fake-browser", width=640, height=360),
+ process=process, calls=calls, profiles=profiles)
+ finally:
+ stack.close()
+ for profile in profiles:
+ if profile.exists():
+ module.shutil.rmtree(profile)
+
+
+class BrowserCleanupContract(unittest.TestCase):
+ def test_windows_tree_termination_failure_reaches_caller_as_oserror(self):
+ module = fixtures.load_script("browser_tools")
+ with patch.object(module.sys, "platform", "win32"), \
+ patch.object(module.subprocess, "run", return_value=subprocess.CompletedProcess([], 1, b"", b"access denied")):
+ with self.assertRaises(OSError):
+ module.kill_process_tree(1100)
+
+ def test_completed_output_does_not_hide_tree_termination_failure(self):
+ with rendering(taskkill_status=1) as rig:
+ with self.assertRaises(OSError):
+ rig.render()
+
+ def test_completed_output_does_not_hide_owned_child_wait_timeout(self):
+ with rendering(wait_timeout=True) as rig:
+ with self.assertRaises(subprocess.TimeoutExpired):
+ rig.render()
+
+ def test_completed_output_does_not_hide_locked_profile(self):
+ with rendering(locked_profile=True) as rig:
+ with self.assertRaises(PermissionError):
+ rig.render()
+
+ def test_normally_exited_completed_card_succeeds_without_numeric_pid_cleanup(self):
+ with rendering(exited=True, taskkill_status=1) as rig:
+ self.assertIsNone(rig.render())
+ self.assertEqual(rig.calls, [])
+ self.assertFalse(rig.profiles[0].exists())
+
+ def test_completed_card_releases_live_browser_and_profile(self):
+ with rendering() as rig:
+ self.assertIsNone(rig.render())
+ self.assertEqual(rig.process.poll(), -9)
+ self.assertEqual(len(rig.calls), 1)
+ self.assertFalse(rig.profiles[0].exists())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/proving-it-works-with-a-movie/test_checker.py b/tests/proving-it-works-with-a-movie/test_checker.py
new file mode 100644
index 0000000..eec81ee
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_checker.py
@@ -0,0 +1,194 @@
+import io
+import json
+import subprocess
+import sys
+import tempfile
+from contextlib import redirect_stdout
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from PIL import Image
+
+import fixtures
+
+
+class CheckerPolicyRegression(unittest.TestCase):
+ def check(self, expected_exit, *options, audio=True, levels=None,
+ embedded=None, sidecar=None, extraction_exit=0):
+ module = fixtures.load_script("check-movie")
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ movie = root / "movie.mp4"
+ movie.write_bytes(b"metadata fixture only")
+ if sidecar is not None:
+ movie.with_suffix(".srt").write_text(sidecar, encoding="utf-8-sig")
+ streams = [{"index": 0, "codec_type": "video", "codec_name": "h264",
+ "width": 640, "height": 360}]
+ if audio:
+ streams.append({"index": 1, "codec_type": "audio", "codec_name": "aac"})
+ if embedded is not None:
+ streams.append({"index": 2, "codec_type": "subtitle", "codec_name": "mov_text"})
+ levels = levels if levels is not None else [-20.0] * 20
+ info = {"format": {"duration": str(len(levels))}, "streams": streams}
+
+ def media_command(cmd, **kwargs):
+ if cmd[0] == "ffprobe":
+ return subprocess.CompletedProcess(cmd, 0, json.dumps(info), "")
+ if cmd[0] == "ffmpeg" and embedded is not None:
+ self.assertEqual(cmd[cmd.index("-i") + 1], str(movie))
+ self.assertEqual(cmd[cmd.index("-map") + 1], "0:s:0")
+ self.assertEqual(cmd[cmd.index("-f") + 1], "srt")
+ return subprocess.CompletedProcess(cmd, extraction_exit, embedded,
+ "subtitle decode failed" if extraction_exit else "")
+ raise AssertionError(f"unexpected media command: {cmd}")
+
+ output = root / "report"
+ argv = ["check-movie", str(movie), "--out", str(output), "--json", *options]
+ stdout = io.StringIO()
+ with patch.object(sys, "argv", argv), \
+ patch.object(module.shutil, "which", return_value="test-tool"), \
+ patch.object(module.subprocess, "run", side_effect=media_command), \
+ patch.object(module, "sample_picture", return_value=([], [0.1] * (len(levels) - 1))), \
+ patch.object(module, "sample_sound", return_value=levels if audio else []), \
+ patch.object(module, "contact_sheet", return_value=[]), \
+ redirect_stdout(stdout):
+ try:
+ code = module.main()
+ except SystemExit as error:
+ code = error.code
+ self.assertEqual(code, expected_exit, stdout.getvalue())
+ report = output / "check.json"
+ failures = json.loads(report.read_text(encoding="utf-8"))["failures"] if report.exists() else []
+ return failures, stdout.getvalue()
+
+ def test_silent_encoded_track_passes_when_audio_is_not_expected(self):
+ failures, _ = self.check(0, "--no-expect-audio", levels=[-120.0] * 20)
+ self.assertEqual(failures, [])
+
+ def test_absent_audio_passes_when_audio_is_not_expected(self):
+ failures, _ = self.check(0, "--no-expect-audio", audio=False)
+ self.assertEqual(failures, [])
+
+ def test_silent_encoded_track_fails_when_audio_is_expected(self):
+ failures, _ = self.check(1, levels=[-120.0] * 20)
+ self.assertTrue(any("silent" in failure for failure in failures))
+
+ def test_audio_opt_out_still_requires_captions_for_audible_speech(self):
+ failures, _ = self.check(1, "--no-expect-audio")
+ self.assertTrue(any("no subtitles" in failure for failure in failures))
+
+ def test_subtitle_opt_out_allows_audible_speech_without_captions(self):
+ failures, _ = self.check(0, "--no-expect-audio", "--no-expect-subtitles")
+ self.assertEqual(failures, [])
+
+ def test_empty_embedded_track_fails_even_for_short_narration(self):
+ for seconds in (2, 20):
+ with self.subTest(seconds=seconds):
+ failures, _ = self.check(1, embedded="", levels=[-20.0] * seconds)
+ self.assertTrue(any("subtitle" in failure for failure in failures))
+
+ def test_sidecar_and_embedded_cues_must_reach_the_end_of_speech(self):
+ for source in ("sidecar", "embedded"):
+ for end, expected_exit in (("06,000", 1), ("10,000", 0)):
+ with self.subTest(source=source, end=end):
+ subtitles = f"1\n00:00:00,000 --> 00:00:{end}\nUnicode λ caption\n"
+ failures, _ = self.check(expected_exit, levels=[-20.0] * 10 + [-120.0] * 10,
+ **{source: subtitles})
+ self.assertEqual(bool(failures), bool(expected_exit))
+
+ def test_embedded_extraction_failure_is_not_accepted(self):
+ _, diagnostics = self.check(2, embedded="", extraction_exit=1)
+ self.assertIn("subtitle", diagnostics)
+
+ def test_malformed_embedded_cue_is_a_reported_failure(self):
+ _, diagnostics = self.check(2, embedded="1\n00:00:00,000 --> invalid\ncaption\n")
+ self.assertIn("subtitle", diagnostics)
+
+
+class CheckerRegression(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls) -> None:
+ missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe")
+ if missing:
+ raise unittest.SkipTest(
+ f"required executable(s) not on PATH: {', '.join(missing)}"
+ )
+ cls._temporary_directory = tempfile.TemporaryDirectory()
+ cls.addClassCleanup(cls._temporary_directory.cleanup)
+ cls.work = Path(cls._temporary_directory.name)
+ cls.movies = fixtures.checker_fixture(cls.work)
+
+ def check(
+ self, expected_exit: int, needle: str, movie: Path, *extra_args: str
+ ) -> Path:
+ output_directory = self.work / f"{movie.stem}-check"
+ result = fixtures.run_tool(
+ "check-movie",
+ [str(movie), "--out", str(output_directory), *extra_args],
+ cwd=self.work,
+ )
+ output = fixtures.output_text(result)
+ self.assertEqual(result.returncode, expected_exit, output)
+ self.assertIn(needle.casefold(), output.casefold())
+ return output_directory
+
+ def test_front_loaded_action_is_rejected(self):
+ self.check(
+ 1,
+ "every visible change happens in the first",
+ self.movies["front-loaded"],
+ )
+
+ def test_paced_with_subtitles_is_accepted(self):
+ self.check(0, "Mechanical checks pass", self.movies["paced"])
+
+ def test_narrated_without_subtitles_is_rejected(self):
+ self.check(
+ 1,
+ "no subtitles",
+ self.movies["paced"],
+ "--subs",
+ str(self.work / "nope.srt"),
+ )
+
+ def test_subtitles_that_stop_early_are_rejected(self):
+ self.check(1, "subtitles stop at", self.movies["short"])
+
+ def test_subtitle_check_is_opt_outable(self):
+ self.check(
+ 0,
+ "Mechanical checks pass",
+ self.movies["paced"],
+ "--subs",
+ str(self.work / "nope.srt"),
+ "--no-expect-subtitles",
+ )
+
+ def test_still_with_audio_is_rejected(self):
+ self.check(1, "never reaches a new state", self.movies["still"])
+
+ def test_missing_narration_is_rejected(self):
+ self.check(1, "no audio stream", self.movies["silent"])
+
+ def test_silent_movie_passes_when_unnarrated(self):
+ self.check(
+ 0,
+ "Mechanical checks pass",
+ self.movies["silent"],
+ "--no-expect-audio",
+ )
+
+ def test_contact_sheet_is_always_written(self):
+ output_directory = self.check(
+ 0, "contact-sheet.png", self.movies["paced"]
+ )
+ sheet = output_directory / "contact-sheet.png"
+ self.assertTrue(sheet.is_file(), f"missing contact sheet: {sheet}")
+ with Image.open(sheet) as image:
+ image.load()
+ self.assertEqual(image.format, "PNG")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/proving-it-works-with-a-movie/test_narration.py b/tests/proving-it-works-with-a-movie/test_narration.py
new file mode 100644
index 0000000..dece4c6
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_narration.py
@@ -0,0 +1,351 @@
+import io
+import json
+import sys
+import tempfile
+from contextlib import redirect_stderr, redirect_stdout
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import fixtures
+
+
+SCRIPT = (
+ "This is smevals studio. Every eval on the shelf is a folder of tasks and "
+ "graders."
+)
+
+
+class NarrationDriftRegression(unittest.TestCase):
+ def drift(self, expected_exit: int, heard: str, script: str = SCRIPT) -> None:
+ missing = fixtures.missing_executables("uv")
+ if missing:
+ self.skipTest(
+ f"required executable(s) not on PATH: {', '.join(missing)}"
+ )
+ with tempfile.TemporaryDirectory() as directory:
+ work = Path(directory)
+ script_path = work / "script.txt"
+ heard_path = work / "heard.txt"
+ script_path.write_text(script, encoding="utf-8")
+ heard_path.write_text(heard, encoding="utf-8")
+ result = fixtures.run_tool(
+ "narrate",
+ ["--drift-check", str(script_path), str(heard_path)],
+ cwd=work,
+ )
+ self.assertEqual(
+ result.returncode, expected_exit, fixtures.output_text(result)
+ )
+
+ def test_mispronounced_jargon_passes(self):
+ self.drift(
+ 0,
+ "This is Mevil studio. Every Yvel on the shelf is a folder of tasks "
+ "and graders.",
+ )
+
+ def test_exact_transcript_passes(self):
+ self.drift(0, SCRIPT)
+
+ def test_dropped_clause_fails(self):
+ self.drift(1, "This is smevals studio.")
+
+ def test_invented_preamble_fails(self):
+ self.drift(
+ 1,
+ "Sure, here it is, happy to help with that. This is smevals studio. "
+ "Every eval on the shelf is a folder of tasks and graders.",
+ )
+
+ def test_empty_clip_fails(self):
+ self.drift(1, "you")
+
+ def test_inserted_runs_fail_even_when_total_length_is_close(self):
+ words = [f"word{i}" for i in range(50)]
+ for position in (0, 25, 50):
+ with self.subTest(position=position):
+ heard = words[:position] + "Before we begin please listen".split() + words[position:]
+ self.drift(1, " ".join(heard), " ".join(words))
+
+ def test_expanded_replacement_counts_the_added_words(self):
+ words = [f"word{i}" for i in range(50)]
+ heard = words[:25] + "Before we begin please listen".split() + words[26:]
+ self.drift(1, " ".join(heard), " ".join(words))
+
+ def test_short_insertions_keep_the_existing_tolerance(self):
+ words = [f"word{i}" for i in range(50)]
+ self.drift(0, "Please listen closely " + " ".join(words), " ".join(words))
+
+ def test_cached_audio_requires_requested_verification(self):
+ import json
+ import sys
+ module = fixtures.load_script("narrate")
+ with tempfile.TemporaryDirectory() as tmp:
+ work = Path(tmp)
+ output = work / "voice"
+ output.mkdir()
+ (output / "clip.wav").write_bytes(b"cached audio fixture")
+ (output / "manifest.json").write_text(json.dumps([
+ {"id": "clip", "text": "Read this sentence.", "wav": "clip.wav",
+ "duration": 1.0, "synthesis": {"engine": "piper",
+ "voice": module.PIPER_VOICE, "model": module.PIPER_VOICE}}
+ ]), encoding="utf-8")
+ scenes = work / "scenes.yaml"
+ scenes.write_text(json.dumps({"scenes": [
+ {"id": "clip", "narration": "Read this sentence."}
+ ]}), encoding="utf-8")
+ argv = ["narrate", str(scenes), str(output),
+ "--engine", "piper", "--verify", "on"]
+ stdout, stderr = io.StringIO(), io.StringIO()
+ with patch.object(sys, "argv", argv), \
+ patch.object(module.shutil, "which", return_value="ffprobe"), \
+ patch.object(module, "openai_key", return_value=None), \
+ patch.object(module, "duration", return_value=1.0), \
+ patch.object(module, "say_piper", side_effect=AssertionError("expected cached clip")), \
+ patch.object(module, "transcribe_local", return_value=None), \
+ redirect_stdout(stdout), redirect_stderr(stderr):
+ self.assertNotEqual(module.main(), 0)
+ self.assertIn("clip: required verification unavailable", stderr.getvalue())
+ self.assertIn("FAILED verbatim delivery: ['clip']", stderr.getvalue())
+ self.assertEqual(
+ json.loads((output / "manifest.json").read_text(encoding="utf-8")),
+ [],
+ )
+ self.assertEqual((output / "clip.wav").read_bytes(), b"cached audio fixture")
+
+ def test_rejected_chat_audio_is_never_cached_but_accepted_audio_is(self):
+ import json
+ import sys
+ module = fixtures.load_script("narrate")
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ scenes = root / "scenes.yaml"
+ scenes.write_text(json.dumps({"scenes": [
+ {"id": "accepted", "narration": "Read this sentence exactly."},
+ {"id": "rejected", "narration": "Keep this evidence out of the manifest."},
+ ]}), encoding="utf-8")
+ output = root / "voice"
+ calls = []
+
+ def synthesize(key, text, wav, voice):
+ calls.append(text)
+ wav.write_bytes(f"render {len(calls)}".encode())
+ if text.startswith("Keep"):
+ return "Unrelated invented preamble with entirely different words here."
+ return text
+
+ argv = ["narrate", str(scenes), str(output), "--engine", "openai-chat",
+ "--verify", "off"]
+ rejected_renders = []
+ with patch.object(sys, "argv", argv), \
+ patch.object(module.shutil, "which", return_value="ffprobe"), \
+ patch.object(module, "openai_key", return_value="test-key"), \
+ patch.object(module, "say_openai_chat", side_effect=synthesize), \
+ patch.object(module, "duration", return_value=1.0):
+ for _ in range(2):
+ with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ self.assertEqual(module.main(), 1)
+ manifest = json.loads(
+ (output / "manifest.json").read_text(encoding="utf-8")
+ )
+ self.assertEqual([entry["id"] for entry in manifest], ["accepted"])
+ rejected_renders.append(sorted(
+ path.read_bytes() for path in output.glob(".rejected.attempt-*.wav")
+ ))
+
+ self.assertEqual(calls.count("Read this sentence exactly."), 1)
+ self.assertEqual(calls.count("Keep this evidence out of the manifest."), 4)
+ self.assertLess(len(rejected_renders[0]), len(rejected_renders[1]))
+
+class TranscriptionProtocolRegression(unittest.TestCase):
+ def test_owned_json_is_used_instead_of_library_stdout(self):
+ import json
+ import subprocess
+ import sys
+ import io
+ module = fixtures.load_script("narrate")
+ def child(argv, **kwargs):
+ self.assertIn("--isolated", argv)
+ self.assertIn("--no-project", argv)
+ self.assertIn("--no-config", argv)
+ self.assertEqual(argv[argv.index("--python") + 1], sys.executable)
+ self.assertNotEqual(Path(kwargs["cwd"]), Path.cwd())
+ Path(argv[-1]).write_text(json.dumps({"text": "Correct λ transcript"}), encoding="utf-8")
+ return subprocess.CompletedProcess(argv, 0, "native library warning", "diagnostic")
+ diagnostics = io.StringIO()
+ with patch.object(module.subprocess, "run", side_effect=child), redirect_stderr(diagnostics):
+ self.assertEqual(module.transcribe_local(Path("clip.wav")), "Correct λ transcript")
+ self.assertIn("native library warning", diagnostics.getvalue())
+ self.assertIn("diagnostic", diagnostics.getvalue())
+
+ def test_failed_absent_and_malformed_child_results_are_unavailable(self):
+ import subprocess
+ module = fixtures.load_script("narrate")
+ for payload, code in ((None, 0), ("garbage", 0), ('{"text": 7}', 0), ('{"text": ""}', 0), ('{"text": "words"}', 1)):
+ with self.subTest(payload=payload, code=code):
+ def child(argv, **kwargs):
+ if payload is not None and "--isolated" in argv:
+ Path(argv[-1]).write_text(payload, encoding="utf-8")
+ return subprocess.CompletedProcess(argv, code, "misleading stdout", "error")
+ diagnostics = io.StringIO()
+ with patch.object(module.subprocess, "run", side_effect=child), \
+ redirect_stderr(diagnostics):
+ self.assertEqual(
+ module.transcribe_local(Path("clip.wav")),
+ "" if payload == '{"text": ""}' and code == 0 else None,
+ )
+ self.assertIn("local ASR", diagnostics.getvalue())
+
+ def test_fresh_and_off_then_on_clips_require_asr(self):
+ import json
+ import sys
+ module = fixtures.load_script("narrate")
+ for cached in (False, True):
+ with self.subTest(cached=cached), tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ scenes = root / "scenes.yaml"
+ scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Read this sentence."}]}), encoding="utf-8-sig")
+ output = root / "voice"
+ def synthesize(text, wav, voice):
+ wav.write_bytes(b"branch policy fixture")
+ argv = ["narrate", str(scenes), str(output), "--engine", "piper", "--verify"]
+ with patch.object(module.shutil, "which", return_value="ffprobe"), patch.object(module, "openai_key", return_value=None), patch.object(module, "say_piper", side_effect=synthesize), patch.object(module, "duration", return_value=1.0), patch.object(module, "transcribe_local", return_value=None):
+ if cached:
+ with patch.object(sys, "argv", [*argv, "off"]), \
+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ self.assertEqual(module.main(), 0)
+ stdout, stderr = io.StringIO(), io.StringIO()
+ with patch.object(sys, "argv", [*argv, "on"]), \
+ redirect_stdout(stdout), redirect_stderr(stderr):
+ self.assertNotEqual(module.main(), 0)
+ self.assertIn("clip: required verification unavailable", stderr.getvalue())
+
+
+class NarrationCacheRegression(unittest.TestCase):
+ def setUp(self):
+ self.module = fixtures.load_script("narrate")
+ directory = tempfile.TemporaryDirectory()
+ self.addCleanup(directory.cleanup)
+ self.root = Path(directory.name)
+ self.scenes = self.root / "scenes.yaml"
+ self.text = "Read this sentence exactly."
+ self.write_scenes(self.text)
+ self.output = self.root / "voice"
+ self.renders = []
+
+ which = patch.object(self.module.shutil, "which", return_value="ffprobe")
+ which.start()
+ self.addCleanup(which.stop)
+
+ def synthesize(*args):
+ text, wav, voice = args[-3:]
+ self.renders.append((text, voice))
+ wav.write_bytes(f"render {len(self.renders)}".encode())
+ return text
+
+ for name, options in (
+ ("openai_key", {"return_value": "test-key"}),
+ ("duration", {"return_value": 1.0}),
+ ("transcribe_local", {"return_value": None}),
+ ("say_piper", {"side_effect": synthesize}),
+ ("say_openai", {"side_effect": synthesize}),
+ ("say_openai_chat", {"side_effect": synthesize}),
+ ):
+ mocked = patch.object(self.module, name, **options)
+ mocked.start()
+ self.addCleanup(mocked.stop)
+
+ def write_scenes(self, text):
+ self.scenes.write_text(json.dumps({"scenes": [
+ {"id": "clip", "narration": text}
+ ]}), encoding="utf-8")
+
+ def narrate(self, *options, verify="off", expected_exit=0):
+ argv = ["narrate", str(self.scenes), str(self.output),
+ "--verify", verify, *options]
+ stdout, stderr = io.StringIO(), io.StringIO()
+ with patch.object(sys, "argv", argv), \
+ redirect_stdout(stdout), redirect_stderr(stderr):
+ self.assertEqual(self.module.main(), expected_exit, stderr.getvalue())
+ return json.loads((self.output / "manifest.json").read_text(encoding="utf-8"))
+
+ def test_engine_and_voice_changes_rerender(self):
+ for index, options in enumerate((
+ ("--engine", "piper", "--voice", "voice-a"),
+ ("--engine", "piper", "--voice", "voice-b"),
+ ("--engine", "openai", "--voice", "voice-b"),
+ ("--engine", "openai-chat", "--voice", "voice-b"),
+ ), 1):
+ with self.subTest(options=options):
+ self.narrate(*options)
+ self.assertEqual(len(self.renders), index)
+ self.assertEqual((self.output / "clip.wav").read_bytes(),
+ f"render {index}".encode())
+
+ def test_cloud_model_changes_rerender(self):
+ for engine, constant in (("openai", "OPENAI_TTS_MODEL"),
+ ("openai-chat", "OPENAI_CHAT_MODEL")):
+ with self.subTest(engine=engine):
+ self.narrate("--engine", engine)
+ count = len(self.renders)
+ with patch.object(self.module, constant, "another-model"):
+ self.narrate("--engine", engine)
+ self.assertEqual(len(self.renders), count + 1)
+
+ def test_implicit_and_explicit_defaults_share_cache(self):
+ for engine, voice in (("piper", self.module.PIPER_VOICE),
+ ("openai", "nova"), ("openai-chat", "nova")):
+ with self.subTest(engine=engine):
+ self.narrate("--engine", engine)
+ count = len(self.renders)
+ self.narrate("--engine", engine, "--voice", voice)
+ self.assertEqual(len(self.renders), count)
+ self.narrate("--engine", "openai")
+ count = len(self.renders)
+ self.narrate()
+ self.assertEqual(len(self.renders), count)
+ with patch.object(self.module, "openai_key", return_value=None):
+ self.narrate()
+ self.assertEqual(len(self.renders), count + 1)
+
+ def test_cache_without_synthesis_settings_rerenders(self):
+ manifest = self.narrate()
+ manifest[0].pop("synthesis", None)
+ (self.output / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
+ self.narrate()
+ self.assertEqual(len(self.renders), 2)
+
+ def test_changed_text_force_and_missing_wav_rerender(self):
+ self.narrate()
+ self.write_scenes("Read a different sentence exactly.")
+ manifest = self.narrate()
+ self.assertEqual(manifest[0]["text"], "Read a different sentence exactly.")
+ self.assertEqual(len(self.renders), 2)
+ self.narrate("--force")
+ self.assertEqual(len(self.renders), 3)
+ (self.output / "clip.wav").unlink()
+ self.narrate()
+ self.assertEqual(len(self.renders), 4)
+
+ def test_cached_clip_is_reverified_with_requested_asr_model(self):
+ self.narrate()
+ with patch.object(self.module, "transcribe_local", return_value=self.text) as asr:
+ self.narrate("--asr-model", "small.en", verify="on")
+ asr.assert_called_once_with(self.output / "clip.wav", "small.en")
+ self.assertEqual(len(self.renders), 1)
+
+ def test_unavailable_asr_respects_each_verification_mode(self):
+ for engine in ("piper", "openai", "openai-chat"):
+ for mode in ("auto", "on", "off"):
+ with self.subTest(engine=engine, mode=mode):
+ self.module.transcribe_local.reset_mock()
+ manifest = self.narrate("--engine", engine, verify=mode,
+ expected_exit=1 if mode == "on" else 0)
+ self.assertEqual(bool(manifest), mode != "on")
+ self.assertEqual(self.module.transcribe_local.call_count,
+ int(mode == "on" or (mode == "auto" and engine != "openai")))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/proving-it-works-with-a-movie/test_narration_contract.py b/tests/proving-it-works-with-a-movie/test_narration_contract.py
new file mode 100644
index 0000000..c5bc56d
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_narration_contract.py
@@ -0,0 +1,546 @@
+import base64
+import io
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from pathlib import Path
+from unittest.mock import patch
+
+import fixtures
+
+
+class ChatResponseContract(unittest.TestCase):
+ def test_malformed_transcripts_cannot_publish_candidates_in_any_asr_mode(self):
+ cases = ({}, {"transcript": None}, {"transcript": 42},
+ {"transcript": ["Two", "words"]}, {"transcript": {}},
+ {"transcript": False}, {"transcript": ""},
+ {"transcript": " "}, {"transcript": "...!?"})
+ for response in cases:
+ for mode in ("off", "auto", "on"):
+ with self.subTest(response=response, mode=mode), tempfile.TemporaryDirectory() as temp:
+ module = fixtures.load_script("narrate")
+ root = Path(temp)
+ scenes, output = root / "scenes.json", root / "narration"
+ scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Two words"}]}))
+ sentinels = []
+
+ def post(*args, **kwargs):
+ sentinel = f"NOT MEDIA: response {len(sentinels) + 1}".encode()
+ sentinels.append(sentinel)
+ audio = dict(response, data=base64.b64encode(sentinel).decode())
+ return {"choices": [{"message": {"audio": audio}}]}
+
+ argv = ["narrate", str(scenes), str(output), "--engine", "openai-chat", "--verify", mode]
+ with patch.object(sys, "argv", argv), \
+ patch.object(module.shutil, "which", return_value="fake-ffprobe"), \
+ patch.object(module, "openai_key", return_value="fake-key"), \
+ patch.object(module, "post", post), \
+ patch.object(module, "duration", return_value=1.25), \
+ patch.object(module, "transcribe_local", side_effect=AssertionError("invalid transcript reached ASR")), \
+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ try:
+ result = module.main()
+ except Exception as error:
+ result = error
+ self.assertEqual(result, 1)
+ self.assertEqual(json.loads((output / "manifest.json").read_text()), [])
+ self.assertEqual(len(sentinels), 2)
+ self.assertEqual({path.read_bytes() for path in output.glob(".clip.attempt-*.wav")}, set(sentinels))
+ self.assertFalse((output / "clip.wav").exists())
+
+ def test_valid_chat_and_cached_reuse_obey_independent_asr_modes(self):
+ for mode in ("off", "auto", "on"):
+ for heard in (None, "Two words"):
+ with self.subTest(mode=mode, heard=heard), tempfile.TemporaryDirectory() as temp:
+ module = fixtures.load_script("narrate")
+ root = Path(temp)
+ scenes, output = root / "scenes.json", root / "narration"
+ scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Two words"}]}))
+ sentinel = b"NOT MEDIA: accepted response"
+ response = {"choices": [{"message": {"audio": {
+ "data": base64.b64encode(sentinel).decode(), "transcript": "Two words",
+ }}}]}
+ argv = ["narrate", str(scenes), str(output), "--engine", "openai-chat", "--verify", "off"]
+ with patch.object(sys, "argv", argv), \
+ patch.object(module.shutil, "which", return_value="fake-ffprobe"), \
+ patch.object(module, "openai_key", return_value="fake-key"), \
+ patch.object(module, "post", return_value=response) as post, \
+ patch.object(module, "duration", return_value=1.25), \
+ patch.object(module, "transcribe_local", return_value=heard) as asr, \
+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ self.assertEqual(module.main(), 0)
+ argv[-1] = mode
+ expected = 1 if mode == "on" and heard is None else 0
+ self.assertEqual(module.main(), expected)
+ self.assertEqual(post.call_count, 1, "accepted cache must not synthesize again")
+ self.assertEqual(asr.call_count, int(mode != "off"))
+ self.assertEqual(bool(json.loads((output / "manifest.json").read_text())), expected == 0)
+ self.assertEqual((output / "clip.wav").read_bytes(), sentinel)
+
+
+class NarrationPublicationContract(unittest.TestCase):
+ def setUp(self):
+ self.module = fixtures.load_script("narrate")
+ self.directory = tempfile.TemporaryDirectory()
+ self.addCleanup(self.directory.cleanup)
+ self.root = Path(self.directory.name)
+ self.scenes = self.root / "scenes.yaml"
+ self.output = self.root / "narration"
+
+ def run_narrate(self, scenes, synthesize, *, verify="off", expected=0, engine="piper",
+ extra_options=(), transcript=None, asr_calls=None):
+ self.scenes.write_text(json.dumps({"scenes": scenes}), encoding="utf-8")
+ argv = ["narrate", str(self.scenes), str(self.output), "--engine", engine,
+ "--verify", verify, *extra_options]
+ stderr = io.StringIO()
+ def transcribe(wav, model):
+ if asr_calls is not None:
+ asr_calls.append((wav, model))
+ return transcript
+ with patch.object(sys, "argv", argv), \
+ patch.object(self.module.shutil, "which", return_value="ffprobe"), \
+ patch.object(self.module, "openai_key", return_value="key" if engine.startswith("openai") else None), \
+ patch.object(self.module, "say_piper", side_effect=synthesize), \
+ patch.object(self.module, "say_openai_chat",
+ side_effect=lambda key, text, wav, voice: synthesize(text, wav, voice)), \
+ patch.object(self.module, "duration", return_value=1.25), \
+ patch.object(self.module, "transcribe_local", side_effect=transcribe), \
+ redirect_stdout(io.StringIO()), redirect_stderr(stderr):
+ self.assertEqual(self.module.main(), expected, stderr.getvalue())
+ manifest = self.output / "manifest.json"
+ return json.loads(manifest.read_text(encoding="utf-8")) if manifest.exists() else []
+
+ def test_acceptance_is_withdrawn_before_replacing_accepted_bytes(self):
+ old_wav = self.output / "first.wav"
+ self.output.mkdir()
+ old_wav.write_bytes(b"accepted bytes")
+ synthesis = {"engine": "piper", "voice": self.module.PIPER_VOICE,
+ "model": self.module.PIPER_VOICE}
+ (self.output / "manifest.json").write_text(json.dumps([{
+ "id": "first", "text": "Old words", "wav": "first.wav",
+ "duration": 1.0, "synthesis": synthesis,
+ }]), encoding="utf-8")
+
+ def synthesize(text, wav, voice):
+ published = json.loads((self.output / "manifest.json").read_text())
+ self.assertNotIn("first", [entry["id"] for entry in published])
+ wav.write_bytes(b"replacement bytes")
+
+ manifest = self.run_narrate(
+ [{"id": "first", "narration": "New words"}], synthesize
+ )
+
+ self.assertEqual(manifest[0]["text"], "New words")
+ self.assertEqual(old_wav.read_bytes(), b"replacement bytes")
+
+ def test_rejected_and_exceptional_takes_are_not_accepted_on_a_rerun(self):
+ attempts = []
+
+ def synthesize(text, wav, voice):
+ attempts.append((text, wav))
+ wav.write_bytes(f"{text}:{len(attempts)}".encode())
+ if text == "Reject this":
+ return "invented words that do not match this script at all"
+ if text == "Raise here":
+ raise RuntimeError("synthesis interrupted")
+
+ scenes = [{"id": "reject", "narration": "Reject this"},
+ {"id": "raise", "narration": "Raise here"}]
+ def accepted(text, wav, voice):
+ wav.write_bytes(b"accepted")
+ return text
+
+ self.run_narrate(scenes, accepted, engine="openai-chat")
+ first = self.run_narrate(scenes, synthesize, engine="openai-chat", expected=1, extra_options=("--force",))
+ attempts_after_rejection = len(attempts)
+ second = self.run_narrate(scenes, synthesize, engine="openai-chat", expected=1)
+
+ self.assertEqual(first, [])
+ self.assertEqual(second, [])
+ self.assertGreater(len(attempts), attempts_after_rejection)
+ rejected_paths = [path for text, path in attempts if text == "Reject this"]
+ self.assertEqual(len({path.name for path in rejected_paths}), len(rejected_paths))
+ self.assertTrue(all(path.exists() for path in rejected_paths))
+ self.assertEqual(len({path.read_bytes() for path in rejected_paths}), len(rejected_paths))
+
+ def test_duration_failure_leaves_only_prior_accepted_scenes_published(self):
+ scenes = [{"id": "accepted", "narration": "Accepted words"},
+ {"id": "broken", "narration": "Broken words"}]
+
+ def synthesize(text, wav, voice):
+ wav.write_bytes(text.encode())
+
+ self.scenes.write_text(json.dumps({"scenes": scenes}), encoding="utf-8")
+ argv = ["narrate", str(self.scenes), str(self.output), "--engine", "piper",
+ "--verify", "off"]
+ durations = iter((1.0, RuntimeError("ffprobe failed")))
+ with patch.object(sys, "argv", argv), \
+ patch.object(self.module.shutil, "which", return_value="ffprobe"), \
+ patch.object(self.module, "openai_key", return_value=None), \
+ patch.object(self.module, "say_piper", side_effect=synthesize), \
+ patch.object(self.module, "duration", side_effect=durations), \
+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ self.assertEqual(self.module.main(), 1)
+
+ manifest = json.loads((self.output / "manifest.json").read_text())
+ self.assertEqual([entry["id"] for entry in manifest], ["accepted"])
+
+ def test_duration_failures_keep_distinct_attempt_bytes_across_reruns(self):
+ scenes = [{"id": "clip", "narration": "Measure this clip"}]
+
+ def synthesize(text, wav, voice):
+ wav.write_bytes(f"attempt {len(list(self.output.glob('.clip.attempt-*.wav'))) + 1}".encode())
+
+ for expected_attempts in (1, 2):
+ self.scenes.write_text(json.dumps({"scenes": scenes}), encoding="utf-8")
+ argv = ["narrate", str(self.scenes), str(self.output), "--engine", "piper", "--verify", "off"]
+ with patch.object(sys, "argv", argv), \
+ patch.object(self.module.shutil, "which", return_value="ffprobe"), \
+ patch.object(self.module, "openai_key", return_value=None), \
+ patch.object(self.module, "say_piper", side_effect=synthesize), \
+ patch.object(self.module, "duration", side_effect=RuntimeError("ffprobe failed")), \
+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ self.assertEqual(self.module.main(), 1)
+ attempts = sorted(self.output.glob(".clip.attempt-*.wav"))
+ self.assertEqual(len(attempts), expected_attempts)
+ self.assertEqual(len({path.read_bytes() for path in attempts}), expected_attempts)
+ self.assertFalse((self.output / "clip.wav").exists())
+
+ def test_two_accepted_scenes_are_published_together(self):
+ def synthesize(text, wav, voice):
+ wav.write_bytes(text.encode())
+
+ manifest = self.run_narrate([
+ {"id": "first", "narration": "First accepted scene"},
+ {"id": "second", "narration": "Second accepted scene"},
+ ], synthesize)
+ self.assertEqual([entry["id"] for entry in manifest], ["first", "second"])
+
+ def test_strict_unavailable_verification_withdraws_cached_acceptance(self):
+ def synthesize(text, wav, voice):
+ wav.write_bytes(b"accepted")
+
+ self.run_narrate([{"id": "clip", "narration": "Read these words"}], synthesize)
+ manifest = self.run_narrate(
+ [{"id": "clip", "narration": "Read these words"}], synthesize,
+ verify="on", expected=1,
+ )
+ self.assertEqual(manifest, [])
+
+ def test_unsupported_asr_comparison_obeys_auto_on_and_off_modes(self):
+ def synthesize(text, wav, voice):
+ wav.write_bytes(b"clip")
+
+ scenes = [{"id": "clip", "narration": "\u77ed\u6587"}]
+ for mode, expected in (("auto", 0), ("on", 1), ("off", 0)):
+ with self.subTest(mode=mode):
+ asr_calls = []
+ manifest = self.run_narrate(scenes, synthesize, verify=mode,
+ expected=expected, transcript="\u77ed\u6587",
+ asr_calls=asr_calls)
+ self.assertEqual(bool(manifest), expected == 0)
+ self.assertEqual(bool(asr_calls), mode != "off")
+
+ def test_missing_ffprobe_stops_before_synthesis(self):
+ self.scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Words"}]}),
+ encoding="utf-8")
+ with patch.object(sys, "argv", ["narrate", str(self.scenes), str(self.output)]), \
+ patch.object(self.module.shutil, "which", return_value=None), \
+ patch.object(self.module, "say_piper", side_effect=AssertionError("synthesized")), \
+ redirect_stderr(io.StringIO()):
+ with self.assertRaises(SystemExit):
+ self.module.main()
+
+ def test_fresh_unsupported_chat_transcript_is_rejected_before_synthesis(self):
+ called = []
+
+ def synthesize(*args):
+ called.append(args)
+
+ manifest = self.run_narrate(
+ [{"id": "clip", "narration": "\U00020000\U00020001"}], synthesize,
+ engine="openai-chat", expected=1,
+ )
+ self.assertEqual(manifest, [])
+ self.assertEqual(called, [])
+
+ def test_cached_unsupported_chat_transcript_withdraws_acceptance_without_asr(self):
+ self.output.mkdir()
+ (self.output / "clip.wav").write_bytes(b"cached bytes")
+ synthesis = {"engine": "openai-chat", "voice": "nova",
+ "model": self.module.OPENAI_CHAT_MODEL}
+ (self.output / "manifest.json").write_text(json.dumps([{
+ "id": "clip", "text": "\u77ed\u6587", "wav": "clip.wav", "duration": 1.0,
+ "synthesis": synthesis,
+ }]), encoding="utf-8")
+ self.scenes.write_text(json.dumps({"scenes": [
+ {"id": "clip", "narration": "\u77ed\u6587"}
+ ]}), encoding="utf-8")
+ for mode in ("off", "auto"):
+ with self.subTest(mode=mode):
+ (self.output / "manifest.json").write_text(json.dumps([{
+ "id": "clip", "text": "\u77ed\u6587", "wav": "clip.wav", "duration": 1.0,
+ "synthesis": synthesis,
+ }]), encoding="utf-8")
+ argv = ["narrate", str(self.scenes), str(self.output), "--engine", "openai-chat",
+ "--verify", mode]
+ with patch.object(sys, "argv", argv), \
+ patch.object(self.module.shutil, "which", return_value="ffprobe"), \
+ patch.object(self.module, "openai_key", return_value="key"), \
+ patch.object(self.module, "say_openai_chat", side_effect=AssertionError("cached")), \
+ patch.object(self.module, "transcribe_local", side_effect=AssertionError("ASR")), \
+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ self.assertEqual(self.module.main(), 1)
+ self.assertEqual(json.loads((self.output / "manifest.json").read_text()), [])
+
+ def test_empty_chat_or_asr_speech_is_rejected(self):
+ def empty_chat_synthesis(text, wav, voice):
+ wav.write_bytes(b"audio")
+ return ""
+
+ empty_chat = self.run_narrate(
+ [{"id": "clip", "narration": "Two words"}],
+ empty_chat_synthesis, engine="openai-chat", expected=1,
+ )
+ self.assertEqual(empty_chat, [])
+
+ def synthesize(text, wav, voice):
+ wav.write_bytes(b"audio")
+
+ self.scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Two words"}]}),
+ encoding="utf-8")
+ argv = ["narrate", str(self.scenes), str(self.output), "--engine", "piper", "--verify", "auto"]
+ with patch.object(sys, "argv", argv), \
+ patch.object(self.module.shutil, "which", return_value="ffprobe"), \
+ patch.object(self.module, "openai_key", return_value=None), \
+ patch.object(self.module, "say_piper", side_effect=synthesize), \
+ patch.object(self.module, "transcribe_local", return_value=""), \
+ patch.object(self.module, "duration", return_value=1.0), \
+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ self.assertEqual(self.module.main(), 1)
+ self.assertEqual(json.loads((self.output / "manifest.json").read_text()), [])
+
+ def test_cached_nested_wav_keeps_its_manifest_path_after_reverification(self):
+ nested = self.output / "takes" / "clip.wav"
+ nested.parent.mkdir(parents=True)
+ nested.write_bytes(b"accepted")
+ synthesis = {"engine": "piper", "voice": self.module.PIPER_VOICE,
+ "model": self.module.PIPER_VOICE}
+ (self.output / "manifest.json").write_text(json.dumps([{
+ "id": "clip", "text": "Nested clip", "wav": "takes/clip.wav", "duration": 1.0,
+ "synthesis": synthesis,
+ }]), encoding="utf-8")
+ manifest = self.run_narrate(
+ [{"id": "clip", "narration": "Nested clip"}],
+ lambda *args: (_ for _ in ()).throw(AssertionError("cached")), verify="on",
+ transcript="Nested clip",
+ )
+ self.assertEqual(manifest[0]["wav"], "takes/clip.wav")
+
+ def test_cached_strict_verification_withdraws_before_interrupt(self):
+ def synthesize(text, wav, voice):
+ wav.write_bytes(b"accepted")
+
+ self.run_narrate([{"id": "clip", "narration": "Interrupt safely"}], synthesize)
+ self.scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Interrupt safely"}]}),
+ encoding="utf-8")
+ argv = ["narrate", str(self.scenes), str(self.output), "--engine", "piper", "--verify", "on"]
+ with patch.object(sys, "argv", argv), \
+ patch.object(self.module.shutil, "which", return_value="ffprobe"), \
+ patch.object(self.module, "openai_key", return_value=None), \
+ patch.object(self.module, "say_piper", side_effect=AssertionError("cached")), \
+ patch.object(self.module, "transcribe_local", side_effect=KeyboardInterrupt), \
+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ with self.assertRaises(KeyboardInterrupt):
+ self.module.main()
+ self.assertEqual(json.loads((self.output / "manifest.json").read_text()), [])
+
+
+class NarrationComparisonContract(unittest.TestCase):
+ def setUp(self):
+ self.module = fixtures.load_script("narrate")
+
+ def test_comparison_marks_scripts_requiring_segmentation_or_no_words_unavailable(self):
+ for script, heard in (
+ ("你好世界", "こんにちは世界"),
+ ("短文", "短文"),
+ ("mixed 日本語 words", "mixed 日本語 words"),
+ ("\U00020000\U00020001", "\U00020002\U00020003"),
+ ("*** !!!", "*** !!!"),
+ ):
+ with self.subTest(script=script):
+ self.assertIsNone(self.module.structural_drift(script, heard))
+
+ def test_comparison_accepts_multiline_accented_latin_and_spaced_cyrillic(self):
+ for script, heard in (
+ ("Caf\u00e9\nna\u00efve", "CAF\u00c9 na\u00efve"),
+ ("\u041f\u0440\u0438\u0432\u0435\u0442 \u043c\u0438\u0440", "\u043f\u0440\u0438\u0432\u0435\u0442 \u043c\u0438\u0440"),
+ ("\uc548\ub155 \uc138\uacc4", "\uc548\ub155 \uc138\uacc4"),
+ ):
+ with self.subTest(script=script):
+ self.assertEqual(self.module.structural_drift(script, heard), (0.0, 0))
+
+ def test_drift_check_reports_unavailable_comparison_as_nonzero(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ script = root / "script.txt"
+ heard = root / "heard.txt"
+ script.write_text("\u77ed\u6587", encoding="utf-8")
+ heard.write_text("\u77ed\u6587", encoding="utf-8")
+ output = io.StringIO()
+ with patch.object(sys, "argv", ["narrate", "--drift-check", str(script), str(heard)]), \
+ redirect_stdout(output):
+ self.assertEqual(self.module.main(), 1)
+ self.assertIn("comparison unavailable", output.getvalue())
+
+
+class AssemblyNarrationContract(unittest.TestCase):
+ def setUp(self):
+ self.module = fixtures.load_script("assemble")
+ self.directory = tempfile.TemporaryDirectory()
+ self.addCleanup(self.directory.cleanup)
+ self.root = Path(self.directory.name)
+ self.scenes = self.root / "scenes.yaml"
+ self.narration = self.root / "narration"
+ self.narration.mkdir()
+
+ def assemble(self, scenes, *, manifest=None, run_side_effect=None):
+ self.scenes.write_text(json.dumps({"scenes": scenes}), encoding="utf-8")
+ if manifest is not None:
+ (self.narration / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
+ calls = []
+
+ def run(command, **kwargs):
+ calls.append(command)
+ if run_side_effect is not None:
+ return run_side_effect(command)
+ return subprocess.CompletedProcess(command, 0, "1.0", "")
+
+ argv = ["assemble", str(self.scenes), str(self.root / "out.mp4"),
+ "--narration", str(self.narration), "--work", str(self.root / "work")]
+ with patch.object(sys, "argv", argv), \
+ patch.object(self.module.shutil, "which", return_value="tool"), \
+ patch.object(self.module, "find_browser", return_value=None), \
+ patch.object(self.module, "run", side_effect=run), \
+ redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()):
+ return self.module.main(), calls
+
+ def test_required_narration_contract_fails_before_encoding(self):
+ with self.assertRaises(SystemExit):
+ self.assemble([{"id": "spoken", "kind": "image", "src": "still.png",
+ "narration": "Expected words"}],
+ run_side_effect=lambda command: (_ for _ in ()).throw(
+ AssertionError("encoding started")
+ ))
+
+ def test_missing_entry_wav_or_matching_text_fails_before_encoding(self):
+ (self.root / "still.png").write_bytes(b"image sentinel")
+ cases = (
+ ([], "missing entry"),
+ ([{"id": "spoken", "text": "Expected words", "wav": "missing.wav"}], "missing WAV"),
+ ([{"id": "spoken", "text": "Changed words", "wav": "accepted.wav"}], "changed text"),
+ )
+ for manifest, label in cases:
+ with self.subTest(label=label):
+ (self.narration / "manifest.json").unlink(missing_ok=True)
+ with self.assertRaises(SystemExit):
+ self.assemble([{"id": "spoken", "kind": "image", "src": "still.png",
+ "narration": "Expected words"}], manifest=manifest,
+ run_side_effect=lambda command: (_ for _ in ()).throw(
+ AssertionError("encoding started")
+ ))
+
+ def test_manifest_wav_name_is_authoritative_and_removed_narration_ignores_leftover_wav(self):
+ selected = self.narration / "accepted-name.wav"
+ selected.write_bytes(b"accepted")
+ leftover = self.narration / "silent.wav"
+ leftover.write_bytes(b"leftover")
+ (self.root / "still.png").write_bytes(b"image sentinel")
+ manifest = [{"id": "spoken", "text": "Expected words",
+ "wav": selected.name, "duration": 1.0, "synthesis": {}}]
+ status, calls = self.assemble([
+ {"id": "spoken", "kind": "image", "src": "still.png", "narration": "Expected words"},
+ {"id": "silent", "kind": "image", "src": "still.png"},
+ ], manifest=manifest)
+ self.assertEqual(status, 0)
+ encoded = [call for call in calls if call and call[0] == "ffmpeg"]
+ self.assertTrue(any(str(selected) in call for call in encoded))
+ self.assertFalse(any(str(leftover) in call for call in encoded))
+ offsets = json.loads((self.root / "work" / "offsets.json").read_text())
+ self.assertEqual(set(offsets), {"spoken"})
+
+ def test_2560x1080_movie_uses_source_audio_and_silent_source_gets_anullsrc(self):
+ source = self.root / "wide-2560x1080.mp4"
+ source.write_bytes(b"source")
+ status, calls = self.assemble(
+ [{"id": "movie", "kind": "movie", "src": source.name,
+ "narration": "Ignore this", "height": 800}],
+ run_side_effect=lambda command: subprocess.CompletedProcess(
+ command, 0,
+ json.dumps({"streams": [{"codec_type": "video", "width": 2560,
+ "height": 1080}]}), ""
+ ) if "-show_streams" in command else subprocess.CompletedProcess(command, 0, "1.0", ""),
+ )
+ self.assertEqual(status, 0)
+ movie_encode = next(call for call in calls if call and call[0] == "ffmpeg")
+ self.assertIn("anullsrc=r=44100:cl=stereo", movie_encode)
+ self.assertIn("0:v:0", movie_encode)
+ self.assertIn("1:a:0", movie_encode)
+ self.assertIn("scale=1920:800:force_original_aspect_ratio=decrease",
+ movie_encode[movie_encode.index("-vf") + 1])
+ offsets = json.loads((self.root / "work" / "offsets.json").read_text())
+ self.assertEqual(offsets, {})
+
+ def test_movie_with_audio_maps_its_source_audio(self):
+ source = self.root / "source-with-audio.mp4"
+ source.write_bytes(b"source")
+ status, calls = self.assemble(
+ [{"id": "movie", "kind": "movie", "src": source.name}],
+ run_side_effect=lambda command: subprocess.CompletedProcess(
+ command, 0,
+ json.dumps({"streams": [{"codec_type": "video"}, {"codec_type": "audio"}]}), ""
+ ) if "-show_streams" in command else subprocess.CompletedProcess(command, 0, "1.0", ""),
+ )
+ self.assertEqual(status, 0)
+ movie_encode = next(call for call in calls if call and call[0] == "ffmpeg")
+ self.assertNotIn("anullsrc=r=44100:cl=stereo", movie_encode)
+ self.assertEqual(movie_encode[movie_encode.index("-map") + 1], "0:v:0")
+ self.assertEqual(movie_encode[movie_encode.index("-map", movie_encode.index("-map") + 1) + 1], "0:a:0")
+
+ def test_movie_geometry_fits_width_and_requested_inner_height_before_padding(self):
+ self.assertEqual(self.module.movie_geometry(1920, 1080, 800), {
+ "scale": (1920, 800),
+ "pad": (1920, 1080),
+ })
+
+
+class PercentPathContract(unittest.TestCase):
+ def test_sequence_pattern_escapes_only_directory_percents(self):
+ module = fixtures.load_script("media_paths")
+ pattern = module.sequence_pattern(Path("folder%name") / "frames", "frame-%08d.png")
+ self.assertEqual(pattern, "folder%%name/frames/frame-%08d.png")
+
+ def test_checker_sampling_escapes_only_output_directory_percents(self):
+ module = fixtures.load_script("check-movie")
+ with tempfile.TemporaryDirectory() as directory:
+ work = Path(directory) / "proof%take"
+ work.mkdir()
+ command = []
+
+ def run(argv, **kwargs):
+ command.extend(argv)
+ return subprocess.CompletedProcess(argv, 0, "", "")
+
+ with patch.object(module.subprocess, "run", side_effect=run), redirect_stdout(io.StringIO()):
+ with self.assertRaises(SystemExit):
+ module.sample_picture(Path("movie.mp4"), work)
+ output = command[-1]
+ self.assertIn("proof%%take", output)
+ self.assertTrue(output.endswith("s%05d.png"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/proving-it-works-with-a-movie/test_paths.py b/tests/proving-it-works-with-a-movie/test_paths.py
new file mode 100644
index 0000000..ed1cb50
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_paths.py
@@ -0,0 +1,104 @@
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+import wave
+from pathlib import Path
+
+
+SCRIPTS = (
+ Path(__file__).resolve().parents[2]
+ / "skills/proving-it-works-with-a-movie/scripts"
+)
+sys.path.insert(0, str(SCRIPTS))
+
+import media_paths
+
+
+class MediaPathRegression(unittest.TestCase):
+ def test_frame_staging_uses_ordinary_ordered_files(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ source = root / "movie O'Brien λ & [take]"
+ source.mkdir()
+ (source / "b.png").write_bytes(b"second")
+ (source / "a.png").write_bytes(b"first")
+
+ staged = media_paths.stage_frames(source, root / "staged")
+
+ self.assertEqual(
+ [path.read_bytes() for path in staged],
+ [b"first", b"second"],
+ )
+ self.assertTrue(all(not path.is_symlink() for path in staged))
+ self.assertEqual(len(list(source.iterdir())), 2)
+
+ def test_frame_staging_rejects_an_empty_source(self):
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ source = root / "empty"
+ source.mkdir()
+
+ with self.assertRaisesRegex(ValueError, "no PNG frames"):
+ media_paths.stage_frames(source, root / "staged")
+
+ self.assertFalse((root / "staged").exists())
+
+ def test_ffconcat_entry_is_accepted_by_ffmpeg(self):
+ ffmpeg = shutil.which("ffmpeg")
+ if ffmpeg is None:
+ self.skipTest("required executable not on PATH: ffmpeg")
+
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory) / "movie O'Brien λ & [take]"
+ nested = root / "subtitles" / "nested"
+ nested.mkdir(parents=True)
+ audio = nested / "tone.wav"
+ with wave.open(str(audio), "wb") as output:
+ output.setnchannels(1)
+ output.setsampwidth(2)
+ output.setframerate(8000)
+ output.writeframes(b"\x00\x00" * 800)
+
+ if sys.platform == "win32":
+ self.assertRegex(str(audio.resolve()), r"^[A-Za-z]:\\")
+
+ listing = root / "concat.txt"
+ listing.write_text(
+ "ffconcat version 1.0\n" + media_paths.ffconcat_entry(audio),
+ encoding="utf-8",
+ )
+ result = subprocess.run(
+ [
+ ffmpeg,
+ "-nostdin",
+ "-v",
+ "error",
+ "-f",
+ "concat",
+ "-safe",
+ "0",
+ "-i",
+ str(listing),
+ "-f",
+ "null",
+ "-",
+ ],
+ cwd=Path(directory),
+ capture_output=True,
+ timeout=30,
+ )
+ self.assertEqual(
+ result.returncode,
+ 0,
+ (result.stdout + result.stderr).decode("utf-8", errors="replace"),
+ )
+
+ def test_ffconcat_entry_rejects_line_breaks(self):
+ with self.assertRaisesRegex(ValueError, "cannot contain line breaks"):
+ media_paths.ffconcat_entry(Path("bad\nname.wav"))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/proving-it-works-with-a-movie/test_recorder_contract.py b/tests/proving-it-works-with-a-movie/test_recorder_contract.py
new file mode 100644
index 0000000..2c213ec
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_recorder_contract.py
@@ -0,0 +1,538 @@
+"""Recorder decisions with fake processes/CDP and byte-token captures only."""
+import contextlib
+import importlib.util
+import io
+import json
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import fixtures
+
+SCRIPT = Path(__file__).resolve().parents[2] / "skills/proving-it-works-with-a-movie/examples/film-terminal.py"
+
+
+def recorder():
+ spec = importlib.util.spec_from_file_location("recorder_contract", SCRIPT)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+class Clock:
+ def __init__(self):
+ self.now = 0.0
+
+ def monotonic(self):
+ return self.now
+
+ def sleep(self, seconds):
+ self.now += seconds
+
+
+class Process:
+ def __init__(self, pid):
+ self.pid = pid
+ self.returncode = None
+
+ def poll(self):
+ return self.returncode
+
+ def wait(self, timeout):
+ if self.returncode is None:
+ raise subprocess.TimeoutExpired("fake child", timeout)
+ return self.returncode
+
+
+@contextlib.contextmanager
+def serving(failure=None, stop_at=None, relative=False):
+ module, clock = recorder(), Clock()
+ with tempfile.TemporaryDirectory() as temp, contextlib.ExitStack() as stack:
+ root = Path(temp)
+ directory = root / "session"
+ args = SimpleNamespace(session=directory, cwd=root, shell="bash", shell_exe=None,
+ ttyd="fake-ttyd", browser="fake-browser")
+ if relative:
+ import os
+ args.session = Path(os.path.relpath(directory))
+ handles, children, launches, tokens = [], [], [], []
+ original_open, original_write = Path.open, module.write_json
+ stopped = False
+
+ def open_file(path, *positional, **kwargs):
+ if positional == ("ab",):
+ if failure == path.name:
+ raise OSError("injected " + path.name)
+ handle = original_open(path, *positional, **kwargs)
+ handles.append(handle)
+ return handle
+ return original_open(path, *positional, **kwargs)
+
+ def popen(argv, **kwargs):
+ index = len(children)
+ if failure == ("ttyd launch" if index == 0 else "browser launch"):
+ raise OSError("injected launch")
+ child = Process(1100 + index)
+ children.append(child)
+ launches.append((argv, kwargs))
+ if index == 1:
+ (directory / "profile").mkdir()
+ return child
+
+ def kill(pid):
+ for child in children:
+ if child.pid == pid:
+ if child.poll() is not None:
+ raise AssertionError("cannot use an exited leader's PID")
+ if failure != "child wait":
+ child.returncode = -9
+
+ def write_json(path, value):
+ if failure == "session metadata" and path.name == "session.json":
+ raise OSError("injected metadata failure")
+ original_write(path, value)
+
+ def stop(phase):
+ nonlocal stopped
+ if stop_at == phase and not stopped:
+ (directory / "stop").write_text("")
+ stopped = True
+
+ class FakeCDP:
+ def __init__(self, url):
+ if failure == "connection":
+ raise RuntimeError("injected connection failure")
+ self.n = 0
+ self.on_event = None
+ self.ws = SimpleNamespace(close=lambda: None)
+
+ def recv(self, timeout):
+ clock.sleep(timeout)
+ if (directory / "ready.json").exists():
+ stop("ready")
+ if failure == "exited leader":
+ children[0].returncode = 0
+ if failure == "later connection":
+ raise ConnectionError("injected later disconnect")
+ else:
+ stop("pump" if self.n == 0 else "quiet")
+ return None
+
+ def call(self, method, params=None, **kwargs):
+ if getattr(self, "before_call", None):
+ self.before_call()
+ if stop_at == "call":
+ stop("call")
+ clock.sleep(10)
+ if method == "Input.dispatchKeyEvent" and params["type"] == "keyDown":
+ self.n += 1
+ return {}
+
+ cdp = None
+
+ def connect(url):
+ nonlocal cdp
+ cdp = FakeCDP(url)
+ return cdp
+
+ def page_url(port):
+ stop("retry")
+ if stop_at == "retry":
+ raise OSError("not listening yet")
+ return "fake-url"
+
+ def tail(path):
+ if cdp and cdp.n:
+ return f"\x1b]0;MOVIE;{cdp.n};1;0;{root}\x07$ ".encode()
+ return b"$ "
+
+ for obj, name, value in (
+ (module, "shell_argv", lambda *a: ["fake-bash"]),
+ (module, "find_browser", lambda *a: "fake-browser"),
+ (module, "free_port", lambda: 1234),
+ (module.subprocess, "Popen", popen),
+ (module, "kill_process_tree", kill),
+ (module, "page_url", page_url),
+ (module, "CDP", connect),
+ (module, "write_json", write_json),
+ (module, "tail", tail),
+ (module, "screenshot", lambda *a: b"capture-token"),
+ (module, "lit_fraction", lambda *a: 0.01),
+ (module.time, "monotonic", clock.monotonic),
+ (module.time, "sleep", clock.sleep),
+ (Path, "open", open_file),
+ (Path, "write_bytes", lambda path, token: tokens.append((path.name, token))),
+ ):
+ stack.enter_context(patch.object(obj, name, value))
+ if failure == "profile removal":
+ stack.enter_context(patch.object(module.shutil, "rmtree", side_effect=PermissionError("locked")))
+ stack.enter_context(contextlib.redirect_stdout(io.StringIO()))
+ stack.enter_context(contextlib.redirect_stderr(io.StringIO()))
+ try:
+ yield SimpleNamespace(module=module, args=args, directory=directory, children=children,
+ handles=handles, launches=launches, clock=clock)
+ finally:
+ # Release the test's log handles even when ownership assertions fail.
+ for handle in handles:
+ handle.close()
+
+
+class ServeLifecycleTests(unittest.TestCase):
+ def test_tree_cleanup_failure_keeps_failure_after_leader_exits_and_cleans_other_resources(self):
+ with serving(stop_at="ready") as rig:
+ browser = fixtures.load_script("browser_tools")
+ terminated = []
+
+ def taskkill(argv, **kwargs):
+ pid = int(argv[-1])
+ terminated.append(pid)
+ child = next(child for child in rig.children if child.pid == pid)
+ child.returncode = 0 if pid == 1100 else -9
+ return subprocess.CompletedProcess(argv, 1 if pid == 1100 else 0, b"", b"tree termination failed")
+
+ with patch.object(rig.module, "kill_process_tree", browser.kill_process_tree), \
+ patch.object(browser.sys, "platform", "win32"), \
+ patch.object(browser.subprocess, "run", taskkill):
+ self.assertEqual(rig.module.serve(rig.args), 1)
+ session = rig.module.read_json(rig.directory / "session.json")
+ self.assertEqual(session["pids"], [1100, 1101])
+ self.assertFalse(session.get("closed", False))
+ self.assertEqual(terminated, [1100, 1101])
+ self.assertTrue(all(child.poll() is not None for child in rig.children))
+ self.assertTrue(all(handle.closed for handle in rig.handles))
+ self.assertFalse((rig.directory / "profile").exists())
+ self.assertFalse((rig.directory / "ready.json").exists())
+ self.assertTrue(all((rig.directory / name).exists()
+ for name in ("ttyd.log", "browser.log", "terminal.log")))
+
+ def test_every_acquisition_failure_releases_owned_resources(self):
+ for failure in ("ttyd.log", "browser.log", "ttyd launch", "browser launch",
+ "session metadata", "terminal.log", "connection", "later connection"):
+ with self.subTest(failure=failure), serving(failure) as rig:
+ try:
+ code = rig.module.serve(rig.args)
+ except Exception as error:
+ code = error
+ self.assertEqual(code, 1)
+ self.assertTrue(all(p.poll() is not None for p in rig.children))
+ self.assertTrue(all(h.closed for h in rig.handles))
+ self.assertFalse((rig.directory / "profile").exists())
+ self.assertFalse((rig.directory / "ready.json").exists())
+ for name in ("ttyd.log", "browser.log", "terminal.log"):
+ if any(Path(h.name).name == name for h in rig.handles):
+ self.assertTrue((rig.directory / name).exists(), "logs are evidence")
+
+ def test_relative_session_path_matches_browser_profile_and_cwd(self):
+ with serving("connection", relative=True) as rig:
+ self.assertEqual(rig.module.serve(rig.args), 1)
+ argv, kwargs = rig.launches[1]
+ profile_arg = next(a.split("=", 1)[1] for a in argv if a.startswith("--user-data-dir="))
+ self.assertEqual(Path(kwargs["cwd"]) / profile_arg, (rig.directory / "profile").resolve())
+ self.assertTrue(Path(kwargs["cwd"]).is_absolute())
+
+ def test_stop_requests_interrupt_startup_and_finish_owned_cleanup(self):
+ for phase in ("retry", "pump", "quiet", "ready"):
+ with self.subTest(phase=phase), serving(stop_at=phase) as rig:
+ self.assertEqual(rig.module.serve(rig.args), 0)
+ self.assertLess(rig.clock.now, 5)
+ self.assertTrue(all(p.poll() is not None for p in rig.children))
+ self.assertTrue(all(h.closed for h in rig.handles))
+ session = rig.module.read_json(rig.directory / "session.json")
+ self.assertEqual(session["pids"], [])
+ self.assertTrue(session["closed"])
+ self.assertFalse((rig.directory / "ready.json").exists())
+ self.assertFalse((rig.directory / "profile").exists())
+
+ def test_stop_during_one_startup_call_prevents_another_bounded_call(self):
+ with serving(stop_at="call") as rig:
+ self.assertEqual(rig.module.serve(rig.args), 0)
+ self.assertLessEqual(rig.clock.now, 10)
+ self.assertTrue(rig.module.read_json(rig.directory / "session.json")["closed"])
+
+ def test_pid_retirement_is_atomic_and_follows_resource_cleanup(self):
+ with serving(stop_at="ready") as rig:
+ replace = Path.replace
+ observed = []
+
+ def publish(path, target):
+ session = rig.module.read_json(target)
+ observed.append(session["pids"])
+ self.assertEqual(session["pids"], [1100, 1101])
+ self.assertTrue(all(p.poll() is not None for p in rig.children))
+ self.assertTrue(all(h.closed for h in rig.handles))
+ self.assertFalse((rig.directory / "profile").exists())
+ self.assertFalse((rig.directory / "ready.json").exists())
+ return replace(path, target)
+
+ with patch.object(Path, "replace", publish):
+ self.assertEqual(rig.module.serve(rig.args), 0)
+ self.assertEqual(observed, [[1100, 1101]])
+ self.assertEqual(rig.module.read_json(rig.directory / "session.json")["pids"], [])
+
+ def test_failed_cleanup_does_not_retire_owned_ids_or_claim_closed(self):
+ for failure in ("child wait", "profile removal"):
+ with self.subTest(failure=failure), serving(failure, stop_at="ready") as rig:
+ self.assertEqual(rig.module.serve(rig.args), 1)
+ session = rig.module.read_json(rig.directory / "session.json")
+ self.assertEqual(session["pids"], [1100, 1101])
+ self.assertFalse(session.get("closed", False))
+ self.assertFalse((rig.directory / "ready.json").exists())
+
+ def test_exited_leader_cannot_confirm_descendant_cleanup(self):
+ with serving("exited leader", stop_at="ready") as rig:
+ self.assertEqual(rig.module.serve(rig.args), 1)
+ session = rig.module.read_json(rig.directory / "session.json")
+ self.assertEqual(session["pids"], [1100, 1101])
+ self.assertFalse(session.get("closed", False))
+ self.assertEqual(rig.children[0].returncode, 0)
+ self.assertEqual(rig.children[1].returncode, -9)
+ self.assertTrue(all(h.closed for h in rig.handles))
+ self.assertFalse((rig.directory / "ready.json").exists())
+
+
+class CloseContractTests(unittest.TestCase):
+ def test_close_waits_for_owner_cleanup_and_repeated_close_never_kills(self):
+ module, clock = recorder(), Clock()
+ with tempfile.TemporaryDirectory() as temp:
+ directory = Path(temp)
+ module.write_json(directory / "session.json", {"pids": [987654]})
+ (directory / "ready.json").write_text("{}")
+ (directory / "profile").mkdir()
+ (directory / "terminal.log").write_text("retain evidence")
+
+ def sleep(seconds):
+ clock.sleep(seconds)
+ self.assertTrue((directory / "stop").exists())
+ if clock.now >= 12:
+ (directory / "profile").rmdir()
+ (directory / "ready.json").unlink()
+ module.write_json(directory / "session.json", {"pids": [], "closed": True})
+
+ with patch.object(module, "kill_process_tree", side_effect=AssertionError("historical PID kill")), \
+ patch.object(module.time, "monotonic", clock.monotonic), \
+ patch.object(module.time, "sleep", sleep), contextlib.redirect_stdout(io.StringIO()):
+ self.assertEqual(module.close(SimpleNamespace(session=directory)), 0)
+ self.assertGreaterEqual(clock.now, 12)
+ self.assertEqual(module.close(SimpleNamespace(session=directory)), 0)
+ self.assertEqual((directory / "terminal.log").read_text(), "retain evidence")
+
+ def test_incomplete_or_unavailable_cleanup_is_not_success(self):
+ for state in ("missing", "unreadable", "unavailable owner", "profile remains", "ready remains"):
+ with self.subTest(state=state), tempfile.TemporaryDirectory() as temp:
+ module, clock, directory = recorder(), Clock(), Path(temp)
+ if state != "missing":
+ module.write_json(directory / "session.json", {"pids": [987654]})
+ if state == "unreadable":
+ (directory / "session.json").write_text("invalid json")
+ if state in ("profile remains", "ready remains"):
+ module.write_json(directory / "session.json", {"pids": [], "closed": True})
+ if state == "profile remains":
+ (directory / "profile").mkdir()
+ else:
+ (directory / "ready.json").write_text("{}")
+ with patch.object(module, "kill_process_tree", lambda pid: None), \
+ patch.object(module.time, "monotonic", clock.monotonic), \
+ patch.object(module.time, "sleep", clock.sleep), \
+ contextlib.redirect_stdout(io.StringIO()) as out, \
+ contextlib.redirect_stderr(io.StringIO()):
+ try:
+ code = module.close(SimpleNamespace(session=directory))
+ except Exception as error:
+ code = error
+ self.assertEqual(code, 1)
+ self.assertNotIn('"closed": true', out.getvalue())
+ self.assertLessEqual(clock.now, 30.1)
+ if state == "unavailable owner":
+ self.assertGreaterEqual(clock.now, 30)
+
+
+class CDPCallTests(unittest.TestCase):
+ def test_stop_during_response_prevents_the_next_call_from_being_sent(self):
+ module, sent, stopped = recorder(), [], []
+
+ def recv():
+ stopped.append(True)
+ return json.dumps({"id": len(sent), "result": {}})
+
+ ws = SimpleNamespace(settimeout=lambda timeout: None, recv=recv,
+ send=lambda data: sent.append(json.loads(data)["method"]))
+ websocket = SimpleNamespace(create_connection=lambda *a, **kw: ws,
+ WebSocketTimeoutException=TimeoutError)
+ def check_active():
+ if stopped:
+ raise InterruptedError("stop requested")
+
+ with patch.dict("sys.modules", websocket=websocket):
+ cdp = module.CDP("fake-url")
+ cdp.before_call = check_active
+ self.assertEqual(cdp.call("Network.enable"), {})
+ with self.assertRaises(InterruptedError):
+ cdp.call("Page.enable")
+ self.assertEqual(sent, ["Network.enable"])
+
+
+class ObservationTests(unittest.TestCase):
+ def observe(self, action, completed=False, seconds=0.2, cp1252=False):
+ module, clock = recorder(), Clock()
+ with tempfile.TemporaryDirectory() as temp:
+ directory = Path(temp)
+ log = directory / "terminal.log"
+ initial = b"\x1b]0;MOVIE;1;1;0;/work\x07"
+ final = "\x1b]0;MOVIE;2;0;7;C:/René/λ\x1b\\".encode()
+ log.write_bytes(initial + (final if completed else b""))
+ (directory / "ready.json").write_text("{}")
+ module.write_json(directory / "session.json", {"pids": [1100, 1101]})
+ calls = []
+
+ def recv(timeout):
+ calls.append(timeout)
+ clock.sleep(timeout)
+ if action == "disconnect":
+ raise ConnectionError("browser connection closed")
+ if action == "session loss":
+ (directory / "ready.json").unlink(missing_ok=True)
+ if action == "prompt arrives":
+ log.write_bytes(initial + final)
+ if action == "prompt then disconnect":
+ log.write_bytes(initial + final)
+ raise ConnectionError("browser connection closed")
+ return None
+
+ cdp = SimpleNamespace(recv=recv)
+ args = SimpleNamespace(session=directory, record=None, seconds=seconds, hold=0)
+ raw = io.BytesIO()
+ out = io.TextIOWrapper(raw, encoding="cp1252" if cp1252 else "utf-8")
+ with patch.object(module.time, "monotonic", clock.monotonic), \
+ patch.object(module.time, "sleep", clock.sleep), contextlib.redirect_stdout(out):
+ code = module.observe(args, cdp, 1)
+ out.flush()
+ result = json.loads(raw.getvalue().decode("ascii" if cp1252 else "utf-8"))
+ return code, result, calls
+
+ def test_disconnected_browser_is_failure_without_recording(self):
+ code, result, calls = self.observe("disconnect")
+ self.assertEqual(code, 1)
+ self.assertEqual(result["outcome"], "failed")
+ self.assertTrue(calls)
+
+ def test_lost_terminal_session_is_failure_even_with_live_browser(self):
+ code, result, calls = self.observe("session loss")
+ self.assertEqual(code, 1)
+ self.assertEqual(result["outcome"], "failed")
+ self.assertTrue(calls)
+
+ def test_only_live_unfinished_command_returns_two(self):
+ for seconds in (0, 0.2):
+ with self.subTest(seconds=seconds):
+ code, result, calls = self.observe("alive", seconds=seconds)
+ self.assertEqual((code, result), (2, {"outcome": "running"}))
+ self.assertTrue(calls, "even an expired observation must establish liveness")
+
+ def test_new_prompt_retains_native_failure_status(self):
+ for action in ("prompt arrives", "prompt then disconnect"):
+ with self.subTest(action=action):
+ code, result, _ = self.observe(action)
+ self.assertEqual(code, 1)
+ self.assertEqual(result, {"outcome": "completed", "ok": False,
+ "exit_code": 7, "cwd": "C:/René/λ"})
+
+ def test_completed_prompt_is_not_consumed_by_later_disconnect(self):
+ code, result, _ = self.observe("disconnect", completed=True)
+ self.assertEqual((code, result["outcome"], result["exit_code"]), (1, "completed", 7))
+
+ def test_stdout_json_round_trips_non_ascii_paths_on_cp1252(self):
+ try:
+ code, result, _ = self.observe("alive", completed=True, cp1252=True)
+ except UnicodeError as error:
+ self.fail(f"stdout JSON was not portable: {error}")
+ self.assertEqual((code, result["cwd"]), (1, "C:/René/λ"))
+
+ def test_capture_disconnect_during_hold_preserves_completed_command_status(self):
+ for ok, exit_code in ((True, 0), (False, 7)):
+ with self.subTest(ok=ok), tempfile.TemporaryDirectory() as temp:
+ module, clock, writes = recorder(), Clock(), []
+ marker = f"\x1b]0;MOVIE;2;{int(ok)};{exit_code};C:/René/λ\x07".encode()
+ args = SimpleNamespace(session=Path(temp), record=Path(temp) / "take",
+ seconds=10, hold=0.6)
+ real_film = module.film
+
+ def film(*args):
+ return real_film(*args, clock=clock.monotonic, sleep=clock.sleep)
+
+ def capture(cdp):
+ if clock.now >= 0.2:
+ raise ConnectionError("capture connection closed")
+ return b"capture-token"
+
+ with patch.object(module, "film", film), \
+ patch.object(module, "screenshot", capture), \
+ patch.object(module, "tail", lambda path: marker), \
+ patch.object(Path, "write_bytes", lambda path, token: writes.append(token)), \
+ patch.object(module, "write_json", side_effect=AssertionError("failed take publication")), \
+ contextlib.redirect_stdout(io.StringIO()) as out:
+ code = module.observe(args, SimpleNamespace(), 1)
+ self.assertEqual(code, 1)
+ self.assertEqual(json.loads(out.getvalue()), {
+ "outcome": "failed", "error": "capture connection closed",
+ "ok": ok, "exit_code": exit_code, "cwd": "C:/René/λ",
+ })
+ self.assertEqual(writes, [b"capture-token"])
+
+
+class VisibleTextTests(unittest.TestCase):
+ def test_visible_prompt_survives_bel_and_st_title_markers(self):
+ module = recorder()
+ for terminator in (b"\x07", b"\x1b\\"):
+ with self.subTest(terminator=terminator):
+ log = b"\x1b[32m/work $ \x1b]0;MOVIE;2;1;0;/work" + terminator
+ self.assertTrue(module.at_prompt(log))
+ self.assertFalse(module.at_prompt(log + b"busy"))
+
+
+class CaptureTimingTests(unittest.TestCase):
+ def film(self, durations, seconds=1, hold=0, complete_at=None):
+ module, clock, writes, shots = recorder(), Clock(), [], []
+ def capture():
+ token = bytes([len(shots) + 1])
+ clock.sleep(durations[len(shots)] if len(shots) < len(durations) else 0)
+ shots.append(token)
+ return token
+ with tempfile.TemporaryDirectory() as temp, \
+ patch.object(Path, "write_bytes", lambda path, token: writes.append((path.name, token))):
+ frames = module.film(Path(temp), seconds, hold, capture,
+ lambda: complete_at is not None and clock.now >= complete_at,
+ clock.monotonic, clock.sleep)
+ self.assertEqual(len(writes), frames)
+ return frames, writes
+
+ def test_capture_crossing_hard_endpoint_fills_exactly_five_slots(self):
+ frames, writes = self.film([1.2])
+ self.assertEqual(frames, 5)
+ self.assertEqual(writes, [("f00000.png", b"\x01"), ("f00001.png", b"\x01"),
+ ("f00002.png", b"\x01"), ("f00003.png", b"\x01"),
+ ("f00004.png", b"\x01")])
+
+ def test_capture_crossing_hold_endpoint_is_bounded(self):
+ frames, writes = self.film([1.2], seconds=10, hold=0.6, complete_at=0)
+ self.assertEqual(frames, 3)
+ self.assertEqual([name for name, _ in writes], ["f00000.png", "f00001.png", "f00002.png"])
+
+ def test_mid_capture_stall_repeats_previous_token_without_missing_slots(self):
+ frames, writes = self.film([0.01, 0.5], seconds=1)
+ self.assertEqual(frames, 5)
+ self.assertEqual([token for _, token in writes], [b"\x01", b"\x02", b"\x02", b"\x03", b"\x04"])
+
+ def test_completion_and_hold_keep_the_normal_grid(self):
+ frames, _ = self.film([], seconds=10, hold=0.4, complete_at=1)
+ self.assertEqual(frames, 7)
+
+ def test_completion_without_hold_and_zero_duration_do_not_add_slots(self):
+ self.assertEqual(self.film([], seconds=10, complete_at=0)[0], 0)
+ self.assertEqual(self.film([], seconds=0)[0], 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/proving-it-works-with-a-movie/test_subtitle_contract.py b/tests/proving-it-works-with-a-movie/test_subtitle_contract.py
new file mode 100644
index 0000000..937d574
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_subtitle_contract.py
@@ -0,0 +1,290 @@
+"""Text and mocked-boundary contracts; this module never processes media."""
+
+import io
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from contextlib import redirect_stderr, redirect_stdout
+from pathlib import Path
+from unittest.mock import patch
+
+import fixtures
+
+
+def milliseconds(timestamp):
+ hours, minutes, rest = timestamp.split(":")
+ seconds, millis = rest.split(",")
+ return ((int(hours) * 60 + int(minutes)) * 60 + int(seconds)) * 1000 + int(millis)
+
+
+def read_cues(path):
+ cues = []
+ for block in path.read_text(encoding="utf-8").strip().split("\n\n"):
+ if block:
+ _, timing, text = block.split("\n", 2)
+ start, end = map(milliseconds, timing.split(" --> "))
+ cues.append((start, end, " ".join(text.split())))
+ return cues
+
+
+class SubtitleTimingContract(unittest.TestCase):
+ def setUp(self):
+ self.module = fixtures.load_script("make-subtitles")
+ temporary = tempfile.TemporaryDirectory()
+ self.addCleanup(temporary.cleanup)
+ self.root = Path(temporary.name)
+ self.output = self.root / "captions.srt"
+
+ def subtitles(self, entries, *options):
+ manifest = self.root / "manifest.json"
+ manifest.write_text(json.dumps(entries), encoding="utf-8")
+ stdout = io.StringIO()
+ with patch.object(sys, "argv", ["make-subtitles", str(manifest), str(self.output), *options]), \
+ redirect_stdout(stdout):
+ self.assertEqual(self.module.main(), 0)
+ return read_cues(self.output), stdout.getvalue()
+
+ def assert_scene(self, cues, start, end, text):
+ self.assertTrue(cues)
+ self.assertEqual(cues[0][0], start)
+ self.assertEqual(cues[-1][1], end)
+ previous = start
+ for a, b, _ in cues:
+ self.assertEqual(a, previous)
+ self.assertLess(a, b)
+ self.assertLessEqual(b, end)
+ previous = b
+ self.assertEqual(" ".join(cue[2] for cue in cues).split(), text.split())
+
+ def test_five_chunks_fit_half_second_before_next_scene(self):
+ text = "one two six ten red"
+ cues, report = self.subtitles([
+ {"id": "short", "duration": 0.5, "text": text},
+ {"id": "next", "duration": 1, "text": "next"},
+ ], "--max-chars", "3")
+ self.assert_scene(cues[:-1], 0, 500, text)
+ self.assertEqual(cues[-1], (500, 1500, "next"))
+ self.assertIn("ends at 00:00:01,500", report)
+
+ def test_one_word_covers_twelve_second_scene(self):
+ cues, report = self.subtitles([{"id": "held", "duration": 12, "text": "Held"}])
+ self.assert_scene(cues, 0, 12000, "Held")
+ self.assertIn("ends at 00:00:12,000", report)
+
+ def test_mixed_chunks_get_proportional_time(self):
+ cues, _ = self.subtitles([{"id": "mix", "duration": 1, "text": "a bbbbbbbbb"}], "--max-chars", "9")
+ self.assertEqual(cues, [(0, 100, "a"), (100, 1000, "bbbbbbbbb")])
+
+ def test_max_seconds_guides_splitting_without_losing_tail_or_words(self):
+ text = "one two six ten red cat dog fox"
+ cues, _ = self.subtitles([{"id": "long", "duration": 12, "text": text}], "--max-secs", "3")
+ self.assert_scene(cues, 0, 12000, text)
+ self.assertGreater(len(cues), 1)
+ self.assertTrue(all(b - a <= 3000 for a, b, _ in cues))
+
+ def test_max_seconds_refines_unequal_chunks_against_allocated_time(self):
+ text = "ab cde f ghi"
+ cues, report = self.subtitles([
+ {"id": "unequal", "duration": 6, "text": text},
+ ], "--max-secs", "3")
+ self.assert_scene(cues, 0, 6000, text)
+ self.assertTrue(all(b - a <= 3000 for a, b, _ in cues), cues)
+ self.assertIn("ends at 00:00:06,000", report)
+
+ def test_chunks_coalesce_to_fit_representable_milliseconds(self):
+ text = "one two six ten red"
+ cues, report = self.subtitles([{"id": "tiny", "duration": 0.002, "text": text}], "--max-chars", "3")
+ self.assert_scene(cues, 0, 2, text)
+ self.assertLessEqual(len(cues), 2)
+ self.assertIn("ends at 00:00:00,002", report)
+
+ def test_unrepresentable_max_seconds_preserves_positive_cues_and_all_words(self):
+ text = "one two six ten red"
+ cues, _ = self.subtitles([
+ {"id": "tiny", "duration": 0.002, "text": text},
+ ], "--max-secs", "0.0001")
+ self.assert_scene(cues, 0, 2, text)
+ self.assertLessEqual(len(cues), 2)
+
+ def test_submillisecond_scene_can_use_its_rounded_interval(self):
+ cues, _ = self.subtitles([{"id": "tiny", "duration": 0.0008, "text": "one two"}])
+ self.assert_scene(cues, 0, 1, "one two")
+
+ def test_invalid_or_unrepresentable_duration_fails_before_writing_srt(self):
+ for duration in (0, -1, 0.0001, float("nan"), float("inf"), "invalid"):
+ with self.subTest(duration=duration), redirect_stderr(io.StringIO()):
+ with self.assertRaises(SystemExit) as caught:
+ self.subtitles([{"id": "invalid", "duration": duration, "text": "words"}])
+ self.assertNotEqual(caught.exception.code, 0)
+ self.assertFalse(self.output.exists())
+
+ def test_invalid_readability_limits_fail_clearly(self):
+ for option, value in (("--max-chars", "0"), ("--max-secs", "0"),
+ ("--max-secs", "nan"), ("--max-secs", "inf")):
+ with self.subTest(option=option, value=value), redirect_stderr(io.StringIO()):
+ with self.assertRaises(SystemExit) as caught:
+ self.subtitles([{"id": "scene", "duration": 1, "text": "words"}], option, value)
+ self.assertNotEqual(caught.exception.code, 0)
+
+ def test_manual_offset_uses_rounded_scene_boundaries(self):
+ cues, report = self.subtitles([{"id": "shifted", "duration": 0.5004, "text": "one two six"}], "--offsets", "shifted=2.1254", "--max-chars", "3")
+ self.assert_scene(cues, 2125, 2626, "one two six")
+ self.assertIn("ends at 00:00:02,626", report)
+
+ def test_empty_cut_and_unknown_offset_keys_do_not_introduce_cues(self):
+ for mapping in ({}, {"unknown": 2}):
+ with self.subTest(mapping=mapping):
+ offsets = self.root / "offsets.json"
+ offsets.write_text(json.dumps(mapping), encoding="utf-8")
+ cues, report = self.subtitles([{"id": "excluded", "duration": 1, "text": "excluded"}], "--offsets-json", str(offsets), "--offsets", "excluded=5")
+ self.assertEqual(cues, [])
+ self.assertIn("0 cues, ends at 00:00:00,000", report)
+
+
+class SubtitleTrackContract(unittest.TestCase):
+ def test_supplied_track_replaces_existing_subtitles_with_optional_audio(self):
+ module = fixtures.load_script("burn-subtitles")
+ for soft in (True, False):
+ for source_audio in (True, False):
+ with self.subTest(soft=soft, source_audio=source_audio), tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ movie, subs = root / "movie.mp4", root / "new.srt"
+ movie.write_bytes(b"source token")
+ subs.write_text("1\n00:00:00,000 --> 00:00:01,000\nNew caption\n", encoding="utf-8")
+ selections = []
+ source = {"v:0": "source video", "s:0": "old caption"}
+ if source_audio:
+ source["a:0"] = "source audio"
+ inputs = [source, {"s:0": "new caption"}]
+
+ def encode(command, **kwargs):
+ if "-vf" in command:
+ return False
+ maps = [command[i + 1] for i, arg in enumerate(command) if arg == "-map"]
+ self.assertEqual(maps, ["0:v:0", "0:a?", "1:s:0"])
+ for spec in maps:
+ index, kind = spec.rstrip("?").split(":", 1)
+ selected = [value for stream, value in inputs[int(index)].items() if stream == kind or stream.startswith(kind + ":")]
+ if not spec.endswith("?"):
+ self.assertTrue(selected)
+ selections.extend(selected)
+ return True
+
+ stdout, stderr = io.StringIO(), io.StringIO()
+ argv = ["burn-subtitles", str(movie), str(subs), str(root / "out.mp4")]
+ if soft:
+ argv.append("--soft")
+ with patch.object(sys, "argv", argv), \
+ patch.object(module.shutil, "which", return_value="mock-ffmpeg"), \
+ patch.object(module, "has_libass", return_value=True), \
+ patch.object(module, "run", side_effect=encode), \
+ redirect_stdout(stdout), \
+ redirect_stderr(stderr):
+ self.assertEqual(module.main(), 0)
+ self.assertEqual(selections, ["source video", *(["source audio"] if source_audio else []), "new caption"])
+ self.assertNotIn("no libass", stdout.getvalue())
+ self.assertEqual("burn failed" in stderr.getvalue(), not soft)
+
+
+class SubtitleParserContract(unittest.TestCase):
+ def setUp(self):
+ self.module = fixtures.load_script("check-movie")
+
+ def test_literal_arrow_in_caption_is_not_a_timing_line(self):
+ self.assertEqual(self.module.subtitle_end("1\n00:00:00,000 --> 00:00:01,250\nFollow source --> destination.\n"), 1.25)
+
+ def test_timestamp_shaped_caption_cannot_extend_coverage(self):
+ text = "1\n00:00:00,000 --> 00:00:01,250\n00:00:00,000 --> 00:59:00,000\n"
+ self.assertEqual(self.module.subtitle_end(text), 1.25)
+
+ def test_malformed_actual_timing_is_rejected(self):
+ for timing in ("00:00:00,000 --> invalid", "not a timing line", "00:00:00,000 --> 00:99:00,000"):
+ with self.subTest(timing=timing), self.assertRaises(ValueError):
+ self.module.subtitle_end(f"1\n{timing}\ncaption\n")
+
+ def test_empty_subtitles_have_no_end(self):
+ self.assertIsNone(self.module.subtitle_end("\n \n"))
+
+ def test_multiple_cues_keep_existing_latest_end_policy(self):
+ text = "1\n00:00:00,000 --> 00:00:10,250\nFirst\n\n2\n00:00:05,000 --> 00:00:06,000\nSecond\n"
+ self.assertEqual(self.module.subtitle_end(text), 10.25)
+
+
+class SubtitleHandoffContract(unittest.TestCase):
+ def test_rerun_removed_opening_narration_keeps_evidence_and_retimes_remaining_caption(self):
+ narrate = fixtures.load_script("narrate")
+ assemble = fixtures.load_script("assemble")
+ subtitles = fixtures.load_script("make-subtitles")
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ scenes_file, narration = root / "scenes.yaml", root / "narration"
+ work, movie, srt = root / "segments", root / "movie.mp4", root / "movie.srt"
+ (root / "still.png").write_bytes(b"still token")
+ scenes = [
+ {"id": "opening", "kind": "image", "src": "still.png", "duration": 2, "narration": "Opening words"},
+ {"id": "body", "kind": "image", "src": "still.png", "duration": 12, "narration": "Body"},
+ ]
+ encoded_inputs = []
+
+ def synthesize(text, wav, voice):
+ wav.write_bytes(text.encode("utf-8"))
+
+ def measure(path):
+ if path.name.startswith(".opening") or path.name == "opening.wav":
+ return 1.0
+ return 12.0
+
+ def media_command(command, **kwargs):
+ if command[0] == "ffprobe":
+ path = Path(command[-1])
+ seconds = {"opening.mp4": 2.375, "body.mp4": 12.0, "movie.mp4": 14.375}.get(path.name)
+ if seconds is None:
+ seconds = measure(path)
+ return subprocess.CompletedProcess(command, 0, str(seconds), "")
+ self.assertEqual(command[0], "ffmpeg")
+ encoded_inputs.append([command[i + 1] for i, item in enumerate(command) if item == "-i"])
+ Path(command[-1]).write_bytes(b"encoded token")
+ return subprocess.CompletedProcess(command, 0, "", "")
+
+ for rerun in (False, True):
+ if rerun:
+ del scenes[0]["narration"]
+ scenes_file.write_text(json.dumps({"scenes": scenes}), encoding="utf-8")
+ with patch.object(sys, "argv", ["narrate", str(scenes_file), str(narration), "--engine", "piper", "--verify", "off"]), \
+ patch.object(narrate.shutil, "which", return_value="mock-tool"), \
+ patch.object(narrate, "openai_key", return_value=None), \
+ patch.object(narrate, "say_piper", side_effect=synthesize), \
+ patch.object(narrate, "duration", side_effect=measure), \
+ redirect_stdout(io.StringIO()), \
+ redirect_stderr(io.StringIO()):
+ self.assertEqual(narrate.main(), 0)
+ manifest = json.loads((narration / "manifest.json").read_text(encoding="utf-8"))
+ self.assertEqual([entry["id"] for entry in manifest], ["body"] if rerun else ["opening", "body"])
+ self.assertEqual(manifest[-1]["text"], "Body")
+ self.assertEqual(manifest[-1]["duration"], 12)
+ self.assertEqual(manifest[-1]["wav"], "body.wav")
+ self.assertEqual((narration / "opening.wav").read_bytes(), b"Opening words")
+ encoded_inputs.clear()
+ with patch.object(sys, "argv", ["assemble", str(scenes_file), str(movie), "--narration", str(narration), "--work", str(work)]), \
+ patch.object(assemble.shutil, "which", return_value="mock-tool"), \
+ patch.object(assemble, "find_browser", return_value=None), \
+ patch.object(assemble, "run", side_effect=media_command), \
+ redirect_stdout(io.StringIO()):
+ self.assertEqual(assemble.main(), 0)
+ offsets = json.loads((work / "offsets.json").read_text(encoding="utf-8"))
+ self.assertEqual(offsets, {"body": 2.375} if rerun else {"opening": 0.0, "body": 2.375})
+ self.assertEqual(str(narration / "opening.wav") in encoded_inputs[0], not rerun)
+ self.assertIn(str(narration / "body.wav"), encoded_inputs[1])
+ with patch.object(sys, "argv", ["make-subtitles", str(narration / "manifest.json"), str(srt), "--offsets-json", str(work / "offsets.json")]), \
+ redirect_stdout(io.StringIO()):
+ self.assertEqual(subtitles.main(), 0)
+ expected = [(2375, 14375, "Body")]
+ if not rerun:
+ expected.insert(0, (0, 1000, "Opening words"))
+ self.assertEqual(read_cues(srt), expected)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/proving-it-works-with-a-movie/test_subtitles.py b/tests/proving-it-works-with-a-movie/test_subtitles.py
new file mode 100644
index 0000000..3f76abf
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_subtitles.py
@@ -0,0 +1,162 @@
+import tempfile
+import json
+import sys
+import io
+from contextlib import redirect_stderr, redirect_stdout
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import fixtures
+
+
+class SubtitlePathRegression(unittest.TestCase):
+ def test_hard_burn_runs_from_safe_directory_with_absolute_media(self):
+ module = fixtures.load_script("burn-subtitles")
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory) / "movie O'Brien λ"
+ root.mkdir()
+ movie, subs, output = root / "in.mp4", root / "nested" / "captions.srt", root / "out.mp4"
+ subs.parent.mkdir()
+ movie.write_bytes(b"movie")
+ subs.write_bytes("\ufeff1\r\n00:00:00,000 --> 00:00:01,000\r\nλ\r\n".encode("utf-8"))
+ calls = []
+
+ def fake_run(cmd, *, cwd=None):
+ calls.append((cmd, cwd))
+ if cwd is not None:
+ self.assertEqual((cwd / "captions.srt").read_bytes(), subs.read_bytes())
+ return True
+
+ stdout, stderr = io.StringIO(), io.StringIO()
+ with patch.object(sys, "argv", ["burn-subtitles", str(movie), str(subs), str(output)]), patch.object(module.shutil, "which", return_value="ffmpeg"), patch.object(module, "has_libass", return_value=True), patch.object(module, "run", side_effect=fake_run), redirect_stdout(stdout), redirect_stderr(stderr):
+ self.assertEqual(module.main(), 0)
+ self.assertIn("burned into the picture", stdout.getvalue())
+ command, cwd = calls[0]
+ self.assertEqual(cwd.name.startswith("movie-subtitles-"), True)
+ self.assertIn(str(movie.resolve()), command)
+ self.assertIn(str(output.resolve()), command)
+ self.assertTrue(any(value.startswith("subtitles=filename=captions.srt") for value in command))
+ self.assertFalse(cwd.exists())
+
+ def test_burn_failure_is_reported_separately_from_missing_libass(self):
+ module = fixtures.load_script("burn-subtitles")
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ movie, subs, output = root / "in.mp4", root / "captions.srt", root / "out.mp4"
+ movie.write_bytes(b"movie")
+ subs.write_text("1\n00:00:00,000 --> 00:00:01,000\ncaption\n", encoding="utf-8")
+ stdout, stderr = io.StringIO(), io.StringIO()
+ with patch.object(sys, "argv", ["burn-subtitles", str(movie), str(subs), str(output)]), patch.object(module.shutil, "which", return_value="ffmpeg"), patch.object(module, "has_libass", return_value=True), patch.object(module, "run", return_value=False), redirect_stdout(stdout), redirect_stderr(stderr):
+ self.assertEqual(module.main(), 1)
+ self.assertIn("burn failed", stderr.getvalue())
+
+class SubtitleOffsetRegression(unittest.TestCase):
+ def subtitles(self, *manual, offsets=None):
+ module = fixtures.load_script("make-subtitles")
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ manifest, output = root / "manifest.json", root / "captions.srt"
+ manifest.write_text(json.dumps([
+ {"id": scene, "text": scene, "duration": 1.0}
+ for scene in ("intro", "body", "end")
+ ]), encoding="utf-8")
+ argv = ["make-subtitles", str(manifest), str(output)]
+ if offsets is not None:
+ path = root / "offsets.json"
+ path.write_text(json.dumps(offsets), encoding="utf-8")
+ argv += ["--offsets-json", str(path)]
+ if manual:
+ argv += ["--offsets", *manual]
+ with patch.object(sys, "argv", argv), redirect_stdout(io.StringIO()):
+ self.assertEqual(module.main(), 0)
+ cues = []
+ for block in output.read_text(encoding="utf-8").strip().split("\n\n"):
+ if not block:
+ continue
+ _, timing, text = block.split("\n", 2)
+ times = []
+ for timestamp in timing.split(" --> "):
+ h, m, s = timestamp.replace(",", ".").split(":")
+ times.append(int(h) * 3600 + int(m) * 60 + float(s))
+ cues.append((*times, text))
+ return cues
+
+ def test_default_scenes_run_back_to_back(self):
+ self.assertEqual(self.subtitles(),
+ [(0, 1, "intro"), (1, 2, "body"), (2, 3, "end")])
+
+ def test_partial_manual_offsets_preserve_other_scenes(self):
+ self.assertEqual(self.subtitles("intro=2"),
+ [(2, 3, "intro"), (3, 4, "body"), (4, 5, "end")])
+ self.assertEqual(self.subtitles("body=4"),
+ [(0, 1, "intro"), (4, 5, "body"), (5, 6, "end")])
+
+ def test_assembly_offsets_select_scenes_in_the_cut(self):
+ self.assertEqual(self.subtitles(offsets={"intro": 2, "end": 8}),
+ [(2, 3, "intro"), (8, 9, "end")])
+
+ def test_manual_offsets_change_timing_without_changing_cut_membership(self):
+ self.assertEqual(self.subtitles("intro=3", "body=5", offsets={"intro": 2, "end": 8}),
+ [(3, 4, "intro"), (8, 9, "end")])
+
+ def test_cut_without_narrated_scenes_has_no_cues(self):
+ for offsets in ({}, {"silent": 2}):
+ with self.subTest(offsets=offsets):
+ self.assertEqual(self.subtitles(offsets=offsets), [])
+
+
+class SubtitleIntegrationRegression(unittest.TestCase):
+ def test_bom_manifest_and_offsets_write_utf8_under_legacy_console(self):
+ import json
+ import os
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ manifest, offsets, out = root / "manifest.json", root / "offsets.json", root / "λ.srt"
+ manifest.write_bytes(('\ufeff' + json.dumps([{"id": "clip", "text": "Unicode λ café", "duration": 1}], ensure_ascii=False) + '\r\n').encode('utf-8'))
+ offsets.write_text('{"clip": 2}', encoding="utf-8-sig")
+ env = dict(os.environ, PYTHONIOENCODING="cp1252", PYTHONUTF8="0")
+ result = fixtures.run_tool("make-subtitles", [str(manifest), str(out), "--offsets-json", str(offsets)], cwd=root, env=env)
+ self.assertEqual(result.returncode, 0, fixtures.output_text(result))
+ text = out.read_text(encoding="utf-8")
+ self.assertIn("Unicode λ café", text)
+ self.assertIn("00:00:02,000 --> 00:00:03,000", text)
+ self.assertFalse(out.read_bytes().startswith(b'\xef\xbb\xbf'))
+
+ def test_hard_subtitles_are_pixels_in_nested_special_path(self):
+ import subprocess
+ missing = fixtures.missing_executables("uv", "ffmpeg")
+ if missing:
+ self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}")
+ module = fixtures.load_script("burn-subtitles")
+ if not module.has_libass():
+ self.skipTest("libass FFmpeg is required for hard subtitle pixels")
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory) / "O'Brien λ # %"
+ root.mkdir()
+ movie, subs, output = root / "in.mp4", root / "nested" / "O'Brien.srt", root / "out.mp4"
+ subs.parent.mkdir()
+ subs.write_bytes("\ufeff1\r\n00:00:00,000 --> 00:00:01,000\r\nVisible caption\r\n".encode("utf-8"))
+ fixtures._run_ffmpeg(["-f", "lavfi", "-i", "color=c=black:s=640x360:d=1", "-c:v", "libx264", str(movie)], cwd=root)
+ result = fixtures.run_tool("burn-subtitles", [str(movie), str(subs), str(output)], cwd=Path(directory))
+ self.assertEqual(result.returncode, 0, fixtures.output_text(result))
+ self.assertIn("burned into the picture", fixtures.output_text(result))
+ frame = subprocess.run(["ffmpeg", "-v", "error", "-ss", "0.5", "-i", str(output), "-frames:v", "1", "-pix_fmt", "gray", "-f", "rawvideo", "-"], capture_output=True, check=True).stdout
+ self.assertGreater(sum(value > 180 for value in frame), 50)
+
+ def test_burn_failure_fallback_does_not_claim_missing_libass(self):
+ module = fixtures.load_script("burn-subtitles")
+ for libass in (True, False):
+ with self.subTest(libass=libass), tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ movie, subs = root / "in.mp4", root / "in.srt"
+ movie.touch(); subs.touch()
+ stdout, stderr = io.StringIO(), io.StringIO()
+ with patch.object(sys, "argv", ["burn-subtitles", str(movie), str(subs), str(root / "out.mp4")]), patch.object(module.shutil, "which", return_value="ffmpeg"), patch.object(module, "has_libass", return_value=libass), patch.object(module, "run", side_effect=[False, True] if libass else [True]), redirect_stdout(stdout), redirect_stderr(stderr):
+ self.assertEqual(module.main(), 0)
+ self.assertEqual("no libass" in stdout.getvalue(), not libass)
+ self.assertEqual("burn failed" in stderr.getvalue(), libass)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/proving-it-works-with-a-movie/test_terminal.py b/tests/proving-it-works-with-a-movie/test_terminal.py
new file mode 100644
index 0000000..9820094
--- /dev/null
+++ b/tests/proving-it-works-with-a-movie/test_terminal.py
@@ -0,0 +1,316 @@
+"""The terminal recorder: prompt parsing and the frame grid anywhere; a real
+ttyd session wherever ttyd and a Chrome-family browser exist."""
+import importlib.util
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import fixtures
+
+SCRIPT = Path(__file__).resolve().parents[2] / "skills/proving-it-works-with-a-movie/examples/film-terminal.py"
+FIXTURE = Path(__file__).resolve().with_name("fixtures") / "terminal_app.py"
+TTYD = os.environ.get("MOVIE_TEST_TTYD") or shutil.which("ttyd")
+BROWSER = fixtures.load_script("browser_tools").find_browser(os.environ.get("MOVIE_TEST_BROWSER"))
+SHELL = os.environ.get("MOVIE_TEST_SHELL") or ("powershell51" if os.name == "nt" else "bash")
+BASH = SHELL in ("bash", "gitbash")
+
+
+def recorder():
+ spec = importlib.util.spec_from_file_location("film_terminal", SCRIPT)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def gone(pid, timeout=5):
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ if os.name == "nt":
+ listed = subprocess.run(["tasklist", "/FI", f"PID eq {pid}", "/NH"],
+ capture_output=True, text=True).stdout
+ if str(pid) not in listed:
+ return True
+ else:
+ try:
+ os.kill(pid, 0)
+ except ProcessLookupError:
+ return True
+ time.sleep(0.1)
+ return False
+
+
+class PromptTests(unittest.TestCase):
+ def test_prompts_parse_both_terminators_and_paths_with_semicolons(self):
+ module = recorder()
+ log = (b"noise\x1b]0;MOVIE;1;1;;C:\\a;b\x07\x1b[0m"
+ b"\x1b]2;MOVIE;2;0;7;/c/x\x1b\\tail"
+ b"\x1b]0;MOVIE;3;1;0;/home/me\x07")
+ self.assertEqual(module.prompts(log), [
+ dict(n=1, ok=True, exit_code=None, cwd="C:\\a;b"),
+ dict(n=2, ok=False, exit_code=7, cwd="/c/x"),
+ dict(n=3, ok=True, exit_code=0, cwd="/home/me"),
+ ])
+ self.assertEqual(module.prompts(b"\x1b]0;something else\x07"), [])
+
+ def test_prompt_install_is_one_typed_line_per_shell(self):
+ module = recorder()
+ cwd = Path("C:/Users/x/movie O'Brien λ")
+ for kind in module.SHELLS:
+ line = module.prompt_command(kind, cwd)
+ self.assertEqual(len(line.splitlines()), 1, kind)
+ self.assertNotIn("MOVIE;", line, "the marker text must not be echoed by the install line")
+ self.assertIn("Brien λ", module.prompt_script(kind, cwd), "the script enters the cwd")
+
+ def test_keys_are_named_or_single_characters(self):
+ module = recorder()
+ self.assertEqual(module.key_params("Ctrl-C")["modifiers"], 2)
+ self.assertEqual(module.key_params("Enter")["text"], "\r")
+ self.assertEqual(module.key_params("q"), dict(key="q", text="q"))
+ with self.assertRaises(SystemExit):
+ module.key_params("Bogus")
+
+
+class FilmGridTests(unittest.TestCase):
+ def test_filming_refuses_a_nonempty_take_without_changing_its_contents(self):
+ module = recorder()
+ with tempfile.TemporaryDirectory() as directory:
+ out = Path(directory) / "take"
+ out.mkdir()
+ (out / "f00000.png").write_bytes(b"old frame")
+ (out / "notes.txt").write_bytes(b"sentinel evidence")
+ before = {path.name: path.read_bytes() for path in out.iterdir()}
+ captures = []
+
+ with self.assertRaisesRegex(SystemExit, "not empty.*new take directory"):
+ module.film(out, seconds=1, hold=0,
+ capture=lambda: captures.append(True) or b"new frame",
+ finished=lambda: True)
+
+ self.assertEqual(captures, [])
+ self.assertEqual(
+ {path.name: path.read_bytes() for path in out.iterdir()}, before
+ )
+
+ def test_a_slow_capture_repeats_the_previous_frame_and_filming_holds_after_the_prompt(self):
+ module = recorder()
+ clock = {"now": 0.0}
+ shots = []
+
+ def capture():
+ shots.append(len(shots) + 1)
+ clock["now"] += 0.5 if len(shots) == 2 else 0.01 # the second screenshot stalls
+ return bytes([len(shots)])
+
+ with tempfile.TemporaryDirectory() as directory:
+ out = Path(directory) / "take"
+ frames = module.film(out, seconds=10, hold=0.4, capture=capture,
+ finished=lambda: clock["now"] >= 1.0,
+ clock=lambda: clock["now"],
+ sleep=lambda s: clock.__setitem__("now", clock["now"] + s))
+ files = sorted(out.glob("f*.png"))
+ self.assertEqual([f.name for f in files], [f"f{i:05d}.png" for i in range(frames)])
+ self.assertEqual(files[2].read_bytes(), files[1].read_bytes(), "missed slot repeats the last frame")
+ self.assertNotEqual(files[3].read_bytes(), files[2].read_bytes())
+ # Captures add 0.01 s and sleeps 0.02 s, so the clock sits on odd
+ # hundredths: the prompt is seen at 1.01 s and the hold ends at 1.41 s.
+ self.assertEqual(frames, 8, "slots 0.0 s through 1.4 s fall before the 1.41 s endpoint")
+
+ def test_filming_stops_at_the_deadline_while_the_command_runs(self):
+ module = recorder()
+ clock = {"now": 0.0}
+ with tempfile.TemporaryDirectory() as directory:
+ frames = module.film(Path(directory), seconds=1.0, hold=5, capture=lambda: b"png",
+ finished=lambda: False, clock=lambda: clock["now"],
+ sleep=lambda s: clock.__setitem__("now", clock["now"] + s))
+ self.assertEqual(frames, 5)
+
+
+class ServeArgumentTests(unittest.TestCase):
+ def test_serve_refuses_a_missing_cwd_before_launching_anything(self):
+ with tempfile.TemporaryDirectory() as directory:
+ result = subprocess.run([sys.executable, str(SCRIPT), "serve", str(Path(directory) / "session"),
+ "--shell", "bash", "--cwd", str(Path(directory) / "missing")],
+ capture_output=True, text=True, timeout=60)
+ self.assertEqual(result.returncode, 1)
+ self.assertIn("--cwd is not a directory", result.stderr)
+ self.assertFalse((Path(directory) / "session" / "session.json").exists())
+
+ def test_recording_verbs_refuse_a_nonempty_take_before_session_side_effects(self):
+ module = recorder()
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ session = root / "session"
+ session.mkdir()
+ (session / "ready.json").write_text("{}", encoding="utf-8")
+ (session / "session.json").write_text(
+ json.dumps({"debug_port": 1}), encoding="utf-8"
+ )
+ (session / "terminal.log").write_bytes(
+ b"\x1b]0;MOVIE;1;1;0;/tmp\x07"
+ )
+ for verb, positional in (("run", ["echo hello"]),
+ ("key", ["Enter"]),
+ ("watch", [])):
+ with self.subTest(verb=verb):
+ take = root / verb
+ take.mkdir()
+ (take / "f00000.png").write_bytes(b"old frame")
+ (take / "sentinel.txt").write_bytes(b"keep me")
+ before = {path.name: path.read_bytes() for path in take.iterdir()}
+ argv = ["film-terminal", verb, str(session), *positional,
+ "--record", str(take)]
+ with patch.object(sys, "argv", argv), \
+ patch.object(module, "connect",
+ side_effect=AssertionError("connected")), \
+ patch.object(module, "type_text",
+ side_effect=AssertionError("typed")), \
+ patch.object(module, "press",
+ side_effect=AssertionError("pressed")), \
+ self.assertRaisesRegex(SystemExit,
+ "not empty.*new take directory"):
+ module.main()
+ self.assertEqual(
+ {path.name: path.read_bytes() for path in take.iterdir()}, before
+ )
+
+
+@unittest.skipUnless(TTYD and BROWSER, "ttyd and a Chrome-family browser are required")
+class SessionTests(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory(prefix="movie-terminal-", ignore_cleanup_errors=True)
+ self.addCleanup(self.tmp.cleanup)
+ self.work = Path(self.tmp.name) / "movie O'Brien λ"
+ self.work.mkdir()
+ self.session = Path(self.tmp.name) / "session"
+ self.log = (Path(self.tmp.name) / "serve.log").open("wb")
+ self.addCleanup(self.log.close)
+ argv = [sys.executable, str(SCRIPT), "serve", str(self.session), "--shell", SHELL,
+ "--cwd", str(self.work), "--ttyd", TTYD, "--browser", BROWSER]
+ if os.environ.get("MOVIE_TEST_SHELL_EXE"):
+ argv += ["--shell-exe", os.environ["MOVIE_TEST_SHELL_EXE"]]
+ self.owned_pids = []
+ self.serve = subprocess.Popen(argv, stdout=self.log, stderr=subprocess.STDOUT)
+ self.addCleanup(self.close_session)
+ deadline = time.monotonic() + 45
+ while not (self.session / "ready.json").exists() and self.serve.poll() is None \
+ and time.monotonic() < deadline:
+ time.sleep(0.1)
+ if (self.session / "session.json").exists():
+ self.owned_pids = json.loads((self.session / "session.json").read_text(encoding="utf-8"))["pids"]
+ if not (self.session / "ready.json").exists():
+ report = "".join(f"--- {name}\n" + path.read_text(errors="replace") if path.exists() else ""
+ for name, path in (("serve.log", Path(self.tmp.name) / "serve.log"),
+ ("ttyd.log", self.session / "ttyd.log"),
+ ("browser.log", self.session / "browser.log")))
+ self.fail(report)
+
+ def close_session(self):
+ if self.serve.poll() is None:
+ # A failed setup may not have session metadata yet, but the
+ # owner still needs its stop request before we wait for cleanup.
+ self.session.mkdir(parents=True, exist_ok=True)
+ (self.session / "stop").write_text("", encoding="utf-8")
+ try:
+ self.serve.wait(35)
+ except subprocess.TimeoutExpired:
+ self.serve.kill()
+ self.serve.wait()
+ for pid in self.owned_pids:
+ self.assertTrue(gone(pid), f"pid {pid} survived close")
+
+ def cli(self, *args, timeout=120):
+ result = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, timeout=timeout)
+ return result.returncode, result.stdout.decode("utf-8", "replace"), result.stderr.decode("utf-8", "replace")
+
+ def run_command(self, command, *extra):
+ code, out, err = self.cli("run", str(self.session), command, *extra)
+ self.assertTrue(out.strip(), err)
+ return code, json.loads(out.strip().splitlines()[-1])
+
+ def quoted(self, *words):
+ if BASH:
+ return " ".join(shlex.quote(w.replace("\\", "/")) for w in words)
+ return "& " + " ".join("'" + w.replace("'", "''") + "'" for w in words)
+
+ def native(self, code):
+ return self.quoted(sys.executable) + f' -c "{code}"'
+
+ def test_commands_report_status_and_the_shell_persists_between_calls(self):
+ code, result = self.run_command("echo hello")
+ self.assertEqual((code, result["outcome"], result["ok"]), (0, "completed", True), result)
+ self.assertTrue(result["cwd"].endswith("movie O'Brien λ"), result["cwd"])
+ if BASH:
+ set_value, check_value, failing = "MOVIE_VALUE=kept", 'test "$MOVIE_VALUE" = kept', "false"
+ else:
+ set_value = "$global:MovieValue = 'kept'"
+ check_value = "if ($global:MovieValue -ne 'kept') { throw 'lost' }"
+ failing = "Get-Item 'Z:\\nowhere'"
+ self.assertEqual(self.run_command(set_value)[0], 0)
+ self.assertEqual(self.run_command(check_value)[0], 0, "state must survive separate calls")
+ code, result = self.run_command(failing)
+ self.assertEqual((code, result["ok"]), (1, False), result)
+ code, result = self.run_command(self.native("import sys; sys.exit(7)"))
+ self.assertEqual((code, result["ok"], result["exit_code"]), (1, False, 7), result)
+
+ def test_a_tui_is_filmed_across_two_takes_and_a_long_command_across_calls(self):
+ from PIL import Image
+
+ take_one, take_two, take_three = (self.work / name for name in ("take-one", "take-two", "take-three"))
+ code, result = self.run_command(self.quoted(sys.executable, str(FIXTURE)),
+ "--record", str(take_one), "--seconds", "7")
+ self.assertEqual((code, result["outcome"]), (2, "running"), result)
+ frames = sorted(take_one.glob("f*.png"))
+ self.assertGreaterEqual(len(frames), 30, "7 s at 5 fps")
+ seen = []
+ for frame in frames:
+ with Image.open(frame) as image:
+ r, g, b = image.convert("RGB").getpixel((300, 120))
+ color = ("red" if r > 150 and g < 100 and b < 100 else
+ "green" if g > 120 and r < 100 and b < 140 else
+ "blue" if b > 150 and r < 100 and g < 140 else None)
+ if color and (not seen or seen[-1] != color):
+ seen.append(color)
+ self.assertEqual(seen, ["red", "green", "blue"], "the three TUI states, in order, in the automatic frames")
+ code, out, err = self.cli("key", str(self.session), "q", "--record", str(take_two), "--seconds", "10")
+ result = json.loads(out.strip().splitlines()[-1])
+ self.assertEqual((code, result["outcome"], result["ok"]), (0, "completed", True), (result, err))
+ self.assertGreaterEqual(result["frames"], 7, "the exit plus the 1.5 s hold")
+ self.assertEqual(len(json.loads((self.work / "states.json").read_text())), 3)
+
+ code, result = self.run_command(self.native("import time; time.sleep(3)"), "--seconds", "1")
+ self.assertEqual(result["outcome"], "running")
+ code, out, err = self.cli("watch", str(self.session), "--record", str(take_three), "--seconds", "15")
+ result = json.loads(out.strip().splitlines()[-1])
+ self.assertEqual((code, result["outcome"], result["ok"]), (0, "completed", True), (result, err))
+ self.assertGreaterEqual(result["frames"], 10, "about 2 s of waiting plus the hold")
+ self.assertEqual(result["scene"], {"kind": "frames", "src": str(take_three.resolve()), "rate": 5})
+
+ def test_close_kills_the_shell_tree_and_spares_unrelated_processes(self):
+ sentinel = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"])
+ self.addCleanup(sentinel.kill)
+ tree = self.work / "tree"
+ code, result = self.run_command(self.quoted(sys.executable, str(FIXTURE), "tree", str(tree)), "--seconds", "2")
+ self.assertEqual(result["outcome"], "running")
+ deadline = time.monotonic() + 20
+ while len(list(tree.glob("*.json"))) < 3 and time.monotonic() < deadline:
+ time.sleep(0.1)
+ pids = [json.loads(path.read_text())["pid"] for path in tree.glob("*.json")]
+ self.assertEqual(len(pids), 3)
+ code, out, err = self.cli("close", str(self.session))
+ self.assertEqual(code, 0, err)
+ self.assertEqual(self.serve.wait(15), 0)
+ for pid in pids:
+ self.assertTrue(gone(pid), f"descendant {pid} survived close")
+ self.assertIsNone(sentinel.poll(), "an unrelated process must survive")
+
+
+if __name__ == "__main__":
+ unittest.main()