diff options
| author | Jesse Vincent <jesse@fsck.com> | 2026-09-18 18:38:59 +0000 |
|---|---|---|
| committer | Jesse Vincent <jesse@fsck.com> | 2026-09-18 18:38:59 +0000 |
| commit | 8412aaf0385cbfd850675093f01b16c88052f268 (patch) | |
| tree | 88ae48d06c4d104d65e35d345e65f7c210189564 /tests | |
| parent | b36e0829c6d0140e93cfef2ca599b1b07d4a7797 (diff) | |
Release v6.4.0: diagnosing-superpowers and movie skills, Native execution, OpenCode 2.0, Muse, and Qwen Code supportrelease/v6.4.0
Everything on dev since the v6.3.0 merge-back (777ceb5), as one commit.
Full notes: RELEASE-NOTES.md, v6.4.0 section.
Claude-Session: https://claude.ai/code/session_01PRUnVUm4g4EcP9eNjAiT2B
Diffstat (limited to 'tests')
26 files changed, 4567 insertions, 0 deletions
diff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.sh index 83217cd..97bce6d 100755 --- a/tests/claude-code/run-skill-tests.sh +++ b/tests/claude-code/run-skill-tests.sh @@ -76,6 +76,7 @@ done tests=( "test-worktree-path-policy.sh" "test-sdd-workspace.sh" + "test-executing-plans-scripts.sh" "test-subagent-driven-development.sh" ) diff --git a/tests/claude-code/test-executing-plans-scripts.sh b/tests/claude-code/test-executing-plans-scripts.sh new file mode 100755 index 0000000..994733b --- /dev/null +++ b/tests/claude-code/test-executing-plans-scripts.sh @@ -0,0 +1,139 @@ +#!/usr/bin/env bash +# Tests for executing-plans' bookkeeping helpers: scripts/task-start extracts +# the brief and records BASE in one call; scripts/task-done runs the task's +# test command, records the result in the ledger, and refuses to record a +# failing task. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +EP_SCRIPTS="$REPO_ROOT/skills/executing-plans/scripts" + +FAILURES=0 +TEST_ROOT="" + +pass() { echo " [PASS] $1"; } +fail() { + echo " [FAIL] $1" + FAILURES=$((FAILURES + 1)) +} + +cleanup() { + if [[ -n "$TEST_ROOT" && -d "$TEST_ROOT" ]]; then + rm -rf "$TEST_ROOT" + fi +} + +main() { + echo "=== Test: executing-plans scripts ===" + + TEST_ROOT="$(mktemp -d)" + trap cleanup EXIT + + git init -q -b main "$TEST_ROOT/repo" + local repo + repo="$(cd "$TEST_ROOT/repo" && git rev-parse --show-toplevel)" + local git_id=(-c user.email=t@example.com -c user.name=t -c commit.gpgsign=false) + + cat > "$repo/plan.md" <<'PLAN' +# Plan + +## Task 1: First thing + +Do the first thing. + +## Task 2: Second thing + +Do the second thing. +PLAN + ( cd "$repo" && git add plan.md && git "${git_id[@]}" commit -qm fixture ) + local base + base="$(cd "$repo" && git rev-parse HEAD)" + + # --- task-start: argument validation --- + local rc=0 + (cd "$repo" && "$EP_SCRIPTS/task-start" plan.md >/dev/null 2>&1) || rc=$? + if [[ "$rc" -eq 2 ]]; then + pass "task-start without a task number errors with exit 2" + else + fail "task-start without a task number errors with exit 2 (got $rc)" + fi + + # --- task-start: brief path + BASE in one call --- + local out + out="$(cd "$repo" && "$EP_SCRIPTS/task-start" plan.md 1)" + if [[ "$out" == *"brief: $repo/.superpowers/sdd/plan/task-1-brief.md"* ]]; then + pass "task-start prints the brief path under the plan's workspace" + else + fail "task-start prints the brief path under the plan's workspace" + echo " got: $out" + fi + if [[ "$out" == *"base: $base"* ]]; then + pass "task-start prints BASE as the current HEAD" + else + fail "task-start prints BASE as the current HEAD" + echo " got: $out" + fi + if [[ -s "$repo/.superpowers/sdd/plan/task-1-brief.md" ]]; then + pass "task-start writes the brief file" + else + fail "task-start writes the brief file" + fi + + # --- task-done: records a passing task --- + ( cd "$repo" && echo x > work.txt && git add work.txt && git "${git_id[@]}" commit -qm "task 1" ) + local head + head="$(cd "$repo" && git rev-parse HEAD)" + out="$(cd "$repo" && "$EP_SCRIPTS/task-done" plan.md 1 "$base" -- sh -c 'echo "Ran 3 tests"; echo OK')" + rc=$? + local ledger="$repo/.superpowers/sdd/plan/progress.md" + local expected="Task 1: complete (commits ${base:0:7}..${head:0:7}, tests: sh -c 'echo \"Ran 3 tests\"; echo OK' → OK)" + if [[ -f "$ledger" ]] && grep -qF "$expected" "$ledger"; then + pass "task-done appends the completion line with commit range and test result" + else + fail "task-done appends the completion line with commit range and test result" + echo " expected: $expected" + echo " ledger:"; sed 's/^/ /' "$ledger" 2>/dev/null || echo " (missing)" + fi + if [[ "$out" == *"OK"* ]]; then + pass "task-done prints the tail of the test output" + else + fail "task-done prints the tail of the test output" + echo " got: $out" + fi + if [[ -s "$repo/.superpowers/sdd/plan/task-1-tests.log" ]]; then + pass "task-done keeps the full test output in the workspace" + else + fail "task-done keeps the full test output in the workspace" + fi + + # --- task-done: refuses to record a failing task --- + rc=0 + out="$(cd "$repo" && "$EP_SCRIPTS/task-done" plan.md 2 "$head" -- sh -c 'echo "FAILED (errors=1)"; exit 1' 2>&1)" || rc=$? + if [[ "$rc" -ne 0 ]]; then + pass "task-done exits non-zero when the test command fails" + else + fail "task-done exits non-zero when the test command fails" + fi + if ! grep -q "Task 2: complete" "$ledger"; then + pass "task-done does not record a failing task as complete" + else + fail "task-done does not record a failing task as complete" + fi + if [[ "$out" == *"FAILED"* ]]; then + pass "task-done shows the failing output" + else + fail "task-done shows the failing output" + echo " got: $out" + fi + + echo + if [[ "$FAILURES" -eq 0 ]]; then + echo "PASS" + else + echo "FAIL ($FAILURES)" + exit 1 + fi +} + +main "$@" diff --git a/tests/claude-code/test-sdd-workspace.sh b/tests/claude-code/test-sdd-workspace.sh index 8417230..2e8c227 100755 --- a/tests/claude-code/test-sdd-workspace.sh +++ b/tests/claude-code/test-sdd-workspace.sh @@ -165,6 +165,30 @@ PLAN echo " got: $rp_explicit" fi + # --- range guards: BASE must be an ancestor of HEAD, range must be non-empty --- + local divergent + divergent="$(cd "$repo" && git "${git_id[@]}" commit-tree 'HEAD~1^{tree}' -p 'HEAD~1' -m divergent)" + rc=0 + local guard_err + guard_err="$(cd "$repo" && "$SDD_SCRIPTS/review-package" plan-a.md "$divergent" HEAD 2>&1 >/dev/null)" || rc=$? + if [[ "$rc" -eq 3 && "$guard_err" == *"not a descendant"* ]]; then + pass "review-package rejects a BASE that is not an ancestor of HEAD with exit 3" + else + fail "review-package rejects a BASE that is not an ancestor of HEAD with exit 3" + echo " exit: $rc" + echo " stderr: $guard_err" + fi + + rc=0 + guard_err="$(cd "$repo" && "$SDD_SCRIPTS/review-package" plan-a.md HEAD HEAD 2>&1 >/dev/null)" || rc=$? + if [[ "$rc" -eq 3 && "$guard_err" == *"empty commit range"* ]]; then + pass "review-package rejects an empty BASE..HEAD range with exit 3" + else + fail "review-package rejects an empty BASE..HEAD range with exit 3" + echo " exit: $rc" + echo " stderr: $guard_err" + fi + # --- Worktree isolation: a linked worktree resolves its own workspace --- local wt="$TEST_ROOT/wt" ( cd "$repo" && git worktree add -q "$wt" -b wt-feature ) @@ -189,6 +213,143 @@ PLAN echo " status: $wt_status" fi + # --- helpers survive a mode-stripping extractor dropping exec bits (#2040) --- + local stripped="$TEST_ROOT/stripped-scripts" + mkdir -p "$stripped" + cp "$SDD_SCRIPTS/sdd-workspace" "$SDD_SCRIPTS/task-brief" "$SDD_SCRIPTS/review-package" "$stripped/" + chmod -x "$stripped"/* + local noexec_out noexec_rc=0 + noexec_out="$(cd "$repo" && bash "$stripped/task-brief" plan-b.md 1 2>&1)" || noexec_rc=$? + if [[ "$noexec_rc" -eq 0 && -f "$repo/.superpowers/sdd/plan-b/task-1-brief.md" ]]; then + pass "task-brief works with no exec bit on sdd-workspace" + else + fail "task-brief works with no exec bit on sdd-workspace" + echo " rc: $noexec_rc" + echo " output: $noexec_out" + fi + + # --- Ownership markers: two plans with the same basename (#2045) --- + mkdir -p "$repo/docs/alpha" "$repo/docs/beta" + cat > "$repo/docs/alpha/plan.md" <<'PLAN' +# Alpha Plan + +## Task 1: Alpha work + +Alpha-only requirement text. +PLAN + cat > "$repo/docs/beta/plan.md" <<'PLAN' +# Beta Plan + +## Task 1: Beta work + +Beta-only requirement text. +PLAN + + local dir_alpha dir_beta + dir_alpha="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" docs/alpha/plan.md)" + dir_beta="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" docs/beta/plan.md)" + if [[ "$dir_alpha" != "$dir_beta" ]]; then + pass "same-basename plans resolve to distinct workspaces" + else + fail "same-basename plans resolve to distinct workspaces" + echo " alpha: $dir_alpha" + echo " beta: $dir_beta" + fi + + ( cd "$repo" && "$SDD_SCRIPTS/task-brief" docs/alpha/plan.md 1 >/dev/null ) + ( cd "$repo" && "$SDD_SCRIPTS/task-brief" docs/beta/plan.md 1 >/dev/null ) + if grep -q "Alpha-only requirement text." "$dir_alpha/task-1-brief.md" 2>/dev/null \ + && grep -q "Beta-only requirement text." "$dir_beta/task-1-brief.md" 2>/dev/null; then + pass "same-basename plans keep both task briefs intact" + else + fail "same-basename plans keep both task briefs intact" + echo " alpha brief: $(cat "$dir_alpha/task-1-brief.md" 2>/dev/null)" + echo " beta brief: $(cat "$dir_beta/task-1-brief.md" 2>/dev/null)" + fi + + # --- Legacy adoption: pre-existing workspace without a marker --- + printf '# Foo\n\n## Task 1: Foo\n\nFoo.\n' > "$repo/foo.md" + mkdir -p "$repo/.superpowers/sdd/foo" + printf 'ledger\n' > "$repo/.superpowers/sdd/foo/progress.md" + local dir_foo + dir_foo="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" foo.md)" + if [[ "$dir_foo" == "$repo/.superpowers/sdd/foo" \ + && -f "$dir_foo/progress.md" \ + && "$(cat "$dir_foo/plan-path" 2>/dev/null)" == "foo.md" ]]; then + pass "legacy markerless workspace is adopted in place and marked" + else + fail "legacy markerless workspace is adopted in place and marked" + echo " dir: $dir_foo" + echo " marker: $(cat "$dir_foo/plan-path" 2>/dev/null)" + fi + + # --- Ownership conflict: marker names a different plan --- + printf '# Bar\n\n## Task 1: Bar\n\nBar.\n' > "$repo/bar.md" + mkdir -p "$repo/.superpowers/sdd/bar" + printf 'somewhere-else/bar.md\n' > "$repo/.superpowers/sdd/bar/plan-path" + printf 'other ledger\n' > "$repo/.superpowers/sdd/bar/progress.md" + local dir_bar + dir_bar="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" bar.md)" + if [[ "$dir_bar" == "$repo/.superpowers/sdd/bar-repo" \ + && "$(cat "$dir_bar/plan-path" 2>/dev/null)" == "bar.md" ]]; then + pass "owned workspace disambiguates with parent-dir suffix" + else + fail "owned workspace disambiguates with parent-dir suffix" + echo " got: $dir_bar" + fi + if [[ "$(cat "$repo/.superpowers/sdd/bar/plan-path")" == "somewhere-else/bar.md" \ + && "$(cat "$repo/.superpowers/sdd/bar/progress.md")" == "other ledger" ]]; then + pass "conflicting plan leaves the original workspace untouched" + else + fail "conflicting plan leaves the original workspace untouched" + fi + + # --- Counter fallback: parent-suffixed workspace is owned too --- + printf '# Baz\n\n## Task 1: Baz\n\nBaz.\n' > "$repo/baz.md" + mkdir -p "$repo/.superpowers/sdd/baz" "$repo/.superpowers/sdd/baz-repo" + printf 'one/baz.md\n' > "$repo/.superpowers/sdd/baz/plan-path" + printf 'two/baz.md\n' > "$repo/.superpowers/sdd/baz-repo/plan-path" + local dir_baz + dir_baz="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" baz.md)" + if [[ "$dir_baz" == "$repo/.superpowers/sdd/baz-repo-2" \ + && "$(cat "$dir_baz/plan-path" 2>/dev/null)" == "baz.md" ]]; then + pass "double conflict falls back to a counter suffix" + else + fail "double conflict falls back to a counter suffix" + echo " got: $dir_baz" + fi + + # --- Same plan spelled differently resolves to one workspace --- + local dir_rel dir_abs dir_dotdot + dir_rel="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" docs/alpha/plan.md)" + dir_abs="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" "$repo/docs/alpha/plan.md")" + dir_dotdot="$(cd "$repo/docs/beta" && "$SDD_SCRIPTS/sdd-workspace" ../alpha/plan.md)" + if [[ "$dir_rel" == "$dir_abs" && "$dir_rel" == "$dir_dotdot" \ + && "$(cat "$dir_rel/plan-path" 2>/dev/null)" == "docs/alpha/plan.md" ]]; then + pass "relative, absolute, and ../ spellings share one workspace and marker" + else + fail "relative, absolute, and ../ spellings share one workspace and marker" + echo " rel: $dir_rel" + echo " abs: $dir_abs" + echo " dotdot: $dir_dotdot" + echo " marker: $(cat "$dir_rel/plan-path" 2>/dev/null)" + fi + + # --- Out-of-repo plans keep working, marker holds the absolute path --- + mkdir -p "$TEST_ROOT/outside" + printf '# Remote\n\n## Task 1: Remote\n\nRemote.\n' > "$TEST_ROOT/outside/remote-plan.md" + local outside_abs dir_out + outside_abs="$(cd "$TEST_ROOT/outside" && pwd -P)/remote-plan.md" + dir_out="$(cd "$repo" && "$SDD_SCRIPTS/sdd-workspace" "$TEST_ROOT/outside/remote-plan.md")" + if [[ "$dir_out" == "$repo/.superpowers/sdd/remote-plan" \ + && "$(cat "$dir_out/plan-path" 2>/dev/null)" == "$outside_abs" ]]; then + pass "out-of-repo plan gets a basename slug and an absolute-path marker" + else + fail "out-of-repo plan gets a basename slug and an absolute-path marker" + echo " dir: $dir_out" + echo " marker: $(cat "$dir_out/plan-path" 2>/dev/null)" + fi + echo "" if [[ "$FAILURES" -ne 0 ]]; then echo "FAILED: $FAILURES assertion(s)." diff --git a/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh b/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh index 01de394..265aea1 100755 --- a/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh +++ b/tests/codex-plugin-sync/test-sync-to-codex-plugin.sh @@ -196,6 +196,10 @@ write_upstream_fixture() { } EOF + cat > "$repo/index.js" <<'EOF' +export { default } from "./.opencode/plugins/superpowers.js"; +EOF + cat > "$repo/.gitignore" <<'EOF' .private-journal/ EOF @@ -303,6 +307,7 @@ EOF hooks/run-hook.cmd \ hooks/session-start \ hooks/session-start-codex \ + index.js \ package.json \ scripts/sync-to-codex-plugin.sh \ skills/example/SKILL.md @@ -664,6 +669,7 @@ main() { assert_not_contains "$preview_section" "evals/" "Preview excludes eval harness" assert_not_contains "$preview_section" ".gitmodules" "Preview excludes repo submodule metadata" assert_not_contains "$preview_section" ".pre-commit-config.yaml" "Preview excludes repo pre-commit config" + assert_not_contains "$preview_section" "index.js" "Preview excludes OpenCode root entrypoint" assert_not_contains "$preview_output" "Overlay file (.codex-plugin/plugin.json) will be regenerated" "Preview omits overlay regeneration note" assert_not_contains "$preview_output" "Assets (superpowers-small.svg, app-icon.png) will be seeded from" "Preview omits assets seeding note" assert_contains "$preview_section" "skills/example/SKILL.md" "Preview reflects dirty tracked destination file" diff --git a/tests/diagnosing-superpowers/test-skill-structure.sh b/tests/diagnosing-superpowers/test-skill-structure.sh new file mode 100755 index 0000000..9c31591 --- /dev/null +++ b/tests/diagnosing-superpowers/test-skill-structure.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +# Structural checks for skills/diagnosing-superpowers. Behavior is tested by +# scenario evals kept by the maintainer; this script only checks the things a +# shell can check: frontmatter, referenced files exist, no local paths or +# names leaked into shipped files, SKILL.md word budget. +set -u + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +SKILL_DIR="$REPO_ROOT/skills/diagnosing-superpowers" +SKILL_MD="$SKILL_DIR/SKILL.md" +WORD_BUDGET=1000 + +PASSES=0 +FAILURES=0 + +pass() { echo " [PASS] $1"; PASSES=$((PASSES + 1)); } +fail() { echo " [FAIL] $1"; FAILURES=$((FAILURES + 1)); } + +echo "diagnosing-superpowers structure" + +# --- SKILL.md frontmatter ------------------------------------------------- +if [ -f "$SKILL_MD" ]; then + pass "SKILL.md exists" + frontmatter="$(awk 'NR==1 && $0!="---"{exit} NR>1 && $0=="---"{exit} NR>1{print}' "$SKILL_MD")" + if printf '%s\n' "$frontmatter" | grep -q '^name: diagnosing-superpowers$'; then + pass "frontmatter name is diagnosing-superpowers" + else + fail "frontmatter name is diagnosing-superpowers" + fi + description="$(printf '%s\n' "$frontmatter" | awk '/^description:/{sub(/^description:[ ]*/,""); print; found=1; next} found && /^[ ]/{print} found && !/^[ ]/{exit}' | tr '\n' ' ')" + if printf '%s' "$description" | grep -q '^Use when'; then + pass "description starts with 'Use when'" + else + fail "description starts with 'Use when' (got: ${description:0:60})" + fi + if [ "${#description}" -le 1024 ]; then + pass "description under 1024 characters" + else + fail "description under 1024 characters (${#description})" + fi + for banned in "dispatch" "then" "step"; do + if printf '%s' "$description" | grep -qiw "$banned"; then + fail "description contains workflow word '$banned'" + else + pass "description avoids workflow word '$banned'" + fi + done + + # --- word budget -------------------------------------------------------- + body_words="$(awk 'BEGIN{fm=0} NR==1 && $0=="---"{fm=1; next} fm==1 && $0=="---"{fm=2; next} fm==2{print}' "$SKILL_MD" | wc -w | tr -d ' ')" + if [ "$body_words" -le "$WORD_BUDGET" ]; then + pass "SKILL.md body within $WORD_BUDGET words ($body_words)" + else + fail "SKILL.md body within $WORD_BUDGET words ($body_words)" + fi + + # --- required sections -------------------------------------------------- + for heading in "## Hard rules" "## Red Flags"; do + if grep -q "^$heading" "$SKILL_MD"; then + pass "SKILL.md has section '$heading'" + else + fail "SKILL.md has section '$heading'" + fi + done + + # --- every referenced skill file exists -------------------------------- + while IFS= read -r ref; do + if [ -f "$SKILL_DIR/$ref" ]; then + pass "referenced file exists: $ref" + else + fail "referenced file exists: $ref" + fi + done < <(grep -o '\(references\|prompts\|templates\)/[A-Za-z0-9._-]*\.md' "$SKILL_MD" | sort -u) +else + fail "SKILL.md exists" +fi + +# --- expected files ------------------------------------------------------- +expected_files=( + references/redaction-policy.md + references/session-discovery.md + references/context-safety.md + references/github-issues.md + prompts/analyst-common.md + prompts/skill-timeline.md + prompts/plan-adherence.md + prompts/repeated-work.md + prompts/stumbles.md + prompts/quality-evidence.md + prompts/request-conflicts.md + prompts/cost-and-time.md + prompts/scrub.md + prompts/scrub-audit.md + prompts/similar-session.md + templates/case.md + templates/report.md + templates/bundle-README.md + templates/issue.md +) +for rel in "${expected_files[@]}"; do + if [ -f "$SKILL_DIR/$rel" ]; then + pass "expected file present: $rel" + else + fail "expected file present: $rel" + fi +done + +# --- removed harness recipes stay removed -------------------------------- +removed_files=( + references/claude-code-sessions.md + references/codex-sessions.md + references/other-harnesses.md +) +for rel in "${removed_files[@]}"; do + if [ ! -e "$SKILL_DIR/$rel" ]; then + pass "removed reference absent: $rel" + else + fail "removed reference absent: $rel" + fi +done + +removed_reference_hits="$(grep -rn -E 'references/(claude-code-sessions|codex-sessions|other-harnesses)\.md' "$SKILL_DIR" --include='*.md' 2>/dev/null || true)" +if [ -z "$removed_reference_hits" ]; then + pass "active skill prose has no references to removed harness recipes" +else + fail "active skill prose has no references to removed harness recipes" + printf '%s\n' "$removed_reference_hits" | head -10 | sed 's/^/ /' +fi + +# --- no local paths or names in shipped files ---------------------------- +leaks="$(grep -rn -E '/Users/|/home/|jesse' "$SKILL_DIR" "$SCRIPT_DIR" --exclude=test-skill-structure.sh 2>/dev/null || true)" +if [ -z "$leaks" ]; then + pass "no machine-specific paths or names in shipped files (skills + tests)" +else + fail "no machine-specific paths or names in shipped files (skills + tests)" + printf '%s\n' "$leaks" | head -10 | sed 's/^/ /' +fi + +# --- "the user" never appears in skill prose ----------------------------- +user_hits="$(grep -rn -i 'the user' "$SKILL_DIR" --include='*.md' 2>/dev/null || true)" +if [ -z "$user_hits" ]; then + pass "skill files say 'your human partner', not 'the user'" +else + fail "skill files say 'your human partner', not 'the user'" + printf '%s\n' "$user_hits" | head -10 | sed 's/^/ /' +fi + +echo +echo "Passed: $PASSES Failed: $FAILURES" +[ "$FAILURES" -eq 0 ] diff --git a/tests/opencode/run-tests.sh b/tests/opencode/run-tests.sh index d9b100e..43e1a3a 100755 --- a/tests/opencode/run-tests.sh +++ b/tests/opencode/run-tests.sh @@ -45,6 +45,8 @@ while [[ $# -gt 0 ]]; do echo "Tests:" echo " test-plugin-loading.sh Verify plugin installation and structure" echo " test-bootstrap-caching.sh Verify bootstrap content caching" + echo " test-session-bootstrap.sh Verify session classification and lookup recovery" + echo " test-skill-registration.sh Verify V2 skill registration contract (2.0.4 path field)" echo " test-tools.sh Test use_skill and find_skills tools (integration)" echo " test-priority.sh Test skill priority resolution (integration)" exit 0 @@ -61,6 +63,8 @@ done tests=( "test-plugin-loading.sh" "test-bootstrap-caching.sh" + "test-session-bootstrap.sh" + "test-skill-registration.sh" ) # Integration tests (require OpenCode) diff --git a/tests/opencode/test-bootstrap-caching.mjs b/tests/opencode/test-bootstrap-caching.mjs index 32149ae..386b7b1 100644 --- a/tests/opencode/test-bootstrap-caching.mjs +++ b/tests/opencode/test-bootstrap-caching.mjs @@ -32,6 +32,10 @@ const mod = await import(pathToFileURL(pluginPath).href); const plugin = await mod.SuperpowersPlugin({ client: {}, directory: '.' }); const transform = plugin['experimental.chat.messages.transform']; +// Mapping constants are flavor-specific (#opencode-v2): V1 keeps the 1.18.x +// tool names, V2 teaches the renamed tools. Assert both directly. +const mappingFailures = assertMappingConstants(mod); + const firstOutput = makeOutput(`${scenario} bootstrap first step`); await transform({}, firstOutput); const afterFirst = { existsCount, readCount }; @@ -40,6 +44,11 @@ const secondOutput = makeOutput(`${scenario} bootstrap second step`); await transform({}, secondOutput); const afterSecond = { existsCount, readCount }; +// Exercise the V2 path (setup() + ctx.session.hook("context")) with a mock +// ctx so the V2_MAPPING wiring is verified, not just the constant. Run after +// the V1 count snapshots: setup() reads SKILL.md files during registration. +const v2Result = await runV2ContextHook(mod); + const result = { scenario, firstBootstrapParts: countBootstrapParts(firstOutput), @@ -52,12 +61,24 @@ const result = { secondReadCount: afterSecond.readCount, firstExistsCount: afterFirst.existsCount, secondExistsCount: afterSecond.existsCount, + v2BootstrapParts: v2Result.bootstrapParts, + mapsV2SubagentTool: v2Result.text.includes('`subagent` with `agent: "general"`'), + mapsV2SessionIDContinuation: v2Result.text.includes('`sessionID` to continue a previous subagent'), + mapsV2NoTodoTool: v2Result.text.includes('no todo tool'), + mapsV2MutationToPatch: v2Result.text.includes('`patch` with `patchText`'), + mapsV2Shell: v2Result.text.includes('`shell`'), + staleV1ToolsInV2: v2Result.text.includes('`apply_patch`') || v2Result.text.includes('`todowrite`') || v2Result.text.includes('`subagent_type`'), }; const failures = scenario === 'present' ? assertPresentBootstrap(result) : assertMissingBootstrap(result); +if (scenario === 'present') { + failures.push(...assertV2Bootstrap(result)); +} +failures.push(...mappingFailures); + if (failures.length > 0) { console.error(JSON.stringify(result, null, 2)); for (const failure of failures) { @@ -144,3 +165,104 @@ function assertMissingBootstrap(result) { } return failures; } + +function assertMappingConstants(mod) { + const failures = []; + if (typeof mod.V1_MAPPING !== 'string' || typeof mod.V2_MAPPING !== 'string') { + failures.push('expected plugin to export V1_MAPPING and V2_MAPPING string constants'); + return failures; + } + for (const needle of ['`todowrite`', '`task` with `subagent_type: "general"`', '`apply_patch`', '`bash`']) { + if (!mod.V1_MAPPING.includes(needle)) { + failures.push(`expected V1_MAPPING to keep the 1.18.x tool name ${needle}`); + } + } + for (const needle of [ + '`subagent` with `agent: "general"`', + '`sessionID` to continue a previous subagent', + 'no todo tool', + '`write`', + '`edit`', + '`patch` with `patchText`', + '`shell`', + '`read`', + '`grep`, `glob`', + '`webfetch`', + '`websearch`', + ]) { + if (!mod.V2_MAPPING.includes(needle)) { + failures.push(`expected V2_MAPPING to teach the V2 tool ${needle}`); + } + } + for (const stale of ['`todowrite`', '`task` with', '`apply_patch`', '`bash`']) { + if (mod.V2_MAPPING.includes(stale)) { + failures.push(`expected V2_MAPPING not to teach the V1-only tool name ${stale}`); + } + } + return failures; +} + +// Drive setup() with a mock V2 ctx and fire the captured "context" hook on a +// top-level (parentID-less) session. Returns the injected-part count and the +// injected bootstrap text ('' when nothing was injected). +async function runV2ContextHook(mod) { + let contextHook = null; + const ctx = { + skill: { + transform: async (fn) => { + fn({ add: () => {} }); + }, + }, + session: { + hook: async (name, cb) => { + if (name === 'context') contextHook = cb; + }, + get: async ({ sessionID }) => ({ id: sessionID }), // top-level: no parentID + }, + }; + try { + await mod.default.setup(ctx); + } catch (err) { + console.error('[test] V2 setup() threw:', err); + return { bootstrapParts: 0, text: '' }; + } + if (typeof contextHook !== 'function') { + return { bootstrapParts: 0, text: '' }; + } + const event = { + sessionID: 'sess-v2-top', + messages: [{ role: 'user', content: [{ type: 'text', text: 'v2 bootstrap step' }] }], + }; + await contextHook(event); + const parts = event.messages[0].content.filter( + (part) => part.type === 'text' && part.text.includes('EXTREMELY_IMPORTANT') + ); + return { bootstrapParts: parts.length, text: parts[0]?.text || '' }; +} + +function assertV2Bootstrap(result) { + const failures = []; + if (result.v2BootstrapParts !== 1) { + failures.push(`expected V2 context hook to inject one bootstrap part, got ${result.v2BootstrapParts}`); + return failures; + } + if (!result.mapsV2SubagentTool) { + failures.push('expected V2 bootstrap to map general-purpose subagents to subagent with agent'); + } + if (!result.mapsV2SessionIDContinuation) { + failures.push('expected V2 bootstrap to teach sessionID continuation for subagents'); + } + if (!result.mapsV2NoTodoTool) { + failures.push('expected V2 bootstrap to state that V2 has no todo tool'); + } + if (!result.mapsV2MutationToPatch) { + failures.push('expected V2 bootstrap to map file mutation to patch with patchText'); + } + if (!result.mapsV2Shell) { + failures.push('expected V2 bootstrap to map shell commands to the shell tool'); + } + if (result.staleV1ToolsInV2) { + failures.push('expected V2 bootstrap not to teach V1-only tool names (apply_patch/todowrite/subagent_type)'); + } + return failures; +} diff --git a/tests/opencode/test-session-bootstrap.mjs b/tests/opencode/test-session-bootstrap.mjs new file mode 100644 index 0000000..31658d5 --- /dev/null +++ b/tests/opencode/test-session-bootstrap.mjs @@ -0,0 +1,224 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +const [, , inputPath] = process.argv; +assert.ok(inputPath, 'pass the plugin module path'); +const pluginURL = pathToFileURL(fs.realpathSync(inputPath)); +const marker = '<EXTREMELY_IMPORTANT>\nYou have superpowers.'; +let generation = 0; + +function reply(flavor, session) { + return flavor === 'v1' ? { data: session } : session; +} + +function makeEvent(flavor, sessionID) { + const text = { type: 'text', text: 'Execute the assigned task' }; + return { + sessionID, + messages: [flavor === 'v1' + ? { info: { role: 'user', sessionID }, parts: [text] } + : { role: 'user', content: [text] }], + }; +} + +function bootstrapCount(event) { + return event.messages.flatMap((message) => message.parts ?? message.content ?? []).filter( + (part) => part.type === 'text' && part.text.startsWith(marker) + ).length; +} + +async function makeHarness(flavor, fetchSession) { + const mod = await import(`${pluginURL.href}?session-test=${++generation}`); + const lookups = []; + const registered = []; + const get = async (id) => { + lookups.push(id); + return fetchSession(id, lookups.length); + }; + let invoke; + if (flavor === 'v1') { + const hooks = await mod.SuperpowersPlugin({ + client: { session: { get: ({ path: { id } }) => get(id) } }, + directory: '.', + }); + invoke = (event) => hooks['experimental.chat.messages.transform']({}, event); + } else { + await mod.default.setup({ + skill: { transform: async (transform) => transform({ add: (skill) => registered.push(skill) }) }, + session: { + get: ({ sessionID }) => get(sessionID), + hook: async (name, callback) => { if (name === 'context') invoke = callback; }, + }, + }); + } + assert.equal(typeof invoke, 'function'); + return { invoke, lookups, registered }; +} + +for (const flavor of ['v1', 'v2']) { + for (const [kind, extra, expected] of [ + ['root', {}, 1], + ['child', { parentID: 'parent' }, 0], + ['fork', { fork: { sessionID: 'origin' } }, 1], + ]) { + const id = `${flavor}-${kind}`; + const h = await makeHarness(flavor, () => reply(flavor, { id, ...extra })); + const event = makeEvent(flavor, id); + await h.invoke(event); + assert.equal(bootstrapCount(event), expected, `${id}: first request`); + await h.invoke(event); + assert.equal(bootstrapCount(event), expected, `${id}: repeated event`); + const fresh = makeEvent(flavor, id); + await h.invoke(fresh); + assert.equal(bootstrapCount(fresh), expected, `${id}: fresh request`); + assert.deepEqual(h.lookups, [id], `${id}: cache successful classification`); + if (flavor === 'v2' && kind === 'child') { + assert.ok(h.registered.some((skill) => skill.id === 'brainstorming')); + } + } + + const failures = [ + ['throws', () => { throw new Error('temporary lookup failure'); }], + ['missing', () => undefined], + ['null', () => null], + ['empty', () => reply(flavor, {})], + ['wrong-id', () => reply(flavor, { id: 'different-session' })], + ['invalid-parent', (id) => reply(flavor, { id, parentID: 42 })], + ]; + if (flavor === 'v1') { + failures.push(['resolved-http-error', () => ({ + data: undefined, + error: { name: 'UnknownError', data: { message: 'temporary 503' } }, + response: { ok: false, status: 503 }, + })]); + } + for (const [kind, firstResult] of failures) { + const id = `${flavor}-${kind}`; + const h = await makeHarness(flavor, (sessionID, call) => call === 1 + ? firstResult(sessionID) + : reply(flavor, { id: sessionID, parentID: 'parent' })); + const counts = []; + for (let step = 0; step < 2; step++) { + const event = makeEvent(flavor, id); + await h.invoke(event); + counts.push(bootstrapCount(event)); + } + assert.deepEqual(counts, [1, 0], `${id}: recover on the next request`); + assert.deepEqual(h.lookups, [id, id], `${id}: never cache the failure`); + } + + const isolated = await makeHarness(flavor, (id) => reply(flavor, + id === 'child-session' ? { id, parentID: 'parent' } : { id })); + for (const [id, expected] of [['root-session', 1], ['child-session', 0], ['root-session', 1], ['child-session', 0]]) { + const event = makeEvent(flavor, id); + await isolated.invoke(event); + assert.equal(bootstrapCount(event), expected); + } + assert.deepEqual(isolated.lookups, ['root-session', 'child-session']); + + const bounded = await makeHarness(flavor, (id) => reply(flavor, { id, parentID: 'parent' })); + for (let index = 0; index <= 512; index++) { + const event = makeEvent(flavor, `eviction-${index}`); + await bounded.invoke(event); + assert.equal(bootstrapCount(event), 0); + } + const evicted = makeEvent(flavor, 'eviction-0'); + await bounded.invoke(evicted); + assert.equal(bootstrapCount(evicted), 0); + assert.equal(bounded.lookups.filter((id) => id === 'eviction-0').length, 2); + + const restarted = await makeHarness(flavor, (id) => reply(flavor, { id, parentID: 'parent' })); + const afterRestart = makeEvent(flavor, 'eviction-0'); + await restarted.invoke(afterRestart); + assert.equal(bootstrapCount(afterRestart), 0); + assert.deepEqual(restarted.lookups, ['eviction-0']); + + const unknown = await makeHarness(flavor, () => { throw new Error('must not look up a missing ID'); }); + const noID = makeEvent(flavor, undefined); + await unknown.invoke(noID); + assert.equal(bootstrapCount(noID), 1); + assert.deepEqual(unknown.lookups, []); +} + +function compactedEvent(sessionID) { + return { + sessionID, + system: [], + messages: [{ + role: 'assistant', + content: [{ type: 'compaction', provider: 'fixture', encrypted: 'opaque-checkpoint' }], + }], + }; +} + +const compactedRoot = await makeHarness('v2', (id) => ({ id })); +const rootEvent = compactedEvent('compacted-root'); +const checkpoint = structuredClone(rootEvent.messages[0]); +await compactedRoot.invoke(rootEvent); +assert.equal(bootstrapCount(rootEvent), 1); +assert.deepEqual(rootEvent.messages[0], checkpoint); +assert.equal(rootEvent.messages.length, 2); +assert.equal(rootEvent.messages[1].role, 'user'); +assert.deepEqual(rootEvent.system, []); +await compactedRoot.invoke(rootEvent); +assert.equal(bootstrapCount(rootEvent), 1); +assert.equal(rootEvent.messages.length, 2); +const freshRootEvent = compactedEvent('compacted-root'); +await compactedRoot.invoke(freshRootEvent); +assert.equal(bootstrapCount(freshRootEvent), 1); +assert.deepEqual(compactedRoot.lookups, ['compacted-root']); + +const compactedChild = await makeHarness('v2', (id) => ({ id, parentID: 'parent' })); +const childEvent = compactedEvent('compacted-child'); +const originalChild = structuredClone(childEvent); +await compactedChild.invoke(childEvent); +assert.equal(bootstrapCount(childEvent), 0); +assert.deepEqual(childEvent, originalChild); +assert.deepEqual(compactedChild.lookups, ['compacted-child']); + +const retryChild = await makeHarness('v2', (id, call) => { + if (call === 1) throw new Error('temporary lookup failure'); + return { id, parentID: 'parent' }; +}); +const unknownChild = compactedEvent('retry-compacted-child'); +await retryChild.invoke(unknownChild); +assert.equal(bootstrapCount(unknownChild), 1); +const recoveredChild = compactedEvent('retry-compacted-child'); +await retryChild.invoke(recoveredChild); +assert.equal(bootstrapCount(recoveredChild), 0); +assert.equal(recoveredChild.messages.length, 1); +assert.equal(retryChild.lookups.length, 2); + +const newPromptAfterCheckpoint = compactedEvent('new-prompt-after-checkpoint-root'); +newPromptAfterCheckpoint.messages.push({ role: 'user', content: [{ type: 'text', text: 'Continue' }] }); +await compactedRoot.invoke(newPromptAfterCheckpoint); +assert.equal(bootstrapCount(newPromptAfterCheckpoint), 1); +assert.equal(newPromptAfterCheckpoint.messages.length, 2); +assert.equal(newPromptAfterCheckpoint.messages[1].content.length, 2); + +const retainedUser = compactedEvent('retained-user-root'); +retainedUser.messages.unshift({ role: 'user', content: [{ type: 'text', text: 'Keep going' }] }); +const retainedCheckpoint = structuredClone(retainedUser.messages[1]); +await compactedRoot.invoke(retainedUser); +assert.equal(bootstrapCount(retainedUser), 1); +assert.equal(retainedUser.messages.length, 2); +assert.equal(retainedUser.messages[0].content.length, 2); +assert.ok(retainedUser.messages[0].content[0].text.startsWith(marker)); +assert.equal(retainedUser.messages[0].content[1].text, 'Keep going'); +assert.deepEqual(retainedUser.messages[1], retainedCheckpoint); +await compactedRoot.invoke(retainedUser); +assert.equal(bootstrapCount(retainedUser), 1); +assert.equal(retainedUser.messages.length, 2); + +const retainedUserChild = compactedEvent('retained-user-child'); +retainedUserChild.messages.unshift({ role: 'user', content: [{ type: 'text', text: 'Keep going' }] }); +const originalRetainedUserChild = structuredClone(retainedUserChild); +await compactedChild.invoke(retainedUserChild); +assert.equal(bootstrapCount(retainedUserChild), 0); +assert.deepEqual(retainedUserChild, originalRetainedUserChild); +const empty = { sessionID: 'empty', messages: [] }; +await compactedRoot.invoke(empty); +assert.deepEqual(empty.messages, []); + +console.log('Session classification, recovery and cache lifetime passed'); diff --git a/tests/opencode/test-session-bootstrap.sh b/tests/opencode/test-session-bootstrap.sh new file mode 100755 index 0000000..accd85b --- /dev/null +++ b/tests/opencode/test-session-bootstrap.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +node "$SCRIPT_DIR/test-session-bootstrap.mjs" "$SCRIPT_DIR/../../.opencode/plugins/superpowers.js" diff --git a/tests/opencode/test-skill-registration.mjs b/tests/opencode/test-skill-registration.mjs new file mode 100644 index 0000000..a31fe4a --- /dev/null +++ b/tests/opencode/test-skill-registration.mjs @@ -0,0 +1,205 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { pathToFileURL } from 'url'; + +// Verifies the V2 skill registration payload matches OpenCode 2.0.4's +// Skill.Info contract (packages/schema/src/skill.ts): +// { id, name, description?, autoinvoke?, path, content } +// Upstream commit 199aabe9e2 (first released in v2.0.4) renamed the required +// file field `location` -> `path`. A wrong field name makes draft.add() +// throw inside the host's transform rebuild, which asynchronously disables +// the whole plugin ("Plugin disabled after skill.transform failed") and +// takes the bootstrap hook down with it — see PR #2106 review by 80avin. + +const [, , inputPath] = process.argv; + +if (!inputPath) { + console.error('Usage: node test-skill-registration.mjs PLUGIN_PATH'); + process.exit(2); +} + +const pluginPath = fs.realpathSync(inputPath); +const skillsDir = path.resolve(path.dirname(pluginPath), '../../skills'); +const mod = await import(pathToFileURL(pluginPath).href); + +const failures = []; + +// --- Run 1: passive capture of every draft.add payload ------------------- +const added = []; +await mod.default.setup(makeCtx({ add: (skill) => added.push(skill) })); + +const expectedIds = fs.existsSync(skillsDir) + ? fs.readdirSync(skillsDir, { withFileTypes: true }) + .filter((e) => e.isDirectory() && !e.name.startsWith('.')) + .filter((e) => fs.existsSync(path.join(skillsDir, e.name, 'SKILL.md'))) + .map((e) => e.name) + .sort() + : []; + +if (added.length === 0) { + failures.push('expected setup() to register at least one skill via draft.add()'); +} +if (JSON.stringify(added.map((s) => s.id).sort()) !== JSON.stringify(expectedIds)) { + failures.push(`expected draft.add() ids to match skills dir contents, got ${JSON.stringify(added.map((s) => s.id))}`); +} + +for (const skill of added) { + if (typeof skill.path !== 'string' || !path.isAbsolute(skill.path)) { + failures.push(`skill "${skill.id}": expected required absolute Skill.Info field "path", got ${JSON.stringify(skill.path)}`); + } else if (skill.path !== path.join(skillsDir, skill.id, 'SKILL.md')) { + failures.push(`skill "${skill.id}": expected path ${path.join(skillsDir, skill.id, 'SKILL.md')}, got ${skill.path}`); + } else if (!fs.existsSync(skill.path)) { + failures.push(`skill "${skill.id}": path does not exist on disk: ${skill.path}`); + } + // Stale 2.0.3-era fields must not leak into the payload: the host strips + // unknown keys, but keeping them would silently mask a future regression + // to a schema that no longer accepts `path`. + if ('location' in skill) { + failures.push(`skill "${skill.id}": payload still carries the pre-2.0.4 field "location"`); + } + if ('slash' in skill) { + failures.push(`skill "${skill.id}": payload carries "slash", removed from Skill.Info in 2.0.4`); + } + if (typeof skill.id !== 'string' || skill.id.length === 0) failures.push(`skill payload missing non-empty "id"`); + if (typeof skill.name !== 'string' || skill.name.length === 0) failures.push(`skill "${skill.id}" missing non-empty "name"`); + if (typeof skill.content !== 'string' || !skill.content.trim()) failures.push(`skill "${skill.id}" missing non-empty "content"`); + if ('description' in skill && typeof skill.description !== 'string') { + failures.push(`skill "${skill.id}": "description" must be a string when present`); + } else if ('description' in skill && /["']$/.test(skill.description)) { + failures.push(`skill "${skill.id}": description ends with a dangling quote: ${JSON.stringify(skill.description)}`); + } + if (typeof skill.content === 'string' && skill.content.startsWith('---')) { + failures.push(`skill "${skill.id}": content still starts with the frontmatter delimiter`); + } +} + +// --- Run 2: hostile draft.add must not abort the remaining registrations -- +// The real host swallows a throw escaping the transform callback and then +// hard-disables the plugin asynchronously. Locally we can only observe the +// synchronous half of that contract: when draft.add() rejects one skill, the +// plugin must keep registering the rest instead of aborting the loop. +const hostileId = added.length > 1 ? added[Math.floor(added.length / 2)].id : null; +const survived = []; +let setupThrew = null; +let survivingContextHook; +try { + await mod.default.setup(makeCtx({ + add: (skill) => { + if (skill.id === hostileId) throw new Error('Simulated Skill.Info decode failure'); + survived.push(skill.id); + }, + onHook: (name, callback) => { + if (name === 'context') survivingContextHook = callback; + }, + })); +} catch (err) { + setupThrew = err; +} +if (setupThrew) { + failures.push(`expected setup() to contain draft.add() failures, but it threw: ${setupThrew.message}`); +} else if (hostileId) { + const expectedSurvivors = added.map((s) => s.id).filter((id) => id !== hostileId); + if (JSON.stringify(survived.sort()) !== JSON.stringify(expectedSurvivors.sort())) { + failures.push(`expected all non-rejected skills to still register when one draft.add() throws, got ${JSON.stringify(survived)}`); + } +} +if (typeof survivingContextHook !== 'function') { + failures.push('expected bootstrap hook to survive a rejected skill'); +} else { + const event = { + sessionID: 'registration-survival-root', + messages: [{ role: 'user', content: [{ type: 'text', text: 'Continue' }] }], + }; + await survivingContextHook(event); + const count = event.messages.flatMap((message) => message.content).filter( + (part) => part.type === 'text' && part.text.startsWith('<EXTREMELY_IMPORTANT>\nYou have superpowers.') + ).length; + if (count !== 1) failures.push(`expected surviving bootstrap once, got ${count}`); +} + +// --- Run 3: quoted and multi-line frontmatter values --------------------- +// The description is what the host shows in its skill list. A quoted value +// that wraps onto indented continuation lines must register as one unquoted +// line, so exercise each layout against a synthetic install: a copy of the +// plugin next to fixture skills, laid out like a real package root. +const frontmatterFixtures = { + 'multi-line-double': { + frontmatter: 'description: "Use when foo happens\n and bar continues\n and baz ends"', + expected: 'Use when foo happens and bar continues and baz ends', + }, + 'multi-line-single': { + frontmatter: "description: 'Use when foo happens\n and bar continues\n and baz ends'", + expected: 'Use when foo happens and bar continues and baz ends', + }, + 'single-line-quoted': { + frontmatter: 'description: "Plain quoted"', + expected: 'Plain quoted', + }, + 'block-scalar': { + frontmatter: 'description: >\n Folded line one\n line two', + expected: 'Folded line one line two', + }, +}; +const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'superpowers-frontmatter-')); +try { + const fixturePlugin = path.join(fixtureRoot, '.opencode', 'plugins', 'superpowers.js'); + fs.mkdirSync(path.dirname(fixturePlugin), { recursive: true }); + fs.copyFileSync(pluginPath, fixturePlugin); + for (const [id, { frontmatter }] of Object.entries(frontmatterFixtures)) { + const skillDir = path.join(fixtureRoot, 'skills', id); + fs.mkdirSync(skillDir, { recursive: true }); + fs.writeFileSync(path.join(skillDir, 'SKILL.md'), `---\nname: ${id}\n${frontmatter}\n---\n# Title\n\nBody.\n`); + } + const fixtureMod = await import(pathToFileURL(fixturePlugin).href); + const fixtureAdded = []; + await fixtureMod.default.setup(makeCtx({ add: (skill) => fixtureAdded.push(skill) })); + for (const [id, { expected }] of Object.entries(frontmatterFixtures)) { + const skill = fixtureAdded.find((s) => s.id === id); + if (!skill) { + failures.push(`fixture "${id}": expected setup() to register it`); + continue; + } + if (skill.description !== expected) { + failures.push(`fixture "${id}": expected description ${JSON.stringify(expected)}, got ${JSON.stringify(skill.description)}`); + } + if (skill.content.startsWith('---')) { + failures.push(`fixture "${id}": content still starts with the frontmatter delimiter`); + } + } +} finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); +} + +const result = { + registered: added.length, + ids: added.map((s) => s.id), + allPathsValid: added.every((s) => s.path === path.join(skillsDir, s.id, 'SKILL.md') && fs.existsSync(s.path)), + staleLocationField: added.some((s) => 'location' in s), + hostileRejectedId: hostileId, + survivedHostileAdd: JSON.stringify(survived.sort()) === JSON.stringify(added.map((s) => s.id).filter((id) => id !== hostileId).sort()), +}; + +if (failures.length > 0) { + console.error(JSON.stringify(result, null, 2)); + for (const failure of failures) { + console.error(`FAIL: ${failure}`); + } + process.exit(1); +} + +console.log(JSON.stringify(result, null, 2)); + +function makeCtx({ add, onHook = () => {} }) { + return { + skill: { + transform: async (fn) => { + await fn({ list: () => [], get: () => undefined, add, update: () => {}, remove: () => {} }); + }, + }, + session: { + hook: async (name, callback) => onHook(name, callback), + get: async ({ sessionID }) => ({ id: sessionID }), // top-level: no parentID + }, + }; +} diff --git a/tests/opencode/test-skill-registration.sh b/tests/opencode/test-skill-registration.sh new file mode 100755 index 0000000..ac882a4 --- /dev/null +++ b/tests/opencode/test-skill-registration.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Test: V2 Skill Registration Contract (#2106 review) +# Verifies setup() registers skills matching OpenCode 2.0.4's Skill.Info +# schema (path field, no stale location/slash) and contains per-skill +# draft.add() failures instead of aborting registration. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" + +echo "=== Test: V2 Skill Registration Contract ===" + +source "$SCRIPT_DIR/setup.sh" +trap cleanup_test_env EXIT + +node "$SCRIPT_DIR/test-skill-registration.mjs" "$SUPERPOWERS_PLUGIN_FILE" +node "$SCRIPT_DIR/test-skill-registration.mjs" "$OPENCODE_CONFIG_DIR/plugins/superpowers.js" + +echo " [PASS] Skill payloads match the 2.0.4 Skill.Info contract" +echo " [PASS] A rejected draft.add() skips one skill without aborting the rest" +echo " [PASS] Quoted and multi-line frontmatter values register unquoted" +echo "" +echo "=== All skill registration tests passed ===" diff --git a/tests/proving-it-works-with-a-movie/README.md b/tests/proving-it-works-with-a-movie/README.md new file mode 100644 index 0000000..2a0eb2d --- /dev/null +++ b/tests/proving-it-works-with-a-movie/README.md @@ -0,0 +1,29 @@ +# Portable movie regressions + +Run a suite from the repository root: + +```sh +uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite assembly +uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite checker +uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite contracts +uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite narration +uv run --script tests/proving-it-works-with-a-movie/run-tests.py --suite all +``` + +By default, unavailable external capabilities are reported as skips. Add +`--require-capabilities` when the selected environment is required to provide +them; any skip then makes the run fail. + +The assembly sine wave is only a synthetic timing fixture. The narration drift +inputs exercise text comparison only. Neither is speech/ASR acceptance. + +The `contracts` suite is the safe entrypoint for mocked process/media boundaries +and text fixtures. It does not run the existing media/session suites, inspect a +finished movie, or replace live acceptance and the required human viewing gate. + +The `terminal` suite starts a real ttyd session and skips where ttyd or a +Chrome-family browser is missing. It films `bash` by default on Unix and +`powershell51` on Windows; set `MOVIE_TEST_SHELL` to `powershell51`, +`powershell7`, `gitbash`, or `bash`, and `MOVIE_TEST_SHELL_EXE`, +`MOVIE_TEST_TTYD`, or `MOVIE_TEST_BROWSER` when those executables are not on +PATH. diff --git a/tests/proving-it-works-with-a-movie/fixtures.py b/tests/proving-it-works-with-a-movie/fixtures.py new file mode 100644 index 0000000..eac965b --- /dev/null +++ b/tests/proving-it-works-with-a-movie/fixtures.py @@ -0,0 +1,299 @@ +"""Portable fixtures for the imported movie regression suites.""" + +import json +import importlib.util +import importlib.machinery +import shutil +import subprocess +import types +import sys +from pathlib import Path + + +TIMEOUT_SECONDS = 900 + + +def missing_executables(*names: str) -> list[str]: + """Return executable names that cannot be resolved on PATH.""" + return [name for name in names if shutil.which(name) is None] + + +def output_text(result: subprocess.CompletedProcess[bytes]) -> str: + """Decode a captured command's combined output as UTF-8 evidence.""" + return (result.stdout + result.stderr).decode("utf-8", errors="replace") + + +def run_tool( + name: str, + args: list[str], + *, + cwd: Path, + env: dict[str, str] | None = None, +) -> subprocess.CompletedProcess[bytes]: + """Invoke one extensionless movie tool through its PEP 723 environment.""" + uv = shutil.which("uv") + if uv is None: + raise RuntimeError("uv is required for movie tool tests") + script = ( + Path(__file__).resolve().parents[2] + / "skills/proving-it-works-with-a-movie/scripts" + / name + ) + return subprocess.run( + [uv, "run", "--script", str(script), *args], + cwd=cwd, + env=env, + capture_output=True, + timeout=TIMEOUT_SECONDS, + ) + + +def load_script(name: str) -> types.ModuleType: + """Load an extensionless movie tool as a test module.""" + script = ( + Path(__file__).resolve().parents[2] + / "skills/proving-it-works-with-a-movie/scripts" + / name + ) + if not script.exists(): + script = script.with_suffix(".py") + sys.path.insert(0, str(script.parent)) + loader = importlib.machinery.SourceFileLoader(f"movie_tool_{name}", str(script)) + spec = importlib.util.spec_from_loader(loader.name, loader) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load movie tool {name!r}") + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + finally: + sys.path.pop(0) + return module + + +def duration(path: Path) -> float: + """Measure a media file's container duration with ffprobe.""" + ffprobe = shutil.which("ffprobe") + if ffprobe is None: + raise RuntimeError("ffprobe is required for movie tool tests") + result = subprocess.run( + [ + ffprobe, + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "csv=p=0", + str(path), + ], + capture_output=True, + timeout=TIMEOUT_SECONDS, + ) + if result.returncode != 0: + raise RuntimeError(f"ffprobe failed for {path}: {output_text(result)}") + return float(result.stdout.decode("utf-8").strip()) + + +def _run_ffmpeg(args: list[str], *, cwd: Path) -> None: + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + raise RuntimeError("ffmpeg is required for movie tool tests") + result = subprocess.run( + [ffmpeg, "-nostdin", "-y", "-v", "error", *args], + cwd=cwd, + capture_output=True, + timeout=TIMEOUT_SECONDS, + ) + if result.returncode != 0: + raise RuntimeError(f"ffmpeg fixture generation failed: {output_text(result)}") + + +def assembly_fixture(work: Path) -> Path: + """Write the imported image/frames/synthetic-timing-audio assembly case.""" + shots = work / "shots" + shots.mkdir(parents=True) + for index in range(1, 5): + _run_ffmpeg( + [ + "-f", + "lavfi", + "-i", + f"color=c=0x{index}0{index}0{index}0:size=320x180:d=0.1", + "-frames:v", + "1", + str(shots / f"s0{index}.png"), + ], + cwd=work, + ) + + narration = work / "narration" + narration.mkdir() + wav = narration / "body.wav" + _run_ffmpeg( + [ + "-f", + "lavfi", + "-i", + "sine=frequency=300:duration=6", + str(wav), + ], + cwd=work, + ) + manifest = [ + { + "id": "body", + "text": "one two three four five six seven eight nine ten", + "wav": "body.wav", + "duration": duration(wav), + } + ] + (narration / "manifest.json").write_text( + json.dumps(manifest), encoding="utf-8" + ) + + scenes = work / "scenes.yaml" + scenes.write_text( + """resolution: { width: 640, height: 360 } +fps: 30 +scenes: + - id: opener + kind: image + src: shots/s01.png + duration: 2 + - id: body + kind: frames + src: shots + rate: 1.0 + narration: one two three four five six seven eight nine ten +""", + encoding="utf-8", + ) + return scenes + + +def checker_fixture(work: Path) -> dict[str, Path]: + """Create the four imported checker movies and their subtitle sidecars.""" + front_loaded = work / "front-loaded.mp4" + _run_ffmpeg( + [ + "-f", + "lavfi", + "-i", + "testsrc2=size=320x240:rate=10:d=2", + "-f", + "lavfi", + "-i", + "color=c=navy:size=320x240:rate=10:d=20", + "-f", + "lavfi", + "-i", + "sine=frequency=300:duration=22", + "-filter_complex", + "[0:v][1:v]concat=n=2:v=1:a=0[v]", + "-map", + "[v]", + "-map", + "2:a", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + str(front_loaded), + ], + cwd=work, + ) + + paced = work / "paced.mp4" + _run_ffmpeg( + [ + "-f", + "lavfi", + "-i", + "testsrc2=size=320x240:rate=10:d=22", + "-f", + "lavfi", + "-i", + "sine=frequency=300:duration=22", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + str(paced), + ], + cwd=work, + ) + + still = work / "still.mp4" + _run_ffmpeg( + [ + "-f", + "lavfi", + "-i", + "color=c=navy:size=320x240:rate=10:d=12", + "-f", + "lavfi", + "-i", + "sine=frequency=300:duration=12", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + str(still), + ], + cwd=work, + ) + + silent = work / "silent.mp4" + _run_ffmpeg( + [ + "-f", + "lavfi", + "-i", + "testsrc2=size=320x240:rate=10:d=12", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + str(silent), + ], + cwd=work, + ) + + subtitles = """1 +00:00:00,000 --> 00:00:07,000 +A narrated movie needs subtitles: +plenty of people watch muted. + +2 +00:00:07,000 --> 00:00:14,000 +The checker treats their absence +as a defect, not a nicety. + +3 +00:00:14,000 --> 00:00:21,500 +And it notices when they stop +before the narration does. +""" + (work / "paced.srt").write_text(subtitles, encoding="utf-8") + (work / "short.srt").write_text( + "\n".join(subtitles.splitlines()[:8]) + "\n", encoding="utf-8" + ) + short = work / "short.mp4" + shutil.copyfile(paced, short) + + return { + "front-loaded": front_loaded, + "paced": paced, + "still": still, + "silent": silent, + "short": short, + } diff --git a/tests/proving-it-works-with-a-movie/fixtures/terminal_app.py b/tests/proving-it-works-with-a-movie/fixtures/terminal_app.py new file mode 100644 index 0000000..0b197d9 --- /dev/null +++ b/tests/proving-it-works-with-a-movie/fixtures/terminal_app.py @@ -0,0 +1,41 @@ +"""TUI fixture: three timed colour states, then wait for `q`. +`tree DIR` instead leaves a three-deep process tree running for cleanup tests.""" +import json, os, subprocess, sys, time +from pathlib import Path + + +def getch(): + if os.name == "nt": + import msvcrt + return msvcrt.getwch() + import termios, tty + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + try: + tty.setcbreak(fd) + return sys.stdin.read(1) + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old) + + +if len(sys.argv) > 1 and sys.argv[1] == "tree": + directory = Path(sys.argv[2]) + directory.mkdir(exist_ok=True) + level = int(sys.argv[3]) if len(sys.argv) > 3 else 0 + if level < 2: + subprocess.Popen([sys.executable, __file__, "tree", str(directory), str(level + 1)]) + (directory / f"{level}.json").write_text(json.dumps(dict(pid=os.getpid(), level=level))) + while True: + time.sleep(1) + +print("WRAPPING OUTPUT " + ("0123456789 wrap proof " * 60), flush=True) +time.sleep(1.4) +states = [] +for label, color in [("STATE ONE RED", 41), ("STATE TWO GREEN", 42), ("STATE THREE BLUE", 44)]: + print("\x1b[2J\x1b[H" + f"\x1b[{color}m" + (label + " " * 60 + "\n") * 12 + "\x1b[0m", end="", flush=True) + states.append(dict(label=label, time=time.monotonic())) + Path("states.json").write_text(json.dumps(states)) + time.sleep(1.5) +while getch() != "q": + pass +print("\nAUTOMATIC GATE COMPLETE", flush=True) diff --git a/tests/proving-it-works-with-a-movie/run-tests.py b/tests/proving-it-works-with-a-movie/run-tests.py new file mode 100644 index 0000000..441d39f --- /dev/null +++ b/tests/proving-it-works-with-a-movie/run-tests.py @@ -0,0 +1,64 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.10" +# dependencies = ["pyyaml", "pillow", "websocket-client==1.9.0"] +# /// +"""Run the proving-it-works-with-a-movie regression suites portably.""" + +import argparse +import sys +import unittest +from pathlib import Path + + +IMPLEMENTED_SUITES = { + "assembly": "test_assembly.py", + "browser": "test_browser.py", + "checker": "test_checker.py", + "contracts": "test_*contract*.py", + "narration": "test_narration.py", + "paths": "test_paths.py", + "subtitles": "test_subtitles.py", + "terminal": "test_terminal.py", +} +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--suite", + required=True, + choices=[*IMPLEMENTED_SUITES, "all"], + ) + parser.add_argument("--require-capabilities", action="store_true") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + patterns = ( + list(IMPLEMENTED_SUITES.values()) + if args.suite == "all" + else [IMPLEMENTED_SUITES[args.suite]] + ) + test_directory = Path(__file__).resolve().parent + loader = unittest.TestLoader() + suite = unittest.TestSuite( + loader.discover( + str(test_directory), + pattern=pattern, + top_level_dir=str(test_directory), + ) + for pattern in patterns + ) + result = unittest.TextTestRunner(verbosity=2).run(suite) + if args.require_capabilities and result.skipped: + print( + f"required capabilities unavailable: {len(result.skipped)} " + "selected test(s) skipped", + file=sys.stderr, + ) + return 1 + return 0 if result.wasSuccessful() else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/proving-it-works-with-a-movie/test_assembly.py b/tests/proving-it-works-with-a-movie/test_assembly.py new file mode 100644 index 0000000..dd771a8 --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_assembly.py @@ -0,0 +1,409 @@ +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import fixtures +from PIL import Image + + +def run_ffmpeg(args: list[str], *, cwd: Path) -> subprocess.CompletedProcess[bytes]: + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + raise RuntimeError("ffmpeg is required for assembly fixtures") + result = subprocess.run( + [ffmpeg, "-nostdin", "-y", "-v", "error", *args], + cwd=cwd, + capture_output=True, + timeout=fixtures.TIMEOUT_SECONDS, + ) + if result.returncode != 0: + raise RuntimeError( + f"ffmpeg fixture generation failed: {fixtures.output_text(result)}" + ) + return result + + +def make_tone(path: Path, duration: float, frequency: int, *, cwd: Path) -> None: + run_ffmpeg( + [ + "-f", + "lavfi", + "-i", + f"sine=frequency={frequency}:sample_rate=8000:duration={duration}", + "-c:a", + "pcm_s16le", + str(path), + ], + cwd=cwd, + ) + + +def make_movie(path: Path, duration: float, frequency: int, *, cwd: Path) -> None: + run_ffmpeg( + [ + "-f", + "lavfi", + "-i", + f"color=c=green:size=160x90:rate=10:duration={duration}", + "-f", + "lavfi", + "-i", + f"sine=frequency={frequency}:sample_rate=8000:duration={duration}", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-shortest", + str(path), + ], + cwd=cwd, + ) + + +def decoded_pixel(path: Path, timestamp: float, *, cwd: Path) -> tuple[int, int, int]: + result = run_ffmpeg( + [ + "-ss", + str(timestamp), + "-i", + str(path), + "-frames:v", + "1", + "-vf", + "scale=1:1", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "-", + ], + cwd=cwd, + ) + if len(result.stdout) < 3: + raise AssertionError(f"no decoded pixel from {path}") + return tuple(result.stdout[:3]) + + +def decoded_frequency(path: Path, *, cwd: Path) -> float: + sample_rate = 8000 + result = run_ffmpeg( + [ + "-ss", + "0.1", + "-t", + "0.5", + "-i", + str(path), + "-map", + "0:a:0", + "-f", + "s16le", + "-acodec", + "pcm_s16le", + "-ac", + "1", + "-ar", + str(sample_rate), + "-", + ], + cwd=cwd, + ) + samples = [ + int.from_bytes(result.stdout[index:index + 2], "little", signed=True) + for index in range(0, len(result.stdout) - 1, 2) + ] + nonzero = [sample for sample in samples if sample] + crossings = sum( + (left < 0 <= right) or (left > 0 >= right) + for left, right in zip(nonzero, nonzero[1:]) + ) + seconds = len(samples) / sample_rate + return crossings / (2 * seconds) + + +def available_browser() -> str | None: + return fixtures.load_script("browser_tools").find_browser(os.environ.get("MOVIE_BROWSER")) + + +class AssemblyRegression(unittest.TestCase): + def test_narration_padding_and_offsets(self): + missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe") + if missing: + self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}") + + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + scenes = fixtures.assembly_fixture(work) + result = fixtures.run_tool( + "assemble", [str(scenes), str(work / "out.mp4")], cwd=work + ) + self.assertEqual(result.returncode, 0, fixtures.output_text(result)) + self.assertAlmostEqual( + fixtures.duration(work / "out.mp4"), 8, delta=0.4 + ) + offsets = json.loads( + (work / "segments/offsets.json").read_text(encoding="utf-8") + ) + self.assertAlmostEqual(offsets["body"], 2, delta=0.3) + + subtitles = fixtures.run_tool( + "make-subtitles", + [ + str(work / "narration/manifest.json"), + str(work / "out.srt"), + "--offsets-json", + str(work / "segments/offsets.json"), + ], + cwd=work, + ) + self.assertEqual( + subtitles.returncode, 0, fixtures.output_text(subtitles) + ) + srt = (work / "out.srt").read_text(encoding="utf-8") + timing_line = next(line for line in srt.splitlines() if "-->" in line) + start = timing_line.partition("-->")[0].strip() + hours, minutes, seconds_millis = start.split(":") + seconds, millis = seconds_millis.split(",") + first_cue_start = ( + int(hours) * 3600 + + int(minutes) * 60 + + int(seconds) + + int(millis) / 1000 + ) + self.assertAlmostEqual( + first_cue_start, + offsets["body"], + delta=0.001, + ) + + def test_image_frames_and_movie_paths_timing_order_and_cleanup(self): + missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe") + if missing: + self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + launch = root / "launch" + launch.mkdir() + project = root / "movie O'Brien λ & [take]" + assets = project / "assets" + frames = assets / "frames" + subtitles = assets / "subtitles" / "nested" + narration = project / "narration" + frames.mkdir(parents=True) + subtitles.mkdir(parents=True) + narration.mkdir() + + still = assets / "still image.png" + Image.new("RGB", (160, 90), (230, 230, 230)).save(still) + Image.new("RGB", (160, 90), (255, 0, 0)).save(frames / "a.png") + Image.new("RGB", (160, 90), (0, 0, 255)).save(frames / "b.png") + source_pngs = { + path: path.read_bytes() for path in [still, *sorted(frames.glob("*.png"))] + } + + source_movie = assets / "source movie.mp4" + make_movie(source_movie, 0.8, 440, cwd=launch) + make_tone(narration / "image.wav", 0.8, 660, cwd=launch) + make_tone(narration / "frames.wav", 1.4, 770, cwd=launch) + make_tone(narration / "movie.wav", 1.6, 880, cwd=launch) + (narration / "manifest.json").write_text(json.dumps([ + {"id": "image", "text": "Image narration", "wav": "image.wav", "duration": 0.8}, + {"id": "frames", "text": "Frames narration", "wav": "frames.wav", "duration": 1.4}, + ]), encoding="utf-8") + + nested_srt = subtitles / "captions.srt" + nested_srt.write_bytes( + b"1\r\n00:00:00,000 --> 00:00:00,500\r\nPortable paths\r\n" + ) + scenes = project / "scenes.yaml" + yaml_text = f"""resolution: {{ width: 160, height: 90 }} +fps: 10 +scenes: + - id: image + kind: image + src: assets/still image.png + duration: 0.3 + narration: Image narration + - id: frames + kind: frames + src: {frames.resolve().as_posix()} + rate: 2.0 + narration: Frames narration + - id: movie + kind: movie + src: assets/source movie.mp4 +""" + scenes.write_bytes(yaml_text.replace("\n", "\r\n").encode("utf-8")) + + work = root / "generated outside launch" + output = project / "assembled output.mp4" + result = fixtures.run_tool( + "assemble", + [ + os.path.relpath(scenes, launch), + os.path.relpath(output, launch), + "--narration", + str(narration.resolve()), + "--work", + str(work.resolve()), + ], + cwd=launch, + ) + self.assertEqual(result.returncode, 0, fixtures.output_text(result)) + + image_duration = fixtures.duration(work / "image.mp4") + frames_duration = fixtures.duration(work / "frames.mp4") + movie_duration = fixtures.duration(work / "movie.mp4") + self.assertAlmostEqual( + image_duration, + max(fixtures.duration(narration / "image.wav"), 0.3), + delta=0.25, + ) + self.assertAlmostEqual( + frames_duration, + max(fixtures.duration(narration / "frames.wav"), 1.0), + delta=0.25, + ) + self.assertAlmostEqual( + movie_duration, + fixtures.duration(source_movie), + delta=0.25, + ) + self.assertLess(movie_duration, fixtures.duration(narration / "movie.wav") - 0.4) + self.assertAlmostEqual( + decoded_frequency(work / "movie.mp4", cwd=launch), + 440, + delta=15, + ) + + offsets = json.loads( + (work / "offsets.json").read_text(encoding="utf-8") + ) + self.assertAlmostEqual(offsets["image"], 0.0, delta=0.001) + self.assertAlmostEqual(offsets["frames"], image_duration, delta=0.001) + self.assertNotIn("movie", offsets) + + first = decoded_pixel(work / "frames.mp4", 0.2, cwd=launch) + second = decoded_pixel(work / "frames.mp4", 0.7, cwd=launch) + self.assertGreater(first[0], first[2] + 100) + self.assertGreater(second[2], second[0] + 100) + self.assertEqual( + {path: path.read_bytes() for path in source_pngs}, + source_pngs, + ) + self.assertEqual(list(work.glob("frames-frames-*")), []) + + subtitled = project / "subtitled output.mp4" + subtitle_result = fixtures.run_tool( + "burn-subtitles", + [ + str(output.resolve()), + os.path.relpath(nested_srt, launch), + str(subtitled.resolve()), + "--soft", + ], + cwd=launch, + ) + self.assertEqual( + subtitle_result.returncode, + 0, + fixtures.output_text(subtitle_result), + ) + self.assertTrue(subtitled.is_file()) + + def test_card_uses_longer_narration_duration(self): + missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe") + if missing: + self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}") + browser = available_browser() + if browser is None: + self.skipTest("required browser unavailable for card assembly") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + narration = root / "narration" + narration.mkdir() + make_tone(narration / "card.wav", 0.8, 550, cwd=root) + (narration / "manifest.json").write_text(json.dumps([ + {"id": "card", "text": "Card narration", "wav": "card.wav", "duration": 0.8} + ]), encoding="utf-8") + scenes = root / "scenes.yaml" + scenes.write_bytes( + b"\xef\xbb\xbfresolution: { width: 160, height: 90 }\r\n" + b"fps: 10\r\n" + b"scenes:\r\n" + b" - id: card\r\n" + b" kind: card\r\n" + b" title: Portable\r\n" + b" subtitle: paths\r\n" + b" duration: 0.3\r\n" + b" narration: Card narration\r\n" + ) + work = root / "work" + result = fixtures.run_tool( + "assemble", + [ + str(scenes), + str(root / "out.mp4"), + "--work", + str(work), + "--browser", + browser, + ], + cwd=root, + ) + self.assertEqual(result.returncode, 0, fixtures.output_text(result)) + self.assertAlmostEqual( + fixtures.duration(work / "card.mp4"), + max(fixtures.duration(narration / "card.wav"), 0.3), + delta=0.25, + ) + offsets = json.loads( + (work / "offsets.json").read_text(encoding="utf-8") + ) + self.assertAlmostEqual(offsets["card"], 0.0, delta=0.001) + + def test_frame_sources_survive_failed_assembly_cleanup(self): + missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe") + if missing: + self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "movie O'Brien λ & [take]" / "frames" + source.mkdir(parents=True) + bad_frame = source / "a.png" + bad_frame.write_bytes(b"not a PNG") + scenes = root / "scenes.yaml" + scenes.write_text( + f"""resolution: {{ width: 160, height: 90 }} +fps: 10 +scenes: + - id: broken + kind: frames + src: {source.resolve().as_posix()} + rate: 1.0 +""", + encoding="utf-8", + ) + work = root / "work" + result = fixtures.run_tool( + "assemble", + [str(scenes), str(root / "out.mp4"), "--work", str(work)], + cwd=root, + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(bad_frame.read_bytes(), b"not a PNG") + self.assertEqual(list(work.glob("frames-broken-*")), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/proving-it-works-with-a-movie/test_browser.py b/tests/proving-it-works-with-a-movie/test_browser.py new file mode 100644 index 0000000..e14e37b --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_browser.py @@ -0,0 +1,73 @@ +import os +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from PIL import Image +import fixtures + + +class BrowserToolsRegression(unittest.TestCase): + def test_explicit_unusable_browser_fails_authoritatively(self): + module = fixtures.load_script("browser_tools") + with self.assertRaises(FileNotFoundError): + module.find_browser("this-browser-does-not-exist") + if os.name != "nt": + with tempfile.NamedTemporaryFile() as file: + with self.assertRaises(FileNotFoundError): + module.find_browser(file.name) + + def test_windows_chrome_edge_discovery(self): + module = fixtures.load_script("browser_tools") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for relative in ("Google/Chrome/Application/chrome.exe", "Microsoft/Edge/Application/msedge.exe"): + browser = root / relative + browser.parent.mkdir(parents=True) + browser.touch() + with patch.object(module.sys, "platform", "win32"), patch.dict(os.environ, {"LOCALAPPDATA": str(root)}, clear=True), patch.object(module.shutil, "which", return_value=None): + self.assertEqual(module.find_browser(None), str(browser.resolve())) + browser.unlink() + with patch.object(module.sys, "platform", "win32"), patch.dict(os.environ, {}, clear=True), patch.object(module.shutil, "which", side_effect=lambda name: "C:/Edge/msedge.exe" if name == "msedge.exe" else None): + self.assertEqual(module.find_browser(None), "C:/Edge/msedge.exe") + + def browser(self, module): + browser = module.find_browser(os.environ.get("MOVIE_BROWSER")) + if not browser: + self.skipTest("Chrome/Edge is required") + return browser + + def test_render_card_special_path_is_a_real_png(self): + module = fixtures.load_script("browser_tools") + browser = self.browser(module) + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) / "special path O'Brien λ # %" + work.mkdir() + html, png = work / "card page.html", work / "card.png" + html.write_text("<meta charset='utf-8'><style>body{margin:0;background:rgb(255,0,0)}</style><p>λ</p>", encoding="utf-8") + module.render_card(html, png, browser=browser, width=640, height=360) + with Image.open(png) as image: + self.assertEqual(image.size, (640, 360)) + self.assertEqual(image.convert("RGB").getpixel((500, 200)), (255, 0, 0)) + + def test_render_card_timeout_preserves_unrelated_process(self): + module = fixtures.load_script("browser_tools") + browser = self.browser(module) + sentinel = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) + try: + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + html = work / "card.html" + html.write_text("<p>timeout</p>", encoding="utf-8") + start = time.monotonic() + with self.assertRaises(TimeoutError): + module.render_card(html, work / "card.png", browser=browser, width=640, height=360, timeout=0) + self.assertLess(time.monotonic() - start, 10) + self.assertIsNone(sentinel.poll()) + finally: + sentinel.terminate() + sentinel.wait(timeout=10) diff --git a/tests/proving-it-works-with-a-movie/test_browser_contract.py b/tests/proving-it-works-with-a-movie/test_browser_contract.py new file mode 100644 index 0000000..64eb6bc --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_browser_contract.py @@ -0,0 +1,112 @@ +"""Browser cleanup decisions with fake processes and a completed-output token.""" +import contextlib +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import fixtures + + +class CompletedOutput: + """Stand in for the completed-output observation without creating an image.""" + def startswith(self, prefix): + return True + + def endswith(self, suffix): + return True + + +@contextlib.contextmanager +def rendering(*, exited=False, taskkill_status=0, wait_timeout=False, locked_profile=False): + module = fixtures.load_script("browser_tools") + with tempfile.TemporaryDirectory() as temp, contextlib.ExitStack() as stack: + html, png = Path(temp).resolve() / "card.html", Path(temp).resolve() / "card.png" + html.write_text("<p>card</p>") + process = SimpleNamespace(pid=1100, returncode=0 if exited else None) + process.poll = lambda: process.returncode + calls, profiles = [], [] + original_is_file, original_unlink = Path.is_file, os.unlink + + def popen(argv, **kwargs): + profiles.append(Path(kwargs["cwd"])) + return process + + def taskkill(argv, **kwargs): + calls.append(argv) + if taskkill_status == 0 and not wait_timeout: + process.returncode = -9 + return subprocess.CompletedProcess(argv, taskkill_status, b"", b"termination failed") + + def wait(timeout): + if process.returncode is None: + raise subprocess.TimeoutExpired("fake browser", timeout) + return process.returncode + + def unlink(path, *args, **kwargs): + if locked_profile and Path(path).name == "browser.log": + raise PermissionError("locked browser profile") + return original_unlink(path, *args, **kwargs) + + process.wait = wait + for obj, name, value in ( + (module.sys, "platform", "win32"), + (module.subprocess, "Popen", popen), + (module.subprocess, "run", taskkill), + (Path, "is_file", lambda path: True if path == png else original_is_file(path)), + (Path, "read_bytes", lambda path: CompletedOutput()), + (os, "unlink", unlink), + ): + stack.enter_context(patch.object(obj, name, value)) + try: + yield SimpleNamespace(render=lambda: module.render_card(html, png, browser="fake-browser", width=640, height=360), + process=process, calls=calls, profiles=profiles) + finally: + stack.close() + for profile in profiles: + if profile.exists(): + module.shutil.rmtree(profile) + + +class BrowserCleanupContract(unittest.TestCase): + def test_windows_tree_termination_failure_reaches_caller_as_oserror(self): + module = fixtures.load_script("browser_tools") + with patch.object(module.sys, "platform", "win32"), \ + patch.object(module.subprocess, "run", return_value=subprocess.CompletedProcess([], 1, b"", b"access denied")): + with self.assertRaises(OSError): + module.kill_process_tree(1100) + + def test_completed_output_does_not_hide_tree_termination_failure(self): + with rendering(taskkill_status=1) as rig: + with self.assertRaises(OSError): + rig.render() + + def test_completed_output_does_not_hide_owned_child_wait_timeout(self): + with rendering(wait_timeout=True) as rig: + with self.assertRaises(subprocess.TimeoutExpired): + rig.render() + + def test_completed_output_does_not_hide_locked_profile(self): + with rendering(locked_profile=True) as rig: + with self.assertRaises(PermissionError): + rig.render() + + def test_normally_exited_completed_card_succeeds_without_numeric_pid_cleanup(self): + with rendering(exited=True, taskkill_status=1) as rig: + self.assertIsNone(rig.render()) + self.assertEqual(rig.calls, []) + self.assertFalse(rig.profiles[0].exists()) + + def test_completed_card_releases_live_browser_and_profile(self): + with rendering() as rig: + self.assertIsNone(rig.render()) + self.assertEqual(rig.process.poll(), -9) + self.assertEqual(len(rig.calls), 1) + self.assertFalse(rig.profiles[0].exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/proving-it-works-with-a-movie/test_checker.py b/tests/proving-it-works-with-a-movie/test_checker.py new file mode 100644 index 0000000..eec81ee --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_checker.py @@ -0,0 +1,194 @@ +import io +import json +import subprocess +import sys +import tempfile +from contextlib import redirect_stdout +import unittest +from pathlib import Path +from unittest.mock import patch + +from PIL import Image + +import fixtures + + +class CheckerPolicyRegression(unittest.TestCase): + def check(self, expected_exit, *options, audio=True, levels=None, + embedded=None, sidecar=None, extraction_exit=0): + module = fixtures.load_script("check-movie") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + movie = root / "movie.mp4" + movie.write_bytes(b"metadata fixture only") + if sidecar is not None: + movie.with_suffix(".srt").write_text(sidecar, encoding="utf-8-sig") + streams = [{"index": 0, "codec_type": "video", "codec_name": "h264", + "width": 640, "height": 360}] + if audio: + streams.append({"index": 1, "codec_type": "audio", "codec_name": "aac"}) + if embedded is not None: + streams.append({"index": 2, "codec_type": "subtitle", "codec_name": "mov_text"}) + levels = levels if levels is not None else [-20.0] * 20 + info = {"format": {"duration": str(len(levels))}, "streams": streams} + + def media_command(cmd, **kwargs): + if cmd[0] == "ffprobe": + return subprocess.CompletedProcess(cmd, 0, json.dumps(info), "") + if cmd[0] == "ffmpeg" and embedded is not None: + self.assertEqual(cmd[cmd.index("-i") + 1], str(movie)) + self.assertEqual(cmd[cmd.index("-map") + 1], "0:s:0") + self.assertEqual(cmd[cmd.index("-f") + 1], "srt") + return subprocess.CompletedProcess(cmd, extraction_exit, embedded, + "subtitle decode failed" if extraction_exit else "") + raise AssertionError(f"unexpected media command: {cmd}") + + output = root / "report" + argv = ["check-movie", str(movie), "--out", str(output), "--json", *options] + stdout = io.StringIO() + with patch.object(sys, "argv", argv), \ + patch.object(module.shutil, "which", return_value="test-tool"), \ + patch.object(module.subprocess, "run", side_effect=media_command), \ + patch.object(module, "sample_picture", return_value=([], [0.1] * (len(levels) - 1))), \ + patch.object(module, "sample_sound", return_value=levels if audio else []), \ + patch.object(module, "contact_sheet", return_value=[]), \ + redirect_stdout(stdout): + try: + code = module.main() + except SystemExit as error: + code = error.code + self.assertEqual(code, expected_exit, stdout.getvalue()) + report = output / "check.json" + failures = json.loads(report.read_text(encoding="utf-8"))["failures"] if report.exists() else [] + return failures, stdout.getvalue() + + def test_silent_encoded_track_passes_when_audio_is_not_expected(self): + failures, _ = self.check(0, "--no-expect-audio", levels=[-120.0] * 20) + self.assertEqual(failures, []) + + def test_absent_audio_passes_when_audio_is_not_expected(self): + failures, _ = self.check(0, "--no-expect-audio", audio=False) + self.assertEqual(failures, []) + + def test_silent_encoded_track_fails_when_audio_is_expected(self): + failures, _ = self.check(1, levels=[-120.0] * 20) + self.assertTrue(any("silent" in failure for failure in failures)) + + def test_audio_opt_out_still_requires_captions_for_audible_speech(self): + failures, _ = self.check(1, "--no-expect-audio") + self.assertTrue(any("no subtitles" in failure for failure in failures)) + + def test_subtitle_opt_out_allows_audible_speech_without_captions(self): + failures, _ = self.check(0, "--no-expect-audio", "--no-expect-subtitles") + self.assertEqual(failures, []) + + def test_empty_embedded_track_fails_even_for_short_narration(self): + for seconds in (2, 20): + with self.subTest(seconds=seconds): + failures, _ = self.check(1, embedded="", levels=[-20.0] * seconds) + self.assertTrue(any("subtitle" in failure for failure in failures)) + + def test_sidecar_and_embedded_cues_must_reach_the_end_of_speech(self): + for source in ("sidecar", "embedded"): + for end, expected_exit in (("06,000", 1), ("10,000", 0)): + with self.subTest(source=source, end=end): + subtitles = f"1\n00:00:00,000 --> 00:00:{end}\nUnicode λ caption\n" + failures, _ = self.check(expected_exit, levels=[-20.0] * 10 + [-120.0] * 10, + **{source: subtitles}) + self.assertEqual(bool(failures), bool(expected_exit)) + + def test_embedded_extraction_failure_is_not_accepted(self): + _, diagnostics = self.check(2, embedded="", extraction_exit=1) + self.assertIn("subtitle", diagnostics) + + def test_malformed_embedded_cue_is_a_reported_failure(self): + _, diagnostics = self.check(2, embedded="1\n00:00:00,000 --> invalid\ncaption\n") + self.assertIn("subtitle", diagnostics) + + +class CheckerRegression(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + missing = fixtures.missing_executables("uv", "ffmpeg", "ffprobe") + if missing: + raise unittest.SkipTest( + f"required executable(s) not on PATH: {', '.join(missing)}" + ) + cls._temporary_directory = tempfile.TemporaryDirectory() + cls.addClassCleanup(cls._temporary_directory.cleanup) + cls.work = Path(cls._temporary_directory.name) + cls.movies = fixtures.checker_fixture(cls.work) + + def check( + self, expected_exit: int, needle: str, movie: Path, *extra_args: str + ) -> Path: + output_directory = self.work / f"{movie.stem}-check" + result = fixtures.run_tool( + "check-movie", + [str(movie), "--out", str(output_directory), *extra_args], + cwd=self.work, + ) + output = fixtures.output_text(result) + self.assertEqual(result.returncode, expected_exit, output) + self.assertIn(needle.casefold(), output.casefold()) + return output_directory + + def test_front_loaded_action_is_rejected(self): + self.check( + 1, + "every visible change happens in the first", + self.movies["front-loaded"], + ) + + def test_paced_with_subtitles_is_accepted(self): + self.check(0, "Mechanical checks pass", self.movies["paced"]) + + def test_narrated_without_subtitles_is_rejected(self): + self.check( + 1, + "no subtitles", + self.movies["paced"], + "--subs", + str(self.work / "nope.srt"), + ) + + def test_subtitles_that_stop_early_are_rejected(self): + self.check(1, "subtitles stop at", self.movies["short"]) + + def test_subtitle_check_is_opt_outable(self): + self.check( + 0, + "Mechanical checks pass", + self.movies["paced"], + "--subs", + str(self.work / "nope.srt"), + "--no-expect-subtitles", + ) + + def test_still_with_audio_is_rejected(self): + self.check(1, "never reaches a new state", self.movies["still"]) + + def test_missing_narration_is_rejected(self): + self.check(1, "no audio stream", self.movies["silent"]) + + def test_silent_movie_passes_when_unnarrated(self): + self.check( + 0, + "Mechanical checks pass", + self.movies["silent"], + "--no-expect-audio", + ) + + def test_contact_sheet_is_always_written(self): + output_directory = self.check( + 0, "contact-sheet.png", self.movies["paced"] + ) + sheet = output_directory / "contact-sheet.png" + self.assertTrue(sheet.is_file(), f"missing contact sheet: {sheet}") + with Image.open(sheet) as image: + image.load() + self.assertEqual(image.format, "PNG") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/proving-it-works-with-a-movie/test_narration.py b/tests/proving-it-works-with-a-movie/test_narration.py new file mode 100644 index 0000000..dece4c6 --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_narration.py @@ -0,0 +1,351 @@ +import io +import json +import sys +import tempfile +from contextlib import redirect_stderr, redirect_stdout +import unittest +from pathlib import Path +from unittest.mock import patch + +import fixtures + + +SCRIPT = ( + "This is smevals studio. Every eval on the shelf is a folder of tasks and " + "graders." +) + + +class NarrationDriftRegression(unittest.TestCase): + def drift(self, expected_exit: int, heard: str, script: str = SCRIPT) -> None: + missing = fixtures.missing_executables("uv") + if missing: + self.skipTest( + f"required executable(s) not on PATH: {', '.join(missing)}" + ) + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + script_path = work / "script.txt" + heard_path = work / "heard.txt" + script_path.write_text(script, encoding="utf-8") + heard_path.write_text(heard, encoding="utf-8") + result = fixtures.run_tool( + "narrate", + ["--drift-check", str(script_path), str(heard_path)], + cwd=work, + ) + self.assertEqual( + result.returncode, expected_exit, fixtures.output_text(result) + ) + + def test_mispronounced_jargon_passes(self): + self.drift( + 0, + "This is Mevil studio. Every Yvel on the shelf is a folder of tasks " + "and graders.", + ) + + def test_exact_transcript_passes(self): + self.drift(0, SCRIPT) + + def test_dropped_clause_fails(self): + self.drift(1, "This is smevals studio.") + + def test_invented_preamble_fails(self): + self.drift( + 1, + "Sure, here it is, happy to help with that. This is smevals studio. " + "Every eval on the shelf is a folder of tasks and graders.", + ) + + def test_empty_clip_fails(self): + self.drift(1, "you") + + def test_inserted_runs_fail_even_when_total_length_is_close(self): + words = [f"word{i}" for i in range(50)] + for position in (0, 25, 50): + with self.subTest(position=position): + heard = words[:position] + "Before we begin please listen".split() + words[position:] + self.drift(1, " ".join(heard), " ".join(words)) + + def test_expanded_replacement_counts_the_added_words(self): + words = [f"word{i}" for i in range(50)] + heard = words[:25] + "Before we begin please listen".split() + words[26:] + self.drift(1, " ".join(heard), " ".join(words)) + + def test_short_insertions_keep_the_existing_tolerance(self): + words = [f"word{i}" for i in range(50)] + self.drift(0, "Please listen closely " + " ".join(words), " ".join(words)) + + def test_cached_audio_requires_requested_verification(self): + import json + import sys + module = fixtures.load_script("narrate") + with tempfile.TemporaryDirectory() as tmp: + work = Path(tmp) + output = work / "voice" + output.mkdir() + (output / "clip.wav").write_bytes(b"cached audio fixture") + (output / "manifest.json").write_text(json.dumps([ + {"id": "clip", "text": "Read this sentence.", "wav": "clip.wav", + "duration": 1.0, "synthesis": {"engine": "piper", + "voice": module.PIPER_VOICE, "model": module.PIPER_VOICE}} + ]), encoding="utf-8") + scenes = work / "scenes.yaml" + scenes.write_text(json.dumps({"scenes": [ + {"id": "clip", "narration": "Read this sentence."} + ]}), encoding="utf-8") + argv = ["narrate", str(scenes), str(output), + "--engine", "piper", "--verify", "on"] + stdout, stderr = io.StringIO(), io.StringIO() + with patch.object(sys, "argv", argv), \ + patch.object(module.shutil, "which", return_value="ffprobe"), \ + patch.object(module, "openai_key", return_value=None), \ + patch.object(module, "duration", return_value=1.0), \ + patch.object(module, "say_piper", side_effect=AssertionError("expected cached clip")), \ + patch.object(module, "transcribe_local", return_value=None), \ + redirect_stdout(stdout), redirect_stderr(stderr): + self.assertNotEqual(module.main(), 0) + self.assertIn("clip: required verification unavailable", stderr.getvalue()) + self.assertIn("FAILED verbatim delivery: ['clip']", stderr.getvalue()) + self.assertEqual( + json.loads((output / "manifest.json").read_text(encoding="utf-8")), + [], + ) + self.assertEqual((output / "clip.wav").read_bytes(), b"cached audio fixture") + + def test_rejected_chat_audio_is_never_cached_but_accepted_audio_is(self): + import json + import sys + module = fixtures.load_script("narrate") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + scenes = root / "scenes.yaml" + scenes.write_text(json.dumps({"scenes": [ + {"id": "accepted", "narration": "Read this sentence exactly."}, + {"id": "rejected", "narration": "Keep this evidence out of the manifest."}, + ]}), encoding="utf-8") + output = root / "voice" + calls = [] + + def synthesize(key, text, wav, voice): + calls.append(text) + wav.write_bytes(f"render {len(calls)}".encode()) + if text.startswith("Keep"): + return "Unrelated invented preamble with entirely different words here." + return text + + argv = ["narrate", str(scenes), str(output), "--engine", "openai-chat", + "--verify", "off"] + rejected_renders = [] + with patch.object(sys, "argv", argv), \ + patch.object(module.shutil, "which", return_value="ffprobe"), \ + patch.object(module, "openai_key", return_value="test-key"), \ + patch.object(module, "say_openai_chat", side_effect=synthesize), \ + patch.object(module, "duration", return_value=1.0): + for _ in range(2): + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertEqual(module.main(), 1) + manifest = json.loads( + (output / "manifest.json").read_text(encoding="utf-8") + ) + self.assertEqual([entry["id"] for entry in manifest], ["accepted"]) + rejected_renders.append(sorted( + path.read_bytes() for path in output.glob(".rejected.attempt-*.wav") + )) + + self.assertEqual(calls.count("Read this sentence exactly."), 1) + self.assertEqual(calls.count("Keep this evidence out of the manifest."), 4) + self.assertLess(len(rejected_renders[0]), len(rejected_renders[1])) + +class TranscriptionProtocolRegression(unittest.TestCase): + def test_owned_json_is_used_instead_of_library_stdout(self): + import json + import subprocess + import sys + import io + module = fixtures.load_script("narrate") + def child(argv, **kwargs): + self.assertIn("--isolated", argv) + self.assertIn("--no-project", argv) + self.assertIn("--no-config", argv) + self.assertEqual(argv[argv.index("--python") + 1], sys.executable) + self.assertNotEqual(Path(kwargs["cwd"]), Path.cwd()) + Path(argv[-1]).write_text(json.dumps({"text": "Correct λ transcript"}), encoding="utf-8") + return subprocess.CompletedProcess(argv, 0, "native library warning", "diagnostic") + diagnostics = io.StringIO() + with patch.object(module.subprocess, "run", side_effect=child), redirect_stderr(diagnostics): + self.assertEqual(module.transcribe_local(Path("clip.wav")), "Correct λ transcript") + self.assertIn("native library warning", diagnostics.getvalue()) + self.assertIn("diagnostic", diagnostics.getvalue()) + + def test_failed_absent_and_malformed_child_results_are_unavailable(self): + import subprocess + module = fixtures.load_script("narrate") + for payload, code in ((None, 0), ("garbage", 0), ('{"text": 7}', 0), ('{"text": ""}', 0), ('{"text": "words"}', 1)): + with self.subTest(payload=payload, code=code): + def child(argv, **kwargs): + if payload is not None and "--isolated" in argv: + Path(argv[-1]).write_text(payload, encoding="utf-8") + return subprocess.CompletedProcess(argv, code, "misleading stdout", "error") + diagnostics = io.StringIO() + with patch.object(module.subprocess, "run", side_effect=child), \ + redirect_stderr(diagnostics): + self.assertEqual( + module.transcribe_local(Path("clip.wav")), + "" if payload == '{"text": ""}' and code == 0 else None, + ) + self.assertIn("local ASR", diagnostics.getvalue()) + + def test_fresh_and_off_then_on_clips_require_asr(self): + import json + import sys + module = fixtures.load_script("narrate") + for cached in (False, True): + with self.subTest(cached=cached), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + scenes = root / "scenes.yaml" + scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Read this sentence."}]}), encoding="utf-8-sig") + output = root / "voice" + def synthesize(text, wav, voice): + wav.write_bytes(b"branch policy fixture") + argv = ["narrate", str(scenes), str(output), "--engine", "piper", "--verify"] + with patch.object(module.shutil, "which", return_value="ffprobe"), patch.object(module, "openai_key", return_value=None), patch.object(module, "say_piper", side_effect=synthesize), patch.object(module, "duration", return_value=1.0), patch.object(module, "transcribe_local", return_value=None): + if cached: + with patch.object(sys, "argv", [*argv, "off"]), \ + redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertEqual(module.main(), 0) + stdout, stderr = io.StringIO(), io.StringIO() + with patch.object(sys, "argv", [*argv, "on"]), \ + redirect_stdout(stdout), redirect_stderr(stderr): + self.assertNotEqual(module.main(), 0) + self.assertIn("clip: required verification unavailable", stderr.getvalue()) + + +class NarrationCacheRegression(unittest.TestCase): + def setUp(self): + self.module = fixtures.load_script("narrate") + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + self.scenes = self.root / "scenes.yaml" + self.text = "Read this sentence exactly." + self.write_scenes(self.text) + self.output = self.root / "voice" + self.renders = [] + + which = patch.object(self.module.shutil, "which", return_value="ffprobe") + which.start() + self.addCleanup(which.stop) + + def synthesize(*args): + text, wav, voice = args[-3:] + self.renders.append((text, voice)) + wav.write_bytes(f"render {len(self.renders)}".encode()) + return text + + for name, options in ( + ("openai_key", {"return_value": "test-key"}), + ("duration", {"return_value": 1.0}), + ("transcribe_local", {"return_value": None}), + ("say_piper", {"side_effect": synthesize}), + ("say_openai", {"side_effect": synthesize}), + ("say_openai_chat", {"side_effect": synthesize}), + ): + mocked = patch.object(self.module, name, **options) + mocked.start() + self.addCleanup(mocked.stop) + + def write_scenes(self, text): + self.scenes.write_text(json.dumps({"scenes": [ + {"id": "clip", "narration": text} + ]}), encoding="utf-8") + + def narrate(self, *options, verify="off", expected_exit=0): + argv = ["narrate", str(self.scenes), str(self.output), + "--verify", verify, *options] + stdout, stderr = io.StringIO(), io.StringIO() + with patch.object(sys, "argv", argv), \ + redirect_stdout(stdout), redirect_stderr(stderr): + self.assertEqual(self.module.main(), expected_exit, stderr.getvalue()) + return json.loads((self.output / "manifest.json").read_text(encoding="utf-8")) + + def test_engine_and_voice_changes_rerender(self): + for index, options in enumerate(( + ("--engine", "piper", "--voice", "voice-a"), + ("--engine", "piper", "--voice", "voice-b"), + ("--engine", "openai", "--voice", "voice-b"), + ("--engine", "openai-chat", "--voice", "voice-b"), + ), 1): + with self.subTest(options=options): + self.narrate(*options) + self.assertEqual(len(self.renders), index) + self.assertEqual((self.output / "clip.wav").read_bytes(), + f"render {index}".encode()) + + def test_cloud_model_changes_rerender(self): + for engine, constant in (("openai", "OPENAI_TTS_MODEL"), + ("openai-chat", "OPENAI_CHAT_MODEL")): + with self.subTest(engine=engine): + self.narrate("--engine", engine) + count = len(self.renders) + with patch.object(self.module, constant, "another-model"): + self.narrate("--engine", engine) + self.assertEqual(len(self.renders), count + 1) + + def test_implicit_and_explicit_defaults_share_cache(self): + for engine, voice in (("piper", self.module.PIPER_VOICE), + ("openai", "nova"), ("openai-chat", "nova")): + with self.subTest(engine=engine): + self.narrate("--engine", engine) + count = len(self.renders) + self.narrate("--engine", engine, "--voice", voice) + self.assertEqual(len(self.renders), count) + self.narrate("--engine", "openai") + count = len(self.renders) + self.narrate() + self.assertEqual(len(self.renders), count) + with patch.object(self.module, "openai_key", return_value=None): + self.narrate() + self.assertEqual(len(self.renders), count + 1) + + def test_cache_without_synthesis_settings_rerenders(self): + manifest = self.narrate() + manifest[0].pop("synthesis", None) + (self.output / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + self.narrate() + self.assertEqual(len(self.renders), 2) + + def test_changed_text_force_and_missing_wav_rerender(self): + self.narrate() + self.write_scenes("Read a different sentence exactly.") + manifest = self.narrate() + self.assertEqual(manifest[0]["text"], "Read a different sentence exactly.") + self.assertEqual(len(self.renders), 2) + self.narrate("--force") + self.assertEqual(len(self.renders), 3) + (self.output / "clip.wav").unlink() + self.narrate() + self.assertEqual(len(self.renders), 4) + + def test_cached_clip_is_reverified_with_requested_asr_model(self): + self.narrate() + with patch.object(self.module, "transcribe_local", return_value=self.text) as asr: + self.narrate("--asr-model", "small.en", verify="on") + asr.assert_called_once_with(self.output / "clip.wav", "small.en") + self.assertEqual(len(self.renders), 1) + + def test_unavailable_asr_respects_each_verification_mode(self): + for engine in ("piper", "openai", "openai-chat"): + for mode in ("auto", "on", "off"): + with self.subTest(engine=engine, mode=mode): + self.module.transcribe_local.reset_mock() + manifest = self.narrate("--engine", engine, verify=mode, + expected_exit=1 if mode == "on" else 0) + self.assertEqual(bool(manifest), mode != "on") + self.assertEqual(self.module.transcribe_local.call_count, + int(mode == "on" or (mode == "auto" and engine != "openai"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/proving-it-works-with-a-movie/test_narration_contract.py b/tests/proving-it-works-with-a-movie/test_narration_contract.py new file mode 100644 index 0000000..c5bc56d --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_narration_contract.py @@ -0,0 +1,546 @@ +import base64 +import io +import json +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +import fixtures + + +class ChatResponseContract(unittest.TestCase): + def test_malformed_transcripts_cannot_publish_candidates_in_any_asr_mode(self): + cases = ({}, {"transcript": None}, {"transcript": 42}, + {"transcript": ["Two", "words"]}, {"transcript": {}}, + {"transcript": False}, {"transcript": ""}, + {"transcript": " "}, {"transcript": "...!?"}) + for response in cases: + for mode in ("off", "auto", "on"): + with self.subTest(response=response, mode=mode), tempfile.TemporaryDirectory() as temp: + module = fixtures.load_script("narrate") + root = Path(temp) + scenes, output = root / "scenes.json", root / "narration" + scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Two words"}]})) + sentinels = [] + + def post(*args, **kwargs): + sentinel = f"NOT MEDIA: response {len(sentinels) + 1}".encode() + sentinels.append(sentinel) + audio = dict(response, data=base64.b64encode(sentinel).decode()) + return {"choices": [{"message": {"audio": audio}}]} + + argv = ["narrate", str(scenes), str(output), "--engine", "openai-chat", "--verify", mode] + with patch.object(sys, "argv", argv), \ + patch.object(module.shutil, "which", return_value="fake-ffprobe"), \ + patch.object(module, "openai_key", return_value="fake-key"), \ + patch.object(module, "post", post), \ + patch.object(module, "duration", return_value=1.25), \ + patch.object(module, "transcribe_local", side_effect=AssertionError("invalid transcript reached ASR")), \ + redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + try: + result = module.main() + except Exception as error: + result = error + self.assertEqual(result, 1) + self.assertEqual(json.loads((output / "manifest.json").read_text()), []) + self.assertEqual(len(sentinels), 2) + self.assertEqual({path.read_bytes() for path in output.glob(".clip.attempt-*.wav")}, set(sentinels)) + self.assertFalse((output / "clip.wav").exists()) + + def test_valid_chat_and_cached_reuse_obey_independent_asr_modes(self): + for mode in ("off", "auto", "on"): + for heard in (None, "Two words"): + with self.subTest(mode=mode, heard=heard), tempfile.TemporaryDirectory() as temp: + module = fixtures.load_script("narrate") + root = Path(temp) + scenes, output = root / "scenes.json", root / "narration" + scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Two words"}]})) + sentinel = b"NOT MEDIA: accepted response" + response = {"choices": [{"message": {"audio": { + "data": base64.b64encode(sentinel).decode(), "transcript": "Two words", + }}}]} + argv = ["narrate", str(scenes), str(output), "--engine", "openai-chat", "--verify", "off"] + with patch.object(sys, "argv", argv), \ + patch.object(module.shutil, "which", return_value="fake-ffprobe"), \ + patch.object(module, "openai_key", return_value="fake-key"), \ + patch.object(module, "post", return_value=response) as post, \ + patch.object(module, "duration", return_value=1.25), \ + patch.object(module, "transcribe_local", return_value=heard) as asr, \ + redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertEqual(module.main(), 0) + argv[-1] = mode + expected = 1 if mode == "on" and heard is None else 0 + self.assertEqual(module.main(), expected) + self.assertEqual(post.call_count, 1, "accepted cache must not synthesize again") + self.assertEqual(asr.call_count, int(mode != "off")) + self.assertEqual(bool(json.loads((output / "manifest.json").read_text())), expected == 0) + self.assertEqual((output / "clip.wav").read_bytes(), sentinel) + + +class NarrationPublicationContract(unittest.TestCase): + def setUp(self): + self.module = fixtures.load_script("narrate") + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.scenes = self.root / "scenes.yaml" + self.output = self.root / "narration" + + def run_narrate(self, scenes, synthesize, *, verify="off", expected=0, engine="piper", + extra_options=(), transcript=None, asr_calls=None): + self.scenes.write_text(json.dumps({"scenes": scenes}), encoding="utf-8") + argv = ["narrate", str(self.scenes), str(self.output), "--engine", engine, + "--verify", verify, *extra_options] + stderr = io.StringIO() + def transcribe(wav, model): + if asr_calls is not None: + asr_calls.append((wav, model)) + return transcript + with patch.object(sys, "argv", argv), \ + patch.object(self.module.shutil, "which", return_value="ffprobe"), \ + patch.object(self.module, "openai_key", return_value="key" if engine.startswith("openai") else None), \ + patch.object(self.module, "say_piper", side_effect=synthesize), \ + patch.object(self.module, "say_openai_chat", + side_effect=lambda key, text, wav, voice: synthesize(text, wav, voice)), \ + patch.object(self.module, "duration", return_value=1.25), \ + patch.object(self.module, "transcribe_local", side_effect=transcribe), \ + redirect_stdout(io.StringIO()), redirect_stderr(stderr): + self.assertEqual(self.module.main(), expected, stderr.getvalue()) + manifest = self.output / "manifest.json" + return json.loads(manifest.read_text(encoding="utf-8")) if manifest.exists() else [] + + def test_acceptance_is_withdrawn_before_replacing_accepted_bytes(self): + old_wav = self.output / "first.wav" + self.output.mkdir() + old_wav.write_bytes(b"accepted bytes") + synthesis = {"engine": "piper", "voice": self.module.PIPER_VOICE, + "model": self.module.PIPER_VOICE} + (self.output / "manifest.json").write_text(json.dumps([{ + "id": "first", "text": "Old words", "wav": "first.wav", + "duration": 1.0, "synthesis": synthesis, + }]), encoding="utf-8") + + def synthesize(text, wav, voice): + published = json.loads((self.output / "manifest.json").read_text()) + self.assertNotIn("first", [entry["id"] for entry in published]) + wav.write_bytes(b"replacement bytes") + + manifest = self.run_narrate( + [{"id": "first", "narration": "New words"}], synthesize + ) + + self.assertEqual(manifest[0]["text"], "New words") + self.assertEqual(old_wav.read_bytes(), b"replacement bytes") + + def test_rejected_and_exceptional_takes_are_not_accepted_on_a_rerun(self): + attempts = [] + + def synthesize(text, wav, voice): + attempts.append((text, wav)) + wav.write_bytes(f"{text}:{len(attempts)}".encode()) + if text == "Reject this": + return "invented words that do not match this script at all" + if text == "Raise here": + raise RuntimeError("synthesis interrupted") + + scenes = [{"id": "reject", "narration": "Reject this"}, + {"id": "raise", "narration": "Raise here"}] + def accepted(text, wav, voice): + wav.write_bytes(b"accepted") + return text + + self.run_narrate(scenes, accepted, engine="openai-chat") + first = self.run_narrate(scenes, synthesize, engine="openai-chat", expected=1, extra_options=("--force",)) + attempts_after_rejection = len(attempts) + second = self.run_narrate(scenes, synthesize, engine="openai-chat", expected=1) + + self.assertEqual(first, []) + self.assertEqual(second, []) + self.assertGreater(len(attempts), attempts_after_rejection) + rejected_paths = [path for text, path in attempts if text == "Reject this"] + self.assertEqual(len({path.name for path in rejected_paths}), len(rejected_paths)) + self.assertTrue(all(path.exists() for path in rejected_paths)) + self.assertEqual(len({path.read_bytes() for path in rejected_paths}), len(rejected_paths)) + + def test_duration_failure_leaves_only_prior_accepted_scenes_published(self): + scenes = [{"id": "accepted", "narration": "Accepted words"}, + {"id": "broken", "narration": "Broken words"}] + + def synthesize(text, wav, voice): + wav.write_bytes(text.encode()) + + self.scenes.write_text(json.dumps({"scenes": scenes}), encoding="utf-8") + argv = ["narrate", str(self.scenes), str(self.output), "--engine", "piper", + "--verify", "off"] + durations = iter((1.0, RuntimeError("ffprobe failed"))) + with patch.object(sys, "argv", argv), \ + patch.object(self.module.shutil, "which", return_value="ffprobe"), \ + patch.object(self.module, "openai_key", return_value=None), \ + patch.object(self.module, "say_piper", side_effect=synthesize), \ + patch.object(self.module, "duration", side_effect=durations), \ + redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertEqual(self.module.main(), 1) + + manifest = json.loads((self.output / "manifest.json").read_text()) + self.assertEqual([entry["id"] for entry in manifest], ["accepted"]) + + def test_duration_failures_keep_distinct_attempt_bytes_across_reruns(self): + scenes = [{"id": "clip", "narration": "Measure this clip"}] + + def synthesize(text, wav, voice): + wav.write_bytes(f"attempt {len(list(self.output.glob('.clip.attempt-*.wav'))) + 1}".encode()) + + for expected_attempts in (1, 2): + self.scenes.write_text(json.dumps({"scenes": scenes}), encoding="utf-8") + argv = ["narrate", str(self.scenes), str(self.output), "--engine", "piper", "--verify", "off"] + with patch.object(sys, "argv", argv), \ + patch.object(self.module.shutil, "which", return_value="ffprobe"), \ + patch.object(self.module, "openai_key", return_value=None), \ + patch.object(self.module, "say_piper", side_effect=synthesize), \ + patch.object(self.module, "duration", side_effect=RuntimeError("ffprobe failed")), \ + redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertEqual(self.module.main(), 1) + attempts = sorted(self.output.glob(".clip.attempt-*.wav")) + self.assertEqual(len(attempts), expected_attempts) + self.assertEqual(len({path.read_bytes() for path in attempts}), expected_attempts) + self.assertFalse((self.output / "clip.wav").exists()) + + def test_two_accepted_scenes_are_published_together(self): + def synthesize(text, wav, voice): + wav.write_bytes(text.encode()) + + manifest = self.run_narrate([ + {"id": "first", "narration": "First accepted scene"}, + {"id": "second", "narration": "Second accepted scene"}, + ], synthesize) + self.assertEqual([entry["id"] for entry in manifest], ["first", "second"]) + + def test_strict_unavailable_verification_withdraws_cached_acceptance(self): + def synthesize(text, wav, voice): + wav.write_bytes(b"accepted") + + self.run_narrate([{"id": "clip", "narration": "Read these words"}], synthesize) + manifest = self.run_narrate( + [{"id": "clip", "narration": "Read these words"}], synthesize, + verify="on", expected=1, + ) + self.assertEqual(manifest, []) + + def test_unsupported_asr_comparison_obeys_auto_on_and_off_modes(self): + def synthesize(text, wav, voice): + wav.write_bytes(b"clip") + + scenes = [{"id": "clip", "narration": "\u77ed\u6587"}] + for mode, expected in (("auto", 0), ("on", 1), ("off", 0)): + with self.subTest(mode=mode): + asr_calls = [] + manifest = self.run_narrate(scenes, synthesize, verify=mode, + expected=expected, transcript="\u77ed\u6587", + asr_calls=asr_calls) + self.assertEqual(bool(manifest), expected == 0) + self.assertEqual(bool(asr_calls), mode != "off") + + def test_missing_ffprobe_stops_before_synthesis(self): + self.scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Words"}]}), + encoding="utf-8") + with patch.object(sys, "argv", ["narrate", str(self.scenes), str(self.output)]), \ + patch.object(self.module.shutil, "which", return_value=None), \ + patch.object(self.module, "say_piper", side_effect=AssertionError("synthesized")), \ + redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit): + self.module.main() + + def test_fresh_unsupported_chat_transcript_is_rejected_before_synthesis(self): + called = [] + + def synthesize(*args): + called.append(args) + + manifest = self.run_narrate( + [{"id": "clip", "narration": "\U00020000\U00020001"}], synthesize, + engine="openai-chat", expected=1, + ) + self.assertEqual(manifest, []) + self.assertEqual(called, []) + + def test_cached_unsupported_chat_transcript_withdraws_acceptance_without_asr(self): + self.output.mkdir() + (self.output / "clip.wav").write_bytes(b"cached bytes") + synthesis = {"engine": "openai-chat", "voice": "nova", + "model": self.module.OPENAI_CHAT_MODEL} + (self.output / "manifest.json").write_text(json.dumps([{ + "id": "clip", "text": "\u77ed\u6587", "wav": "clip.wav", "duration": 1.0, + "synthesis": synthesis, + }]), encoding="utf-8") + self.scenes.write_text(json.dumps({"scenes": [ + {"id": "clip", "narration": "\u77ed\u6587"} + ]}), encoding="utf-8") + for mode in ("off", "auto"): + with self.subTest(mode=mode): + (self.output / "manifest.json").write_text(json.dumps([{ + "id": "clip", "text": "\u77ed\u6587", "wav": "clip.wav", "duration": 1.0, + "synthesis": synthesis, + }]), encoding="utf-8") + argv = ["narrate", str(self.scenes), str(self.output), "--engine", "openai-chat", + "--verify", mode] + with patch.object(sys, "argv", argv), \ + patch.object(self.module.shutil, "which", return_value="ffprobe"), \ + patch.object(self.module, "openai_key", return_value="key"), \ + patch.object(self.module, "say_openai_chat", side_effect=AssertionError("cached")), \ + patch.object(self.module, "transcribe_local", side_effect=AssertionError("ASR")), \ + redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertEqual(self.module.main(), 1) + self.assertEqual(json.loads((self.output / "manifest.json").read_text()), []) + + def test_empty_chat_or_asr_speech_is_rejected(self): + def empty_chat_synthesis(text, wav, voice): + wav.write_bytes(b"audio") + return "" + + empty_chat = self.run_narrate( + [{"id": "clip", "narration": "Two words"}], + empty_chat_synthesis, engine="openai-chat", expected=1, + ) + self.assertEqual(empty_chat, []) + + def synthesize(text, wav, voice): + wav.write_bytes(b"audio") + + self.scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Two words"}]}), + encoding="utf-8") + argv = ["narrate", str(self.scenes), str(self.output), "--engine", "piper", "--verify", "auto"] + with patch.object(sys, "argv", argv), \ + patch.object(self.module.shutil, "which", return_value="ffprobe"), \ + patch.object(self.module, "openai_key", return_value=None), \ + patch.object(self.module, "say_piper", side_effect=synthesize), \ + patch.object(self.module, "transcribe_local", return_value=""), \ + patch.object(self.module, "duration", return_value=1.0), \ + redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertEqual(self.module.main(), 1) + self.assertEqual(json.loads((self.output / "manifest.json").read_text()), []) + + def test_cached_nested_wav_keeps_its_manifest_path_after_reverification(self): + nested = self.output / "takes" / "clip.wav" + nested.parent.mkdir(parents=True) + nested.write_bytes(b"accepted") + synthesis = {"engine": "piper", "voice": self.module.PIPER_VOICE, + "model": self.module.PIPER_VOICE} + (self.output / "manifest.json").write_text(json.dumps([{ + "id": "clip", "text": "Nested clip", "wav": "takes/clip.wav", "duration": 1.0, + "synthesis": synthesis, + }]), encoding="utf-8") + manifest = self.run_narrate( + [{"id": "clip", "narration": "Nested clip"}], + lambda *args: (_ for _ in ()).throw(AssertionError("cached")), verify="on", + transcript="Nested clip", + ) + self.assertEqual(manifest[0]["wav"], "takes/clip.wav") + + def test_cached_strict_verification_withdraws_before_interrupt(self): + def synthesize(text, wav, voice): + wav.write_bytes(b"accepted") + + self.run_narrate([{"id": "clip", "narration": "Interrupt safely"}], synthesize) + self.scenes.write_text(json.dumps({"scenes": [{"id": "clip", "narration": "Interrupt safely"}]}), + encoding="utf-8") + argv = ["narrate", str(self.scenes), str(self.output), "--engine", "piper", "--verify", "on"] + with patch.object(sys, "argv", argv), \ + patch.object(self.module.shutil, "which", return_value="ffprobe"), \ + patch.object(self.module, "openai_key", return_value=None), \ + patch.object(self.module, "say_piper", side_effect=AssertionError("cached")), \ + patch.object(self.module, "transcribe_local", side_effect=KeyboardInterrupt), \ + redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + with self.assertRaises(KeyboardInterrupt): + self.module.main() + self.assertEqual(json.loads((self.output / "manifest.json").read_text()), []) + + +class NarrationComparisonContract(unittest.TestCase): + def setUp(self): + self.module = fixtures.load_script("narrate") + + def test_comparison_marks_scripts_requiring_segmentation_or_no_words_unavailable(self): + for script, heard in ( + ("你好世界", "こんにちは世界"), + ("短文", "短文"), + ("mixed 日本語 words", "mixed 日本語 words"), + ("\U00020000\U00020001", "\U00020002\U00020003"), + ("*** !!!", "*** !!!"), + ): + with self.subTest(script=script): + self.assertIsNone(self.module.structural_drift(script, heard)) + + def test_comparison_accepts_multiline_accented_latin_and_spaced_cyrillic(self): + for script, heard in ( + ("Caf\u00e9\nna\u00efve", "CAF\u00c9 na\u00efve"), + ("\u041f\u0440\u0438\u0432\u0435\u0442 \u043c\u0438\u0440", "\u043f\u0440\u0438\u0432\u0435\u0442 \u043c\u0438\u0440"), + ("\uc548\ub155 \uc138\uacc4", "\uc548\ub155 \uc138\uacc4"), + ): + with self.subTest(script=script): + self.assertEqual(self.module.structural_drift(script, heard), (0.0, 0)) + + def test_drift_check_reports_unavailable_comparison_as_nonzero(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + script = root / "script.txt" + heard = root / "heard.txt" + script.write_text("\u77ed\u6587", encoding="utf-8") + heard.write_text("\u77ed\u6587", encoding="utf-8") + output = io.StringIO() + with patch.object(sys, "argv", ["narrate", "--drift-check", str(script), str(heard)]), \ + redirect_stdout(output): + self.assertEqual(self.module.main(), 1) + self.assertIn("comparison unavailable", output.getvalue()) + + +class AssemblyNarrationContract(unittest.TestCase): + def setUp(self): + self.module = fixtures.load_script("assemble") + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.scenes = self.root / "scenes.yaml" + self.narration = self.root / "narration" + self.narration.mkdir() + + def assemble(self, scenes, *, manifest=None, run_side_effect=None): + self.scenes.write_text(json.dumps({"scenes": scenes}), encoding="utf-8") + if manifest is not None: + (self.narration / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") + calls = [] + + def run(command, **kwargs): + calls.append(command) + if run_side_effect is not None: + return run_side_effect(command) + return subprocess.CompletedProcess(command, 0, "1.0", "") + + argv = ["assemble", str(self.scenes), str(self.root / "out.mp4"), + "--narration", str(self.narration), "--work", str(self.root / "work")] + with patch.object(sys, "argv", argv), \ + patch.object(self.module.shutil, "which", return_value="tool"), \ + patch.object(self.module, "find_browser", return_value=None), \ + patch.object(self.module, "run", side_effect=run), \ + redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + return self.module.main(), calls + + def test_required_narration_contract_fails_before_encoding(self): + with self.assertRaises(SystemExit): + self.assemble([{"id": "spoken", "kind": "image", "src": "still.png", + "narration": "Expected words"}], + run_side_effect=lambda command: (_ for _ in ()).throw( + AssertionError("encoding started") + )) + + def test_missing_entry_wav_or_matching_text_fails_before_encoding(self): + (self.root / "still.png").write_bytes(b"image sentinel") + cases = ( + ([], "missing entry"), + ([{"id": "spoken", "text": "Expected words", "wav": "missing.wav"}], "missing WAV"), + ([{"id": "spoken", "text": "Changed words", "wav": "accepted.wav"}], "changed text"), + ) + for manifest, label in cases: + with self.subTest(label=label): + (self.narration / "manifest.json").unlink(missing_ok=True) + with self.assertRaises(SystemExit): + self.assemble([{"id": "spoken", "kind": "image", "src": "still.png", + "narration": "Expected words"}], manifest=manifest, + run_side_effect=lambda command: (_ for _ in ()).throw( + AssertionError("encoding started") + )) + + def test_manifest_wav_name_is_authoritative_and_removed_narration_ignores_leftover_wav(self): + selected = self.narration / "accepted-name.wav" + selected.write_bytes(b"accepted") + leftover = self.narration / "silent.wav" + leftover.write_bytes(b"leftover") + (self.root / "still.png").write_bytes(b"image sentinel") + manifest = [{"id": "spoken", "text": "Expected words", + "wav": selected.name, "duration": 1.0, "synthesis": {}}] + status, calls = self.assemble([ + {"id": "spoken", "kind": "image", "src": "still.png", "narration": "Expected words"}, + {"id": "silent", "kind": "image", "src": "still.png"}, + ], manifest=manifest) + self.assertEqual(status, 0) + encoded = [call for call in calls if call and call[0] == "ffmpeg"] + self.assertTrue(any(str(selected) in call for call in encoded)) + self.assertFalse(any(str(leftover) in call for call in encoded)) + offsets = json.loads((self.root / "work" / "offsets.json").read_text()) + self.assertEqual(set(offsets), {"spoken"}) + + def test_2560x1080_movie_uses_source_audio_and_silent_source_gets_anullsrc(self): + source = self.root / "wide-2560x1080.mp4" + source.write_bytes(b"source") + status, calls = self.assemble( + [{"id": "movie", "kind": "movie", "src": source.name, + "narration": "Ignore this", "height": 800}], + run_side_effect=lambda command: subprocess.CompletedProcess( + command, 0, + json.dumps({"streams": [{"codec_type": "video", "width": 2560, + "height": 1080}]}), "" + ) if "-show_streams" in command else subprocess.CompletedProcess(command, 0, "1.0", ""), + ) + self.assertEqual(status, 0) + movie_encode = next(call for call in calls if call and call[0] == "ffmpeg") + self.assertIn("anullsrc=r=44100:cl=stereo", movie_encode) + self.assertIn("0:v:0", movie_encode) + self.assertIn("1:a:0", movie_encode) + self.assertIn("scale=1920:800:force_original_aspect_ratio=decrease", + movie_encode[movie_encode.index("-vf") + 1]) + offsets = json.loads((self.root / "work" / "offsets.json").read_text()) + self.assertEqual(offsets, {}) + + def test_movie_with_audio_maps_its_source_audio(self): + source = self.root / "source-with-audio.mp4" + source.write_bytes(b"source") + status, calls = self.assemble( + [{"id": "movie", "kind": "movie", "src": source.name}], + run_side_effect=lambda command: subprocess.CompletedProcess( + command, 0, + json.dumps({"streams": [{"codec_type": "video"}, {"codec_type": "audio"}]}), "" + ) if "-show_streams" in command else subprocess.CompletedProcess(command, 0, "1.0", ""), + ) + self.assertEqual(status, 0) + movie_encode = next(call for call in calls if call and call[0] == "ffmpeg") + self.assertNotIn("anullsrc=r=44100:cl=stereo", movie_encode) + self.assertEqual(movie_encode[movie_encode.index("-map") + 1], "0:v:0") + self.assertEqual(movie_encode[movie_encode.index("-map", movie_encode.index("-map") + 1) + 1], "0:a:0") + + def test_movie_geometry_fits_width_and_requested_inner_height_before_padding(self): + self.assertEqual(self.module.movie_geometry(1920, 1080, 800), { + "scale": (1920, 800), + "pad": (1920, 1080), + }) + + +class PercentPathContract(unittest.TestCase): + def test_sequence_pattern_escapes_only_directory_percents(self): + module = fixtures.load_script("media_paths") + pattern = module.sequence_pattern(Path("folder%name") / "frames", "frame-%08d.png") + self.assertEqual(pattern, "folder%%name/frames/frame-%08d.png") + + def test_checker_sampling_escapes_only_output_directory_percents(self): + module = fixtures.load_script("check-movie") + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) / "proof%take" + work.mkdir() + command = [] + + def run(argv, **kwargs): + command.extend(argv) + return subprocess.CompletedProcess(argv, 0, "", "") + + with patch.object(module.subprocess, "run", side_effect=run), redirect_stdout(io.StringIO()): + with self.assertRaises(SystemExit): + module.sample_picture(Path("movie.mp4"), work) + output = command[-1] + self.assertIn("proof%%take", output) + self.assertTrue(output.endswith("s%05d.png")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/proving-it-works-with-a-movie/test_paths.py b/tests/proving-it-works-with-a-movie/test_paths.py new file mode 100644 index 0000000..ed1cb50 --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_paths.py @@ -0,0 +1,104 @@ +import shutil +import subprocess +import sys +import tempfile +import unittest +import wave +from pathlib import Path + + +SCRIPTS = ( + Path(__file__).resolve().parents[2] + / "skills/proving-it-works-with-a-movie/scripts" +) +sys.path.insert(0, str(SCRIPTS)) + +import media_paths + + +class MediaPathRegression(unittest.TestCase): + def test_frame_staging_uses_ordinary_ordered_files(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "movie O'Brien λ & [take]" + source.mkdir() + (source / "b.png").write_bytes(b"second") + (source / "a.png").write_bytes(b"first") + + staged = media_paths.stage_frames(source, root / "staged") + + self.assertEqual( + [path.read_bytes() for path in staged], + [b"first", b"second"], + ) + self.assertTrue(all(not path.is_symlink() for path in staged)) + self.assertEqual(len(list(source.iterdir())), 2) + + def test_frame_staging_rejects_an_empty_source(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / "empty" + source.mkdir() + + with self.assertRaisesRegex(ValueError, "no PNG frames"): + media_paths.stage_frames(source, root / "staged") + + self.assertFalse((root / "staged").exists()) + + def test_ffconcat_entry_is_accepted_by_ffmpeg(self): + ffmpeg = shutil.which("ffmpeg") + if ffmpeg is None: + self.skipTest("required executable not on PATH: ffmpeg") + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "movie O'Brien λ & [take]" + nested = root / "subtitles" / "nested" + nested.mkdir(parents=True) + audio = nested / "tone.wav" + with wave.open(str(audio), "wb") as output: + output.setnchannels(1) + output.setsampwidth(2) + output.setframerate(8000) + output.writeframes(b"\x00\x00" * 800) + + if sys.platform == "win32": + self.assertRegex(str(audio.resolve()), r"^[A-Za-z]:\\") + + listing = root / "concat.txt" + listing.write_text( + "ffconcat version 1.0\n" + media_paths.ffconcat_entry(audio), + encoding="utf-8", + ) + result = subprocess.run( + [ + ffmpeg, + "-nostdin", + "-v", + "error", + "-f", + "concat", + "-safe", + "0", + "-i", + str(listing), + "-f", + "null", + "-", + ], + cwd=Path(directory), + capture_output=True, + timeout=30, + ) + self.assertEqual( + result.returncode, + 0, + (result.stdout + result.stderr).decode("utf-8", errors="replace"), + ) + + def test_ffconcat_entry_rejects_line_breaks(self): + with self.assertRaisesRegex(ValueError, "cannot contain line breaks"): + media_paths.ffconcat_entry(Path("bad\nname.wav")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/proving-it-works-with-a-movie/test_recorder_contract.py b/tests/proving-it-works-with-a-movie/test_recorder_contract.py new file mode 100644 index 0000000..2c213ec --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_recorder_contract.py @@ -0,0 +1,538 @@ +"""Recorder decisions with fake processes/CDP and byte-token captures only.""" +import contextlib +import importlib.util +import io +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import fixtures + +SCRIPT = Path(__file__).resolve().parents[2] / "skills/proving-it-works-with-a-movie/examples/film-terminal.py" + + +def recorder(): + spec = importlib.util.spec_from_file_location("recorder_contract", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Clock: + def __init__(self): + self.now = 0.0 + + def monotonic(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +class Process: + def __init__(self, pid): + self.pid = pid + self.returncode = None + + def poll(self): + return self.returncode + + def wait(self, timeout): + if self.returncode is None: + raise subprocess.TimeoutExpired("fake child", timeout) + return self.returncode + + +@contextlib.contextmanager +def serving(failure=None, stop_at=None, relative=False): + module, clock = recorder(), Clock() + with tempfile.TemporaryDirectory() as temp, contextlib.ExitStack() as stack: + root = Path(temp) + directory = root / "session" + args = SimpleNamespace(session=directory, cwd=root, shell="bash", shell_exe=None, + ttyd="fake-ttyd", browser="fake-browser") + if relative: + import os + args.session = Path(os.path.relpath(directory)) + handles, children, launches, tokens = [], [], [], [] + original_open, original_write = Path.open, module.write_json + stopped = False + + def open_file(path, *positional, **kwargs): + if positional == ("ab",): + if failure == path.name: + raise OSError("injected " + path.name) + handle = original_open(path, *positional, **kwargs) + handles.append(handle) + return handle + return original_open(path, *positional, **kwargs) + + def popen(argv, **kwargs): + index = len(children) + if failure == ("ttyd launch" if index == 0 else "browser launch"): + raise OSError("injected launch") + child = Process(1100 + index) + children.append(child) + launches.append((argv, kwargs)) + if index == 1: + (directory / "profile").mkdir() + return child + + def kill(pid): + for child in children: + if child.pid == pid: + if child.poll() is not None: + raise AssertionError("cannot use an exited leader's PID") + if failure != "child wait": + child.returncode = -9 + + def write_json(path, value): + if failure == "session metadata" and path.name == "session.json": + raise OSError("injected metadata failure") + original_write(path, value) + + def stop(phase): + nonlocal stopped + if stop_at == phase and not stopped: + (directory / "stop").write_text("") + stopped = True + + class FakeCDP: + def __init__(self, url): + if failure == "connection": + raise RuntimeError("injected connection failure") + self.n = 0 + self.on_event = None + self.ws = SimpleNamespace(close=lambda: None) + + def recv(self, timeout): + clock.sleep(timeout) + if (directory / "ready.json").exists(): + stop("ready") + if failure == "exited leader": + children[0].returncode = 0 + if failure == "later connection": + raise ConnectionError("injected later disconnect") + else: + stop("pump" if self.n == 0 else "quiet") + return None + + def call(self, method, params=None, **kwargs): + if getattr(self, "before_call", None): + self.before_call() + if stop_at == "call": + stop("call") + clock.sleep(10) + if method == "Input.dispatchKeyEvent" and params["type"] == "keyDown": + self.n += 1 + return {} + + cdp = None + + def connect(url): + nonlocal cdp + cdp = FakeCDP(url) + return cdp + + def page_url(port): + stop("retry") + if stop_at == "retry": + raise OSError("not listening yet") + return "fake-url" + + def tail(path): + if cdp and cdp.n: + return f"\x1b]0;MOVIE;{cdp.n};1;0;{root}\x07$ ".encode() + return b"$ " + + for obj, name, value in ( + (module, "shell_argv", lambda *a: ["fake-bash"]), + (module, "find_browser", lambda *a: "fake-browser"), + (module, "free_port", lambda: 1234), + (module.subprocess, "Popen", popen), + (module, "kill_process_tree", kill), + (module, "page_url", page_url), + (module, "CDP", connect), + (module, "write_json", write_json), + (module, "tail", tail), + (module, "screenshot", lambda *a: b"capture-token"), + (module, "lit_fraction", lambda *a: 0.01), + (module.time, "monotonic", clock.monotonic), + (module.time, "sleep", clock.sleep), + (Path, "open", open_file), + (Path, "write_bytes", lambda path, token: tokens.append((path.name, token))), + ): + stack.enter_context(patch.object(obj, name, value)) + if failure == "profile removal": + stack.enter_context(patch.object(module.shutil, "rmtree", side_effect=PermissionError("locked"))) + stack.enter_context(contextlib.redirect_stdout(io.StringIO())) + stack.enter_context(contextlib.redirect_stderr(io.StringIO())) + try: + yield SimpleNamespace(module=module, args=args, directory=directory, children=children, + handles=handles, launches=launches, clock=clock) + finally: + # Release the test's log handles even when ownership assertions fail. + for handle in handles: + handle.close() + + +class ServeLifecycleTests(unittest.TestCase): + def test_tree_cleanup_failure_keeps_failure_after_leader_exits_and_cleans_other_resources(self): + with serving(stop_at="ready") as rig: + browser = fixtures.load_script("browser_tools") + terminated = [] + + def taskkill(argv, **kwargs): + pid = int(argv[-1]) + terminated.append(pid) + child = next(child for child in rig.children if child.pid == pid) + child.returncode = 0 if pid == 1100 else -9 + return subprocess.CompletedProcess(argv, 1 if pid == 1100 else 0, b"", b"tree termination failed") + + with patch.object(rig.module, "kill_process_tree", browser.kill_process_tree), \ + patch.object(browser.sys, "platform", "win32"), \ + patch.object(browser.subprocess, "run", taskkill): + self.assertEqual(rig.module.serve(rig.args), 1) + session = rig.module.read_json(rig.directory / "session.json") + self.assertEqual(session["pids"], [1100, 1101]) + self.assertFalse(session.get("closed", False)) + self.assertEqual(terminated, [1100, 1101]) + self.assertTrue(all(child.poll() is not None for child in rig.children)) + self.assertTrue(all(handle.closed for handle in rig.handles)) + self.assertFalse((rig.directory / "profile").exists()) + self.assertFalse((rig.directory / "ready.json").exists()) + self.assertTrue(all((rig.directory / name).exists() + for name in ("ttyd.log", "browser.log", "terminal.log"))) + + def test_every_acquisition_failure_releases_owned_resources(self): + for failure in ("ttyd.log", "browser.log", "ttyd launch", "browser launch", + "session metadata", "terminal.log", "connection", "later connection"): + with self.subTest(failure=failure), serving(failure) as rig: + try: + code = rig.module.serve(rig.args) + except Exception as error: + code = error + self.assertEqual(code, 1) + self.assertTrue(all(p.poll() is not None for p in rig.children)) + self.assertTrue(all(h.closed for h in rig.handles)) + self.assertFalse((rig.directory / "profile").exists()) + self.assertFalse((rig.directory / "ready.json").exists()) + for name in ("ttyd.log", "browser.log", "terminal.log"): + if any(Path(h.name).name == name for h in rig.handles): + self.assertTrue((rig.directory / name).exists(), "logs are evidence") + + def test_relative_session_path_matches_browser_profile_and_cwd(self): + with serving("connection", relative=True) as rig: + self.assertEqual(rig.module.serve(rig.args), 1) + argv, kwargs = rig.launches[1] + profile_arg = next(a.split("=", 1)[1] for a in argv if a.startswith("--user-data-dir=")) + self.assertEqual(Path(kwargs["cwd"]) / profile_arg, (rig.directory / "profile").resolve()) + self.assertTrue(Path(kwargs["cwd"]).is_absolute()) + + def test_stop_requests_interrupt_startup_and_finish_owned_cleanup(self): + for phase in ("retry", "pump", "quiet", "ready"): + with self.subTest(phase=phase), serving(stop_at=phase) as rig: + self.assertEqual(rig.module.serve(rig.args), 0) + self.assertLess(rig.clock.now, 5) + self.assertTrue(all(p.poll() is not None for p in rig.children)) + self.assertTrue(all(h.closed for h in rig.handles)) + session = rig.module.read_json(rig.directory / "session.json") + self.assertEqual(session["pids"], []) + self.assertTrue(session["closed"]) + self.assertFalse((rig.directory / "ready.json").exists()) + self.assertFalse((rig.directory / "profile").exists()) + + def test_stop_during_one_startup_call_prevents_another_bounded_call(self): + with serving(stop_at="call") as rig: + self.assertEqual(rig.module.serve(rig.args), 0) + self.assertLessEqual(rig.clock.now, 10) + self.assertTrue(rig.module.read_json(rig.directory / "session.json")["closed"]) + + def test_pid_retirement_is_atomic_and_follows_resource_cleanup(self): + with serving(stop_at="ready") as rig: + replace = Path.replace + observed = [] + + def publish(path, target): + session = rig.module.read_json(target) + observed.append(session["pids"]) + self.assertEqual(session["pids"], [1100, 1101]) + self.assertTrue(all(p.poll() is not None for p in rig.children)) + self.assertTrue(all(h.closed for h in rig.handles)) + self.assertFalse((rig.directory / "profile").exists()) + self.assertFalse((rig.directory / "ready.json").exists()) + return replace(path, target) + + with patch.object(Path, "replace", publish): + self.assertEqual(rig.module.serve(rig.args), 0) + self.assertEqual(observed, [[1100, 1101]]) + self.assertEqual(rig.module.read_json(rig.directory / "session.json")["pids"], []) + + def test_failed_cleanup_does_not_retire_owned_ids_or_claim_closed(self): + for failure in ("child wait", "profile removal"): + with self.subTest(failure=failure), serving(failure, stop_at="ready") as rig: + self.assertEqual(rig.module.serve(rig.args), 1) + session = rig.module.read_json(rig.directory / "session.json") + self.assertEqual(session["pids"], [1100, 1101]) + self.assertFalse(session.get("closed", False)) + self.assertFalse((rig.directory / "ready.json").exists()) + + def test_exited_leader_cannot_confirm_descendant_cleanup(self): + with serving("exited leader", stop_at="ready") as rig: + self.assertEqual(rig.module.serve(rig.args), 1) + session = rig.module.read_json(rig.directory / "session.json") + self.assertEqual(session["pids"], [1100, 1101]) + self.assertFalse(session.get("closed", False)) + self.assertEqual(rig.children[0].returncode, 0) + self.assertEqual(rig.children[1].returncode, -9) + self.assertTrue(all(h.closed for h in rig.handles)) + self.assertFalse((rig.directory / "ready.json").exists()) + + +class CloseContractTests(unittest.TestCase): + def test_close_waits_for_owner_cleanup_and_repeated_close_never_kills(self): + module, clock = recorder(), Clock() + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + module.write_json(directory / "session.json", {"pids": [987654]}) + (directory / "ready.json").write_text("{}") + (directory / "profile").mkdir() + (directory / "terminal.log").write_text("retain evidence") + + def sleep(seconds): + clock.sleep(seconds) + self.assertTrue((directory / "stop").exists()) + if clock.now >= 12: + (directory / "profile").rmdir() + (directory / "ready.json").unlink() + module.write_json(directory / "session.json", {"pids": [], "closed": True}) + + with patch.object(module, "kill_process_tree", side_effect=AssertionError("historical PID kill")), \ + patch.object(module.time, "monotonic", clock.monotonic), \ + patch.object(module.time, "sleep", sleep), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(module.close(SimpleNamespace(session=directory)), 0) + self.assertGreaterEqual(clock.now, 12) + self.assertEqual(module.close(SimpleNamespace(session=directory)), 0) + self.assertEqual((directory / "terminal.log").read_text(), "retain evidence") + + def test_incomplete_or_unavailable_cleanup_is_not_success(self): + for state in ("missing", "unreadable", "unavailable owner", "profile remains", "ready remains"): + with self.subTest(state=state), tempfile.TemporaryDirectory() as temp: + module, clock, directory = recorder(), Clock(), Path(temp) + if state != "missing": + module.write_json(directory / "session.json", {"pids": [987654]}) + if state == "unreadable": + (directory / "session.json").write_text("invalid json") + if state in ("profile remains", "ready remains"): + module.write_json(directory / "session.json", {"pids": [], "closed": True}) + if state == "profile remains": + (directory / "profile").mkdir() + else: + (directory / "ready.json").write_text("{}") + with patch.object(module, "kill_process_tree", lambda pid: None), \ + patch.object(module.time, "monotonic", clock.monotonic), \ + patch.object(module.time, "sleep", clock.sleep), \ + contextlib.redirect_stdout(io.StringIO()) as out, \ + contextlib.redirect_stderr(io.StringIO()): + try: + code = module.close(SimpleNamespace(session=directory)) + except Exception as error: + code = error + self.assertEqual(code, 1) + self.assertNotIn('"closed": true', out.getvalue()) + self.assertLessEqual(clock.now, 30.1) + if state == "unavailable owner": + self.assertGreaterEqual(clock.now, 30) + + +class CDPCallTests(unittest.TestCase): + def test_stop_during_response_prevents_the_next_call_from_being_sent(self): + module, sent, stopped = recorder(), [], [] + + def recv(): + stopped.append(True) + return json.dumps({"id": len(sent), "result": {}}) + + ws = SimpleNamespace(settimeout=lambda timeout: None, recv=recv, + send=lambda data: sent.append(json.loads(data)["method"])) + websocket = SimpleNamespace(create_connection=lambda *a, **kw: ws, + WebSocketTimeoutException=TimeoutError) + def check_active(): + if stopped: + raise InterruptedError("stop requested") + + with patch.dict("sys.modules", websocket=websocket): + cdp = module.CDP("fake-url") + cdp.before_call = check_active + self.assertEqual(cdp.call("Network.enable"), {}) + with self.assertRaises(InterruptedError): + cdp.call("Page.enable") + self.assertEqual(sent, ["Network.enable"]) + + +class ObservationTests(unittest.TestCase): + def observe(self, action, completed=False, seconds=0.2, cp1252=False): + module, clock = recorder(), Clock() + with tempfile.TemporaryDirectory() as temp: + directory = Path(temp) + log = directory / "terminal.log" + initial = b"\x1b]0;MOVIE;1;1;0;/work\x07" + final = "\x1b]0;MOVIE;2;0;7;C:/René/λ\x1b\\".encode() + log.write_bytes(initial + (final if completed else b"")) + (directory / "ready.json").write_text("{}") + module.write_json(directory / "session.json", {"pids": [1100, 1101]}) + calls = [] + + def recv(timeout): + calls.append(timeout) + clock.sleep(timeout) + if action == "disconnect": + raise ConnectionError("browser connection closed") + if action == "session loss": + (directory / "ready.json").unlink(missing_ok=True) + if action == "prompt arrives": + log.write_bytes(initial + final) + if action == "prompt then disconnect": + log.write_bytes(initial + final) + raise ConnectionError("browser connection closed") + return None + + cdp = SimpleNamespace(recv=recv) + args = SimpleNamespace(session=directory, record=None, seconds=seconds, hold=0) + raw = io.BytesIO() + out = io.TextIOWrapper(raw, encoding="cp1252" if cp1252 else "utf-8") + with patch.object(module.time, "monotonic", clock.monotonic), \ + patch.object(module.time, "sleep", clock.sleep), contextlib.redirect_stdout(out): + code = module.observe(args, cdp, 1) + out.flush() + result = json.loads(raw.getvalue().decode("ascii" if cp1252 else "utf-8")) + return code, result, calls + + def test_disconnected_browser_is_failure_without_recording(self): + code, result, calls = self.observe("disconnect") + self.assertEqual(code, 1) + self.assertEqual(result["outcome"], "failed") + self.assertTrue(calls) + + def test_lost_terminal_session_is_failure_even_with_live_browser(self): + code, result, calls = self.observe("session loss") + self.assertEqual(code, 1) + self.assertEqual(result["outcome"], "failed") + self.assertTrue(calls) + + def test_only_live_unfinished_command_returns_two(self): + for seconds in (0, 0.2): + with self.subTest(seconds=seconds): + code, result, calls = self.observe("alive", seconds=seconds) + self.assertEqual((code, result), (2, {"outcome": "running"})) + self.assertTrue(calls, "even an expired observation must establish liveness") + + def test_new_prompt_retains_native_failure_status(self): + for action in ("prompt arrives", "prompt then disconnect"): + with self.subTest(action=action): + code, result, _ = self.observe(action) + self.assertEqual(code, 1) + self.assertEqual(result, {"outcome": "completed", "ok": False, + "exit_code": 7, "cwd": "C:/René/λ"}) + + def test_completed_prompt_is_not_consumed_by_later_disconnect(self): + code, result, _ = self.observe("disconnect", completed=True) + self.assertEqual((code, result["outcome"], result["exit_code"]), (1, "completed", 7)) + + def test_stdout_json_round_trips_non_ascii_paths_on_cp1252(self): + try: + code, result, _ = self.observe("alive", completed=True, cp1252=True) + except UnicodeError as error: + self.fail(f"stdout JSON was not portable: {error}") + self.assertEqual((code, result["cwd"]), (1, "C:/René/λ")) + + def test_capture_disconnect_during_hold_preserves_completed_command_status(self): + for ok, exit_code in ((True, 0), (False, 7)): + with self.subTest(ok=ok), tempfile.TemporaryDirectory() as temp: + module, clock, writes = recorder(), Clock(), [] + marker = f"\x1b]0;MOVIE;2;{int(ok)};{exit_code};C:/René/λ\x07".encode() + args = SimpleNamespace(session=Path(temp), record=Path(temp) / "take", + seconds=10, hold=0.6) + real_film = module.film + + def film(*args): + return real_film(*args, clock=clock.monotonic, sleep=clock.sleep) + + def capture(cdp): + if clock.now >= 0.2: + raise ConnectionError("capture connection closed") + return b"capture-token" + + with patch.object(module, "film", film), \ + patch.object(module, "screenshot", capture), \ + patch.object(module, "tail", lambda path: marker), \ + patch.object(Path, "write_bytes", lambda path, token: writes.append(token)), \ + patch.object(module, "write_json", side_effect=AssertionError("failed take publication")), \ + contextlib.redirect_stdout(io.StringIO()) as out: + code = module.observe(args, SimpleNamespace(), 1) + self.assertEqual(code, 1) + self.assertEqual(json.loads(out.getvalue()), { + "outcome": "failed", "error": "capture connection closed", + "ok": ok, "exit_code": exit_code, "cwd": "C:/René/λ", + }) + self.assertEqual(writes, [b"capture-token"]) + + +class VisibleTextTests(unittest.TestCase): + def test_visible_prompt_survives_bel_and_st_title_markers(self): + module = recorder() + for terminator in (b"\x07", b"\x1b\\"): + with self.subTest(terminator=terminator): + log = b"\x1b[32m/work $ \x1b]0;MOVIE;2;1;0;/work" + terminator + self.assertTrue(module.at_prompt(log)) + self.assertFalse(module.at_prompt(log + b"busy")) + + +class CaptureTimingTests(unittest.TestCase): + def film(self, durations, seconds=1, hold=0, complete_at=None): + module, clock, writes, shots = recorder(), Clock(), [], [] + def capture(): + token = bytes([len(shots) + 1]) + clock.sleep(durations[len(shots)] if len(shots) < len(durations) else 0) + shots.append(token) + return token + with tempfile.TemporaryDirectory() as temp, \ + patch.object(Path, "write_bytes", lambda path, token: writes.append((path.name, token))): + frames = module.film(Path(temp), seconds, hold, capture, + lambda: complete_at is not None and clock.now >= complete_at, + clock.monotonic, clock.sleep) + self.assertEqual(len(writes), frames) + return frames, writes + + def test_capture_crossing_hard_endpoint_fills_exactly_five_slots(self): + frames, writes = self.film([1.2]) + self.assertEqual(frames, 5) + self.assertEqual(writes, [("f00000.png", b"\x01"), ("f00001.png", b"\x01"), + ("f00002.png", b"\x01"), ("f00003.png", b"\x01"), + ("f00004.png", b"\x01")]) + + def test_capture_crossing_hold_endpoint_is_bounded(self): + frames, writes = self.film([1.2], seconds=10, hold=0.6, complete_at=0) + self.assertEqual(frames, 3) + self.assertEqual([name for name, _ in writes], ["f00000.png", "f00001.png", "f00002.png"]) + + def test_mid_capture_stall_repeats_previous_token_without_missing_slots(self): + frames, writes = self.film([0.01, 0.5], seconds=1) + self.assertEqual(frames, 5) + self.assertEqual([token for _, token in writes], [b"\x01", b"\x02", b"\x02", b"\x03", b"\x04"]) + + def test_completion_and_hold_keep_the_normal_grid(self): + frames, _ = self.film([], seconds=10, hold=0.4, complete_at=1) + self.assertEqual(frames, 7) + + def test_completion_without_hold_and_zero_duration_do_not_add_slots(self): + self.assertEqual(self.film([], seconds=10, complete_at=0)[0], 0) + self.assertEqual(self.film([], seconds=0)[0], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/proving-it-works-with-a-movie/test_subtitle_contract.py b/tests/proving-it-works-with-a-movie/test_subtitle_contract.py new file mode 100644 index 0000000..937d574 --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_subtitle_contract.py @@ -0,0 +1,290 @@ +"""Text and mocked-boundary contracts; this module never processes media.""" + +import io +import json +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest.mock import patch + +import fixtures + + +def milliseconds(timestamp): + hours, minutes, rest = timestamp.split(":") + seconds, millis = rest.split(",") + return ((int(hours) * 60 + int(minutes)) * 60 + int(seconds)) * 1000 + int(millis) + + +def read_cues(path): + cues = [] + for block in path.read_text(encoding="utf-8").strip().split("\n\n"): + if block: + _, timing, text = block.split("\n", 2) + start, end = map(milliseconds, timing.split(" --> ")) + cues.append((start, end, " ".join(text.split()))) + return cues + + +class SubtitleTimingContract(unittest.TestCase): + def setUp(self): + self.module = fixtures.load_script("make-subtitles") + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.output = self.root / "captions.srt" + + def subtitles(self, entries, *options): + manifest = self.root / "manifest.json" + manifest.write_text(json.dumps(entries), encoding="utf-8") + stdout = io.StringIO() + with patch.object(sys, "argv", ["make-subtitles", str(manifest), str(self.output), *options]), \ + redirect_stdout(stdout): + self.assertEqual(self.module.main(), 0) + return read_cues(self.output), stdout.getvalue() + + def assert_scene(self, cues, start, end, text): + self.assertTrue(cues) + self.assertEqual(cues[0][0], start) + self.assertEqual(cues[-1][1], end) + previous = start + for a, b, _ in cues: + self.assertEqual(a, previous) + self.assertLess(a, b) + self.assertLessEqual(b, end) + previous = b + self.assertEqual(" ".join(cue[2] for cue in cues).split(), text.split()) + + def test_five_chunks_fit_half_second_before_next_scene(self): + text = "one two six ten red" + cues, report = self.subtitles([ + {"id": "short", "duration": 0.5, "text": text}, + {"id": "next", "duration": 1, "text": "next"}, + ], "--max-chars", "3") + self.assert_scene(cues[:-1], 0, 500, text) + self.assertEqual(cues[-1], (500, 1500, "next")) + self.assertIn("ends at 00:00:01,500", report) + + def test_one_word_covers_twelve_second_scene(self): + cues, report = self.subtitles([{"id": "held", "duration": 12, "text": "Held"}]) + self.assert_scene(cues, 0, 12000, "Held") + self.assertIn("ends at 00:00:12,000", report) + + def test_mixed_chunks_get_proportional_time(self): + cues, _ = self.subtitles([{"id": "mix", "duration": 1, "text": "a bbbbbbbbb"}], "--max-chars", "9") + self.assertEqual(cues, [(0, 100, "a"), (100, 1000, "bbbbbbbbb")]) + + def test_max_seconds_guides_splitting_without_losing_tail_or_words(self): + text = "one two six ten red cat dog fox" + cues, _ = self.subtitles([{"id": "long", "duration": 12, "text": text}], "--max-secs", "3") + self.assert_scene(cues, 0, 12000, text) + self.assertGreater(len(cues), 1) + self.assertTrue(all(b - a <= 3000 for a, b, _ in cues)) + + def test_max_seconds_refines_unequal_chunks_against_allocated_time(self): + text = "ab cde f ghi" + cues, report = self.subtitles([ + {"id": "unequal", "duration": 6, "text": text}, + ], "--max-secs", "3") + self.assert_scene(cues, 0, 6000, text) + self.assertTrue(all(b - a <= 3000 for a, b, _ in cues), cues) + self.assertIn("ends at 00:00:06,000", report) + + def test_chunks_coalesce_to_fit_representable_milliseconds(self): + text = "one two six ten red" + cues, report = self.subtitles([{"id": "tiny", "duration": 0.002, "text": text}], "--max-chars", "3") + self.assert_scene(cues, 0, 2, text) + self.assertLessEqual(len(cues), 2) + self.assertIn("ends at 00:00:00,002", report) + + def test_unrepresentable_max_seconds_preserves_positive_cues_and_all_words(self): + text = "one two six ten red" + cues, _ = self.subtitles([ + {"id": "tiny", "duration": 0.002, "text": text}, + ], "--max-secs", "0.0001") + self.assert_scene(cues, 0, 2, text) + self.assertLessEqual(len(cues), 2) + + def test_submillisecond_scene_can_use_its_rounded_interval(self): + cues, _ = self.subtitles([{"id": "tiny", "duration": 0.0008, "text": "one two"}]) + self.assert_scene(cues, 0, 1, "one two") + + def test_invalid_or_unrepresentable_duration_fails_before_writing_srt(self): + for duration in (0, -1, 0.0001, float("nan"), float("inf"), "invalid"): + with self.subTest(duration=duration), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as caught: + self.subtitles([{"id": "invalid", "duration": duration, "text": "words"}]) + self.assertNotEqual(caught.exception.code, 0) + self.assertFalse(self.output.exists()) + + def test_invalid_readability_limits_fail_clearly(self): + for option, value in (("--max-chars", "0"), ("--max-secs", "0"), + ("--max-secs", "nan"), ("--max-secs", "inf")): + with self.subTest(option=option, value=value), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as caught: + self.subtitles([{"id": "scene", "duration": 1, "text": "words"}], option, value) + self.assertNotEqual(caught.exception.code, 0) + + def test_manual_offset_uses_rounded_scene_boundaries(self): + cues, report = self.subtitles([{"id": "shifted", "duration": 0.5004, "text": "one two six"}], "--offsets", "shifted=2.1254", "--max-chars", "3") + self.assert_scene(cues, 2125, 2626, "one two six") + self.assertIn("ends at 00:00:02,626", report) + + def test_empty_cut_and_unknown_offset_keys_do_not_introduce_cues(self): + for mapping in ({}, {"unknown": 2}): + with self.subTest(mapping=mapping): + offsets = self.root / "offsets.json" + offsets.write_text(json.dumps(mapping), encoding="utf-8") + cues, report = self.subtitles([{"id": "excluded", "duration": 1, "text": "excluded"}], "--offsets-json", str(offsets), "--offsets", "excluded=5") + self.assertEqual(cues, []) + self.assertIn("0 cues, ends at 00:00:00,000", report) + + +class SubtitleTrackContract(unittest.TestCase): + def test_supplied_track_replaces_existing_subtitles_with_optional_audio(self): + module = fixtures.load_script("burn-subtitles") + for soft in (True, False): + for source_audio in (True, False): + with self.subTest(soft=soft, source_audio=source_audio), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + movie, subs = root / "movie.mp4", root / "new.srt" + movie.write_bytes(b"source token") + subs.write_text("1\n00:00:00,000 --> 00:00:01,000\nNew caption\n", encoding="utf-8") + selections = [] + source = {"v:0": "source video", "s:0": "old caption"} + if source_audio: + source["a:0"] = "source audio" + inputs = [source, {"s:0": "new caption"}] + + def encode(command, **kwargs): + if "-vf" in command: + return False + maps = [command[i + 1] for i, arg in enumerate(command) if arg == "-map"] + self.assertEqual(maps, ["0:v:0", "0:a?", "1:s:0"]) + for spec in maps: + index, kind = spec.rstrip("?").split(":", 1) + selected = [value for stream, value in inputs[int(index)].items() if stream == kind or stream.startswith(kind + ":")] + if not spec.endswith("?"): + self.assertTrue(selected) + selections.extend(selected) + return True + + stdout, stderr = io.StringIO(), io.StringIO() + argv = ["burn-subtitles", str(movie), str(subs), str(root / "out.mp4")] + if soft: + argv.append("--soft") + with patch.object(sys, "argv", argv), \ + patch.object(module.shutil, "which", return_value="mock-ffmpeg"), \ + patch.object(module, "has_libass", return_value=True), \ + patch.object(module, "run", side_effect=encode), \ + redirect_stdout(stdout), \ + redirect_stderr(stderr): + self.assertEqual(module.main(), 0) + self.assertEqual(selections, ["source video", *(["source audio"] if source_audio else []), "new caption"]) + self.assertNotIn("no libass", stdout.getvalue()) + self.assertEqual("burn failed" in stderr.getvalue(), not soft) + + +class SubtitleParserContract(unittest.TestCase): + def setUp(self): + self.module = fixtures.load_script("check-movie") + + def test_literal_arrow_in_caption_is_not_a_timing_line(self): + self.assertEqual(self.module.subtitle_end("1\n00:00:00,000 --> 00:00:01,250\nFollow source --> destination.\n"), 1.25) + + def test_timestamp_shaped_caption_cannot_extend_coverage(self): + text = "1\n00:00:00,000 --> 00:00:01,250\n00:00:00,000 --> 00:59:00,000\n" + self.assertEqual(self.module.subtitle_end(text), 1.25) + + def test_malformed_actual_timing_is_rejected(self): + for timing in ("00:00:00,000 --> invalid", "not a timing line", "00:00:00,000 --> 00:99:00,000"): + with self.subTest(timing=timing), self.assertRaises(ValueError): + self.module.subtitle_end(f"1\n{timing}\ncaption\n") + + def test_empty_subtitles_have_no_end(self): + self.assertIsNone(self.module.subtitle_end("\n \n")) + + def test_multiple_cues_keep_existing_latest_end_policy(self): + text = "1\n00:00:00,000 --> 00:00:10,250\nFirst\n\n2\n00:00:05,000 --> 00:00:06,000\nSecond\n" + self.assertEqual(self.module.subtitle_end(text), 10.25) + + +class SubtitleHandoffContract(unittest.TestCase): + def test_rerun_removed_opening_narration_keeps_evidence_and_retimes_remaining_caption(self): + narrate = fixtures.load_script("narrate") + assemble = fixtures.load_script("assemble") + subtitles = fixtures.load_script("make-subtitles") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + scenes_file, narration = root / "scenes.yaml", root / "narration" + work, movie, srt = root / "segments", root / "movie.mp4", root / "movie.srt" + (root / "still.png").write_bytes(b"still token") + scenes = [ + {"id": "opening", "kind": "image", "src": "still.png", "duration": 2, "narration": "Opening words"}, + {"id": "body", "kind": "image", "src": "still.png", "duration": 12, "narration": "Body"}, + ] + encoded_inputs = [] + + def synthesize(text, wav, voice): + wav.write_bytes(text.encode("utf-8")) + + def measure(path): + if path.name.startswith(".opening") or path.name == "opening.wav": + return 1.0 + return 12.0 + + def media_command(command, **kwargs): + if command[0] == "ffprobe": + path = Path(command[-1]) + seconds = {"opening.mp4": 2.375, "body.mp4": 12.0, "movie.mp4": 14.375}.get(path.name) + if seconds is None: + seconds = measure(path) + return subprocess.CompletedProcess(command, 0, str(seconds), "") + self.assertEqual(command[0], "ffmpeg") + encoded_inputs.append([command[i + 1] for i, item in enumerate(command) if item == "-i"]) + Path(command[-1]).write_bytes(b"encoded token") + return subprocess.CompletedProcess(command, 0, "", "") + + for rerun in (False, True): + if rerun: + del scenes[0]["narration"] + scenes_file.write_text(json.dumps({"scenes": scenes}), encoding="utf-8") + with patch.object(sys, "argv", ["narrate", str(scenes_file), str(narration), "--engine", "piper", "--verify", "off"]), \ + patch.object(narrate.shutil, "which", return_value="mock-tool"), \ + patch.object(narrate, "openai_key", return_value=None), \ + patch.object(narrate, "say_piper", side_effect=synthesize), \ + patch.object(narrate, "duration", side_effect=measure), \ + redirect_stdout(io.StringIO()), \ + redirect_stderr(io.StringIO()): + self.assertEqual(narrate.main(), 0) + manifest = json.loads((narration / "manifest.json").read_text(encoding="utf-8")) + self.assertEqual([entry["id"] for entry in manifest], ["body"] if rerun else ["opening", "body"]) + self.assertEqual(manifest[-1]["text"], "Body") + self.assertEqual(manifest[-1]["duration"], 12) + self.assertEqual(manifest[-1]["wav"], "body.wav") + self.assertEqual((narration / "opening.wav").read_bytes(), b"Opening words") + encoded_inputs.clear() + with patch.object(sys, "argv", ["assemble", str(scenes_file), str(movie), "--narration", str(narration), "--work", str(work)]), \ + patch.object(assemble.shutil, "which", return_value="mock-tool"), \ + patch.object(assemble, "find_browser", return_value=None), \ + patch.object(assemble, "run", side_effect=media_command), \ + redirect_stdout(io.StringIO()): + self.assertEqual(assemble.main(), 0) + offsets = json.loads((work / "offsets.json").read_text(encoding="utf-8")) + self.assertEqual(offsets, {"body": 2.375} if rerun else {"opening": 0.0, "body": 2.375}) + self.assertEqual(str(narration / "opening.wav") in encoded_inputs[0], not rerun) + self.assertIn(str(narration / "body.wav"), encoded_inputs[1]) + with patch.object(sys, "argv", ["make-subtitles", str(narration / "manifest.json"), str(srt), "--offsets-json", str(work / "offsets.json")]), \ + redirect_stdout(io.StringIO()): + self.assertEqual(subtitles.main(), 0) + expected = [(2375, 14375, "Body")] + if not rerun: + expected.insert(0, (0, 1000, "Opening words")) + self.assertEqual(read_cues(srt), expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/proving-it-works-with-a-movie/test_subtitles.py b/tests/proving-it-works-with-a-movie/test_subtitles.py new file mode 100644 index 0000000..3f76abf --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_subtitles.py @@ -0,0 +1,162 @@ +import tempfile +import json +import sys +import io +from contextlib import redirect_stderr, redirect_stdout +import unittest +from pathlib import Path +from unittest.mock import patch + +import fixtures + + +class SubtitlePathRegression(unittest.TestCase): + def test_hard_burn_runs_from_safe_directory_with_absolute_media(self): + module = fixtures.load_script("burn-subtitles") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "movie O'Brien λ" + root.mkdir() + movie, subs, output = root / "in.mp4", root / "nested" / "captions.srt", root / "out.mp4" + subs.parent.mkdir() + movie.write_bytes(b"movie") + subs.write_bytes("\ufeff1\r\n00:00:00,000 --> 00:00:01,000\r\nλ\r\n".encode("utf-8")) + calls = [] + + def fake_run(cmd, *, cwd=None): + calls.append((cmd, cwd)) + if cwd is not None: + self.assertEqual((cwd / "captions.srt").read_bytes(), subs.read_bytes()) + return True + + stdout, stderr = io.StringIO(), io.StringIO() + with patch.object(sys, "argv", ["burn-subtitles", str(movie), str(subs), str(output)]), patch.object(module.shutil, "which", return_value="ffmpeg"), patch.object(module, "has_libass", return_value=True), patch.object(module, "run", side_effect=fake_run), redirect_stdout(stdout), redirect_stderr(stderr): + self.assertEqual(module.main(), 0) + self.assertIn("burned into the picture", stdout.getvalue()) + command, cwd = calls[0] + self.assertEqual(cwd.name.startswith("movie-subtitles-"), True) + self.assertIn(str(movie.resolve()), command) + self.assertIn(str(output.resolve()), command) + self.assertTrue(any(value.startswith("subtitles=filename=captions.srt") for value in command)) + self.assertFalse(cwd.exists()) + + def test_burn_failure_is_reported_separately_from_missing_libass(self): + module = fixtures.load_script("burn-subtitles") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + movie, subs, output = root / "in.mp4", root / "captions.srt", root / "out.mp4" + movie.write_bytes(b"movie") + subs.write_text("1\n00:00:00,000 --> 00:00:01,000\ncaption\n", encoding="utf-8") + stdout, stderr = io.StringIO(), io.StringIO() + with patch.object(sys, "argv", ["burn-subtitles", str(movie), str(subs), str(output)]), patch.object(module.shutil, "which", return_value="ffmpeg"), patch.object(module, "has_libass", return_value=True), patch.object(module, "run", return_value=False), redirect_stdout(stdout), redirect_stderr(stderr): + self.assertEqual(module.main(), 1) + self.assertIn("burn failed", stderr.getvalue()) + +class SubtitleOffsetRegression(unittest.TestCase): + def subtitles(self, *manual, offsets=None): + module = fixtures.load_script("make-subtitles") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, output = root / "manifest.json", root / "captions.srt" + manifest.write_text(json.dumps([ + {"id": scene, "text": scene, "duration": 1.0} + for scene in ("intro", "body", "end") + ]), encoding="utf-8") + argv = ["make-subtitles", str(manifest), str(output)] + if offsets is not None: + path = root / "offsets.json" + path.write_text(json.dumps(offsets), encoding="utf-8") + argv += ["--offsets-json", str(path)] + if manual: + argv += ["--offsets", *manual] + with patch.object(sys, "argv", argv), redirect_stdout(io.StringIO()): + self.assertEqual(module.main(), 0) + cues = [] + for block in output.read_text(encoding="utf-8").strip().split("\n\n"): + if not block: + continue + _, timing, text = block.split("\n", 2) + times = [] + for timestamp in timing.split(" --> "): + h, m, s = timestamp.replace(",", ".").split(":") + times.append(int(h) * 3600 + int(m) * 60 + float(s)) + cues.append((*times, text)) + return cues + + def test_default_scenes_run_back_to_back(self): + self.assertEqual(self.subtitles(), + [(0, 1, "intro"), (1, 2, "body"), (2, 3, "end")]) + + def test_partial_manual_offsets_preserve_other_scenes(self): + self.assertEqual(self.subtitles("intro=2"), + [(2, 3, "intro"), (3, 4, "body"), (4, 5, "end")]) + self.assertEqual(self.subtitles("body=4"), + [(0, 1, "intro"), (4, 5, "body"), (5, 6, "end")]) + + def test_assembly_offsets_select_scenes_in_the_cut(self): + self.assertEqual(self.subtitles(offsets={"intro": 2, "end": 8}), + [(2, 3, "intro"), (8, 9, "end")]) + + def test_manual_offsets_change_timing_without_changing_cut_membership(self): + self.assertEqual(self.subtitles("intro=3", "body=5", offsets={"intro": 2, "end": 8}), + [(3, 4, "intro"), (8, 9, "end")]) + + def test_cut_without_narrated_scenes_has_no_cues(self): + for offsets in ({}, {"silent": 2}): + with self.subTest(offsets=offsets): + self.assertEqual(self.subtitles(offsets=offsets), []) + + +class SubtitleIntegrationRegression(unittest.TestCase): + def test_bom_manifest_and_offsets_write_utf8_under_legacy_console(self): + import json + import os + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest, offsets, out = root / "manifest.json", root / "offsets.json", root / "λ.srt" + manifest.write_bytes(('\ufeff' + json.dumps([{"id": "clip", "text": "Unicode λ café", "duration": 1}], ensure_ascii=False) + '\r\n').encode('utf-8')) + offsets.write_text('{"clip": 2}', encoding="utf-8-sig") + env = dict(os.environ, PYTHONIOENCODING="cp1252", PYTHONUTF8="0") + result = fixtures.run_tool("make-subtitles", [str(manifest), str(out), "--offsets-json", str(offsets)], cwd=root, env=env) + self.assertEqual(result.returncode, 0, fixtures.output_text(result)) + text = out.read_text(encoding="utf-8") + self.assertIn("Unicode λ café", text) + self.assertIn("00:00:02,000 --> 00:00:03,000", text) + self.assertFalse(out.read_bytes().startswith(b'\xef\xbb\xbf')) + + def test_hard_subtitles_are_pixels_in_nested_special_path(self): + import subprocess + missing = fixtures.missing_executables("uv", "ffmpeg") + if missing: + self.skipTest(f"required executable(s) not on PATH: {', '.join(missing)}") + module = fixtures.load_script("burn-subtitles") + if not module.has_libass(): + self.skipTest("libass FFmpeg is required for hard subtitle pixels") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "O'Brien λ # %" + root.mkdir() + movie, subs, output = root / "in.mp4", root / "nested" / "O'Brien.srt", root / "out.mp4" + subs.parent.mkdir() + subs.write_bytes("\ufeff1\r\n00:00:00,000 --> 00:00:01,000\r\nVisible caption\r\n".encode("utf-8")) + fixtures._run_ffmpeg(["-f", "lavfi", "-i", "color=c=black:s=640x360:d=1", "-c:v", "libx264", str(movie)], cwd=root) + result = fixtures.run_tool("burn-subtitles", [str(movie), str(subs), str(output)], cwd=Path(directory)) + self.assertEqual(result.returncode, 0, fixtures.output_text(result)) + self.assertIn("burned into the picture", fixtures.output_text(result)) + frame = subprocess.run(["ffmpeg", "-v", "error", "-ss", "0.5", "-i", str(output), "-frames:v", "1", "-pix_fmt", "gray", "-f", "rawvideo", "-"], capture_output=True, check=True).stdout + self.assertGreater(sum(value > 180 for value in frame), 50) + + def test_burn_failure_fallback_does_not_claim_missing_libass(self): + module = fixtures.load_script("burn-subtitles") + for libass in (True, False): + with self.subTest(libass=libass), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + movie, subs = root / "in.mp4", root / "in.srt" + movie.touch(); subs.touch() + stdout, stderr = io.StringIO(), io.StringIO() + with patch.object(sys, "argv", ["burn-subtitles", str(movie), str(subs), str(root / "out.mp4")]), patch.object(module.shutil, "which", return_value="ffmpeg"), patch.object(module, "has_libass", return_value=libass), patch.object(module, "run", side_effect=[False, True] if libass else [True]), redirect_stdout(stdout), redirect_stderr(stderr): + self.assertEqual(module.main(), 0) + self.assertEqual("no libass" in stdout.getvalue(), not libass) + self.assertEqual("burn failed" in stderr.getvalue(), libass) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/proving-it-works-with-a-movie/test_terminal.py b/tests/proving-it-works-with-a-movie/test_terminal.py new file mode 100644 index 0000000..9820094 --- /dev/null +++ b/tests/proving-it-works-with-a-movie/test_terminal.py @@ -0,0 +1,316 @@ +"""The terminal recorder: prompt parsing and the frame grid anywhere; a real +ttyd session wherever ttyd and a Chrome-family browser exist.""" +import importlib.util +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +import fixtures + +SCRIPT = Path(__file__).resolve().parents[2] / "skills/proving-it-works-with-a-movie/examples/film-terminal.py" +FIXTURE = Path(__file__).resolve().with_name("fixtures") / "terminal_app.py" +TTYD = os.environ.get("MOVIE_TEST_TTYD") or shutil.which("ttyd") +BROWSER = fixtures.load_script("browser_tools").find_browser(os.environ.get("MOVIE_TEST_BROWSER")) +SHELL = os.environ.get("MOVIE_TEST_SHELL") or ("powershell51" if os.name == "nt" else "bash") +BASH = SHELL in ("bash", "gitbash") + + +def recorder(): + spec = importlib.util.spec_from_file_location("film_terminal", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def gone(pid, timeout=5): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if os.name == "nt": + listed = subprocess.run(["tasklist", "/FI", f"PID eq {pid}", "/NH"], + capture_output=True, text=True).stdout + if str(pid) not in listed: + return True + else: + try: + os.kill(pid, 0) + except ProcessLookupError: + return True + time.sleep(0.1) + return False + + +class PromptTests(unittest.TestCase): + def test_prompts_parse_both_terminators_and_paths_with_semicolons(self): + module = recorder() + log = (b"noise\x1b]0;MOVIE;1;1;;C:\\a;b\x07\x1b[0m" + b"\x1b]2;MOVIE;2;0;7;/c/x\x1b\\tail" + b"\x1b]0;MOVIE;3;1;0;/home/me\x07") + self.assertEqual(module.prompts(log), [ + dict(n=1, ok=True, exit_code=None, cwd="C:\\a;b"), + dict(n=2, ok=False, exit_code=7, cwd="/c/x"), + dict(n=3, ok=True, exit_code=0, cwd="/home/me"), + ]) + self.assertEqual(module.prompts(b"\x1b]0;something else\x07"), []) + + def test_prompt_install_is_one_typed_line_per_shell(self): + module = recorder() + cwd = Path("C:/Users/x/movie O'Brien λ") + for kind in module.SHELLS: + line = module.prompt_command(kind, cwd) + self.assertEqual(len(line.splitlines()), 1, kind) + self.assertNotIn("MOVIE;", line, "the marker text must not be echoed by the install line") + self.assertIn("Brien λ", module.prompt_script(kind, cwd), "the script enters the cwd") + + def test_keys_are_named_or_single_characters(self): + module = recorder() + self.assertEqual(module.key_params("Ctrl-C")["modifiers"], 2) + self.assertEqual(module.key_params("Enter")["text"], "\r") + self.assertEqual(module.key_params("q"), dict(key="q", text="q")) + with self.assertRaises(SystemExit): + module.key_params("Bogus") + + +class FilmGridTests(unittest.TestCase): + def test_filming_refuses_a_nonempty_take_without_changing_its_contents(self): + module = recorder() + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) / "take" + out.mkdir() + (out / "f00000.png").write_bytes(b"old frame") + (out / "notes.txt").write_bytes(b"sentinel evidence") + before = {path.name: path.read_bytes() for path in out.iterdir()} + captures = [] + + with self.assertRaisesRegex(SystemExit, "not empty.*new take directory"): + module.film(out, seconds=1, hold=0, + capture=lambda: captures.append(True) or b"new frame", + finished=lambda: True) + + self.assertEqual(captures, []) + self.assertEqual( + {path.name: path.read_bytes() for path in out.iterdir()}, before + ) + + def test_a_slow_capture_repeats_the_previous_frame_and_filming_holds_after_the_prompt(self): + module = recorder() + clock = {"now": 0.0} + shots = [] + + def capture(): + shots.append(len(shots) + 1) + clock["now"] += 0.5 if len(shots) == 2 else 0.01 # the second screenshot stalls + return bytes([len(shots)]) + + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) / "take" + frames = module.film(out, seconds=10, hold=0.4, capture=capture, + finished=lambda: clock["now"] >= 1.0, + clock=lambda: clock["now"], + sleep=lambda s: clock.__setitem__("now", clock["now"] + s)) + files = sorted(out.glob("f*.png")) + self.assertEqual([f.name for f in files], [f"f{i:05d}.png" for i in range(frames)]) + self.assertEqual(files[2].read_bytes(), files[1].read_bytes(), "missed slot repeats the last frame") + self.assertNotEqual(files[3].read_bytes(), files[2].read_bytes()) + # Captures add 0.01 s and sleeps 0.02 s, so the clock sits on odd + # hundredths: the prompt is seen at 1.01 s and the hold ends at 1.41 s. + self.assertEqual(frames, 8, "slots 0.0 s through 1.4 s fall before the 1.41 s endpoint") + + def test_filming_stops_at_the_deadline_while_the_command_runs(self): + module = recorder() + clock = {"now": 0.0} + with tempfile.TemporaryDirectory() as directory: + frames = module.film(Path(directory), seconds=1.0, hold=5, capture=lambda: b"png", + finished=lambda: False, clock=lambda: clock["now"], + sleep=lambda s: clock.__setitem__("now", clock["now"] + s)) + self.assertEqual(frames, 5) + + +class ServeArgumentTests(unittest.TestCase): + def test_serve_refuses_a_missing_cwd_before_launching_anything(self): + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run([sys.executable, str(SCRIPT), "serve", str(Path(directory) / "session"), + "--shell", "bash", "--cwd", str(Path(directory) / "missing")], + capture_output=True, text=True, timeout=60) + self.assertEqual(result.returncode, 1) + self.assertIn("--cwd is not a directory", result.stderr) + self.assertFalse((Path(directory) / "session" / "session.json").exists()) + + def test_recording_verbs_refuse_a_nonempty_take_before_session_side_effects(self): + module = recorder() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + session = root / "session" + session.mkdir() + (session / "ready.json").write_text("{}", encoding="utf-8") + (session / "session.json").write_text( + json.dumps({"debug_port": 1}), encoding="utf-8" + ) + (session / "terminal.log").write_bytes( + b"\x1b]0;MOVIE;1;1;0;/tmp\x07" + ) + for verb, positional in (("run", ["echo hello"]), + ("key", ["Enter"]), + ("watch", [])): + with self.subTest(verb=verb): + take = root / verb + take.mkdir() + (take / "f00000.png").write_bytes(b"old frame") + (take / "sentinel.txt").write_bytes(b"keep me") + before = {path.name: path.read_bytes() for path in take.iterdir()} + argv = ["film-terminal", verb, str(session), *positional, + "--record", str(take)] + with patch.object(sys, "argv", argv), \ + patch.object(module, "connect", + side_effect=AssertionError("connected")), \ + patch.object(module, "type_text", + side_effect=AssertionError("typed")), \ + patch.object(module, "press", + side_effect=AssertionError("pressed")), \ + self.assertRaisesRegex(SystemExit, + "not empty.*new take directory"): + module.main() + self.assertEqual( + {path.name: path.read_bytes() for path in take.iterdir()}, before + ) + + +@unittest.skipUnless(TTYD and BROWSER, "ttyd and a Chrome-family browser are required") +class SessionTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(prefix="movie-terminal-", ignore_cleanup_errors=True) + self.addCleanup(self.tmp.cleanup) + self.work = Path(self.tmp.name) / "movie O'Brien λ" + self.work.mkdir() + self.session = Path(self.tmp.name) / "session" + self.log = (Path(self.tmp.name) / "serve.log").open("wb") + self.addCleanup(self.log.close) + argv = [sys.executable, str(SCRIPT), "serve", str(self.session), "--shell", SHELL, + "--cwd", str(self.work), "--ttyd", TTYD, "--browser", BROWSER] + if os.environ.get("MOVIE_TEST_SHELL_EXE"): + argv += ["--shell-exe", os.environ["MOVIE_TEST_SHELL_EXE"]] + self.owned_pids = [] + self.serve = subprocess.Popen(argv, stdout=self.log, stderr=subprocess.STDOUT) + self.addCleanup(self.close_session) + deadline = time.monotonic() + 45 + while not (self.session / "ready.json").exists() and self.serve.poll() is None \ + and time.monotonic() < deadline: + time.sleep(0.1) + if (self.session / "session.json").exists(): + self.owned_pids = json.loads((self.session / "session.json").read_text(encoding="utf-8"))["pids"] + if not (self.session / "ready.json").exists(): + report = "".join(f"--- {name}\n" + path.read_text(errors="replace") if path.exists() else "" + for name, path in (("serve.log", Path(self.tmp.name) / "serve.log"), + ("ttyd.log", self.session / "ttyd.log"), + ("browser.log", self.session / "browser.log"))) + self.fail(report) + + def close_session(self): + if self.serve.poll() is None: + # A failed setup may not have session metadata yet, but the + # owner still needs its stop request before we wait for cleanup. + self.session.mkdir(parents=True, exist_ok=True) + (self.session / "stop").write_text("", encoding="utf-8") + try: + self.serve.wait(35) + except subprocess.TimeoutExpired: + self.serve.kill() + self.serve.wait() + for pid in self.owned_pids: + self.assertTrue(gone(pid), f"pid {pid} survived close") + + def cli(self, *args, timeout=120): + result = subprocess.run([sys.executable, str(SCRIPT), *args], capture_output=True, timeout=timeout) + return result.returncode, result.stdout.decode("utf-8", "replace"), result.stderr.decode("utf-8", "replace") + + def run_command(self, command, *extra): + code, out, err = self.cli("run", str(self.session), command, *extra) + self.assertTrue(out.strip(), err) + return code, json.loads(out.strip().splitlines()[-1]) + + def quoted(self, *words): + if BASH: + return " ".join(shlex.quote(w.replace("\\", "/")) for w in words) + return "& " + " ".join("'" + w.replace("'", "''") + "'" for w in words) + + def native(self, code): + return self.quoted(sys.executable) + f' -c "{code}"' + + def test_commands_report_status_and_the_shell_persists_between_calls(self): + code, result = self.run_command("echo hello") + self.assertEqual((code, result["outcome"], result["ok"]), (0, "completed", True), result) + self.assertTrue(result["cwd"].endswith("movie O'Brien λ"), result["cwd"]) + if BASH: + set_value, check_value, failing = "MOVIE_VALUE=kept", 'test "$MOVIE_VALUE" = kept', "false" + else: + set_value = "$global:MovieValue = 'kept'" + check_value = "if ($global:MovieValue -ne 'kept') { throw 'lost' }" + failing = "Get-Item 'Z:\\nowhere'" + self.assertEqual(self.run_command(set_value)[0], 0) + self.assertEqual(self.run_command(check_value)[0], 0, "state must survive separate calls") + code, result = self.run_command(failing) + self.assertEqual((code, result["ok"]), (1, False), result) + code, result = self.run_command(self.native("import sys; sys.exit(7)")) + self.assertEqual((code, result["ok"], result["exit_code"]), (1, False, 7), result) + + def test_a_tui_is_filmed_across_two_takes_and_a_long_command_across_calls(self): + from PIL import Image + + take_one, take_two, take_three = (self.work / name for name in ("take-one", "take-two", "take-three")) + code, result = self.run_command(self.quoted(sys.executable, str(FIXTURE)), + "--record", str(take_one), "--seconds", "7") + self.assertEqual((code, result["outcome"]), (2, "running"), result) + frames = sorted(take_one.glob("f*.png")) + self.assertGreaterEqual(len(frames), 30, "7 s at 5 fps") + seen = [] + for frame in frames: + with Image.open(frame) as image: + r, g, b = image.convert("RGB").getpixel((300, 120)) + color = ("red" if r > 150 and g < 100 and b < 100 else + "green" if g > 120 and r < 100 and b < 140 else + "blue" if b > 150 and r < 100 and g < 140 else None) + if color and (not seen or seen[-1] != color): + seen.append(color) + self.assertEqual(seen, ["red", "green", "blue"], "the three TUI states, in order, in the automatic frames") + code, out, err = self.cli("key", str(self.session), "q", "--record", str(take_two), "--seconds", "10") + result = json.loads(out.strip().splitlines()[-1]) + self.assertEqual((code, result["outcome"], result["ok"]), (0, "completed", True), (result, err)) + self.assertGreaterEqual(result["frames"], 7, "the exit plus the 1.5 s hold") + self.assertEqual(len(json.loads((self.work / "states.json").read_text())), 3) + + code, result = self.run_command(self.native("import time; time.sleep(3)"), "--seconds", "1") + self.assertEqual(result["outcome"], "running") + code, out, err = self.cli("watch", str(self.session), "--record", str(take_three), "--seconds", "15") + result = json.loads(out.strip().splitlines()[-1]) + self.assertEqual((code, result["outcome"], result["ok"]), (0, "completed", True), (result, err)) + self.assertGreaterEqual(result["frames"], 10, "about 2 s of waiting plus the hold") + self.assertEqual(result["scene"], {"kind": "frames", "src": str(take_three.resolve()), "rate": 5}) + + def test_close_kills_the_shell_tree_and_spares_unrelated_processes(self): + sentinel = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(120)"]) + self.addCleanup(sentinel.kill) + tree = self.work / "tree" + code, result = self.run_command(self.quoted(sys.executable, str(FIXTURE), "tree", str(tree)), "--seconds", "2") + self.assertEqual(result["outcome"], "running") + deadline = time.monotonic() + 20 + while len(list(tree.glob("*.json"))) < 3 and time.monotonic() < deadline: + time.sleep(0.1) + pids = [json.loads(path.read_text())["pid"] for path in tree.glob("*.json")] + self.assertEqual(len(pids), 3) + code, out, err = self.cli("close", str(self.session)) + self.assertEqual(code, 0, err) + self.assertEqual(self.serve.wait(15), 0) + for pid in pids: + self.assertTrue(gone(pid), f"descendant {pid} survived close") + self.assertIsNone(sentinel.poll(), "an unrelated process must survive") + + +if __name__ == "__main__": + unittest.main() |
