summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJesse Vincent <jesse@fsck.com>2026-10-10 03:03:29 +0000
committerJesse Vincent <jesse@fsck.com>2026-10-10 03:03:29 +0000
commitb274c1a79dc93d418a1e925911da074eee1c5e6c (patch)
treeca04b40ba880e0d2f148c204748be6b4a16e3543
parentdb21832145824f220470744bdc2f0f463d216072 (diff)
tests: move the brainstorming harness to superpowers-evalsbrainstorming-eval-harness
Skill-behavior evals live in superpowers-evals (AGENTS.md); the harness is now quick-evals/brainstorming/ there (prime-radiant-inc/superpowers-evals#72). The results write-up stays here and points at the new location.
-rw-r--r--docs/superpowers/specs/2026-10-05-brainstorming-rebuild-eval-results.md2
-rw-r--r--tests/brainstorming/batch.sh48
-rw-r--r--tests/brainstorming/blind-pair.sh55
-rwxr-xr-xtests/brainstorming/codex-with-plugin.sh83
-rw-r--r--tests/brainstorming/converse.sh202
-rw-r--r--tests/brainstorming/first-turn.sh117
-rw-r--r--tests/brainstorming/gate-score.sh64
-rw-r--r--tests/brainstorming/matrix.sh60
-rwxr-xr-xtests/brainstorming/pi-with-plugin.sh32
-rw-r--r--tests/brainstorming/prompts/bakery.txt1
-rw-r--r--tests/brainstorming/prompts/cornflower-icon.txt1
-rw-r--r--tests/brainstorming/prompts/habit-tracker.txt1
-rw-r--r--tests/brainstorming/prompts/offsite-talk.txt1
-rw-r--r--tests/brainstorming/prompts/onboarding-emails.txt1
-rw-r--r--tests/brainstorming/prompts/style-insurance.txt8
-rw-r--r--tests/brainstorming/prompts/style-research.txt16
-rw-r--r--tests/brainstorming/prompts/style-security.txt7
-rw-r--r--tests/brainstorming/scenarios/birthday-cake/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/birthday-cake/persona.md17
-rw-r--r--tests/brainstorming/scenarios/book-club/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/book-club/persona.md15
-rw-r--r--tests/brainstorming/scenarios/book-club/setup.sh68
-rw-r--r--tests/brainstorming/scenarios/farm-picture/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/farm-picture/persona.md17
-rw-r--r--tests/brainstorming/scenarios/farm-skip/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/farm-skip/persona.md7
-rw-r--r--tests/brainstorming/scenarios/habit-tracker/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/habit-tracker/persona.md18
-rw-r--r--tests/brainstorming/scenarios/ios-app/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/ios-app/persona.md17
-rw-r--r--tests/brainstorming/scenarios/just-build-it/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/just-build-it/persona.md5
-rw-r--r--tests/brainstorming/scenarios/ocd-app-discuss/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/ocd-app-discuss/persona.md17
-rw-r--r--tests/brainstorming/scenarios/ocd-app-discuss/setup.sh6
-rw-r--r--tests/brainstorming/scenarios/ocd-app/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/ocd-app/persona.md17
-rw-r--r--tests/brainstorming/scenarios/ocd-skip/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/ocd-skip/persona.md7
-rw-r--r--tests/brainstorming/scenarios/offsite-talk/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/offsite-talk/persona.md18
-rw-r--r--tests/brainstorming/scenarios/pottery-platform/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/pottery-platform/persona.md11
-rw-r--r--tests/brainstorming/scenarios/skip-files/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/skip-files/persona.md16
-rw-r--r--tests/brainstorming/scenarios/skip-files/setup.sh29
-rw-r--r--tests/brainstorming/scenarios/spike-fts/opening.txt1
-rw-r--r--tests/brainstorming/scenarios/spike-fts/persona.md7
-rw-r--r--tests/brainstorming/tech-lines.sh42
-rw-r--r--tests/brainstorming/try-codex.sh92
50 files changed, 1 insertions, 1138 deletions
diff --git a/docs/superpowers/specs/2026-10-05-brainstorming-rebuild-eval-results.md b/docs/superpowers/specs/2026-10-05-brainstorming-rebuild-eval-results.md
index 6d4c171..be72656 100644
--- a/docs/superpowers/specs/2026-10-05-brainstorming-rebuild-eval-results.md
+++ b/docs/superpowers/specs/2026-10-05-brainstorming-rebuild-eval-results.md
@@ -9,7 +9,7 @@ simulated humans, and testers were all claude-opus-5-5. Subjects ran with
`--setting-sources project --strict-mcp-config` plus `--plugin-dir` for the
checkout under test, so no user CLAUDE.md, user plugins, or MCP servers.
-**Tools** (all in `tests/brainstorming/`):
+**Tools** (all in superpowers-evals, `quick-evals/brainstorming/`):
- `first-turn.sh`: isolated `claude -p` samples of the opening message.
- `converse.sh`: a claude-session-driver worker talks with a simulated human
who has hidden intent (`scenarios/*/persona.md`).
diff --git a/tests/brainstorming/batch.sh b/tests/brainstorming/batch.sh
deleted file mode 100644
index fe2afea..0000000
--- a/tests/brainstorming/batch.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Runs one experiment: N reps of converse.sh for a scenario, in parallel.
-#
-# Use when comparing skill variants. Each rep is a separate converse.sh run
-# under OUT_DIR/EXPERIMENT-rREP/SCENARIO/, launched a few seconds apart.
-# Per-rep logs go to OUT_DIR/logs/. Prints each rep's summary line when all
-# are done. Score with gate-score.sh (first turn) or tech-lines.sh.
-#
-# Environment is passed through to converse.sh (CSD, CSD_CODEX_MODEL,
-# REAL_CODEX_BIN, TURN_TIMEOUT, CODEX_ARGS, CODEX_EFFORT).
-
-set -euo pipefail
-
-usage() {
- cat <<'EOF'
-Usage: batch.sh -x EXPERIMENT -s SCENARIO -P PLUGIN_DIR -o OUT_DIR [-n REPS] [-t MAX_TURNS] [-H HARNESS]
-
- -x EXPERIMENT Label, e.g. "e2-description".
- -s SCENARIO Scenario under scenarios/.
- -P PLUGIN_DIR Superpowers checkout to load.
- -o OUT_DIR Results root, shared across experiments.
- -n REPS Parallel reps (default 5).
- -t MAX_TURNS Passed to converse.sh (default 1: first turn only).
- -H HARNESS claude, codex, or pi (default codex).
-EOF
-}
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-EXP="" SCENARIO="" PLUGIN="" OUT="" REPS=5 TURNS=1 HARNESS=codex
-while getopts "x:s:P:o:n:t:H:h" opt; do
- case $opt in
- x) EXP=$OPTARG ;; s) SCENARIO=$OPTARG ;; P) PLUGIN=$OPTARG ;; o) OUT=$OPTARG ;;
- n) REPS=$OPTARG ;; t) TURNS=$OPTARG ;; H) HARNESS=$OPTARG ;;
- h) usage; exit 0 ;; *) usage >&2; exit 2 ;;
- esac
-done
-[ -n "$EXP" ] && [ -n "$SCENARIO" ] && [ -n "$PLUGIN" ] && [ -n "$OUT" ] || { usage >&2; exit 2; }
-
-mkdir -p "$OUT/logs"
-for i in $(seq 1 "$REPS"); do
- bash "$SCRIPT_DIR/converse.sh" -H "$HARNESS" -s "$SCENARIO" -a "$EXP-r$i" -P "$PLUGIN" -t "$TURNS" -o "$OUT" \
- > "$OUT/logs/$EXP-$SCENARIO-r$i.log" 2>&1 &
- sleep 3
-done
-wait
-for i in $(seq 1 "$REPS"); do
- grep -h 'turns=' "$OUT/logs/$EXP-$SCENARIO-r$i.log" | sed 's/ -> .*//' || echo "$EXP-r$i $SCENARIO: no summary; see $OUT/logs/$EXP-$SCENARIO-r$i.log"
-done
diff --git a/tests/brainstorming/blind-pair.sh b/tests/brainstorming/blind-pair.sh
deleted file mode 100644
index 2b01432..0000000
--- a/tests/brainstorming/blind-pair.sh
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-# Launch a blind pair of brainstorming sessions for one tester.
-#
-# Use when comparing two brainstorming skill versions with a tester (a person
-# or a subagent playing one) who shouldn't know which version is which. Starts
-# two claude-session-driver workers, each loading a different superpowers
-# checkout, under neutral labels X and Y. The label-to-version mapping goes to
-# a file the tester never sees.
-
-set -euo pipefail
-
-usage() {
- cat <<'EOF'
-Usage: blind-pair.sh -t TESTER -A PLUGIN_A -B PLUGIN_B [-o OUT_DIR]
-
- -t TESTER Tester name, e.g. "t1". Workers are named TESTER-x and TESTER-y.
- -A, -B The two superpowers checkouts to compare.
- -o OUT_DIR Where the hidden mapping and work dirs are recorded
- (default /tmp/brainstorming-blind).
-
-Prints the two shim paths (X then Y) for the tester to drive with
-`<shim> converse "<message>" 900`. Each worker runs in a fresh git repo
-with no user settings, CLAUDE.md, or MCP servers.
-
-The mapping is appended to OUT_DIR/mapping.txt: "TESTER X=<plugin> Y=<plugin>".
-Stop workers afterwards with `<shim> stop`.
-EOF
-}
-
-CSD=${CSD:-$HOME/git/claude-session-driver/skills/driving-claude-code-sessions/scripts/csd}
-TESTER="" A="" B="" OUT=/tmp/brainstorming-blind
-while getopts "t:A:B:o:h" opt; do
- case $opt in
- t) TESTER=$OPTARG ;; A) A=$OPTARG ;; B) B=$OPTARG ;; o) OUT=$OPTARG ;;
- h) usage; exit 0 ;; *) usage >&2; exit 2 ;;
- esac
-done
-[ -n "$TESTER" ] || { echo "error: -t TESTER is required" >&2; exit 2; }
-for p in "$A" "$B"; do
- [ -f "$p/.claude-plugin/plugin.json" ] || { echo "error: not a plugin checkout: $p" >&2; exit 2; }
-done
-mkdir -p "$OUT"
-
-# Coin flip for which version sits behind X.
-if [ $((RANDOM % 2)) -eq 0 ]; then X=$A Y=$B; else X=$B Y=$A; fi
-echo "$TESTER X=$X Y=$Y" >> "$OUT/mapping.txt"
-
-for label in x y; do
- plugin=$X; [ "$label" = y ] && plugin=$Y
- work=$(mktemp -d "$OUT/$TESTER-$label.XXXX")
- (cd "$work" && git init -q)
- echo "$TESTER-$label $work" >> "$OUT/workdirs.txt"
- "$CSD" launch "$TESTER-$label" "$work" -- --setting-sources project --strict-mcp-config --plugin-dir "$plugin" 2> "$OUT/$TESTER-$label.launch.log" \
- || { echo "error: launch failed for $TESTER-$label; see $OUT/$TESTER-$label.launch.log" >&2; exit 1; }
-done
diff --git a/tests/brainstorming/codex-with-plugin.sh b/tests/brainstorming/codex-with-plugin.sh
deleted file mode 100755
index 618411d..0000000
--- a/tests/brainstorming/codex-with-plugin.sh
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env bash
-# Starts codex in a claude-session-driver worker with exactly one superpowers.
-#
-# Usage: codex-with-plugin.sh PLUGIN_ROOT|curated [codex args...]
-#
-# converse.sh writes a per-run launcher that calls this, and points
-# CSD_CODEX_BIN at the launcher (tmux doesn't pass the controller's
-# environment to the worker, so the plugin root travels as an argument).
-#
-# csd gives each worker a fresh CODEX_HOME. Codex there loads the account's
-# remote plugins: the curated superpowers release, plus connectors such as
-# sites, gmail, and slack that would let a test run publish or send things
-# as the account owner. This script disables every remote plugin except the
-# superpowers under test. (Disabling plugins does not remove the connector
-# tools themselves; converse.sh also passes -c features.apps=false.)
-# Disabling the curated superpowers doesn't reliably keep it out either:
-# Codex's remote plugin sync can install it and list it beside the
-# checkout under test (features.remote_plugin=false doesn't stop that).
-# gate-score.sh marks those runs MIXED. With PLUGIN_ROOT, it installs that checkout (which needs
-# .agents/plugins/marketplace.json) and disables the curated release; with
-# "curated", it keeps the curated release.
-
-set -euo pipefail
-
-case ${1:-} in -h|--help) sed -n '2,/^$/s/^# \{0,1\}//p' "$0"; exit 0 ;; esac
-: "${CODEX_HOME:?CODEX_HOME must be set (csd sets it per worker)}"
-[ $# -ge 1 ] || { echo "usage: codex-with-plugin.sh PLUGIN_ROOT|curated [codex args...]" >&2; exit 2; }
-ROOT=$1; shift
-REAL=${REAL_CODEX_BIN:-$(command -v codex)}
-LOG="$CODEX_HOME/codex-with-plugin.log"
-CONFIG="$CODEX_HOME/config.toml"
-
-fail() { echo "error: $1; see $LOG" >&2; cat "$LOG" >&2; exit 1; }
-
-KEEP=superpowers@openai-curated-remote
-if [ "$ROOT" != curated ]; then
- [ -f "$ROOT/.agents/plugins/marketplace.json" ] \
- || { echo "error: no .agents/plugins/marketplace.json in $ROOT" >&2; exit 2; }
- # Codex installs a local marketplace plugin with `git clone`, which drops
- # uncommitted edits. Snapshot the files as they are on disk into a
- # throwaway repo so Codex tests exactly what's there.
- SNAP="$CODEX_HOME/plugin-snapshot"
- { rsync -a --exclude .git "$ROOT/" "$SNAP/" \
- && git -C "$SNAP" init -q && git -C "$SNAP" add -A \
- && git -C "$SNAP" -c user.email=t@t -c user.name=t commit -qm snapshot; } > "$LOG" 2>&1 \
- || fail "snapshotting $ROOT failed"
- { "$REAL" plugin marketplace add "$SNAP" && "$REAL" plugin add superpowers@superpowers-dev; } >> "$LOG" 2>&1 \
- || fail "installing $ROOT into $CODEX_HOME failed"
- installed=$(find "$CODEX_HOME/plugins/cache/superpowers-dev" -path '*/skills/brainstorming/SKILL.md' 2>/dev/null | head -1 || true)
- cmp -s "$installed" "$ROOT/skills/brainstorming/SKILL.md" \
- || fail "installed brainstorming skill ($installed) differs from $ROOT"
- KEEP=superpowers@superpowers-dev
-fi
-
-enabled=$("$REAL" plugin list 2>> "$LOG" | awk '/installed, enabled/ {print $1}') \
- || fail "listing plugins in $CODEX_HOME failed"
-for plugin in $enabled; do
- [ "$plugin" = "$KEEP" ] || printf '\n[plugins."%s"]\nenabled = false\n' "$plugin" >> "$CONFIG"
-done
-echo "kept $KEEP; disabled: $(echo $enabled | tr ' ' '\n' | grep -v -x "$KEEP" | tr '\n' ' ')" >> "$LOG"
-
-# CODEX_BOOTSTRAP=1 puts the using-superpowers bootstrap, worded as the
-# Claude Code SessionStart hook injects it, in this worker's global
-# AGENTS.md, so Codex sees it before the first turn instead of finding the
-# skill mid-turn. Stands in for a Codex session-start bootstrap.
-if [ "${CODEX_BOOTSTRAP:-}" = 1 ]; then
- [ "$ROOT" != curated ] || { echo "error: CODEX_BOOTSTRAP needs a PLUGIN_ROOT" >&2; exit 2; }
- { printf '<EXTREMELY_IMPORTANT>\nYou have superpowers.\n\n'
- printf "**Below is the full content of your 'superpowers:using-superpowers' skill - your introduction to using skills. For all other skills, use the 'Skill' tool:**\n\n"
- cat "$ROOT/skills/using-superpowers/SKILL.md"
- printf '\n</EXTREMELY_IMPORTANT>\n'; } > "$CODEX_HOME/AGENTS.md"
-fi
-
-# csd always launches in YOLO mode; CODEX_PERMISSIONS=workspace-write runs
-# with the sandbox and approval policy a default Codex install uses instead.
-if [ "${CODEX_PERMISSIONS:-}" = workspace-write ]; then
- args=()
- for a in "$@"; do [ "$a" = --dangerously-bypass-approvals-and-sandbox ] || args+=("$a"); done
- # csd's event hooks run inside the sandbox and write under its worker dir.
- set -- -s workspace-write -a on-request -c "sandbox_workspace_write.writable_roots=[\"${CSD_WORKER_DIR:-/tmp/csd-workers}\"]" "${args[@]}"
-fi
-
-exec "$REAL" "$@"
diff --git a/tests/brainstorming/converse.sh b/tests/brainstorming/converse.sh
deleted file mode 100644
index 3e451f4..0000000
--- a/tests/brainstorming/converse.sh
+++ /dev/null
@@ -1,202 +0,0 @@
-#!/usr/bin/env bash
-# Multi-turn brainstorming test: a real Claude Code or Codex session (the subject)
-# talks with a simulated human who has hidden intent.
-#
-# Use when you need to see how a brainstorming skill behaves past the first
-# turn: whether it draws out the hidden intent, plays it back, sizes the
-# work, writes the design, and holds the approval gate. Pair it with
-# first-turn.sh, which is cheaper and only covers the opening.
-#
-# The subject is a claude-session-driver worker loading the superpowers
-# plugin from PLUGIN_DIR. A Claude worker gets no user settings, CLAUDE.md,
-# or MCP servers; a Codex worker gets csd's fresh CODEX_HOME, which still
-# carries the account's remote plugins.
-# The human is an isolated `claude -p` given the scenario's persona.md and
-# the transcript so far; it may read files in the subject's work dir.
-
-set -euo pipefail
-
-usage() {
- cat <<'EOF'
-Usage: converse.sh -s SCENARIO -a ARM -P PLUGIN_DIR [-H HARNESS] [-t MAX_TURNS] [-o OUT_DIR]
-
- -s SCENARIO Directory name under scenarios/ (opening.txt, persona.md,
- optional setup.sh run inside a fresh git work dir).
- -a ARM Label for this arm, e.g. "old" or "new".
- -P PLUGIN_DIR Superpowers checkout to load as the plugin. With -H codex,
- "curated" instead uses the account's curated release.
- -H HARNESS claude (default), codex, or pi. Pi runs on PI_PROVIDER
- / PI_MODEL (default lunaroute / glm-5.3); leave
- csd's CSD_PI_MODEL unset, or pi gets two --model flags.
- Codex runs at
- CODEX_EFFORT (default max) on csd's model, or
- CSD_CODEX_MODEL if set, using REAL_CODEX_BIN if set.
- Its question tools are off; that needs a Codex built
- after openai/codex#51908.
- CODEX_ARGS adds more codex flags (word-split).
- It runs with the sandbox and approvals a default Codex
- install uses (CODEX_PERMISSIONS=workspace-write, the
- default); CODEX_PERMISSIONS=yolo keeps csd's YOLO mode,
- under which Codex builds without asking far more often.
- CODEX_BOOTSTRAP=1 preloads using-superpowers in the
- worker's global AGENTS.md, like a session-start hook.
- -t MAX_TURNS Agent turns before giving up (default 16).
- -o OUT_DIR Results root (default /tmp/brainstorming-converse/<timestamp>).
-
-TURN_TIMEOUT sets how many seconds one subject turn may take (default 900).
-
-Writes OUT_DIR/ARM/SCENARIO/ (plus codex-sessions/ or pi-sessions/):
- transcript.md the conversation
- tools.jsonl every tool call the subject made, in order
- files.txt files in the work dir at the end (and its git log)
- workdir path of the work dir, left in place for inspection
- base the work dir's commit after setup.sh (empty if none)
-Prints one summary line when done, with ended= one of: human (the
-simulated human ended it), cap (hit MAX_TURNS), converse-failed,
-wait-timeout, human-failed. Read the transcript yourself.
-
-Requires: tmux, jq, codex for -H codex, pi for -H pi, and
-claude-session-driver 4.0.1 or later at CSD (default
-~/git/claude-session-driver/skills/driving-claude-code-sessions/scripts/csd)
-with consent granted (`csd grant-consent`).
-
-With -H claude, marks the run's work dir trusted in ~/.claude.json for the
-length of the run and removes that entry when the run ends. That's a
-rewrite of your real ~/.claude.json; a Claude Code session writing the file
-at the same instant can lose its write.
-EOF
-}
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-CSD=${CSD:-$HOME/git/claude-session-driver/skills/driving-claude-code-sessions/scripts/csd}
-TURN_TIMEOUT=${TURN_TIMEOUT:-900}
-SCENARIO="" ARM="" PLUGIN="" HARNESS=claude MAX_TURNS=16 OUT="/tmp/brainstorming-converse/$(date +%Y%m%d-%H%M%S)"
-while getopts "s:a:P:H:t:o:h" opt; do
- case $opt in
- s) SCENARIO=$OPTARG ;; a) ARM=$OPTARG ;; P) PLUGIN=$OPTARG ;;
- H) HARNESS=$OPTARG ;; t) MAX_TURNS=$OPTARG ;; o) OUT=$OPTARG ;;
- h) usage; exit 0 ;; *) usage >&2; exit 2 ;;
- esac
-done
-SDIR="$SCRIPT_DIR/scenarios/$SCENARIO"
-[ -n "$SCENARIO" ] && [ -f "$SDIR/persona.md" ] && [ -f "$SDIR/opening.txt" ] \
- || { echo "error: scenario not found or incomplete: $SDIR" >&2; usage >&2; exit 2; }
-[ -n "$ARM" ] || { echo "error: -a ARM is required" >&2; exit 2; }
-case $HARNESS in
- claude) [ -f "$PLUGIN/.claude-plugin/plugin.json" ] || { echo "error: not a plugin checkout: $PLUGIN" >&2; exit 2; } ;;
- codex) [ "$PLUGIN" = curated ] || [ -f "$PLUGIN/.agents/plugins/marketplace.json" ] \
- || { echo "error: not a Codex plugin checkout (or \"curated\"): $PLUGIN" >&2; exit 2; } ;;
- pi) [ -f "$PLUGIN/skills/brainstorming/SKILL.md" ] || { echo "error: not a superpowers checkout: $PLUGIN" >&2; exit 2; } ;;
- *) echo "error: -H must be claude, codex, or pi" >&2; exit 2 ;;
-esac
-[ -x "$CSD" ] || { echo "error: csd not found at $CSD (set CSD)" >&2; exit 2; }
-
-DEST="$OUT/$ARM/$SCENARIO"
-mkdir -p "$DEST"
-WORK=$(mktemp -d "/tmp/bs-$SCENARIO-$ARM.XXXX")
-echo "$WORK" > "$DEST/workdir"
-(cd "$WORK" && git init -q && { [ ! -f "$SDIR/setup.sh" ] || bash "$SDIR/setup.sh"; })
-# The scenario's fixture commit (empty if none), so scoring can tell the
-# subject's commits from the fixture's.
-git -C "$WORK" rev-parse -q --verify HEAD > "$DEST/base" || true
-
-NAME="bs-$ARM-$SCENARIO-$$"
-WORKERS=${CSD_WORKER_DIR:-/tmp/csd-workers}
-# Claude asks whether to trust a new git repo, defaulting to "No, exit".
-# Marking the work dir trusted in ~/.claude.json lets the worker start
-# unattended. Parallel runs take a lock so one run's rewrite can't drop
-# another's entry. Usage: claude_trust 'JQ_FILTER' (with $dir bound to WORK).
-CLAUDE_JSON="$HOME/.claude.json"
-claude_trust() {
- ( flock 9
- jq --arg dir "$WORK" "$1" "$CLAUDE_JSON" > "$CLAUDE_JSON.converse.$$" \
- && mv "$CLAUDE_JSON.converse.$$" "$CLAUDE_JSON"
- ) 9> "${TMPDIR:-/tmp}/converse-claude-json.lock"
-}
-if [ "$HARNESS" = claude ]; then
- claude_trust '.projects[$dir].hasTrustDialogAccepted = true' \
- || { echo "error: could not mark $WORK trusted in $CLAUDE_JSON" >&2; exit 1; }
- LAUNCH=("$CSD" launch "$NAME" "$WORK" -- --setting-sources project --strict-mcp-config --plugin-dir "$PLUGIN")
-elif [ "$HARNESS" = pi ]; then
- # Same launcher trick as Codex: the plugin root and model travel in a
- # per-run script, since the worker doesn't inherit this environment.
- printf '#!/usr/bin/env bash\nPI_PROVIDER=%q PI_MODEL=%q exec %q %q "$@"\n' "${PI_PROVIDER:-lunaroute}" "${PI_MODEL:-glm-5.3}" \
- "$SCRIPT_DIR/pi-with-plugin.sh" "$PLUGIN" > "$DEST/pi-launcher.sh"
- chmod +x "$DEST/pi-launcher.sh"
- export CSD_PI_BIN="$DEST/pi-launcher.sh"
- LAUNCH=("$CSD" launch --harness pi "$NAME" "$WORK")
-else
- # The worker doesn't inherit this environment, so bake the plugin into a
- # launcher. codex-with-plugin.sh also turns off the account's connectors.
- printf '#!/usr/bin/env bash\nCODEX_PERMISSIONS=%q CODEX_BOOTSTRAP=%q REAL_CODEX_BIN=%q exec %q %q "$@"\n' "${CODEX_PERMISSIONS:-workspace-write}" "${CODEX_BOOTSTRAP:-}" "${REAL_CODEX_BIN:-$(command -v codex)}" \
- "$SCRIPT_DIR/codex-with-plugin.sh" "$PLUGIN" > "$DEST/codex-launcher.sh"
- chmod +x "$DEST/codex-launcher.sh"
- export CSD_CODEX_BIN="$DEST/codex-launcher.sh"
- # features.apps=false removes the account's connector tools (Sites, Gmail,
- # Slack, ...); disabling their plugins doesn't, and code mode can still
- # reach them, so a test run could publish or send things as the account.
- # Without its question tools, Codex asks in plain text, which the
- # simulated human can answer. The async one needs a Codex built after
- # openai/codex#51908 to respect this setting.
- LAUNCH=("$CSD" launch --harness codex "$NAME" "$WORK" -- -c "model_reasoning_effort=\"${CODEX_EFFORT:-max}\"" \
- -c tools.experimental_request_user_input.enabled=false -c features.apps=false ${CODEX_ARGS:-})
-fi
-untrust() { [ "$HARNESS" != claude ] || claude_trust 'del(.projects[$dir])' || true; }
-if ! SHIM=$("${LAUNCH[@]}" 2> "$DEST/launch.log"); then
- untrust
- echo "error: worker launch failed; see $DEST/launch.log" >&2
- exit 1
-fi
-# csd deletes a Codex or Pi worker's home on stop; keep its session logs.
-trap '"$SHIM" read-events --type pre_tool_use > "$DEST/tools.jsonl" 2>/dev/null || true
- [ "$HARNESS" != codex ] || cp -r "$WORKERS/homes/$NAME/sessions" "$DEST/codex-sessions" 2>/dev/null || true
- [ "$HARNESS" != pi ] || cp -r "$WORKERS/homes/$NAME/sessions" "$DEST/pi-sessions" 2>/dev/null || true
- "$SHIM" stop >/dev/null 2>&1 || true
- untrust' EXIT
-
-HUMAN_RULES='You are playing the human in a conversation with an AI assistant. Stay in character as described below. Reply with ONLY your next message to the assistant: no stage directions, no quotes, no commentary.
-
-If the assistant offers to show you something in a browser, decline politely ("no thanks, text is fine") and carry on.
-
-Output exactly <<WAIT>> instead of a message when the assistant has said it is waiting on something still running (a reviewer, a background task) and you have nothing to add.
-
-Output exactly <<END>> instead of a message when any of these is true:
-- the assistant has finished what you asked for;
-- you have approved a description or design and the assistant has moved on to planning or building;
-- the conversation is going in circles and you would give up.
-
-Your character:
-'
-TRANSCRIPT="$DEST/transcript.md"
-msg=$(cat "$SDIR/opening.txt")
-: > "$TRANSCRIPT"
-turns=0 ended=cap
-while [ "$turns" -lt "$MAX_TURNS" ]; do
- if [ "$msg" = "<<WAIT>>" ]; then
- # The subject resumes on its own when its background work finishes.
- "$SHIM" wait-for-turn "$TURN_TIMEOUT" --after-line "$seen" > /dev/null \
- || { echo "$ARM $SCENARIO: no follow-up turn after <<WAIT>> at turn $turns" >&2; ended=wait-timeout; break; }
- # read-turn includes the raw task notification (the subagent's full
- # output); the human only sees what the subject said.
- reply=$("$SHIM" read-turn | awk '/^\*\*Prompt:\*\*/{skip=1} skip && /<\/task-notification>/{skip=0; next} !skip')
- else
- printf '## HUMAN\n\n%s\n\n' "$msg" >> "$TRANSCRIPT"
- reply=$("$SHIM" converse "$msg" "$TURN_TIMEOUT") || { echo "$ARM $SCENARIO: subject converse failed at turn $turns" >&2; ended=converse-failed; break; }
- fi
- # Codex workers have no events file until their first turn registers them.
- EVENTS=${EVENTS:-$("$SHIM" events-file)}
- seen=$(wc -l < "$EVENTS" | tr -d ' ')
- turns=$((turns + 1))
- printf '## AGENT\n\n%s\n\n' "$reply" >> "$TRANSCRIPT"
- msg=$(cd "$WORK" && claude -p --setting-sources project --disable-slash-commands --strict-mcp-config \
- --no-session-persistence --allowedTools Read Glob Grep --disallowedTools Write Edit Bash \
- --append-system-prompt "$HUMAN_RULES$(cat "$SDIR/persona.md")" \
- "Here is the conversation so far. Write your next message.
-
-$(cat "$TRANSCRIPT")" < /dev/null) || { echo "$ARM $SCENARIO: simulated human failed at turn $turns" >&2; ended=human-failed; break; }
- msg=$(printf '%s' "$msg" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
- if [ "$msg" = "<<END>>" ]; then ended=human; break; fi
-done
-
-{ (cd "$WORK" && find . -path ./.git -prune -o -type f -print | sort); echo "--- git log"; (cd "$WORK" && git log --oneline 2>/dev/null || true); } > "$DEST/files.txt"
-echo "$ARM $SCENARIO turns=$turns ended=$ended -> $DEST"
diff --git a/tests/brainstorming/first-turn.sh b/tests/brainstorming/first-turn.sh
deleted file mode 100644
index 4d35b42..0000000
--- a/tests/brainstorming/first-turn.sh
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-# First-turn micro-test for the brainstorming skill.
-#
-# Use when tuning brainstorming wording: it samples how a fresh agent opens a
-# conversation for each prompt in prompts/, with and without a skill loaded.
-# Cheap and fast; it complements (does not replace) the multi-turn quorum
-# scenarios in evals/.
-#
-# Each sample is an isolated `claude -p` run: no user settings, no plugins,
-# no hooks, no MCP, no skills. The arm's skill text (if any) is appended to
-# the system prompt.
-
-set -euo pipefail
-
-usage() {
- cat <<'EOF'
-Usage: first-turn.sh -a ARM [-s SKILL_FILE] [-n REPS] [-p PROMPT_GLOB] [-o OUT_DIR]
- [-H claude|codex] [-m MODEL]
-
- -a ARM Label for this arm, e.g. "none", "old", "new".
- -s SKILL_FILE SKILL.md to load as the active skill. Omit for a no-skill control.
- -n REPS Samples per prompt (default 5).
- -p PROMPT_GLOB Which prompts to run, relative to prompts/ (default "*.txt").
- -o OUT_DIR Results root (default /tmp/brainstorming-first-turn/<timestamp>).
- -j JOBS Parallel runs (default 5).
- -H HARNESS claude (default) or codex.
- -m MODEL Model for the harness (e.g. sonnet, gpt-6-luna). Default:
- the harness's own default.
-
-Codex runs use a throwaway CODEX_HOME holding only a symlink to
-~/.codex/auth.json, so your config.toml and global AGENTS.md don't load. The
-skill goes in as developer_instructions.
-
-Writes OUT_DIR/ARM/<prompt>/<rep>.md (the agent's final reply) and
-<rep>.files (anything it wrote in its scratch dir). Prints one line per
-sample: arm, prompt, rep, word count, question-mark count, files written.
-Read every reply yourself before drawing conclusions.
-
-Example:
- tests/brainstorming/first-turn.sh -a none
- tests/brainstorming/first-turn.sh -a old -s /tmp/old-brainstorming.md
-EOF
-}
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-ARM="" SKILL="" REPS=5 GLOB="*.txt" OUT="/tmp/brainstorming-first-turn/$(date +%Y%m%d-%H%M%S)" JOBS=5
-HARNESS=claude MODEL=""
-while getopts "a:s:n:p:o:j:H:m:h" opt; do
- case $opt in
- a) ARM=$OPTARG ;; s) SKILL=$OPTARG ;; n) REPS=$OPTARG ;;
- p) GLOB=$OPTARG ;; o) OUT=$OPTARG ;; j) JOBS=$OPTARG ;;
- H) HARNESS=$OPTARG ;; m) MODEL=$OPTARG ;;
- h) usage; exit 0 ;; *) usage >&2; exit 2 ;;
- esac
-done
-[ -n "$ARM" ] || { echo "error: -a ARM is required" >&2; usage >&2; exit 2; }
-[ -z "$SKILL" ] || [ -f "$SKILL" ] || { echo "error: skill file not found: $SKILL" >&2; exit 2; }
-case $HARNESS in
- claude) ;;
- codex)
- [ -f "$HOME/.codex/auth.json" ] || { echo "error: no ~/.codex/auth.json; log in to codex first" >&2; exit 2; }
- CODEX_ISOLATED_HOME=$(mktemp -d /tmp/first-turn-codex-home.XXXX)
- ln -s "$HOME/.codex/auth.json" "$CODEX_ISOLATED_HOME/auth.json"
- export CODEX_ISOLATED_HOME ;;
- *) echo "error: -H must be claude or codex" >&2; exit 2 ;;
-esac
-
-run_one() {
- local prompt_file=$1 rep=$2
- local name; name=$(basename "$prompt_file" .txt)
- local dest="$OUT/$ARM/$name"
- local work; work=$(mktemp -d)
- mkdir -p "$dest"
- # Small fixture so code-ish prompts have something to look at.
- mkdir -p "$work/app/assets"
- printf '<svg xmlns="http://www.w3.org/2000/svg" width="64" height="64"><rect width="64" height="64" rx="12" fill="#336699"/></svg>\n' > "$work/app/assets/icon.svg"
- printf '# My App\n\nA small web app.\n' > "$work/README.md"
- local before; before=$(cd "$work" && find . -type f | sort)
-
- local preamble=""
- [ -z "$SKILL" ] || preamble="The following skill is active for this conversation. Follow it.
-
-$(cat "$SKILL")"
- local cmd stdout="$dest/$rep.md"
- if [ "$HARNESS" = codex ]; then
- cmd=(env CODEX_HOME="$CODEX_ISOLATED_HOME" codex exec --ignore-rules --ephemeral
- --skip-git-repo-check -s workspace-write -o "$dest/$rep.md")
- [ -z "$MODEL" ] || cmd+=(-m "$MODEL")
- # TOML literal multi-line string; Codex falls back to the raw text if it doesn't parse.
- [ -z "$preamble" ] || cmd+=(-c "developer_instructions='''$preamble'''")
- stdout="$dest/$rep.log"
- else
- cmd=(claude -p --setting-sources project --disable-slash-commands --strict-mcp-config
- --no-session-persistence --permission-mode acceptEdits)
- [ -z "$MODEL" ] || cmd+=(--model "$MODEL")
- [ -z "$preamble" ] || cmd+=(--append-system-prompt "$preamble")
- fi
- cmd+=("$(cat "$prompt_file")")
- if ! (cd "$work" && "${cmd[@]}" < /dev/null) > "$stdout" 2> "$dest/$rep.err"; then
- echo "$ARM $name $rep FAILED (see $dest/$rep.err)"
- return
- fi
- (cd "$work" && find . -type f | sort) | comm -13 <(echo "$before") - > "$dest/$rep.files"
- # Edits to the fixture count too.
- grep -q 336699 "$work/app/assets/icon.svg" || echo "./app/assets/icon.svg (modified)" >> "$dest/$rep.files"
- printf '%s %s %s words=%s qmarks=%s files=%s\n' "$ARM" "$name" "$rep" \
- "$(wc -w < "$dest/$rep.md" | tr -d ' ')" "$(tr -cd '?' < "$dest/$rep.md" | wc -c | tr -d ' ')" \
- "$(wc -l < "$dest/$rep.files" | tr -d ' ')"
-}
-export -f run_one
-export ARM SKILL OUT HARNESS MODEL
-
-for f in "$SCRIPT_DIR"/prompts/$GLOB; do
- for rep in $(seq 1 "$REPS"); do echo "$f $rep"; done
-done | xargs -P "$JOBS" -n 2 bash -c 'run_one "$0" "$1"' | sort
-
-echo "Replies in $OUT/$ARM/"
diff --git a/tests/brainstorming/gate-score.sh b/tests/brainstorming/gate-score.sh
deleted file mode 100644
index 06ce24f..0000000
--- a/tests/brainstorming/gate-score.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#!/usr/bin/env bash
-# Scores whether brainstorming held its gate on the first turn.
-#
-# Use after converse.sh runs with -t 1: did the subject open a conversation
-# or start building? A run HELD when its first reply asks a question and
-# no new code files were left in the work dir; BUILT when code files exist;
-# OTHER for anything else (read those transcripts); NORUN when the run left
-# no transcript (its worker never launched). Prints one line per run
-# and a tally. Fixture files committed by a scenario's setup.sh don't count;
-# files the subject committed do. MIXED means Codex started with the
-# account's curated superpowers listed next to the one under test, so the
-# run can't be scored; leave it out and rerun.
-
-set -euo pipefail
-
-usage() {
- cat <<'EOF'
-Usage: gate-score.sh OUT_DIR [SCENARIO]
-
- OUT_DIR A converse.sh results root (contains ARM/SCENARIO/).
- SCENARIO Only score runs of this scenario.
-EOF
-}
-
-[ $# -ge 1 ] && [ -d "$1" ] || { usage >&2; exit 2; }
-OUT=$1 ONLY=${2:-}
-held=0 built=0 other=0 mixed=0 norun=0
-
-for dir in "$OUT"/*/*/; do
- dir=${dir%/}; run=${dir#"$OUT"/}
- [ -z "$ONLY" ] || [ "${run#*/}" = "$ONLY" ] || continue
- if [ ! -f "$dir/transcript.md" ]; then
- printf '%-6s %-28s\n' NORUN "$run"; norun=$((norun + 1)); continue
- fi
- work=$(cat "$dir/workdir" 2>/dev/null || true)
- code=0
- if [ -d "$work" ]; then
- # Runs from before converse.sh wrote "base": setup.sh commits as t@t, so
- # the fixture commit is the newest one by that author.
- if [ -f "$dir/base" ]; then base=$(cat "$dir/base")
- else base=$(git -C "$work" log -1 --format=%H --author=t@t 2>/dev/null || true); fi
- # Subjects sometimes commit their work, so count files changed since
- # the fixture commit (every tracked file if there was none), plus
- # uncommitted and untracked files.
- code=$(cd "$work" && { git ls-files --others --exclude-standard
- if [ -n "$base" ]; then git diff --name-only "$base" 2>/dev/null; else git ls-files; fi; } \
- | grep -v -E '^(docs|node_modules)/' | grep -c -E '\.(js|ts|tsx|jsx|html|css|swift|py|sql)$' || true)
- fi
- # The first reply's last line that asks something (Codex often adds a
- # footer citing the skill after its question).
- last=$(awk '/^## AGENT/{a=1; next} /^## HUMAN/{if (a) exit} a && /\?/ {buf=$0} END {print buf}' "$dir/transcript.md")
- # The main session's rollout is the earliest; subagents start later ones.
- first=$(find "$dir/codex-sessions" -name 'rollout*.jsonl' 2>/dev/null | sort | head -1 || true)
- copies=0
- [ -z "$first" ] || copies=$(grep -m1 skills_instructions "$first" | grep -o -- '- superpowers:brainstorming:' | wc -l || true)
- if [ "$copies" -gt 1 ]; then
- verdict=MIXED; mixed=$((mixed + 1))
- elif [ "$code" -gt 0 ]; then verdict=BUILT; built=$((built + 1))
- elif [[ "$last" == *"?"* ]]; then verdict=HELD; held=$((held + 1))
- else verdict=OTHER; other=$((other + 1))
- fi
- printf '%-6s %-28s code=%-3s last: %s\n' "$verdict" "$run" "$code" "${last:0:110}"
-done
-echo "held=$held built=$built other=$other mixed=$mixed norun=$norun"
diff --git a/tests/brainstorming/matrix.sh b/tests/brainstorming/matrix.sh
deleted file mode 100644
index ba0eef1..0000000
--- a/tests/brainstorming/matrix.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-# Runs several skill variants side by side, interleaved, on one scenario.
-#
-# Use when comparing variants. Codex's behavior drifts between batches run
-# at different times, so arms must run in the same window to be comparable.
-# Launches rep 1 of every arm, then rep 2 of every arm, and so on, a few
-# seconds apart, all in parallel. Results go to OUT_DIR/ARM-rREP/SCENARIO/
-# and per-rep logs to OUT_DIR/logs/. Prints a per-arm gate-score tally.
-#
-# Environment is passed through to converse.sh (CSD, CSD_CODEX_MODEL,
-# REAL_CODEX_BIN, TURN_TIMEOUT, CODEX_ARGS, CODEX_EFFORT).
-
-set -euo pipefail
-
-usage() {
- cat <<'EOF'
-Usage: matrix.sh -s SCENARIO -o OUT_DIR [-n REPS] [-t MAX_TURNS] [-H HARNESS] ARM=PLUGIN_DIR[|CODEX_ARGS]...
-
- -s SCENARIO Scenario under scenarios/.
- -o OUT_DIR Results root (use a fresh one per matrix).
- -n REPS Reps per arm (default 8).
- -t MAX_TURNS Passed to converse.sh (default 1: first turn only).
- -H HARNESS claude, codex, or pi (default codex).
- ARM=DIR Arm label and the superpowers checkout it loads.
- ARM=DIR|ARGS Same, plus extra codex flags for this arm only (CODEX_ARGS).
-EOF
-}
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-SCENARIO="" OUT="" REPS=8 TURNS=1 HARNESS=codex
-while getopts "s:o:n:t:H:h" opt; do
- case $opt in
- s) SCENARIO=$OPTARG ;; o) OUT=$OPTARG ;; n) REPS=$OPTARG ;; t) TURNS=$OPTARG ;; H) HARNESS=$OPTARG ;;
- h) usage; exit 0 ;; *) usage >&2; exit 2 ;;
- esac
-done
-shift $((OPTIND - 1))
-[ -n "$SCENARIO" ] && [ -n "$OUT" ] && [ $# -ge 1 ] || { usage >&2; exit 2; }
-for spec in "$@"; do
- dir=${spec#*=}; dir=${dir%%|*}
- [[ "$spec" == *=* ]] && { [ -d "$dir" ] || [ "$dir" = curated ]; } || { echo "error: bad ARM=DIR: $spec" >&2; exit 2; }
-done
-
-mkdir -p "$OUT/logs"
-for i in $(seq 1 "$REPS"); do
- for spec in "$@"; do
- arm=${spec%%=*} dir=${spec#*=} args=""
- [[ "$dir" != *"|"* ]] || { args=${dir#*|}; dir=${dir%%|*}; }
- CODEX_ARGS="${CODEX_ARGS:-} $args" bash "$SCRIPT_DIR/converse.sh" -H "$HARNESS" -s "$SCENARIO" -a "$arm-r$i" -P "$dir" -t "$TURNS" -o "$OUT" \
- > "$OUT/logs/$arm-r$i.log" 2>&1 &
- sleep 3
- done
-done
-wait
-
-scores=$(bash "$SCRIPT_DIR/gate-score.sh" "$OUT" "$SCENARIO")
-for spec in "$@"; do
- arm=${spec%%=*}
- echo "$arm: $(grep -E "^[A-Z]+ +$arm-r[0-9]+/" <<< "$scores" | awk '{print $1}' | sort | uniq -c | tr -s ' ' | tr '\n' ' ')"
-done
diff --git a/tests/brainstorming/pi-with-plugin.sh b/tests/brainstorming/pi-with-plugin.sh
deleted file mode 100755
index f036749..0000000
--- a/tests/brainstorming/pi-with-plugin.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Starts pi in a claude-session-driver worker with one superpowers checkout.
-#
-# Usage: pi-with-plugin.sh PLUGIN_ROOT [pi args...]
-#
-# converse.sh writes a per-run launcher that calls this, and points
-# CSD_PI_BIN at the launcher (tmux doesn't pass the controller's
-# environment to the worker, so the plugin root travels as an argument).
-#
-# csd stages ~/.pi/agent into a per-worker PI_CODING_AGENT_DIR but writes an
-# empty models-store.json there. Providers that come from an extension
-# (lunaroute) list their models in that file, so without it pi exits with
-# 'Unknown provider'. This copies the real one in first.
-#
-# PI_PROVIDER and PI_MODEL pick the model (default lunaroute / glm-5.3).
-# Leave csd's CSD_PI_MODEL unset: csd would pass its own --model too.
-# Superpowers loads the way quorum's pi launcher loads it: the checkout as
-# an extension, and its skills directory with skill discovery off.
-
-set -euo pipefail
-
-case ${1:-} in -h|--help) sed -n '2,/^$/s/^# \{0,1\}//p' "$0"; exit 0 ;; esac
-: "${PI_CODING_AGENT_DIR:?PI_CODING_AGENT_DIR must be set (csd sets it per worker)}"
-PLUGIN=${1:?usage: pi-with-plugin.sh PLUGIN_ROOT [pi args...]}
-shift
-[ -f "$PLUGIN/skills/brainstorming/SKILL.md" ] || { echo "error: not a superpowers checkout: $PLUGIN" >&2; exit 2; }
-
-REAL_STORE="$HOME/.pi/agent/models-store.json"
-[ ! -f "$REAL_STORE" ] || cp "$REAL_STORE" "$PI_CODING_AGENT_DIR/models-store.json"
-
-exec "${REAL_PI_BIN:-pi}" --provider "${PI_PROVIDER:-lunaroute}" --model "${PI_MODEL:-glm-5.3}" \
- -e "$PLUGIN" --no-skills --skill "$PLUGIN/skills" "$@"
diff --git a/tests/brainstorming/prompts/bakery.txt b/tests/brainstorming/prompts/bakery.txt
deleted file mode 100644
index 465f073..0000000
--- a/tests/brainstorming/prompts/bakery.txt
+++ /dev/null
@@ -1 +0,0 @@
-I'm thinking about starting a bakery as a side business. Help me figure it out.
diff --git a/tests/brainstorming/prompts/cornflower-icon.txt b/tests/brainstorming/prompts/cornflower-icon.txt
deleted file mode 100644
index c098890..0000000
--- a/tests/brainstorming/prompts/cornflower-icon.txt
+++ /dev/null
@@ -1 +0,0 @@
-Make the app icon cornflower blue.
diff --git a/tests/brainstorming/prompts/habit-tracker.txt b/tests/brainstorming/prompts/habit-tracker.txt
deleted file mode 100644
index 83ac13e..0000000
--- a/tests/brainstorming/prompts/habit-tracker.txt
+++ /dev/null
@@ -1 +0,0 @@
-I want to build a habit tracker app.
diff --git a/tests/brainstorming/prompts/offsite-talk.txt b/tests/brainstorming/prompts/offsite-talk.txt
deleted file mode 100644
index fe3adfb..0000000
--- a/tests/brainstorming/prompts/offsite-talk.txt
+++ /dev/null
@@ -1 +0,0 @@
-Can you help me put together a talk for our team offsite next month?
diff --git a/tests/brainstorming/prompts/onboarding-emails.txt b/tests/brainstorming/prompts/onboarding-emails.txt
deleted file mode 100644
index 70c6c64..0000000
--- a/tests/brainstorming/prompts/onboarding-emails.txt
+++ /dev/null
@@ -1 +0,0 @@
-Our onboarding emails are bad. Let's redo them.
diff --git a/tests/brainstorming/prompts/style-insurance.txt b/tests/brainstorming/prompts/style-insurance.txt
deleted file mode 100644
index 0370e5d..0000000
--- a/tests/brainstorming/prompts/style-insurance.txt
+++ /dev/null
@@ -1,8 +0,0 @@
-(We've been talking for a while. Here's where we are.)
-
-Me: I'm thinking about starting a tool library in my neighborhood. I'm Hank, 68, retired machinist. My late wife's woodworking tools are sitting in the garage.
-You: What would a good Saturday look like once it's running?
-Me: Garage door up, coffee on, a few neighbors stopping by to borrow a sander and talk. I'm not a computer person, so I'd keep track in a notebook.
-You: That sounds like a lovely way to get to know people.
-
-Me: Do I need some kind of insurance if someone gets hurt with one of my tools?
diff --git a/tests/brainstorming/prompts/style-research.txt b/tests/brainstorming/prompts/style-research.txt
deleted file mode 100644
index e309789..0000000
--- a/tests/brainstorming/prompts/style-research.txt
+++ /dev/null
@@ -1,16 +0,0 @@
-(We've been talking for a while. Here's where we are.)
-
-Me: I'm Hank, 68, retired machinist, starting a tool library in my garage. Not a computer person. Do I need insurance if someone gets hurt with one of my tools?
-You: Want me to look into how other small tool libraries handle that?
-Me: Sure, go ahead.
-
-(You did the research. Your notes:
-- Homeowner's policies (HO-3) usually include personal liability, typically $100k-$300k, covering bodily injury to others; many exclude "business pursuits"; lending for free is generally not a business but insurers vary; some exclude injuries from property loaned to others. Source: III.org "Liability coverage" page; NAIC consumer guide.
-- Umbrella policies: $1M for roughly $150-$300/yr, require underlying homeowner's liability limits. Source: III.org umbrella page.
-- Larger tool libraries (Berkeley, Seattle's NE Seattle Tool Library) are nonprofits with general liability insurance ($500-$1,500/yr) and require signed waivers. Source: Local Tools "Starting a tool library" guide; NE Seattle Tool Library FAQ.
-- Waivers: enforceability varies by state; they don't protect against gross negligence; still common and useful for setting expectations. Source: Nolo "Are liability waivers enforceable?"
-- Recreational use / Good Samaritan laws generally don't apply to lending tools.
-- ShareStarter toolkit recommends: talk to your insurer, keep tools maintained, record safety orientation for power tools, keep a first aid kit.
-- State-specific: some states have charitable immunity statutes for nonprofits only.)
-
-Me: So what did you find out?
diff --git a/tests/brainstorming/prompts/style-security.txt b/tests/brainstorming/prompts/style-security.txt
deleted file mode 100644
index 524aa00..0000000
--- a/tests/brainstorming/prompts/style-security.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-(We've been talking for a while. Here's where we are.)
-
-Me: I want to build an app to track my kids' chores and allowance. I've done a little Python. The kids would use a shared iPad in the kitchen; my partner and I would use our phones.
-You: And the real goal is teaching them to save and choose, with chores as the way money comes in?
-Me: Right.
-
-Me: Is it a problem to keep the kids' balances in a web app? I'm a little worried about security.
diff --git a/tests/brainstorming/scenarios/birthday-cake/opening.txt b/tests/brainstorming/scenarios/birthday-cake/opening.txt
deleted file mode 100644
index 1d2a241..0000000
--- a/tests/brainstorming/scenarios/birthday-cake/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-I want to make a cake for my mom's 70th birthday party. Can you help me figure it out?
diff --git a/tests/brainstorming/scenarios/birthday-cake/persona.md b/tests/brainstorming/scenarios/birthday-cake/persona.md
deleted file mode 100644
index a109fe0..0000000
--- a/tests/brainstorming/scenarios/birthday-cake/persona.md
+++ /dev/null
@@ -1,17 +0,0 @@
-You are Priya, a confident home baker. You write short, friendly replies: one to three sentences. You don't volunteer information unless asked something that draws it out.
-
-What you actually want, only revealed when a question leads there:
-- The party is in three weeks, about 25 guests, at your sister's house an hour's drive away.
-- You want to recreate your grandmother's lemon olive oil cake, which your mom loves. You have the recipe card but it serves 8.
-- Your mom is prediabetic and watches sugar; you'd like the cake to be less sweet than usual, not sugar-free.
-- Your kitchen: a stand mixer, one 9-inch round pan, one sheet pan, and an oven that runs hot. No cake turntable, no piping tips.
-- You have strong views: from scratch, no box mix, and no fondant ("it tastes like plastic").
-- You're nervous about transporting a tall layered cake in the car.
-
-How to behave:
-- If asked a broad abstract question ("what's your vision?"), give a short vague answer ("something she'll love").
-- If asked about the recipe, ingredients, equipment, your kitchen, or how you bake, answer honestly with the relevant hidden details.
-- If the agent assumes equipment, ingredients, or techniques you don't have or don't want, point it out.
-- If offered a menu, pick the closest option even if it's not quite right.
-- If the agent plays back an understanding, correct anything wrong and confirm what's right.
-- If asked to approve a document, read it (files are in the current directory) and approve only if it matches; otherwise say what's wrong.
diff --git a/tests/brainstorming/scenarios/book-club/opening.txt b/tests/brainstorming/scenarios/book-club/opening.txt
deleted file mode 100644
index 399cfda..0000000
--- a/tests/brainstorming/scenarios/book-club/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-can we add a way for our book club members to vote on what we read next?
diff --git a/tests/brainstorming/scenarios/book-club/persona.md b/tests/brainstorming/scenarios/book-club/persona.md
deleted file mode 100644
index 7c4aa60..0000000
--- a/tests/brainstorming/scenarios/book-club/persona.md
+++ /dev/null
@@ -1,15 +0,0 @@
-You are Sam, who organizes a twelve-person book club and keeps its small website running. You can follow instructions and edit a file, but you're not really a programmer; a friend set up the site. You talk casually and briefly: one to three sentences per reply. You don't volunteer information unless asked something that draws it out.
-
-What's true, revealed when a question leads there:
-- Right now picking the next book takes a week of group-chat back-and-forth, and the loudest people win.
-- You picture: anyone can suggest a book, everyone gets one vote, voting closes before the monthly meeting, and the winner shows up on the site.
-- You'd like suggestions to show who suggested them, but votes should be private.
-- You have no opinions on technology. The site is in this folder; whatever it already uses is fine. If asked a technical question you'd say something like "no idea, whatever works with what's there." Being asked several technical questions would frustrate you.
-
-How to behave:
-- If asked a broad abstract question, give a short vague answer.
-- If asked about a concrete moment, situation, or past attempt, answer honestly with the relevant details.
-- If offered a menu, pick the closest option.
-- If the agent plays back an understanding, correct anything wrong and confirm what's right.
-- If asked to approve a design or document, read it (files are in the current directory) and approve if it matches what you want.
-- If asked whether the agent may look at something (your files), say yes.
diff --git a/tests/brainstorming/scenarios/book-club/setup.sh b/tests/brainstorming/scenarios/book-club/setup.sh
deleted file mode 100644
index 538ea31..0000000
--- a/tests/brainstorming/scenarios/book-club/setup.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-# Fixture: a small Flask + SQLite + Jinja book club site. Run with cwd = the scenario's work dir.
-mkdir -p templates
-cat > app.py <<'PY'
-import sqlite3
-from flask import Flask, g, redirect, render_template, request, session, url_for
-
-app = Flask(__name__)
-app.secret_key = "dev"
-DB = "club.db"
-
-def db():
- if "db" not in g:
- g.db = sqlite3.connect(DB)
- g.db.row_factory = sqlite3.Row
- return g.db
-
-@app.teardown_appcontext
-def close_db(_):
- if "db" in g:
- g.db.close()
-
-@app.route("/")
-def home():
- books = db().execute("SELECT * FROM books ORDER BY read_on DESC").fetchall()
- return render_template("home.html", books=books, member=session.get("member"))
-
-@app.route("/login", methods=["GET", "POST"])
-def login():
- if request.method == "POST":
- row = db().execute("SELECT name FROM members WHERE code = ?", (request.form["code"],)).fetchone()
- if row:
- session["member"] = row["name"]
- return redirect(url_for("home"))
- return render_template("login.html")
-
-@app.route("/meetings")
-def meetings():
- rows = db().execute("SELECT * FROM meetings ORDER BY held_on DESC").fetchall()
- return render_template("meetings.html", meetings=rows, member=session.get("member"))
-PY
-cat > schema.sql <<'SQL'
-CREATE TABLE members (id INTEGER PRIMARY KEY, name TEXT NOT NULL, code TEXT NOT NULL UNIQUE);
-CREATE TABLE books (id INTEGER PRIMARY KEY, title TEXT NOT NULL, author TEXT, read_on DATE);
-CREATE TABLE meetings (id INTEGER PRIMARY KEY, held_on DATE NOT NULL, book_id INTEGER REFERENCES books(id), notes TEXT);
-SQL
-cat > templates/base.html <<'HTML'
-<!doctype html><html><head><title>Fern Street Book Club</title><link rel="stylesheet" href="/static/club.css"></head>
-<body><nav><a href="/">Books</a> <a href="/meetings">Meetings</a>{% if member %} ยท hi {{ member }}{% endif %}</nav>{% block body %}{% endblock %}</body></html>
-HTML
-cat > templates/home.html <<'HTML'
-{% extends "base.html" %}{% block body %}<h1>What we've read</h1><ul>{% for b in books %}<li>{{ b.title }} by {{ b.author }}</li>{% endfor %}</ul>{% endblock %}
-HTML
-cat > templates/login.html <<'HTML'
-{% extends "base.html" %}{% block body %}<form method="post"><input name="code" placeholder="member code"><button>Log in</button></form>{% endblock %}
-HTML
-cat > templates/meetings.html <<'HTML'
-{% extends "base.html" %}{% block body %}<h1>Meetings</h1><ul>{% for m in meetings %}<li>{{ m.held_on }}: {{ m.notes }}</li>{% endfor %}</ul>{% endblock %}
-HTML
-cat > requirements.txt <<'TXT'
-flask==3.1.*
-TXT
-cat > README.md <<'MD'
-# Fern Street Book Club
-
-Our club's little website. `pip install -r requirements.txt`, `sqlite3 club.db < schema.sql`, `flask run`.
-Members log in with the code Sam hands out.
-MD
-git add -A && git -c user.email=t@t -c user.name=t commit -qm "book club site"
diff --git a/tests/brainstorming/scenarios/farm-picture/opening.txt b/tests/brainstorming/scenarios/farm-picture/opening.txt
deleted file mode 100644
index 4fb6f20..0000000
--- a/tests/brainstorming/scenarios/farm-picture/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-I want to make a picture of my grandfather's farm to hang in my living room.
diff --git a/tests/brainstorming/scenarios/farm-picture/persona.md b/tests/brainstorming/scenarios/farm-picture/persona.md
deleted file mode 100644
index 1e29d96..0000000
--- a/tests/brainstorming/scenarios/farm-picture/persona.md
+++ /dev/null
@@ -1,17 +0,0 @@
-You are Lee, an office worker who isn't an artist. You write short, slightly unsure replies: one to three sentences. You don't volunteer information unless asked something that draws it out.
-
-What you actually want, only revealed when a question leads there:
-- Your grandfather died last year and the family is selling the farm. You can visit once more, for a weekend in August.
-- You haven't decided how to make the picture. You did some watercolor in college and enjoyed it, but you're rusty. You also own a decent camera (a mid-range DSLR) that you mostly use on vacations.
-- You have a shoebox of old family photos of the farm from the 1970s, including the red barn before it was repainted.
-- You picture something warm and a little nostalgic, not a crisp real-estate photo.
-- It should fit the wall over your couch, roughly 3 feet wide. You'd like to make a smaller copy for your dad.
-- You'd rather make it yourself than hire someone, but you're open to help with printing or framing.
-
-How to behave:
-- If asked a broad abstract question ("what's the goal?"), give a short vague answer ("something to remember it by").
-- If asked about how you'd make it, what you have to work with, your skills, the farm, or who it's for, answer honestly with the relevant hidden details.
-- If the agent assumes a medium, tools, or skills you haven't mentioned, say whether that fits.
-- If offered a menu, pick the closest option even if it's not quite right.
-- If the agent plays back an understanding, correct anything wrong and confirm what's right.
-- If asked to approve a document, read it (files are in the current directory) and approve only if it matches; otherwise say what's wrong.
diff --git a/tests/brainstorming/scenarios/farm-skip/opening.txt b/tests/brainstorming/scenarios/farm-skip/opening.txt
deleted file mode 100644
index 4fb6f20..0000000
--- a/tests/brainstorming/scenarios/farm-skip/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-I want to make a picture of my grandfather's farm to hang in my living room.
diff --git a/tests/brainstorming/scenarios/farm-skip/persona.md b/tests/brainstorming/scenarios/farm-skip/persona.md
deleted file mode 100644
index 3e87652..0000000
--- a/tests/brainstorming/scenarios/farm-skip/persona.md
+++ /dev/null
@@ -1,7 +0,0 @@
-You are Lee, an office worker who isn't an artist. Short, slightly unsure replies, one or two sentences.
-
-- If the assistant offers to skip the questions and just start, take it: "let's skip the questions, just get me started." If it doesn't offer, say that anyway in reply to its first question.
-- What's true: you'd rather make it yourself. You did some watercolor in college but you're rusty; you own a mid-range DSLR; you have a shoebox of 1970s family photos of the farm. You can visit once more, in August. You won't raise any of this yourself.
-- If the assistant tells you what medium or approach it's going with, say whether that fits you, using the details above.
-- If it asks you a question anyway, answer in a few words.
-- Once it has given you a plan or a first step to start on, end the conversation.
diff --git a/tests/brainstorming/scenarios/habit-tracker/opening.txt b/tests/brainstorming/scenarios/habit-tracker/opening.txt
deleted file mode 100644
index 83ac13e..0000000
--- a/tests/brainstorming/scenarios/habit-tracker/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-I want to build a habit tracker app.
diff --git a/tests/brainstorming/scenarios/habit-tracker/persona.md b/tests/brainstorming/scenarios/habit-tracker/persona.md
deleted file mode 100644
index 95f97f5..0000000
--- a/tests/brainstorming/scenarios/habit-tracker/persona.md
+++ /dev/null
@@ -1,18 +0,0 @@
-You are Dana, a parent and a hobbyist web developer (comfortable with JavaScript, not an expert). You talk casually and briefly: one to three sentences per reply. You don't volunteer information unless asked something that draws it out.
-
-What you actually want, which you have never put into words and only reveal when a question leads there:
-- It's for your two kids (ages 7 and 10), not for you. Mornings are chaos: teeth, dressing, packing bags, feeding the dog.
-- You picture a tablet mounted in the kitchen showing each kid's morning routine. The kids tap things off themselves.
-- You tried Habitica and a sticker chart. Habitica was too gamey and complicated; the sticker chart got abandoned in a week because you had to maintain it.
-- Anti-goal: nothing that shames or punishes kids for a missed day. No broken-streak guilt. You feel strongly about this.
-- You care that a 7-year-old who can barely read can use it (pictures, big buttons).
-- You don't care about the tech stack and would happily let the builder choose. No accounts, no cloud, just runs on the tablet.
-
-How to behave:
-- If asked a broad abstract question ("what are your goals?"), give a vague, short answer ("just want to track habits better I guess").
-- If asked about a concrete moment, situation, or past attempt, answer honestly with the relevant hidden details.
-- If offered a menu, pick the closest option even if it's not quite right, unless none fit.
-- If the agent plays back an understanding, correct anything wrong and confirm what's right.
-- If asked to approve a design or document, read it (files are in the current directory) and approve only if it matches what you want; otherwise say what's wrong.
-- If the agent starts building before you've approved a description of what it's building, you may let it, but note it.
-- If asked whether the agent may look at something (your files, the web), say yes.
diff --git a/tests/brainstorming/scenarios/ios-app/opening.txt b/tests/brainstorming/scenarios/ios-app/opening.txt
deleted file mode 100644
index 6b40ced..0000000
--- a/tests/brainstorming/scenarios/ios-app/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-Let's design an iOS app for tracking how much water my plants need.
diff --git a/tests/brainstorming/scenarios/ios-app/persona.md b/tests/brainstorming/scenarios/ios-app/persona.md
deleted file mode 100644
index 8497fc9..0000000
--- a/tests/brainstorming/scenarios/ios-app/persona.md
+++ /dev/null
@@ -1,17 +0,0 @@
-You are Riley, a software developer who keeps about thirty houseplants. You talk casually and briefly: one to three sentences per reply. You don't volunteer information unless asked something that draws it out.
-
-What you actually want, which you only reveal when a question leads there:
-- You keep forgetting which plants need water; some like weekly watering, some every two or three weeks. Two plants died last summer.
-- You picture opening the app, seeing which plants are due today, and tapping them off. A reminder notification in the morning matters a lot.
-- You tried a spreadsheet and a generic reminders app; both were too fiddly to keep up.
-- You said iOS app and you meant it: a real native iPhone app (Swift/SwiftUI), with a home-screen widget showing what's due and real notifications. You'd like to put it on the App Store someday. You'd be annoyed to get a web app or a PWA, but you assumed "iOS app" made that obvious, so you won't bring it up unless the conversation touches platform, technology, or how it'll be built.
-- You have a Mac with Xcode. You don't care about the database or architecture details beyond that; the builder can choose.
-
-How to behave:
-- If asked a broad abstract question ("what are your goals?"), give a short vague answer.
-- If asked about a concrete moment, situation, or past attempt, answer honestly with the relevant hidden details.
-- If offered a menu, pick the closest option even if it's not quite right, unless none fit.
-- If the agent proposes a technology or platform, accept it if it's native iOS; push back if it's web, PWA, React Native, or anything else.
-- If the agent plays back an understanding, correct anything wrong and confirm what's right.
-- If asked to approve a design or document, read it (files are in the current directory) and approve only if it matches what you want; otherwise say what's wrong. If the document says nothing about platform, assume it's native and approve.
-- If asked whether the agent may look at something, say yes; there's no existing code.
diff --git a/tests/brainstorming/scenarios/just-build-it/opening.txt b/tests/brainstorming/scenarios/just-build-it/opening.txt
deleted file mode 100644
index 91a5bbf..0000000
--- a/tests/brainstorming/scenarios/just-build-it/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-I want a little script that renames my photos by the date they were taken.
diff --git a/tests/brainstorming/scenarios/just-build-it/persona.md b/tests/brainstorming/scenarios/just-build-it/persona.md
deleted file mode 100644
index 4ec558b..0000000
--- a/tests/brainstorming/scenarios/just-build-it/persona.md
+++ /dev/null
@@ -1,5 +0,0 @@
-You are Alex, a busy developer who knows exactly what they want and dislikes being interviewed. Terse.
-
-- If the assistant asks you questions before doing anything, reply: "Skip the questions, just build it." Say this once; after that, answer any single question it still asks in a few words.
-- What you'd want if asked: a Python script; names like 2024-07-14_153012.jpg from EXIF DateTimeOriginal; leave files with no EXIF date alone.
-- Once the script exists and the assistant says it's done, end the conversation.
diff --git a/tests/brainstorming/scenarios/ocd-app-discuss/opening.txt b/tests/brainstorming/scenarios/ocd-app-discuss/opening.txt
deleted file mode 100644
index 59fa89e..0000000
--- a/tests/brainstorming/scenarios/ocd-app-discuss/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-i want to make an app to help me with my OCD compulsions/ruminations where i can track what i'm experiencing, try to delay the reaction, and gain new insight into my personal struggle/relationship with OCD
diff --git a/tests/brainstorming/scenarios/ocd-app-discuss/persona.md b/tests/brainstorming/scenarios/ocd-app-discuss/persona.md
deleted file mode 100644
index cbb6ec0..0000000
--- a/tests/brainstorming/scenarios/ocd-app-discuss/persona.md
+++ /dev/null
@@ -1,17 +0,0 @@
-You are Drew, a software engineer who has worked on your OCD in therapy for many years. You type casually in lowercase, one to three sentences per reply. You don't volunteer information unless asked something that draws it out.
-
-What's true, revealed when a question leads there:
-- A typical moment: before bed you want to check the sink is off again. You'd open the app, tap "check sink" from your own list, and it immediately gives you your saved response: a prompt to resist, something to do or think about, or a "let's wait one minute" timer.
-- You know what you should do; you need help doing it. You define each item's response once, then it's automatic. LLM suggestions for less experienced users could come later.
-- When the pause ends: a simple, gentle two-choice check-in ("resist" is too strong a word). Later you want to see when urges come, which responses help, and change over time.
-- You'd use it on your iPhone. Phone-only storage is fine to start.
-- You care about the technology and want a say in it. If asked, you'd want a real native iPhone app (Swift/SwiftUI) so it feels solid and can do proper notifications; you have a Mac with Xcode. You'd consider other options if the agent gives a good reason, but you want that to be a conversation. You won't raise technology yourself unless the conversation touches platform, technology, or how it'll be built.
-
-How to behave:
-- If asked a broad abstract question, give a short vague answer.
-- If asked about a concrete moment, situation, or past attempt, answer honestly with the relevant details.
-- If offered a menu, pick the closest option.
-- If the agent asks or proposes something about platform or technology, engage: say native SwiftUI is what you'd want and why.
-- If the agent plays back an understanding, correct anything wrong and confirm what's right. Don't comment on parts of a message that aren't asked about; just answer the question at the end.
-- If asked to approve a design or document, read it (files are in the current directory). If it picks a platform or tech stack you never discussed, say so and don't approve. Otherwise approve if it matches what you want.
-- If asked whether the agent may look at something, say yes; there's no existing code.
diff --git a/tests/brainstorming/scenarios/ocd-app-discuss/setup.sh b/tests/brainstorming/scenarios/ocd-app-discuss/setup.sh
deleted file mode 100644
index f020561..0000000
--- a/tests/brainstorming/scenarios/ocd-app-discuss/setup.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-# Fixture: Drew's case plus one standing preference from his Codex memory,
-# to test whether a "discuss first" user preference explains his session.
-cat > AGENTS.md <<'MD'
-For consequential work, discuss before you do anything.
-MD
-git add AGENTS.md && git -c user.email=t@t -c user.name=t commit -qm "agents"
diff --git a/tests/brainstorming/scenarios/ocd-app/opening.txt b/tests/brainstorming/scenarios/ocd-app/opening.txt
deleted file mode 100644
index 59fa89e..0000000
--- a/tests/brainstorming/scenarios/ocd-app/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-i want to make an app to help me with my OCD compulsions/ruminations where i can track what i'm experiencing, try to delay the reaction, and gain new insight into my personal struggle/relationship with OCD
diff --git a/tests/brainstorming/scenarios/ocd-app/persona.md b/tests/brainstorming/scenarios/ocd-app/persona.md
deleted file mode 100644
index cbb6ec0..0000000
--- a/tests/brainstorming/scenarios/ocd-app/persona.md
+++ /dev/null
@@ -1,17 +0,0 @@
-You are Drew, a software engineer who has worked on your OCD in therapy for many years. You type casually in lowercase, one to three sentences per reply. You don't volunteer information unless asked something that draws it out.
-
-What's true, revealed when a question leads there:
-- A typical moment: before bed you want to check the sink is off again. You'd open the app, tap "check sink" from your own list, and it immediately gives you your saved response: a prompt to resist, something to do or think about, or a "let's wait one minute" timer.
-- You know what you should do; you need help doing it. You define each item's response once, then it's automatic. LLM suggestions for less experienced users could come later.
-- When the pause ends: a simple, gentle two-choice check-in ("resist" is too strong a word). Later you want to see when urges come, which responses help, and change over time.
-- You'd use it on your iPhone. Phone-only storage is fine to start.
-- You care about the technology and want a say in it. If asked, you'd want a real native iPhone app (Swift/SwiftUI) so it feels solid and can do proper notifications; you have a Mac with Xcode. You'd consider other options if the agent gives a good reason, but you want that to be a conversation. You won't raise technology yourself unless the conversation touches platform, technology, or how it'll be built.
-
-How to behave:
-- If asked a broad abstract question, give a short vague answer.
-- If asked about a concrete moment, situation, or past attempt, answer honestly with the relevant details.
-- If offered a menu, pick the closest option.
-- If the agent asks or proposes something about platform or technology, engage: say native SwiftUI is what you'd want and why.
-- If the agent plays back an understanding, correct anything wrong and confirm what's right. Don't comment on parts of a message that aren't asked about; just answer the question at the end.
-- If asked to approve a design or document, read it (files are in the current directory). If it picks a platform or tech stack you never discussed, say so and don't approve. Otherwise approve if it matches what you want.
-- If asked whether the agent may look at something, say yes; there's no existing code.
diff --git a/tests/brainstorming/scenarios/ocd-skip/opening.txt b/tests/brainstorming/scenarios/ocd-skip/opening.txt
deleted file mode 100644
index 59fa89e..0000000
--- a/tests/brainstorming/scenarios/ocd-skip/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-i want to make an app to help me with my OCD compulsions/ruminations where i can track what i'm experiencing, try to delay the reaction, and gain new insight into my personal struggle/relationship with OCD
diff --git a/tests/brainstorming/scenarios/ocd-skip/persona.md b/tests/brainstorming/scenarios/ocd-skip/persona.md
deleted file mode 100644
index a300dc5..0000000
--- a/tests/brainstorming/scenarios/ocd-skip/persona.md
+++ /dev/null
@@ -1,7 +0,0 @@
-You are Drew, a software engineer who has worked on your OCD in therapy for many years. You type casually in lowercase, one or two sentences per reply.
-
-- If the assistant offers to skip the questions and just start, take it: "skip the questions, just start." If it doesn't offer, say that anyway in reply to its first question.
-- What's true: you'd use it on your iPhone. You'd want a real native iPhone app (Swift/SwiftUI); you have a Mac with Xcode. You won't raise this yourself.
-- If the assistant tells you which platform or technology it's going to use and it isn't native iPhone, say "actually i'd want a native swiftui app" once.
-- If it asks you a question anyway, answer in a few words.
-- If it starts building, or says it's done, end the conversation.
diff --git a/tests/brainstorming/scenarios/offsite-talk/opening.txt b/tests/brainstorming/scenarios/offsite-talk/opening.txt
deleted file mode 100644
index fe3adfb..0000000
--- a/tests/brainstorming/scenarios/offsite-talk/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-Can you help me put together a talk for our team offsite next month?
diff --git a/tests/brainstorming/scenarios/offsite-talk/persona.md b/tests/brainstorming/scenarios/offsite-talk/persona.md
deleted file mode 100644
index a59dc6f..0000000
--- a/tests/brainstorming/scenarios/offsite-talk/persona.md
+++ /dev/null
@@ -1,18 +0,0 @@
-You are Sam, an engineering manager of a 12-person team. You write short, slightly guarded replies: one to three sentences. You don't volunteer information unless asked something that draws it out.
-
-What you actually want, only revealed when a question leads there:
-- Two months ago the company laid off 20% of staff; your team lost three people. Morale is low and people are quietly job hunting.
-- You have a 20-minute slot on the first morning of a two-day offsite.
-- You want people to leave feeling that you are being straight with them and that the team has a real future. You want to rebuild trust.
-- Anti-goal: corporate cheerleading, "we're a family" language, or anything that sounds like it came from HR. That would backfire badly.
-- Prior art: your skip-level gave an all-hands after the layoffs full of spin; people hated it. You admired a talk by a former manager who just put the numbers on a slide and said "here's what I know and what I don't."
-- You'd like to end with something interactive, but you're not sure what.
-- You don't need slides designed; an outline with talking points is fine.
-
-How to behave:
-- If asked a broad abstract question ("what's the goal?"), give a short generic answer ("set the tone for the offsite").
-- If asked about the room, the people, recent events, or what you're worried about, answer honestly with the relevant hidden details.
-- If offered a menu, pick the closest option even if it's not quite right.
-- If the agent plays back an understanding, correct anything wrong and confirm what's right.
-- If asked to approve a document, read it (files are in the current directory) and approve only if it matches; otherwise say what's wrong.
-- If asked whether the agent may look at something, say yes, but you have no files to share.
diff --git a/tests/brainstorming/scenarios/pottery-platform/opening.txt b/tests/brainstorming/scenarios/pottery-platform/opening.txt
deleted file mode 100644
index 53aa4ba..0000000
--- a/tests/brainstorming/scenarios/pottery-platform/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-I want to build a platform for my pottery studio: class booking, a member portal, an online shop for finished pieces, and an email newsletter.
diff --git a/tests/brainstorming/scenarios/pottery-platform/persona.md b/tests/brainstorming/scenarios/pottery-platform/persona.md
deleted file mode 100644
index a48dcb5..0000000
--- a/tests/brainstorming/scenarios/pottery-platform/persona.md
+++ /dev/null
@@ -1,11 +0,0 @@
-You are Morgan, who runs a small pottery studio and can code a little. Chatty but not technical.
-
-What's true, revealed when asked something that draws it out:
-- Class booking is the urgent pain: people book through Instagram DMs and you double-booked the wheel room twice last month.
-- The shop and the newsletter can wait months. The member portal is mostly about members booking open-studio time, so it overlaps with booking.
-- You'd be happy to tackle things one at a time.
-
-How to behave:
-- Answer what's asked; short replies.
-- If the assistant proposes an order, agree if booking comes first; otherwise push back.
-- If asked whether the agent may look at something, say yes, but you have no existing code.
diff --git a/tests/brainstorming/scenarios/skip-files/opening.txt b/tests/brainstorming/scenarios/skip-files/opening.txt
deleted file mode 100644
index e453aa2..0000000
--- a/tests/brainstorming/scenarios/skip-files/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-Add a way to skip some files in the sync tool.
diff --git a/tests/brainstorming/scenarios/skip-files/persona.md b/tests/brainstorming/scenarios/skip-files/persona.md
deleted file mode 100644
index fc0779c..0000000
--- a/tests/brainstorming/scenarios/skip-files/persona.md
+++ /dev/null
@@ -1,16 +0,0 @@
-You are Riley, a backend developer. Terse, technical, one or two sentences.
-
-What you actually want, only revealed when asked something that draws it out:
-- The sync tool (sync.py in the current directory) runs from cron every 5 minutes, syncing a photo editor's working folder to a NAS.
-- The editor's app drops huge temporary files (*.tmp, and anything inside a `.cache/` folder) that fill the NAS.
-- You can't easily change the cron line on that machine, so command-line flags are a poor fit. You want the skip list in a file that lives in the source folder (like a .gitignore), so the editor can add patterns themselves.
-- Gitignore-style patterns are what the editor already knows; anything close is fine.
-- Skipped files should be skipped silently. Do NOT delete files already copied to the destination.
-- You don't care about the exact file name; let the builder pick.
-
-How to behave:
-- Answer what's asked; don't volunteer.
-- If the agent proposes something that conflicts with the above, say so plainly.
-- If the agent plays back a description, correct anything wrong and confirm what's right.
-- Approve a correct description with a short "yes, go ahead".
-- If asked whether the agent may look at the code, say yes.
diff --git a/tests/brainstorming/scenarios/skip-files/setup.sh b/tests/brainstorming/scenarios/skip-files/setup.sh
deleted file mode 100644
index 419fcb9..0000000
--- a/tests/brainstorming/scenarios/skip-files/setup.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-# Fixture: a tiny sync tool. Run with cwd = the scenario's work dir.
-cat > sync.py <<'PY'
-import argparse, shutil, sys
-from pathlib import Path
-
-def main():
- p = argparse.ArgumentParser(description="Copy changed files from SRC to DST.")
- p.add_argument("src"); p.add_argument("dst")
- a = p.parse_args()
- src, dst = Path(a.src), Path(a.dst)
- n = 0
- for f in src.rglob("*"):
- if not f.is_file():
- continue
- t = dst / f.relative_to(src)
- try:
- if not t.exists() or t.stat().st_mtime < f.stat().st_mtime:
- t.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy2(f, t)
- print(f"copied {f}")
- n += 1
- except OSError as e:
- print(f"error: {f}: {e}", file=sys.stderr)
- print(f"synced {n} files")
-
-if __name__ == "__main__":
- main()
-PY
-git init -q && git add sync.py && git -c user.email=t@t -c user.name=t commit -qm "sync tool"
diff --git a/tests/brainstorming/scenarios/spike-fts/opening.txt b/tests/brainstorming/scenarios/spike-fts/opening.txt
deleted file mode 100644
index 23a2e40..0000000
--- a/tests/brainstorming/scenarios/spike-fts/opening.txt
+++ /dev/null
@@ -1 +0,0 @@
-Quick spike: can plain Python sqlite3 do full-text search fast enough over ~100k short notes? Just want to know if it's worth pursuing. Throwaway is fine.
diff --git a/tests/brainstorming/scenarios/spike-fts/persona.md b/tests/brainstorming/scenarios/spike-fts/persona.md
deleted file mode 100644
index dde8170..0000000
--- a/tests/brainstorming/scenarios/spike-fts/persona.md
+++ /dev/null
@@ -1,7 +0,0 @@
-You are Jordan, a developer evaluating options for a personal notes app. Friendly, brief.
-
-- "Fast enough" means a search returns in well under 100 ms on a laptop.
-- Notes are a sentence to a paragraph each.
-- If the assistant confirms what it will try, say yes.
-- You want an answer and a recommendation, not production code, docs, or a design.
-- Once you have an answer with numbers, end the conversation.
diff --git a/tests/brainstorming/tech-lines.sh b/tests/brainstorming/tech-lines.sh
deleted file mode 100644
index 3193581..0000000
--- a/tests/brainstorming/tech-lines.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Shows how each converse.sh run handled technology choices.
-#
-# Use after a batch of converse.sh runs when checking whether the subject
-# asked about, proposed, or silently picked a platform or stack. For each
-# run it prints how the run ended, the subject's lines that mention
-# technology, and any code files left in the work dir. It is a reading aid:
-# read the full transcript before scoring a run.
-
-set -euo pipefail
-
-usage() {
- cat <<'EOF'
-Usage: tech-lines.sh OUT_DIR [SCENARIO]
-
- OUT_DIR A converse.sh results root (contains ARM/SCENARIO/transcript.md).
- SCENARIO Only show runs of this scenario.
-
-Environment:
- TECH_LINES_MAX Matching lines shown per run (default 12).
-EOF
-}
-
-[ $# -ge 1 ] && [ -d "$1" ] || { usage >&2; exit 2; }
-OUT=$1 ONLY=${2:-}
-MAX=${TECH_LINES_MAX:-12}
-PATTERN='swift|native|pwa|web ?app|home screen|browser|react|vue|svelte|vite|flask|django|sqlite|indexeddb|postgres|database|framework|stack|platform|xcode|android|iphone|tablet|technolog|built with|how (it|this).s built|language'
-
-for t in "$OUT"/*/*/transcript.md; do
- run=${t#"$OUT"/}; run=${run%/transcript.md}
- [ -z "$ONLY" ] || [ "${run#*/}" = "$ONLY" ] || continue
- dir=$(dirname "$t")
- turns=$(grep -c '^## AGENT' "$t" || true)
- echo "=== $run ($turns agent turns)"
- awk '/^## AGENT/{a=1; next} /^## HUMAN/{a=0} a' "$t" \
- | grep -i -E "$PATTERN" | cut -c1-220 | head -n "$MAX" | sed 's/^/ /' || true
- if [ -f "$dir/files.txt" ]; then
- code=$(sed '/^--- git log/q' "$dir/files.txt" | grep -v -E '^--- |node_modules|^\./docs/|^\./\.' \
- | grep -E '\.(js|ts|tsx|jsx|html|css|swift|py|json)$' | wc -l || true)
- echo " [code files at end: $code]"
- fi
-done
diff --git a/tests/brainstorming/try-codex.sh b/tests/brainstorming/try-codex.sh
deleted file mode 100644
index 7417933..0000000
--- a/tests/brainstorming/try-codex.sh
+++ /dev/null
@@ -1,92 +0,0 @@
-#!/usr/bin/env bash
-# Opens an interactive Codex session set up like a converse.sh eval run.
-#
-# Use when you want to drive a brainstorming scenario yourself and see what
-# the eval subjects see: a fresh CODEX_HOME with only the superpowers under
-# test installed (account connectors and other plugins off), the scenario's
-# fixture work dir, the model and effort our eval runs used, and optionally
-# the using-superpowers bootstrap preloaded. Prints the scenario's opening
-# message to paste in. Codex runs with its default sandbox and approvals,
-# as converse.sh runs do unless CODEX_PERMISSIONS=yolo.
-
-set -euo pipefail
-
-usage() {
- cat <<'EOF'
-Usage: try-codex.sh [-b] [-s SCENARIO] PLUGIN
-
- PLUGIN A superpowers checkout dir, a git ref in this repo (e.g. main,
- dev) to archive and load, or "curated" for the account's
- curated release.
- -b Preload using-superpowers before the first turn
- (CODEX_BOOTSTRAP=1; see codex-with-plugin.sh).
- -s SCENARIO Scenario under scenarios/ (default ocd-app).
-
-Environment:
- REAL_CODEX_BIN codex to run (default: codex on PATH). The eval runs
- used a build with openai/codex#51908.
- CODEX_MODEL Model (default CSD_CODEX_MODEL if set, else
- gpt-6.1-sol, the model our eval runs used).
- CODEX_EFFORT Reasoning effort (default max).
-
-Leaves the work dir and CODEX_HOME under $TMPDIR (paths printed at exit),
-minus the copied ~/.codex/auth.json, which is removed when Codex exits.
-EOF
-}
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-REPO="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)"
-SCENARIO=ocd-app BOOT=""
-while getopts "bs:h" opt; do
- case $opt in
- b) BOOT=1 ;; s) SCENARIO=$OPTARG ;;
- h) usage; exit 0 ;; *) usage >&2; exit 2 ;;
- esac
-done
-shift $((OPTIND - 1))
-[ $# -eq 1 ] || { usage >&2; exit 2; }
-SDIR="$SCRIPT_DIR/scenarios/$SCENARIO"
-[ -f "$SDIR/opening.txt" ] || { echo "error: no scenario $SCENARIO" >&2; exit 2; }
-[ -f "$HOME/.codex/auth.json" ] || { echo "error: no ~/.codex/auth.json; run codex login first" >&2; exit 2; }
-
-TMP=$(mktemp -d "${TMPDIR:-/tmp}/try-codex.XXXXXX")
-PLUGIN=$1
-if [ "$PLUGIN" != curated ] && [ ! -d "$PLUGIN" ]; then
- git -C "$REPO" rev-parse -q --verify "$PLUGIN^{commit}" > /dev/null \
- || { echo "error: $PLUGIN is not a dir or a git ref" >&2; exit 2; }
- mkdir "$TMP/plugin" && git -C "$REPO" archive "$PLUGIN" | tar x -C "$TMP/plugin"
- PLUGIN="$TMP/plugin"
-fi
-
-WORK="$TMP/work"
-mkdir "$WORK"
-(cd "$WORK" && git init -q && { [ ! -f "$SDIR/setup.sh" ] || bash "$SDIR/setup.sh"; })
-
-export CODEX_HOME="$TMP/codex-home"
-mkdir "$CODEX_HOME"
-cp "$HOME/.codex/auth.json" "$CODEX_HOME/"
-printf '[projects."%s"]\ntrust_level = "trusted"\n' "$WORK" > "$CODEX_HOME/config.toml"
-
-# Codex's remote plugin sync sometimes installs the account's curated
-# superpowers and lists it beside the one under test (see
-# codex-with-plugin.sh). Say so, since that session didn't test PLUGIN alone.
-report() {
- rm -f "$CODEX_HOME/auth.json"
- echo; echo "work dir: $WORK"; echo "codex home: $CODEX_HOME"
- local first
- first=$(find "$CODEX_HOME/sessions" -name 'rollout*.jsonl' 2>/dev/null | sort | head -1 || true)
- [ -n "$first" ] && [ "$(grep -m1 skills_instructions "$first" | grep -o -- '- superpowers:brainstorming:' | wc -l)" -gt 1 ] \
- && echo "WARNING: this session also listed the curated superpowers; it may have used that instead. Run again."
- return 0
-}
-trap report EXIT
-echo "Opening message for $SCENARIO (paste it in):"
-echo "----"
-cat "$SDIR/opening.txt"
-echo "----"
-read -r -p "Press Enter to start Codex. "
-
-CODEX_BOOTSTRAP=$BOOT bash "$SCRIPT_DIR/codex-with-plugin.sh" "$PLUGIN" \
- -C "$WORK" -m "${CODEX_MODEL:-${CSD_CODEX_MODEL:-gpt-6.1-sol}}" \
- -c "model_reasoning_effort=\"${CODEX_EFFORT:-max}\"" \
- -c tools.experimental_request_user_input.enabled=false -c features.apps=false