diff options
| author | Jesse Vincent <jesse@fsck.com> | 2026-07-16 04:00:15 +0000 |
|---|---|---|
| committer | Jesse Vincent <jesse@fsck.com> | 2026-07-16 04:00:15 +0000 |
| commit | cda4b15fd68cec5991ab46c1bc8d5bc2b8ec2677 (patch) | |
| tree | e1120321b08acf97139b06cdb6fa0a1f7f75d74b | |
| parent | 93b250b15527103afbb34f6e3f1cec64f94337ad (diff) | |
fix(tests): stop the SDD skill test flaking on timing and prose casefix/test-suite-portability
tests/claude-code/test-subagent-driven-development.sh failed
intermittently for two independent reasons:
- Budget mismatch: the file runs 9 prompts with a 90s timeout each
(810s worst case) inside the runner's 600s per-file ceiling, so slow
backend days produced spurious timeouts. Raise the runner default to
900s and fix the help text, which claimed the default was 300.
- Case-sensitive prose matching: the assert helpers grepped free-form
model output case-sensitively, but models capitalize the skill's own
headings — observed failures include "Do Not Trust the Report"
missing pattern "not trust" and a structured answer missing
"First:.*spec.*compliance". Match case-insensitively in
assert_contains/assert_not_contains/assert_count/assert_order, widen
two Test 5 keyword patterns to phrasings observed in real runs, and
make assert_order dump the output on failure the way assert_contains
already does, so the next flake is diagnosable.
Observed 3 failures across 4 runs before the change (timeout, two
distinct pattern misses); 3/3 consecutive full runs pass after it.
| -rwxr-xr-x | tests/claude-code/run-skill-tests.sh | 5 | ||||
| -rwxr-xr-x | tests/claude-code/test-helpers.sh | 16 | ||||
| -rwxr-xr-x | tests/claude-code/test-subagent-driven-development.sh | 4 |
3 files changed, 16 insertions, 9 deletions
diff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.sh index c94cfec..83217cd 100755 --- a/tests/claude-code/run-skill-tests.sh +++ b/tests/claude-code/run-skill-tests.sh @@ -25,7 +25,8 @@ fi # Parse command line arguments VERBOSE=false SPECIFIC_TEST="" -TIMEOUT=600 # Default 10 minute timeout per test +TIMEOUT=900 # Per-test-file budget; must exceed the file's worst case + # (test-subagent-driven-development.sh: 9 prompts x 90s each) RUN_INTEGRATION=false while [[ $# -gt 0 ]]; do @@ -52,7 +53,7 @@ while [[ $# -gt 0 ]]; do echo "Options:" echo " --verbose, -v Show verbose output" echo " --test, -t NAME Run only the specified test" - echo " --timeout SECONDS Set timeout per test (default: 300)" + echo " --timeout SECONDS Set timeout per test (default: 900)" echo " --integration, -i Run integration tests (slow, 10-30 min)" echo " --help, -h Show this help" echo "" diff --git a/tests/claude-code/test-helpers.sh b/tests/claude-code/test-helpers.sh index 1b5ead3..9e18761 100755 --- a/tests/claude-code/test-helpers.sh +++ b/tests/claude-code/test-helpers.sh @@ -30,12 +30,14 @@ run_claude() { # Check if output contains a pattern # Usage: assert_contains "output" "pattern" "test name" +# Matching is case-insensitive: patterns are prose keywords, and models +# freely capitalize skill terms ("Do Not Trust", "Spec Compliance"). assert_contains() { local output="$1" local pattern="$2" local test_name="${3:-test}" - if echo "$output" | grep -q "$pattern"; then + if echo "$output" | grep -qi "$pattern"; then echo " [PASS] $test_name" return 0 else @@ -54,7 +56,7 @@ assert_not_contains() { local pattern="$2" local test_name="${3:-test}" - if echo "$output" | grep -q "$pattern"; then + if echo "$output" | grep -qi "$pattern"; then echo " [FAIL] $test_name" echo " Did not expect to find: $pattern" echo " In output:" @@ -74,7 +76,7 @@ assert_count() { local expected="$3" local test_name="${4:-test}" - local actual=$(echo "$output" | grep -c "$pattern" || echo "0") + local actual=$(echo "$output" | grep -ci "$pattern" || echo "0") if [ "$actual" -eq "$expected" ]; then echo " [PASS] $test_name (found $actual instances)" @@ -98,16 +100,20 @@ assert_order() { local test_name="${4:-test}" # Get line numbers where patterns appear - local line_a=$(echo "$output" | grep -n "$pattern_a" | head -1 | cut -d: -f1) - local line_b=$(echo "$output" | grep -n "$pattern_b" | head -1 | cut -d: -f1) + local line_a=$(echo "$output" | grep -ni "$pattern_a" | head -1 | cut -d: -f1) + local line_b=$(echo "$output" | grep -ni "$pattern_b" | head -1 | cut -d: -f1) if [ -z "$line_a" ]; then echo " [FAIL] $test_name: pattern A not found: $pattern_a" + echo " In output:" + echo "$output" | sed 's/^/ /' return 1 fi if [ -z "$line_b" ]; then echo " [FAIL] $test_name: pattern B not found: $pattern_b" + echo " In output:" + echo "$output" | sed 's/^/ /' return 1 fi diff --git a/tests/claude-code/test-subagent-driven-development.sh b/tests/claude-code/test-subagent-driven-development.sh index d8f3e10..151fc64 100755 --- a/tests/claude-code/test-subagent-driven-development.sh +++ b/tests/claude-code/test-subagent-driven-development.sh @@ -96,13 +96,13 @@ echo "Test 5: Spec compliance reviewer mindset..." output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" "$CLAUDE_PROMPT_TIMEOUT") -if assert_contains "$output" "not trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then +if assert_contains "$output" "not.*trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then : # pass else exit 1 fi -if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code" "Reviewer reads code"; then +if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code\|read.*diff\|trust.*diff" "Reviewer reads code"; then : # pass else exit 1 |
