summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJesse Vincent <jesse@fsck.com>2026-07-16 04:00:15 +0000
committerJesse Vincent <jesse@fsck.com>2026-07-16 04:00:15 +0000
commitcda4b15fd68cec5991ab46c1bc8d5bc2b8ec2677 (patch)
treee1120321b08acf97139b06cdb6fa0a1f7f75d74b
parent93b250b15527103afbb34f6e3f1cec64f94337ad (diff)
fix(tests): stop the SDD skill test flaking on timing and prose casefix/test-suite-portability
tests/claude-code/test-subagent-driven-development.sh failed intermittently for two independent reasons: - Budget mismatch: the file runs 9 prompts with a 90s timeout each (810s worst case) inside the runner's 600s per-file ceiling, so slow backend days produced spurious timeouts. Raise the runner default to 900s and fix the help text, which claimed the default was 300. - Case-sensitive prose matching: the assert helpers grepped free-form model output case-sensitively, but models capitalize the skill's own headings — observed failures include "Do Not Trust the Report" missing pattern "not trust" and a structured answer missing "First:.*spec.*compliance". Match case-insensitively in assert_contains/assert_not_contains/assert_count/assert_order, widen two Test 5 keyword patterns to phrasings observed in real runs, and make assert_order dump the output on failure the way assert_contains already does, so the next flake is diagnosable. Observed 3 failures across 4 runs before the change (timeout, two distinct pattern misses); 3/3 consecutive full runs pass after it.
-rwxr-xr-xtests/claude-code/run-skill-tests.sh5
-rwxr-xr-xtests/claude-code/test-helpers.sh16
-rwxr-xr-xtests/claude-code/test-subagent-driven-development.sh4
3 files changed, 16 insertions, 9 deletions
diff --git a/tests/claude-code/run-skill-tests.sh b/tests/claude-code/run-skill-tests.sh
index c94cfec..83217cd 100755
--- a/tests/claude-code/run-skill-tests.sh
+++ b/tests/claude-code/run-skill-tests.sh
@@ -25,7 +25,8 @@ fi
# Parse command line arguments
VERBOSE=false
SPECIFIC_TEST=""
-TIMEOUT=600 # Default 10 minute timeout per test
+TIMEOUT=900 # Per-test-file budget; must exceed the file's worst case
+ # (test-subagent-driven-development.sh: 9 prompts x 90s each)
RUN_INTEGRATION=false
while [[ $# -gt 0 ]]; do
@@ -52,7 +53,7 @@ while [[ $# -gt 0 ]]; do
echo "Options:"
echo " --verbose, -v Show verbose output"
echo " --test, -t NAME Run only the specified test"
- echo " --timeout SECONDS Set timeout per test (default: 300)"
+ echo " --timeout SECONDS Set timeout per test (default: 900)"
echo " --integration, -i Run integration tests (slow, 10-30 min)"
echo " --help, -h Show this help"
echo ""
diff --git a/tests/claude-code/test-helpers.sh b/tests/claude-code/test-helpers.sh
index 1b5ead3..9e18761 100755
--- a/tests/claude-code/test-helpers.sh
+++ b/tests/claude-code/test-helpers.sh
@@ -30,12 +30,14 @@ run_claude() {
# Check if output contains a pattern
# Usage: assert_contains "output" "pattern" "test name"
+# Matching is case-insensitive: patterns are prose keywords, and models
+# freely capitalize skill terms ("Do Not Trust", "Spec Compliance").
assert_contains() {
local output="$1"
local pattern="$2"
local test_name="${3:-test}"
- if echo "$output" | grep -q "$pattern"; then
+ if echo "$output" | grep -qi "$pattern"; then
echo " [PASS] $test_name"
return 0
else
@@ -54,7 +56,7 @@ assert_not_contains() {
local pattern="$2"
local test_name="${3:-test}"
- if echo "$output" | grep -q "$pattern"; then
+ if echo "$output" | grep -qi "$pattern"; then
echo " [FAIL] $test_name"
echo " Did not expect to find: $pattern"
echo " In output:"
@@ -74,7 +76,7 @@ assert_count() {
local expected="$3"
local test_name="${4:-test}"
- local actual=$(echo "$output" | grep -c "$pattern" || echo "0")
+ local actual=$(echo "$output" | grep -ci "$pattern" || echo "0")
if [ "$actual" -eq "$expected" ]; then
echo " [PASS] $test_name (found $actual instances)"
@@ -98,16 +100,20 @@ assert_order() {
local test_name="${4:-test}"
# Get line numbers where patterns appear
- local line_a=$(echo "$output" | grep -n "$pattern_a" | head -1 | cut -d: -f1)
- local line_b=$(echo "$output" | grep -n "$pattern_b" | head -1 | cut -d: -f1)
+ local line_a=$(echo "$output" | grep -ni "$pattern_a" | head -1 | cut -d: -f1)
+ local line_b=$(echo "$output" | grep -ni "$pattern_b" | head -1 | cut -d: -f1)
if [ -z "$line_a" ]; then
echo " [FAIL] $test_name: pattern A not found: $pattern_a"
+ echo " In output:"
+ echo "$output" | sed 's/^/ /'
return 1
fi
if [ -z "$line_b" ]; then
echo " [FAIL] $test_name: pattern B not found: $pattern_b"
+ echo " In output:"
+ echo "$output" | sed 's/^/ /'
return 1
fi
diff --git a/tests/claude-code/test-subagent-driven-development.sh b/tests/claude-code/test-subagent-driven-development.sh
index d8f3e10..151fc64 100755
--- a/tests/claude-code/test-subagent-driven-development.sh
+++ b/tests/claude-code/test-subagent-driven-development.sh
@@ -96,13 +96,13 @@ echo "Test 5: Spec compliance reviewer mindset..."
output=$(run_claude "What is the spec compliance reviewer's attitude toward the implementer's report in subagent-driven-development?" "$CLAUDE_PROMPT_TIMEOUT")
-if assert_contains "$output" "not trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then
+if assert_contains "$output" "not.*trust\|don't trust\|skeptical\|verify.*independently\|suspiciously" "Reviewer is skeptical"; then
: # pass
else
exit 1
fi
-if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code" "Reviewer reads code"; then
+if assert_contains "$output" "read.*code\|inspect.*code\|verify.*code\|read.*diff\|trust.*diff" "Reviewer reads code"; then
: # pass
else
exit 1