blob: ff4c65f36b01cd891a2251fe1c2f707314cfd47d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
|
#!/usr/bin/env bash
# Integration Test: Document Review System
# Actually runs spec/plan review and verifies reviewers catch issues
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "$SCRIPT_DIR/test-helpers.sh"
echo "========================================"
echo " Integration Test: Document Review System"
echo "========================================"
echo ""
echo "This test verifies the document review system by:"
echo " 1. Creating a spec with intentional errors"
echo " 2. Running the spec document reviewer"
echo " 3. Verifying the reviewer catches the errors"
echo ""
# Create test project
TEST_PROJECT=$(create_test_project)
echo "Test project: $TEST_PROJECT"
# Trap to cleanup
trap "cleanup_test_project $TEST_PROJECT" EXIT
cd "$TEST_PROJECT"
# Create directory structure
mkdir -p docs/superpowers/specs
# Create a spec document WITH INTENTIONAL ERRORS for the reviewer to catch
cat > docs/superpowers/specs/test-feature-design.md <<'EOF'
# Test Feature Design
## Overview
This is a test feature that does something useful.
## Requirements
1. The feature should work correctly
2. It should be fast
3. TODO: Add more requirements here
## Architecture
The feature will use a simple architecture with:
- A frontend component
- A backend service
- Error handling will be specified later once we understand the failure modes better
## Data Flow
Data flows from the frontend to the backend.
## Testing Strategy
Tests will be written to cover the main functionality.
EOF
# Initialize git repo
git init --quiet
git config user.email "test@test.com"
git config user.name "Test User"
git add .
git commit -m "Initial commit with test spec" --quiet
echo ""
echo "Created test spec with intentional errors:"
echo " - TODO placeholder in Requirements section"
echo " - 'specified later' deferral in Architecture section"
echo ""
echo "Running spec document reviewer..."
echo ""
# Run Claude to review the spec
OUTPUT_FILE="$TEST_PROJECT/claude-output.txt"
PROMPT="You are testing the spec document reviewer.
Read the spec-document-reviewer-prompt.md template in skills/brainstorming/ to understand the review format.
Then review the spec at $TEST_PROJECT/docs/superpowers/specs/test-feature-design.md using the criteria from that template.
Look for:
- TODOs, placeholders, 'TBD', incomplete sections
- Sections saying 'to be defined later' or 'will spec when X is done'
- Sections noticeably less detailed than others
Output your review in the format specified in the template."
echo "================================================================================"
cd "$SCRIPT_DIR/../.." && timeout 120 claude -p "$PROMPT" --permission-mode bypassPermissions 2>&1 | tee "$OUTPUT_FILE" || {
echo ""
echo "================================================================================"
echo "EXECUTION FAILED (exit code: $?)"
exit 1
}
echo "================================================================================"
echo ""
echo "Analyzing reviewer output..."
echo ""
# Verification tests
FAILED=0
echo "=== Verification Tests ==="
echo ""
# Test 1: Reviewer found the TODO
echo "Test 1: Reviewer found TODO..."
if grep -qi "TODO" "$OUTPUT_FILE" && grep -qi "requirements\|Requirements" "$OUTPUT_FILE"; then
echo " [PASS] Reviewer identified TODO in Requirements section"
else
echo " [FAIL] Reviewer did not identify TODO"
FAILED=$((FAILED + 1))
fi
echo ""
# Test 2: Reviewer found the "specified later" deferral
echo "Test 2: Reviewer found 'specified later' deferral..."
if grep -qi "specified later\|later\|defer\|incomplete\|error handling" "$OUTPUT_FILE"; then
echo " [PASS] Reviewer identified deferred content"
else
echo " [FAIL] Reviewer did not identify deferred content"
FAILED=$((FAILED + 1))
fi
echo ""
# Test 3: Reviewer output includes Issues section
echo "Test 3: Review output format..."
if grep -qi "issues\|Issues" "$OUTPUT_FILE"; then
echo " [PASS] Review includes Issues section"
else
echo " [FAIL] Review missing Issues section"
FAILED=$((FAILED + 1))
fi
echo ""
# Test 4: Reviewer did NOT approve (found issues)
echo "Test 4: Reviewer verdict..."
if grep -qi "Issues Found\|❌\|not approved\|issues found" "$OUTPUT_FILE"; then
echo " [PASS] Reviewer correctly found issues (not approved)"
elif grep -qi "Approved\|✅" "$OUTPUT_FILE" && ! grep -qi "Issues Found\|❌" "$OUTPUT_FILE"; then
echo " [FAIL] Reviewer incorrectly approved spec with errors"
FAILED=$((FAILED + 1))
else
echo " [PASS] Reviewer identified problems (ambiguous format but found issues)"
fi
echo ""
# Summary
echo "========================================"
echo " Test Summary"
echo "========================================"
echo ""
if [ $FAILED -eq 0 ]; then
echo "STATUS: PASSED"
echo "All verification tests passed!"
echo ""
echo "The spec document reviewer correctly:"
echo " ✓ Found TODO placeholder"
echo " ✓ Found 'specified later' deferral"
echo " ✓ Produced properly formatted review"
echo " ✓ Did not approve spec with errors"
exit 0
else
echo "STATUS: FAILED"
echo "Failed $FAILED verification tests"
echo ""
echo "Output saved to: $OUTPUT_FILE"
echo ""
echo "Review the output to see what went wrong."
exit 1
fi
|