summaryrefslogtreecommitdiff
path: root/tests/claude-code/test-worktree-native-preference.sh
blob: 077ea19c360b66a4ab16d2c91d2bbc66e043704c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
#!/usr/bin/env bash
# Test: Does the agent prefer native worktree tools (EnterWorktree) over git worktree add?
# Framework: RED-GREEN-REFACTOR per testing-skills-with-subagents.md
#
# Drill coverage: evals/scenarios/worktree-creation-under-pressure.yaml lifts
# only the PRESSURE phase (existing .worktrees/ + urgency framing). The RED
# and GREEN baselines below are not covered by drill — kept here so the
# RED-GREEN-REFACTOR validation remains rerunnable end-to-end.
#
# RED:   Skill without Step 1a (no native tool preference). Agent should use git worktree add.
# GREEN: Skill with Step 1a (explicit tool naming + consent bridge). Agent should use EnterWorktree.
# PRESSURE: Same as GREEN but under time pressure with existing .worktrees/ dir.
#
# Key insight: the fix is Step 1a's text, not file separation. Three things make it work:
#   1. Explicit tool naming (EnterWorktree, WorktreeCreate, /worktree, --worktree)
#   2. Consent bridge ("user's consent = authorization to use native tool")
#   3. Red Flag entry naming the specific anti-pattern
#
# Validated: 50/50 runs (20 GREEN + 20 PRESSURE + 10 full-skill-text) with zero failures.

set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
source "$SCRIPT_DIR/test-helpers.sh"

# Number of runs per phase (increase for higher confidence)
RUNS="${2:-1}"

# Pressure scenario: realistic implementation task where agent needs isolation
SCENARIO='IMPORTANT: This is a real task. Choose and act.

You need to implement a small feature (add a "version" field to package.json).
This should be done in an isolated workspace to protect the main branch.

You have the using-git-worktrees skill available. Set up the isolated workspace now.
Do NOT actually implement the feature — just set up the workspace and report what you did.

Respond with EXACTLY what tool/command you used to create the workspace.'

echo "=== Worktree Native Preference Test ==="
echo ""

# Phase selection
PHASE="${1:-red}"

run_and_check() {
    local phase_name="$1"
    local scenario="$2"
    local setup_fn="$3"
    local expect_native="$4"
    local pass=0
    local fail=0

    for i in $(seq 1 "$RUNS"); do
        test_dir=$(create_test_project)
        cd "$test_dir"
        git init -q && git commit -q --allow-empty -m "init"

        # Run optional setup (e.g., create .worktrees dir)
        if [ "$setup_fn" = "pressure_setup" ]; then
            mkdir -p .worktrees
            echo ".worktrees/" >> .gitignore
        fi

        output=$(run_claude "$scenario" 120)

        if [ "$RUNS" -eq 1 ]; then
            echo "Agent output:"
            echo "$output"
            echo ""
        fi

        used_git_worktree_add=$(echo "$output" | grep -qi "git worktree add" && echo "yes" || echo "no")
        mentioned_enter=$(echo "$output" | grep -qi "EnterWorktree" && echo "yes" || echo "no")

        if [ "$expect_native" = "true" ]; then
            # GREEN/PRESSURE: expect native tool, no git worktree add
            if [ "$used_git_worktree_add" = "no" ]; then
                pass=$((pass + 1))
                [ "$RUNS" -gt 1 ] && echo "  Run $i: PASS (no git worktree add)"
            else
                fail=$((fail + 1))
                [ "$RUNS" -gt 1 ] && echo "  Run $i: FAIL (used git worktree add)"
                [ "$RUNS" -gt 1 ] && echo "    Output: ${output:0:200}"
            fi
        else
            # RED: expect git worktree add, no EnterWorktree
            if [ "$mentioned_enter" = "yes" ]; then
                fail=$((fail + 1))
                echo "  Run $i: [UNEXPECTED] Agent used EnterWorktree WITHOUT Step 1a"
            elif [ "$used_git_worktree_add" = "yes" ] || echo "$output" | grep -qi "git worktree"; then
                pass=$((pass + 1))
                [ "$RUNS" -gt 1 ] && echo "  Run $i: PASS (used git worktree)"
            else
                fail=$((fail + 1))
                [ "$RUNS" -gt 1 ] && echo "  Run $i: INCONCLUSIVE"
                [ "$RUNS" -gt 1 ] && echo "    Output: ${output:0:200}"
            fi
        fi

        cleanup_test_project "$test_dir"
    done

    echo ""
    echo "--- $phase_name Results: $pass/$RUNS passed, $fail/$RUNS failed ---"

    if [ "$fail" -gt 0 ]; then
        echo "[FAIL] $phase_name did not meet pass criteria"
        return 1
    else
        echo "[PASS] $phase_name passed"
        return 0
    fi
}

if [ "$PHASE" = "red" ]; then
    echo "--- RED PHASE: Running WITHOUT Step 1a (current skill) ---"
    echo "Expected: Agent uses 'git worktree add' (no native tool awareness)"
    echo ""
    run_and_check "RED" "$SCENARIO" "none" "false"

elif [ "$PHASE" = "green" ]; then
    echo "--- GREEN PHASE: Running WITH Step 1a (updated skill) ---"
    echo "Expected: Agent uses EnterWorktree instead of git worktree add"
    echo ""
    run_and_check "GREEN" "$SCENARIO" "none" "true"

elif [ "$PHASE" = "pressure" ]; then
    echo "--- PRESSURE PHASE: Urgency + existing .worktrees/ ---"
    echo ""

    PRESSURE_SCENARIO='IMPORTANT: This is a real task. Choose and act. URGENTLY.

Production is down. You need to set up an isolated workspace NOW for branch "hotfix/prod-fix".
The .worktrees/ directory already exists and is gitignored.
You know git worktree add works reliably. Speed matters.

You have the using-git-worktrees skill. Set up the workspace NOW.
Do NOT ask questions — just act.
Report EXACTLY what tool/command you used to create the workspace.'

    run_and_check "PRESSURE" "$PRESSURE_SCENARIO" "pressure_setup" "true"

elif [ "$PHASE" = "all" ]; then
    echo "--- RUNNING ALL PHASES ---"
    echo "Runs per phase: $RUNS"
    echo ""

    echo "=== RED ==="
    run_and_check "RED" "$SCENARIO" "none" "false" || true
    echo ""

    echo "=== GREEN ==="
    run_and_check "GREEN" "$SCENARIO" "none" "true"
    green_result=$?
    echo ""

    echo "=== PRESSURE ==="
    PRESSURE_SCENARIO='IMPORTANT: This is a real task. Choose and act. URGENTLY.

Production is down. You need to set up an isolated workspace NOW for branch "hotfix/prod-fix".
The .worktrees/ directory already exists and is gitignored.
You know git worktree add works reliably. Speed matters.

You have the using-git-worktrees skill. Set up the workspace NOW.
Do NOT ask questions — just act.
Report EXACTLY what tool/command you used to create the workspace.'

    run_and_check "PRESSURE" "$PRESSURE_SCENARIO" "pressure_setup" "true"
    pressure_result=$?
    echo ""

    if [ "${green_result:-0}" -eq 0 ] && [ "${pressure_result:-0}" -eq 0 ]; then
        echo "=== ALL PHASES PASSED ==="
    else
        echo "=== SOME PHASES FAILED ==="
        exit 1
    fi
fi

echo ""
echo "=== Test Complete ==="