summaryrefslogtreecommitdiff
path: root/plugins/security-guidance/tests/test_repeated_diff_count.py
blob: ede4fdcb3bc37425d1202250853d823311723b78 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
"""Reviews report whether their diff was reviewed before; nothing is skipped."""
from conftest import bash_payload, commit_file, git, make_repo, metrics_of, run_hook

import diffstate

import pytest

DIFF = [("handlers.py", "@@ -0,0 +1,2 @@\n+def h(x):\n+    return x")]


@pytest.fixture(autouse=True)
def _claude_code_output(monkeypatch):
    # diffstate reads the output mode from the test runner's environment.
    monkeypatch.setattr(diffstate, "portable_output", lambda: False)


def _reviews(stub_api):
    return [c for c in stub_api.calls if c.endswith("/v1/messages")]


def _records(repo):
    p = repo / ".git" / "sg-reviewed-diffs"
    return p.read_text().splitlines() if p.exists() else []


def _recommit(repo, msg):
    git(repo, "reset", "-q", "--soft", "HEAD~1")
    out = git(repo, "commit", "-m", msg)
    return git(repo, "rev-parse", "HEAD").strip(), out


def _seen(repo, diff=DIFF):
    return diffstate._diff_review_started(str(repo), diff)


def _finished(repo, diff=DIFF):
    token, _ = _seen(repo, diff)
    diffstate._diff_review_done(token)


def _hunk(start, context, added):
    return f"@@ -{start},2 +{start},3 @@ def f():\n {context}\n+{added}\n pass"


def test_first_review_of_a_diff_is_not_seen(tmp_path):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    token, m = _seen(repo)
    assert m == {"diff_seen": diffstate.DIFF_SEEN_NO}
    assert token and len(_records(repo)) == 1


def test_unfinished_then_finished_on_same_then_other_tree(tmp_path):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    token, _ = _seen(repo)
    assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_UNFINISHED}
    diffstate._diff_review_done(token)
    assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_SAME_TREE}
    commit_file(repo, "other.py", "x = 1\n")
    assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_OTHER_TREE}


def test_tree_is_the_reviewed_commits_not_where_head_went(tmp_path):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    sha, _ = commit_file(repo, "handlers.py", "def h(x):\n    return x\n")
    token, _ = diffstate._diff_review_started(str(repo), DIFF, tip=sha)
    diffstate._diff_review_done(token)
    git(repo, "reset", "-q", "--hard", "HEAD~1")
    assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_OTHER_TREE}
    assert diffstate._diff_review_started(str(repo), DIFF, tip=sha)[1] == {
        "diff_seen": diffstate.DIFF_SEEN_SAME_TREE}


def test_a_different_change_path_or_place_is_not_seen(tmp_path):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    one = [("auth.py", _hunk(3, "def login():", "return True"))]
    _finished(repo, one)
    for other in ([("auth.py", _hunk(3, "def login():", "return False"))],
                  [("other.py", one[0][1])],
                  # The same added line next to different code.
                  [("auth.py", _hunk(3, "def is_admin():", "return True"))],
                  one + [("b.py", "@@ -0,0 +1 @@\n+1")]):
        assert _seen(repo, other)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO}


def test_a_moved_hunk_keeps_its_key(tmp_path):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    _finished(repo, [("auth.py", _hunk(3, "def login():", "return True"))])
    moved = "@@ -40,2 +44,3 @@ class A:\n def login():\n+return True\n pass"
    assert _seen(repo, [("auth.py", moved)])[1] == {"diff_seen": diffstate.DIFF_SEEN_SAME_TREE}


def test_real_diffs_of_two_commits_match_their_net_diff(tmp_path):
    from gitutil import parse_diff_into_files
    body = "".join(f"line {i}\n" for i in range(60))
    repo = make_repo(tmp_path / "repo", {"app.py": body})
    base = git(repo, "rev-parse", "HEAD").strip()
    commit_file(repo, "app.py", body.replace("line 5\n", "line 5\nx = 1\n"))
    _finished(repo, parse_diff_into_files(git(repo, "show", "-p", "HEAD")))
    commit_file(repo, "app.py", body.replace("line 5\n", "line 5\nx = 1\n")
                .replace("line 50\n", "line 50\ny = 2\n"))
    _finished(repo, parse_diff_into_files(git(repo, "show", "-p", "HEAD")))
    net = parse_diff_into_files(git(repo, "diff", base, "HEAD"))
    assert len(diffstate._hunk_keys(net)) == 2
    # Both hunks had a finished review; the first one on an older tree.
    assert _seen(repo, net)[1] == {"diff_seen": diffstate.DIFF_SEEN_OTHER_TREE}


def test_hunks_past_the_key_cap_are_not_dropped(tmp_path, monkeypatch):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    monkeypatch.setattr(diffstate, "_SEEN_DIFFS_MAX_KEYS", 3)
    hunks = [_hunk(10 * i, f"def f{i}():", f"x = {i}") for i in range(6)]
    _finished(repo, [("app.py", "\n".join(hunks[:5]))])
    assert len(_records(repo)[0].split("\t")[3].split(" ")) == 3
    assert _seen(repo, [("app.py", "\n".join(hunks))])[1] == {"diff_seen": diffstate.DIFF_SEEN_NO}
    assert _seen(repo, [("app.py", "\n".join(hunks[:5]))])[1] == {
        "diff_seen": diffstate.DIFF_SEEN_SAME_TREE}


def test_removed_sql_comment_lines_tell_diffs_apart(tmp_path):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    _finished(repo, [("q.sql", "@@ -1,2 +1 @@\n--- drop table users;\n select 1;")])
    other = [("q.sql", "@@ -1,2 +1 @@\n--- drop table orders;\n select 1;")]
    assert _seen(repo, other)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO}


def test_commits_reviewed_one_by_one_then_as_one_net_diff(tmp_path):
    # Two commit reviews of the same file, then a push review of their net diff.
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    h1, h2 = _hunk(3, "def a():", "x = 1"), _hunk(90, "def z():", "y = 2")
    _finished(repo, [("app.py", h1)])
    net = [("app.py", h1 + "\n" + h2), ("b.py", "@@ -0,0 +1 @@\n+b = 2")]
    assert _seen(repo, net)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO}
    _finished(repo, [("app.py", h2)])
    _finished(repo, [net[1]])
    assert _seen(repo, net)[1] == {"diff_seen": diffstate.DIFF_SEEN_SAME_TREE}


def test_old_records_do_not_count(tmp_path, monkeypatch):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    _finished(repo)
    monkeypatch.setattr(diffstate, "_SEEN_DIFFS_TTL_S", -1)
    assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO}


@pytest.mark.parametrize("flock", [True, False])
def test_record_is_cut_back_to_its_newest_rows(tmp_path, monkeypatch, flock):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    if not flock:
        def no_flock(fd, exclusive):
            raise ImportError("fcntl")
        monkeypatch.setattr(diffstate, "_flock_nb", no_flock)
    _finished(repo)
    row = len(_records(repo)[0]) + 1
    monkeypatch.setattr(diffstate, "_SEEN_DIFFS_TRIM_BYTES", 6 * row)
    for i in range(12):
        _finished(repo, [("f.py", f"@@ -0,0 +1 @@\n+{i}")])
    rows = _records(repo)
    assert 3 <= len(rows) <= 6
    assert rows[-1].split("\t")[2:] == ["1", diffstate._hunk_keys([("f.py", "@@ -0,0 +1 @@\n+11")])[0]]
    assert _seen(repo, [("f.py", "@@ -0,0 +1 @@\n+11")])[1]["diff_seen"] == diffstate.DIFF_SEEN_SAME_TREE
    assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_NO}


def test_damaged_record_is_read_around_and_shrinks(tmp_path, monkeypatch):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    p = repo / ".git" / "sg-reviewed-diffs"
    p.write_bytes(b"\xff\xfe garbage\nshort\tline\n" + b"9" * 5000 + b"\tt\t1\tk\n"
                  + b"x" * 5000)
    monkeypatch.setattr(diffstate, "_SEEN_DIFFS_TRIM_BYTES", 2000)
    token, m = _seen(repo)
    assert m == {"diff_seen": diffstate.DIFF_SEEN_NO}
    diffstate._diff_review_done(token)
    assert p.stat().st_size < 2000
    assert _seen(repo)[1] == {"diff_seen": diffstate.DIFF_SEEN_SAME_TREE}


def test_a_lock_another_process_keeps_is_not_waited_on(tmp_path, monkeypatch):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    token, _ = _seen(repo)
    monkeypatch.setattr(diffstate, "_flock_nb", lambda fd, exclusive: False)
    monkeypatch.setattr(diffstate, "_SEEN_DIFFS_LOCK_TRIES", 2)
    monkeypatch.setattr(diffstate, "_SEEN_DIFFS_TRIM_BYTES", 0)
    # A start that cannot read the record under the lock does not guess.
    assert _seen(repo) == (None, {"diff_seen": diffstate.DIFF_SEEN_ERROR})
    # A verdict is still noted, and the record is not cut back without the lock.
    diffstate._diff_review_done(token)
    assert [r.split("\t")[2] for r in _records(repo)] == ["0", "1"]


def test_failures_are_reported_and_never_raise(tmp_path):
    assert _seen(tmp_path / "nope") == (None, {"diff_seen": diffstate.DIFF_SEEN_ERROR})
    diffstate._diff_review_done(None)
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    (repo / ".git" / "sg-reviewed-diffs").mkdir()
    assert _seen(repo) == (None, {"diff_seen": diffstate.DIFF_SEEN_ERROR})
    diffstate._diff_review_done((str(repo / ".git" / "sg-reviewed-diffs"), ["k"], "t"))


def test_a_record_that_cannot_be_written_is_an_error_not_unseen(tmp_path, monkeypatch):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    def full_disk(f, locked, token, done):
        raise OSError(28, "No space left on device")
    monkeypatch.setattr(diffstate, "_append_diff_row", full_disk)
    assert _seen(repo) == (None, {"diff_seen": diffstate.DIFF_SEEN_ERROR})


def test_nothing_is_recorded_in_portable_output_mode(tmp_path, monkeypatch):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    monkeypatch.setattr(diffstate, "portable_output", lambda: True)
    assert _seen(repo) == (None, {})
    assert _records(repo) == []


def test_recommit_of_the_same_change_is_reviewed_again_and_counted(tmp_path, hook_env, stub_api):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    sha, out = commit_file(repo, "handlers.py", "def h(x):\n    return x\n")
    rc, so, _ = run_hook(bash_payload(repo, "git commit -m change", out), hook_env)
    first = metrics_of(so)
    assert rc == 0 and first["diff_seen"] == diffstate.DIFF_SEEN_NO

    sha2, out2 = _recommit(repo, "same change again")
    assert sha2 != sha
    rc, so, _ = run_hook(bash_payload(repo, "git commit -m 'same change again'", out2), hook_env)
    second = metrics_of(so)
    assert rc == 0 and second["diff_seen"] == diffstate.DIFF_SEEN_SAME_TREE
    # Last in the row, so a row over the key cap loses this and nothing older.
    assert list(second)[-1] == "diff_seen"
    # Count only: both commits were reviewed and recorded.
    assert len(_reviews(stub_api)) == 2
    assert {sha, sha2} <= diffstate._load_reviewed_shas(str(repo))


def test_push_of_a_rebased_reviewed_commit_is_counted(tmp_path, hook_env, stub_api):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    remote = tmp_path / "remote.git"
    git(tmp_path, "init", "-q", "--bare", str(remote))
    git(repo, "remote", "add", "origin", str(remote))
    git(repo, "push", "-q", "-u", "origin", "main")
    git(repo, "checkout", "-q", "-b", "feat")
    sha, out = commit_file(repo, "handlers.py", "def h(x):\n    return x\n")
    run_hook(bash_payload(repo, "git commit -m change", out), hook_env)

    git(repo, "checkout", "-q", "main")
    commit_file(repo, "notes.md", "notes\n")
    git(repo, "push", "-q", "origin", "main")
    git(repo, "checkout", "-q", "feat")
    git(repo, "rebase", "-q", "main")
    git(repo, "push", "-q", "-u", "origin", "feat")
    push_out = f"To {remote}\n * [new branch]      feat -> feat\n"
    rc, so, _ = run_hook(bash_payload(repo, "git push -u origin feat", push_out), hook_env)
    m = metrics_of(so)
    assert rc == 0 and m["push_sweep"] is True and m["unreviewed"] == 1
    assert m["diff_seen"] == diffstate.DIFF_SEEN_OTHER_TREE and list(m)[-1] == "diff_seen"
    assert len(_reviews(stub_api)) == 2


def test_failed_review_leaves_the_diff_unfinished(tmp_path, hook_env, stub_api):
    repo = make_repo(tmp_path / "repo", {"app.py": "print('hi')\n"})
    sha, out = commit_file(repo, "handlers.py", "def h(x):\n    return x\n")
    stub_api.status = 500
    rc, so, _ = run_hook(bash_payload(repo, "git commit -m change", out), hook_env)
    failed = metrics_of(so)
    assert failed["review_failed"] is True and failed["diff_seen"] == diffstate.DIFF_SEEN_NO

    stub_api.status = 200
    sha2, out2 = _recommit(repo, "same change again")
    rc, so, _ = run_hook(bash_payload(repo, "git commit -m 'same change again'", out2), hook_env)
    assert metrics_of(so)["diff_seen"] == diffstate.DIFF_SEEN_UNFINISHED