diff options
| author | Claude <noreply@anthropic.com> | 2026-11-21 02:21:35 +0000 |
|---|---|---|
| committer | Claude <noreply@anthropic.com> | 2026-11-21 02:21:35 +0000 |
| commit | dc9735bede9ba62d9bacb9c16671471be8f058a6 (patch) | |
| tree | 62e7e6b4dddabc4e0e58a581c54924757f2c562e | |
| parent | 1a40e473d74a8a9e337b6e631e8ce6d59a034ed0 (diff) | |
security-guidance: a fenced reply without findings does not stand beside an unread onemhegazy/sg-fenced-reply
Where the first model's reply, or the other look's, could not be read, a
reply out of a code fence that says "nothing found" no longer settles the
review. It may have said otherwise, and "nothing" fits the schema more
easily than a finding. Replies of other odd shapes fail as they did before.
Co-authored-by: mhegazy <mhegazy@anthropic.com>
| -rw-r--r-- | plugins/security-guidance/hooks/llm.py | 35 | ||||
| -rw-r--r-- | plugins/security-guidance/tests/test_json_reply.py | 57 |
2 files changed, 82 insertions, 10 deletions
diff --git a/plugins/security-guidance/hooks/llm.py b/plugins/security-guidance/hooks/llm.py index 5ab9934b..b7264257 100644 --- a/plugins/security-guidance/hooks/llm.py +++ b/plugins/security-guidance/hooks/llm.py @@ -1371,17 +1371,32 @@ def _whole_reply_object(content, schema, stop_reason): return None, "the object does not fit the schema" if value[flag] != bool(value[findings]): return None, "the object contradicts itself" - return value, "" + return _Unheld(value), "" + + +class _Unheld(dict): + """An object from a reply that nothing held to the schema.""" + + +def _without_lone_clean(result, flag, unread): + """None for an _Unheld `result` that says "nothing found" when another + reply of the same review could not be read: that one may have said + otherwise, and "nothing" fits the schema more easily than a finding.""" + if unread and isinstance(result, _Unheld) and not result.get(flag): + debug_log("a reply of this review could not be read, so one without findings does not count") + return None + return result def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000, model=None, - retry_5xx=True, cacheable_prefix=None): + retry_5xx=True, cacheable_prefix=None, unread=None): """ Call the configured LLM model with extended thinking and structured outputs. cacheable_prefix: static text sent ahead of `prompt` as its own content block with cache_control, so repeated calls share a cached request prefix. Model defaults to default_review_model(); SECURITY_REVIEW_MODEL overrides it. Returns parsed JSON response or None on failure. + unread: a list that gets an entry for a reply that could not be read. On failure, sets module-level _last_call_claude_http_error to the HTTP status (or -1 for network/timeout) so callers can distinguish API failure from an empty-result success. @@ -1530,10 +1545,12 @@ def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000, if block.get("type") == "text": try: return json.loads(block["text"]) - except (ValueError, RecursionError, TypeError) as e: + except json.JSONDecodeError as e: value, why = _whole_reply_object(content, output_schema, response_data.get("stop_reason")) if value is None: debug_log(f"JSON parse error: {e}" + (f"; {why}" if why else "")) + if unread is not None: + unread.append(why) return value debug_log("No text block in response") @@ -1579,34 +1596,36 @@ def _call_claude_dual_or(prompt, output_schema, *, bool_key: str, list_key: str, # Not looked up for a pinned model, which is never retried on another. fallback = None if explicit else retry_review_model() + unread: list = [] if not _dual_or_enabled(): # Single-call path. Reuse the same sonnet-fallback retry as a dual_or # leg so a 529/400 on the primary doesn't drop recall to zero. r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget, max_tokens=max_tokens, model=primary, retry_5xx=False, - cacheable_prefix=cacheable_prefix) + cacheable_prefix=cacheable_prefix, unread=unread) if r is None and not explicit and not _gateway_review_timed_out: debug_log(f"single: {primary} failed, falling back to sonnet") r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget, max_tokens=max_tokens, model=fallback, - retry_5xx=True, cacheable_prefix=cacheable_prefix) - return r + retry_5xx=True, cacheable_prefix=cacheable_prefix, unread=unread) + return _without_lone_clean(r, bool_key, unread) def _leg(): r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget, max_tokens=max_tokens, model=primary, retry_5xx=False, - cacheable_prefix=cacheable_prefix) + cacheable_prefix=cacheable_prefix, unread=unread) if r is None and not explicit and not _gateway_review_timed_out: debug_log(f"dual_or: {primary} leg failed, falling back to sonnet") r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget, max_tokens=max_tokens, model=fallback, - retry_5xx=True, cacheable_prefix=cacheable_prefix) + retry_5xx=True, cacheable_prefix=cacheable_prefix, unread=unread) return r with ThreadPoolExecutor(max_workers=2) as ex: fa = ex.submit(_leg) fb = ex.submit(_leg) ra, rb = fa.result(), fb.result() + ra, rb = (_without_lone_clean(r, bool_key, unread) for r in (ra, rb)) if ra is None and rb is None: return None diff --git a/plugins/security-guidance/tests/test_json_reply.py b/plugins/security-guidance/tests/test_json_reply.py index 2f8a6ce0..786f4a34 100644 --- a/plugins/security-guidance/tests/test_json_reply.py +++ b/plugins/security-guidance/tests/test_json_reply.py @@ -1,7 +1,10 @@ """A verdict that a gateway's model puts in a code fence is read (#6928), as the whole reply only: in a longer text an object may be quoted from the code.""" +import io import json +import threading import time +import urllib.error import pytest @@ -231,8 +234,58 @@ def test_odd_block_beside_a_fence_is_no_verdict(monkeypatch, odd, verdict): assert _through_call(monkeypatch, [{"type": "text", "text": "\n"}, odd, {"type": "text", "text": verdict}]) is None -def test_null_text_first_is_no_crash(monkeypatch): - assert _through_call(monkeypatch, [{"type": "text", "text": None}, {"type": "text", "text": F % VULN}]) is None +REPLIES = {"plain clean": CLEAN, "plain vuln": VULN, "fenced clean": F % CLEAN, "fenced vuln": F % VULN, + "bad fence": F % VULN.replace('"high"', '"HIGH"'), "prose": "a.py is injectable.", "error": None} + + +def _review(monkeypatch, replies, dual=False): + """(findings, failed, calls) of a review whose calls get `replies` in turn.""" + left, calls, lock = list(replies), [], threading.Lock() + + class Response: + def __init__(self, text): self.text = text + def __enter__(self): return self + def __exit__(self, *a): return False + def read(self): return json.dumps({"content": [{"type": "text", "text": self.text}], + "stop_reason": "end_turn", "usage": {}}).encode() + + def answer(*a, **k): + with lock: + name = left.pop(0) if left else "prose" + calls.append(name) + if REPLIES[name] is None: + raise urllib.error.HTTPError("u", 400, "x", {}, io.BytesIO(b"{}")) + return Response(REPLIES[name]) + for name, value in (("_urlopen", answer), ("ANTHROPIC_API_KEY", "k"), ("HAS_API_CREDENTIALS", True), + ("default_review_model", lambda: "m"), ("retry_review_model", lambda: "n")): + monkeypatch.setattr(llm, name, value) + monkeypatch.delenv("SECURITY_REVIEW_MODEL", raising=False) + monkeypatch.setenv("SG_DUAL_OR", "on" if dual else "off") + found = llm.analyze_code_security([("a.py", "x")], is_diff=True)[1] + return len(found), llm.last_review_failed(), len(calls) + + +@pytest.mark.parametrize("replies,expected", [ + (["plain clean"], (0, False, 1)), (["plain vuln"], (1, False, 1)), + (["fenced clean"], (0, False, 1)), (["fenced vuln"], (1, False, 1)), + (["bad fence", "fenced clean"], (0, True, 2)), (["prose", "fenced clean"], (0, True, 2)), + (["bad fence", "fenced vuln"], (1, False, 2)), (["prose", "fenced vuln"], (1, False, 2)), + (["error", "fenced clean"], (0, False, 2)), + # As before: a reply the API held to the schema. + (["prose", "plain clean"], (0, False, 2)), (["bad fence", "plain vuln"], (1, False, 2)), +]) +def test_retry_after_an_unread_reply(monkeypatch, replies, expected): + assert _review(monkeypatch, replies) == expected + + +@pytest.mark.parametrize("replies,expected", [ + (["fenced clean", "prose", "prose"], (0, True)), (["fenced clean", "bad fence", "fenced clean"], (0, True)), + (["fenced clean", "fenced clean"], (0, False)), (["fenced clean", "fenced vuln"], (1, False)), + (["fenced vuln", "prose", "prose"], (1, False)), (["plain clean", "plain clean"], (0, False)), + (["fenced clean", "plain vuln"], (1, False)), (["prose", "prose", "fenced clean", "fenced clean"], (0, True)), +]) +def test_two_legs(monkeypatch, replies, expected): + assert _review(monkeypatch, replies, dual=True)[:2] == expected def test_the_reason_is_logged(monkeypatch): |
