summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorClaude <noreply@anthropic.com>2026-11-21 02:21:35 +0000
committerClaude <noreply@anthropic.com>2026-11-21 02:21:35 +0000
commitdc9735bede9ba62d9bacb9c16671471be8f058a6 (patch)
tree62e7e6b4dddabc4e0e58a581c54924757f2c562e
parent1a40e473d74a8a9e337b6e631e8ce6d59a034ed0 (diff)
security-guidance: a fenced reply without findings does not stand beside an unread onemhegazy/sg-fenced-reply
Where the first model's reply, or the other look's, could not be read, a reply out of a code fence that says "nothing found" no longer settles the review. It may have said otherwise, and "nothing" fits the schema more easily than a finding. Replies of other odd shapes fail as they did before. Co-authored-by: mhegazy <mhegazy@anthropic.com>
-rw-r--r--plugins/security-guidance/hooks/llm.py35
-rw-r--r--plugins/security-guidance/tests/test_json_reply.py57
2 files changed, 82 insertions, 10 deletions
diff --git a/plugins/security-guidance/hooks/llm.py b/plugins/security-guidance/hooks/llm.py
index 5ab9934b..b7264257 100644
--- a/plugins/security-guidance/hooks/llm.py
+++ b/plugins/security-guidance/hooks/llm.py
@@ -1371,17 +1371,32 @@ def _whole_reply_object(content, schema, stop_reason):
return None, "the object does not fit the schema"
if value[flag] != bool(value[findings]):
return None, "the object contradicts itself"
- return value, ""
+ return _Unheld(value), ""
+
+
+class _Unheld(dict):
+ """An object from a reply that nothing held to the schema."""
+
+
+def _without_lone_clean(result, flag, unread):
+ """None for an _Unheld `result` that says "nothing found" when another
+ reply of the same review could not be read: that one may have said
+ otherwise, and "nothing" fits the schema more easily than a finding."""
+ if unread and isinstance(result, _Unheld) and not result.get(flag):
+ debug_log("a reply of this review could not be read, so one without findings does not count")
+ return None
+ return result
def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000, model=None,
- retry_5xx=True, cacheable_prefix=None):
+ retry_5xx=True, cacheable_prefix=None, unread=None):
"""
Call the configured LLM model with extended thinking and structured outputs.
cacheable_prefix: static text sent ahead of `prompt` as its own content block
with cache_control, so repeated calls share a cached request prefix.
Model defaults to default_review_model(); SECURITY_REVIEW_MODEL overrides it.
Returns parsed JSON response or None on failure.
+ unread: a list that gets an entry for a reply that could not be read.
On failure, sets module-level _last_call_claude_http_error to the HTTP status
(or -1 for network/timeout) so callers can distinguish API failure from an
empty-result success.
@@ -1530,10 +1545,12 @@ def _call_claude(prompt, output_schema, thinking_budget=10000, max_tokens=16000,
if block.get("type") == "text":
try:
return json.loads(block["text"])
- except (ValueError, RecursionError, TypeError) as e:
+ except json.JSONDecodeError as e:
value, why = _whole_reply_object(content, output_schema, response_data.get("stop_reason"))
if value is None:
debug_log(f"JSON parse error: {e}" + (f"; {why}" if why else ""))
+ if unread is not None:
+ unread.append(why)
return value
debug_log("No text block in response")
@@ -1579,34 +1596,36 @@ def _call_claude_dual_or(prompt, output_schema, *, bool_key: str, list_key: str,
# Not looked up for a pinned model, which is never retried on another.
fallback = None if explicit else retry_review_model()
+ unread: list = []
if not _dual_or_enabled():
# Single-call path. Reuse the same sonnet-fallback retry as a dual_or
# leg so a 529/400 on the primary doesn't drop recall to zero.
r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget,
max_tokens=max_tokens, model=primary, retry_5xx=False,
- cacheable_prefix=cacheable_prefix)
+ cacheable_prefix=cacheable_prefix, unread=unread)
if r is None and not explicit and not _gateway_review_timed_out:
debug_log(f"single: {primary} failed, falling back to sonnet")
r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget,
max_tokens=max_tokens, model=fallback,
- retry_5xx=True, cacheable_prefix=cacheable_prefix)
- return r
+ retry_5xx=True, cacheable_prefix=cacheable_prefix, unread=unread)
+ return _without_lone_clean(r, bool_key, unread)
def _leg():
r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget,
max_tokens=max_tokens, model=primary, retry_5xx=False,
- cacheable_prefix=cacheable_prefix)
+ cacheable_prefix=cacheable_prefix, unread=unread)
if r is None and not explicit and not _gateway_review_timed_out:
debug_log(f"dual_or: {primary} leg failed, falling back to sonnet")
r = _call_claude(prompt, output_schema, thinking_budget=thinking_budget,
max_tokens=max_tokens, model=fallback,
- retry_5xx=True, cacheable_prefix=cacheable_prefix)
+ retry_5xx=True, cacheable_prefix=cacheable_prefix, unread=unread)
return r
with ThreadPoolExecutor(max_workers=2) as ex:
fa = ex.submit(_leg)
fb = ex.submit(_leg)
ra, rb = fa.result(), fb.result()
+ ra, rb = (_without_lone_clean(r, bool_key, unread) for r in (ra, rb))
if ra is None and rb is None:
return None
diff --git a/plugins/security-guidance/tests/test_json_reply.py b/plugins/security-guidance/tests/test_json_reply.py
index 2f8a6ce0..786f4a34 100644
--- a/plugins/security-guidance/tests/test_json_reply.py
+++ b/plugins/security-guidance/tests/test_json_reply.py
@@ -1,7 +1,10 @@
"""A verdict that a gateway's model puts in a code fence is read (#6928), as the
whole reply only: in a longer text an object may be quoted from the code."""
+import io
import json
+import threading
import time
+import urllib.error
import pytest
@@ -231,8 +234,58 @@ def test_odd_block_beside_a_fence_is_no_verdict(monkeypatch, odd, verdict):
assert _through_call(monkeypatch, [{"type": "text", "text": "\n"}, odd, {"type": "text", "text": verdict}]) is None
-def test_null_text_first_is_no_crash(monkeypatch):
- assert _through_call(monkeypatch, [{"type": "text", "text": None}, {"type": "text", "text": F % VULN}]) is None
+REPLIES = {"plain clean": CLEAN, "plain vuln": VULN, "fenced clean": F % CLEAN, "fenced vuln": F % VULN,
+ "bad fence": F % VULN.replace('"high"', '"HIGH"'), "prose": "a.py is injectable.", "error": None}
+
+
+def _review(monkeypatch, replies, dual=False):
+ """(findings, failed, calls) of a review whose calls get `replies` in turn."""
+ left, calls, lock = list(replies), [], threading.Lock()
+
+ class Response:
+ def __init__(self, text): self.text = text
+ def __enter__(self): return self
+ def __exit__(self, *a): return False
+ def read(self): return json.dumps({"content": [{"type": "text", "text": self.text}],
+ "stop_reason": "end_turn", "usage": {}}).encode()
+
+ def answer(*a, **k):
+ with lock:
+ name = left.pop(0) if left else "prose"
+ calls.append(name)
+ if REPLIES[name] is None:
+ raise urllib.error.HTTPError("u", 400, "x", {}, io.BytesIO(b"{}"))
+ return Response(REPLIES[name])
+ for name, value in (("_urlopen", answer), ("ANTHROPIC_API_KEY", "k"), ("HAS_API_CREDENTIALS", True),
+ ("default_review_model", lambda: "m"), ("retry_review_model", lambda: "n")):
+ monkeypatch.setattr(llm, name, value)
+ monkeypatch.delenv("SECURITY_REVIEW_MODEL", raising=False)
+ monkeypatch.setenv("SG_DUAL_OR", "on" if dual else "off")
+ found = llm.analyze_code_security([("a.py", "x")], is_diff=True)[1]
+ return len(found), llm.last_review_failed(), len(calls)
+
+
+@pytest.mark.parametrize("replies,expected", [
+ (["plain clean"], (0, False, 1)), (["plain vuln"], (1, False, 1)),
+ (["fenced clean"], (0, False, 1)), (["fenced vuln"], (1, False, 1)),
+ (["bad fence", "fenced clean"], (0, True, 2)), (["prose", "fenced clean"], (0, True, 2)),
+ (["bad fence", "fenced vuln"], (1, False, 2)), (["prose", "fenced vuln"], (1, False, 2)),
+ (["error", "fenced clean"], (0, False, 2)),
+ # As before: a reply the API held to the schema.
+ (["prose", "plain clean"], (0, False, 2)), (["bad fence", "plain vuln"], (1, False, 2)),
+])
+def test_retry_after_an_unread_reply(monkeypatch, replies, expected):
+ assert _review(monkeypatch, replies) == expected
+
+
+@pytest.mark.parametrize("replies,expected", [
+ (["fenced clean", "prose", "prose"], (0, True)), (["fenced clean", "bad fence", "fenced clean"], (0, True)),
+ (["fenced clean", "fenced clean"], (0, False)), (["fenced clean", "fenced vuln"], (1, False)),
+ (["fenced vuln", "prose", "prose"], (1, False)), (["plain clean", "plain clean"], (0, False)),
+ (["fenced clean", "plain vuln"], (1, False)), (["prose", "prose", "fenced clean", "fenced clean"], (0, True)),
+])
+def test_two_legs(monkeypatch, replies, expected):
+ assert _review(monkeypatch, replies, dual=True)[:2] == expected
def test_the_reason_is_logged(monkeypatch):