From 8b797f7a7b0c12141f0d67997e43cfb2e16d3fc7 Mon Sep 17 00:00:00 2001 From: Lavya Tandel Date: Sat, 4 Jul 2026 00:05:52 +0530 Subject: [PATCH] fix(prompt-caching): skip invalid top-level cache_control on empty assistant/tool messages on OpenRouter - role:tool no longer gets top-level cache_control on OpenRouter - empty/None assistant turns skip useless marker - non-empty tool content wrapped so marker lands on a content part - preserves native Anthropic behavior --- agent/prompt_caching.py | 44 ++++++++++++++-- tests/agent/test_prompt_caching.py | 83 ++++++++++++++++++++++++++---- 2 files changed, 114 insertions(+), 13 deletions(-) diff --git a/agent/prompt_caching.py b/agent/prompt_caching.py index a73d6e113d9..91b368c86b0 100644 --- a/agent/prompt_caching.py +++ b/agent/prompt_caching.py @@ -17,12 +17,23 @@ def _apply_cache_marker(msg: dict, cache_marker: dict, native_anthropic: bool = role = msg.get("role", "") content = msg.get("content") - if role == "tool": - if native_anthropic: - msg["cache_control"] = cache_marker + if role == "tool" and native_anthropic: + # Native Anthropic layout: top-level marker; the adapter moves it + # inside the tool_result block. + msg["cache_control"] = cache_marker return if content is None or content == "": + if role == "tool" and not native_anthropic: + # OpenRouter rejects top-level cache_control on role:tool (silent + # hang) and an empty message has no content part to carry the + # marker — skip. Non-empty tool content falls through below and + # gets the marker on a content part, which OpenRouter honors. + return + if role == "assistant" and not native_anthropic: + # Empty assistant turns are pure tool_calls. A top-level marker + # here is ignored on the envelope layout, so skip. + return msg["cache_control"] = cache_marker return @@ -38,6 +49,26 @@ def _apply_cache_marker(msg: dict, cache_marker: dict, native_anthropic: bool = last["cache_control"] = cache_marker +def _can_carry_marker(msg: dict, native_anthropic: bool) -> bool: + """True if a marker on this message is actually honored by the provider. + + On the native Anthropic layout every message works (top-level markers are + relocated by the adapter). On the envelope layout (OpenRouter et al.) only + markers inside content parts are honored: empty-content messages (e.g. + assistant turns that are pure tool_calls) and empty tool messages would + receive a top-level marker the provider ignores — wasting one of the four + breakpoints. Skip those so the breakpoints land on messages that count. + """ + if native_anthropic: + return True + content = msg.get("content") + if content is None or content == "": + return False + if isinstance(content, list): + return any(isinstance(part, dict) for part in content) + return isinstance(content, str) + + def _build_marker(ttl: str) -> Dict[str, str]: """Build a cache_control marker dict for the given TTL ('5m' or '1h').""" marker: Dict[str, str] = {"type": "ephemeral"} @@ -72,7 +103,12 @@ def apply_anthropic_cache_control( breakpoints_used += 1 remaining = 4 - breakpoints_used - non_sys = [i for i in range(len(messages)) if messages[i].get("role") != "system"] + non_sys = [ + i + for i in range(len(messages)) + if messages[i].get("role") != "system" + and _can_carry_marker(messages[i], native_anthropic=native_anthropic) + ] for idx in non_sys[-remaining:]: _apply_cache_marker(messages[idx], marker, native_anthropic=native_anthropic) diff --git a/tests/agent/test_prompt_caching.py b/tests/agent/test_prompt_caching.py index 499ffc765a4..8af18c02793 100644 --- a/tests/agent/test_prompt_caching.py +++ b/tests/agent/test_prompt_caching.py @@ -3,6 +3,7 @@ from agent.prompt_caching import ( _apply_cache_marker, + _can_carry_marker, apply_anthropic_cache_control, ) @@ -23,18 +24,37 @@ class TestApplyCacheMarker: _apply_cache_marker(msg, MARKER, native_anthropic=False) assert "cache_control" not in msg - def test_none_content_gets_top_level_marker(self): - msg = {"role": "assistant", "content": None} - _apply_cache_marker(msg, MARKER) + def test_tool_message_wraps_non_empty_content_on_openrouter(self): + """Non-empty tool content should be wrapped so the marker lands on a content part.""" + msg = {"role": "tool", "content": "tool result bytes"} + _apply_cache_marker(msg, MARKER, native_anthropic=False) + assert "cache_control" not in msg + assert isinstance(msg["content"], list) + assert msg["content"][0]["cache_control"] == MARKER + + def test_empty_assistant_message_skips_marker_on_openrouter(self): + """OpenRouter path: empty assistant turns are pure tool_calls, marker would be ignored.""" + msg = {"role": "assistant", "content": ""} + _apply_cache_marker(msg, MARKER, native_anthropic=False) + assert "cache_control" not in msg + + def test_native_anthropic_empty_assistant_gets_top_level_marker(self): + """Native Anthropic layout can still carry top-level marker on empty content.""" + msg = {"role": "assistant", "content": ""} + _apply_cache_marker(msg, MARKER, native_anthropic=True) assert msg["cache_control"] == MARKER - def test_empty_string_content_gets_top_level_marker(self): - """Empty text blocks cannot have cache_control (Anthropic rejects them).""" - msg = {"role": "assistant", "content": ""} - _apply_cache_marker(msg, MARKER) + def test_none_content_skips_marker_on_openrouter(self): + """OpenRouter path: None-content assistant turns are ignored.""" + msg = {"role": "assistant", "content": None} + _apply_cache_marker(msg, MARKER, native_anthropic=False) + assert "cache_control" not in msg + + def test_none_content_gets_top_level_marker_on_native_anthropic(self): + """Native Anthropic path: None content still gets top-level marker.""" + msg = {"role": "assistant", "content": None} + _apply_cache_marker(msg, MARKER, native_anthropic=True) assert msg["cache_control"] == MARKER - # Must NOT wrap into [{"type": "text", "text": "", "cache_control": ...}] - assert msg["content"] == "" def test_string_content_wrapped_in_list(self): msg = {"role": "user", "content": "Hello"} @@ -63,6 +83,22 @@ class TestApplyCacheMarker: _apply_cache_marker(msg, MARKER) +class TestCanCarryMarker: + def test_native_anthropic_always_true(self): + assert _can_carry_marker({"role": "assistant", "content": ""}, native_anthropic=True) is True + assert _can_carry_marker({"role": "tool", "content": ""}, native_anthropic=True) is True + + def test_openrouter_content_parts_carry_marker(self): + assert _can_carry_marker({"role": "user", "content": "text"}, native_anthropic=False) is True + assert _can_carry_marker({"role": "user", "content": [{"type": "text", "text": "a"}]}, native_anthropic=False) is True + + def test_openrouter_empty_or_none_does_not_carry_marker(self): + assert _can_carry_marker({"role": "assistant", "content": ""}, native_anthropic=False) is False + assert _can_carry_marker({"role": "assistant", "content": None}, native_anthropic=False) is False + assert _can_carry_marker({"role": "tool", "content": "result"}, native_anthropic=False) is True + assert _can_carry_marker({"role": "tool", "content": ""}, native_anthropic=False) is False + + class TestApplyAnthropicCacheControl: def test_empty_messages(self): result = apply_anthropic_cache_control([]) @@ -139,3 +175,32 @@ class TestApplyAnthropicCacheControl: elif "cache_control" in msg: count += 1 assert count <= 4 + + def test_tool_loop_empty_assistant_and_tool_messages_do_not_consume_breakpoints(self): + """Tool loops should keep breakpoints on messages that can carry markers.""" + msgs = [ + {"role": "system", "content": "You are helpful"}, + {"role": "user", "content": "run tool 1", "cache_control": MARKER}, + {"role": "assistant", "content": "", "tool_calls": [{"type": "function"}]}, + {"role": "tool", "content": "tool result 1"}, + {"role": "user", "content": "run tool 2", "cache_control": MARKER}, + {"role": "assistant", "content": "", "tool_calls": [{"type": "function"}]}, + {"role": "tool", "content": "tool result 2"}, + ] + result = apply_anthropic_cache_control(msgs, native_anthropic=False) + # Empty assistant/tool turns should not get broken markers + assert "cache_control" not in result[2] + assert "cache_control" not in result[3] + assert "cache_control" not in result[5] + assert "cache_control" not in result[6] + + def test_tool_message_marker_lands_on_content_part_on_openrouter(self): + """Non-empty tool content should be wrapped so the marker lands on a content part.""" + msgs = [ + {"role": "user", "content": "hello"}, + {"role": "tool", "content": "tool output"}, + ] + result = apply_anthropic_cache_control(msgs, native_anthropic=False) + assert isinstance(result[1]["content"], list) + assert result[1]["content"][0]["cache_control"] == {"type": "ephemeral"} + assert "cache_control" not in result[1]