mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-26 17:38:36 +00:00
fix(codex): send prompt_cache_retention 24h for the GPT-5.5 family
OpenAI documents GPT-5.5 / GPT-5.5 Pro as extended-cache-only: in-memory prompt cache retention is not available for them, and only prompt_cache_retention: "24h" is supported. Responses requests that omit the field see near-zero cached_tokens even with a stable prompt_cache_key and identical prefixes (observed on an OpenAI-compatible Responses relay: 0 cached across repeated identical calls before; 97% cache reads after). Send the field for the gpt-5.5 model family (bare and namespaced ids like openai.gpt-5.5) on OpenAI-compatible Responses routes, mirrored in the auxiliary Codex adapter, and pass it through preflight normalization. Skipped for xAI, GitHub/Copilot, and the chatgpt.com Codex backend, which reject or ignore body-level cache fields.
This commit is contained in:
parent
98470ae33b
commit
339be21542
5 changed files with 86 additions and 4 deletions
|
|
@ -1058,7 +1058,10 @@ class _CodexCompletionsAdapter:
|
|||
# key in extra_body (not top-level) and GitHub/Copilot Responses opts
|
||||
# out of cache-key routing entirely — for those hosts, skip it here.
|
||||
try:
|
||||
from agent.transports.codex import _content_cache_key
|
||||
from agent.transports.codex import (
|
||||
_content_cache_key,
|
||||
_prompt_cache_retention_for_model,
|
||||
)
|
||||
from utils import base_url_host_matches
|
||||
|
||||
_host_src = str(getattr(self._client, "base_url", "") or "")
|
||||
|
|
@ -1068,6 +1071,10 @@ class _CodexCompletionsAdapter:
|
|||
_cache_key = _content_cache_key(instructions, resp_kwargs.get("tools"))
|
||||
if _cache_key:
|
||||
resp_kwargs["prompt_cache_key"] = _cache_key
|
||||
if not _is_xai and not _is_github and "prompt_cache_retention" not in resp_kwargs:
|
||||
_cache_retention = _prompt_cache_retention_for_model(model)
|
||||
if _cache_retention:
|
||||
resp_kwargs["prompt_cache_retention"] = _cache_retention
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"Codex auxiliary: prompt_cache_key derivation skipped", exc_info=True
|
||||
|
|
|
|||
|
|
@ -912,7 +912,8 @@ def _preflight_codex_api_kwargs(
|
|||
allowed_keys = {
|
||||
"model", "instructions", "input", "tools", "store",
|
||||
"reasoning", "include", "max_output_tokens", "temperature",
|
||||
"tool_choice", "parallel_tool_calls", "prompt_cache_key", "service_tier",
|
||||
"tool_choice", "parallel_tool_calls", "prompt_cache_key",
|
||||
"prompt_cache_retention", "service_tier",
|
||||
"extra_headers", "extra_body", "timeout",
|
||||
}
|
||||
normalized: Dict[str, Any] = {
|
||||
|
|
@ -950,8 +951,13 @@ def _preflight_codex_api_kwargs(
|
|||
if isinstance(temperature, (int, float)):
|
||||
normalized["temperature"] = float(temperature)
|
||||
|
||||
# Pass through tool_choice, parallel_tool_calls, prompt_cache_key
|
||||
for passthrough_key in ("tool_choice", "parallel_tool_calls", "prompt_cache_key"):
|
||||
# Pass through cache routing/retention and tool-dispatch hints.
|
||||
for passthrough_key in (
|
||||
"tool_choice",
|
||||
"parallel_tool_calls",
|
||||
"prompt_cache_key",
|
||||
"prompt_cache_retention",
|
||||
):
|
||||
val = api_kwargs.get(passthrough_key)
|
||||
if val is not None:
|
||||
normalized[passthrough_key] = val
|
||||
|
|
|
|||
|
|
@ -27,6 +27,23 @@ def _bounded_prompt_cache_key(value: Any) -> Optional[str]:
|
|||
return f"pck_{digest}"
|
||||
|
||||
|
||||
def _prompt_cache_retention_for_model(model: str) -> Optional[str]:
|
||||
"""Return the OpenAI Responses prompt-cache retention policy for models
|
||||
that require an explicit policy.
|
||||
|
||||
OpenAI documents GPT-5.5 / GPT-5.5 Pro as extended-cache-only
|
||||
(``prompt_cache_retention: "24h"``). Sending the field only for
|
||||
those model families keeps older/OpenAI-compatible relays on their
|
||||
default behavior.
|
||||
"""
|
||||
normalized = str(model or "").lower().replace("_", "-")
|
||||
# Custom relays commonly prefix provider namespaces, e.g.
|
||||
# ``openai.gpt-5.5``. Match both bare and namespaced model ids.
|
||||
if normalized.endswith("gpt-5.5") or "gpt-5.5-" in normalized:
|
||||
return "24h"
|
||||
return None
|
||||
|
||||
|
||||
def _content_cache_key(instructions: str, tools: Optional[List[Dict[str, Any]]]) -> Optional[str]:
|
||||
"""Content-address the prompt cache key from the static request prefix.
|
||||
|
||||
|
|
@ -284,6 +301,15 @@ class ResponsesApiTransport(ProviderTransport):
|
|||
if not is_github_responses and not is_xai_responses and cache_key:
|
||||
kwargs["prompt_cache_key"] = cache_key
|
||||
|
||||
cache_retention = _prompt_cache_retention_for_model(model)
|
||||
if (
|
||||
cache_retention
|
||||
and not is_github_responses
|
||||
and not is_xai_responses
|
||||
and not is_codex_backend
|
||||
):
|
||||
kwargs.setdefault("prompt_cache_retention", cache_retention)
|
||||
|
||||
if reasoning_enabled and is_xai_responses:
|
||||
from agent.model_metadata import grok_supports_reasoning_effort
|
||||
|
||||
|
|
|
|||
|
|
@ -4958,6 +4958,29 @@ class TestCodexAdapterPromptCacheKey:
|
|||
])
|
||||
assert "prompt_cache_key" not in captured
|
||||
|
||||
def test_gpt55_sets_prompt_cache_retention(self):
|
||||
adapter, captured = self._build_adapter(base_url="https://bedrock-mantle.us-east-1.api.aws/openai/v1")
|
||||
adapter.create(messages=[
|
||||
{"role": "system", "content": "SYS"},
|
||||
{"role": "user", "content": "hi"},
|
||||
])
|
||||
assert captured["prompt_cache_retention"] == "24h"
|
||||
|
||||
def test_prompt_cache_retention_skipped_for_xai_and_github_hosts(self):
|
||||
adapter, captured = self._build_adapter(base_url="https://api.x.ai/v1")
|
||||
adapter.create(messages=[
|
||||
{"role": "system", "content": "SYS"},
|
||||
{"role": "user", "content": "hi"},
|
||||
])
|
||||
assert "prompt_cache_retention" not in captured
|
||||
|
||||
adapter, captured = self._build_adapter(base_url="https://api.githubcopilot.com")
|
||||
adapter.create(messages=[
|
||||
{"role": "system", "content": "SYS"},
|
||||
{"role": "user", "content": "hi"},
|
||||
])
|
||||
assert "prompt_cache_retention" not in captured
|
||||
|
||||
|
||||
class TestCodexAdapterGithubResponsesMessageIdDrop:
|
||||
"""_CodexCompletionsAdapter must drop codex_message_items ``id`` when
|
||||
|
|
|
|||
|
|
@ -243,6 +243,26 @@ class TestCodexBuildKwargs:
|
|||
message_item = next(item for item in kw["input"] if item.get("type") == "message")
|
||||
assert message_item["id"] == "msg_short_id"
|
||||
|
||||
def test_gpt55_sets_prompt_cache_retention(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
kw = transport.build_kwargs(
|
||||
model="openai.gpt-5.5", messages=messages, tools=[],
|
||||
session_id="test-session",
|
||||
)
|
||||
assert kw["prompt_cache_retention"] == "24h"
|
||||
|
||||
def test_gpt55_prompt_cache_retention_skipped_for_known_incompatible_backends(self, transport):
|
||||
messages = [{"role": "user", "content": "Hi"}]
|
||||
assert "prompt_cache_retention" not in transport.build_kwargs(
|
||||
model="gpt-5.5", messages=messages, tools=[], is_xai_responses=True
|
||||
)
|
||||
assert "prompt_cache_retention" not in transport.build_kwargs(
|
||||
model="gpt-5.5", messages=messages, tools=[], is_github_responses=True
|
||||
)
|
||||
assert "prompt_cache_retention" not in transport.build_kwargs(
|
||||
model="gpt-5.5", messages=messages, tools=[], is_codex_backend=True
|
||||
)
|
||||
|
||||
def test_xai_responses_sends_cache_key_via_extra_body(self, transport):
|
||||
"""xAI's Responses API documents ``prompt_cache_key`` as the
|
||||
body-level cache-routing key (the ``x-grok-conv-id`` header is
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue