mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-20 15:33:54 +00:00
fix(usage): read DeepSeek's native prompt_cache_hit_tokens cache field (#65678)
Some checks are pending
CI / Detect affected areas (push) Waiting to run
CI / Python tests (push) Blocked by required conditions
CI / Python lints (push) Blocked by required conditions
CI / JS & TS checks (push) Blocked by required conditions
CI / Docs Site (push) Blocked by required conditions
CI / Deny unrelated histories (push) Blocked by required conditions
CI / Check contributors (push) Blocked by required conditions
CI / Check uv.lock (push) Blocked by required conditions
CI / package-lock.json diff (push) Blocked by required conditions
CI / Lint Docker scripts (push) Blocked by required conditions
CI / Build&Test Docker image (push) Blocked by required conditions
CI / Supply-chain scan (push) Blocked by required conditions
CI / OSV scan (push) Waiting to run
CI / All required checks pass (push) Blocked by required conditions
CI / CI timing report (push) Blocked by required conditions
Deploy Site / deploy-vercel (push) Waiting to run
Deploy Site / deploy-docs (push) Waiting to run
auto-fix lint issues & formatting / Generate eslint --fix patch (push) Waiting to run
auto-fix lint issues & formatting / Apply patch (push) Blocked by required conditions
Some checks are pending
CI / Detect affected areas (push) Waiting to run
CI / Python tests (push) Blocked by required conditions
CI / Python lints (push) Blocked by required conditions
CI / JS & TS checks (push) Blocked by required conditions
CI / Docs Site (push) Blocked by required conditions
CI / Deny unrelated histories (push) Blocked by required conditions
CI / Check contributors (push) Blocked by required conditions
CI / Check uv.lock (push) Blocked by required conditions
CI / package-lock.json diff (push) Blocked by required conditions
CI / Lint Docker scripts (push) Blocked by required conditions
CI / Build&Test Docker image (push) Blocked by required conditions
CI / Supply-chain scan (push) Blocked by required conditions
CI / OSV scan (push) Waiting to run
CI / All required checks pass (push) Blocked by required conditions
CI / CI timing report (push) Blocked by required conditions
Deploy Site / deploy-vercel (push) Waiting to run
Deploy Site / deploy-docs (push) Waiting to run
auto-fix lint issues & formatting / Generate eslint --fix patch (push) Waiting to run
auto-fix lint issues & formatting / Apply patch (push) Blocked by required conditions
DeepSeek's own API (api.deepseek.com) reports context-cache hits as top-level usage.prompt_cache_hit_tokens / prompt_cache_miss_tokens (prompt_tokens = hit + miss), not the OpenAI nested prompt_tokens_details.cached_tokens shape. Neither normalize_usage() nor the chat_completions transport's extract_cache_stats() read those fields, so direct DeepSeek sessions always showed 0 cache-hit tokens: invisible in accounting, mis-billed at the full input rate, and 0% cache display. Both layers now fall back to prompt_cache_hit_tokens when the nested shape is absent; the nested value wins when both are present (proxies). Fixes #61871.
This commit is contained in:
parent
7edaaf4682
commit
03c0b00f45
4 changed files with 82 additions and 5 deletions
|
|
@ -776,15 +776,18 @@ class ChatCompletionsTransport(ProviderTransport):
|
|||
return True
|
||||
|
||||
def extract_cache_stats(self, response: Any) -> dict[str, int] | None:
|
||||
"""Extract OpenRouter/OpenAI cache stats from prompt_tokens_details."""
|
||||
"""Extract cache stats from prompt_tokens_details (OpenRouter/OpenAI)
|
||||
or DeepSeek's native top-level prompt_cache_hit_tokens field."""
|
||||
usage = getattr(response, "usage", None)
|
||||
if usage is None:
|
||||
return None
|
||||
details = getattr(usage, "prompt_tokens_details", None)
|
||||
if details is None:
|
||||
return None
|
||||
cached = getattr(details, "cached_tokens", 0) or 0
|
||||
written = getattr(details, "cache_write_tokens", 0) or 0
|
||||
cached = getattr(details, "cached_tokens", 0) or 0 if details else 0
|
||||
written = getattr(details, "cache_write_tokens", 0) or 0 if details else 0
|
||||
if not cached:
|
||||
# DeepSeek native API shape (api.deepseek.com): top-level
|
||||
# prompt_cache_hit_tokens / prompt_cache_miss_tokens (#61871).
|
||||
cached = getattr(usage, "prompt_cache_hit_tokens", 0) or 0
|
||||
if cached or written:
|
||||
return {"cached_tokens": cached, "creation_tokens": written}
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1074,6 +1074,15 @@ def normalize_usage(
|
|||
cache_read_tokens = _to_int(getattr(details, "cached_tokens", 0) if details else 0)
|
||||
if not cache_read_tokens:
|
||||
cache_read_tokens = _to_int(getattr(response_usage, "cache_read_input_tokens", 0))
|
||||
if not cache_read_tokens:
|
||||
# DeepSeek's native API (api.deepseek.com) reports context-cache
|
||||
# hits as top-level prompt_cache_hit_tokens (+ the complementary
|
||||
# prompt_cache_miss_tokens; prompt_tokens = hit + miss), not the
|
||||
# OpenAI nested shape. Without this, direct DeepSeek sessions
|
||||
# always showed 0 cache-hit tokens (#61871).
|
||||
cache_read_tokens = _to_int(
|
||||
getattr(response_usage, "prompt_cache_hit_tokens", 0)
|
||||
)
|
||||
cache_write_tokens = _to_int(
|
||||
getattr(details, "cache_write_tokens", 0) if details else 0
|
||||
)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,44 @@ def test_normalize_usage_openai_subtracts_cached_prompt_tokens():
|
|||
assert normalized.output_tokens == 700
|
||||
|
||||
|
||||
def test_normalize_usage_reads_deepseek_native_cache_hit_tokens():
|
||||
"""DeepSeek's native API (api.deepseek.com) reports context-cache hits as
|
||||
top-level prompt_cache_hit_tokens / prompt_cache_miss_tokens (with
|
||||
prompt_tokens = hit + miss), not OpenAI's nested
|
||||
prompt_tokens_details.cached_tokens. Before this fix, direct DeepSeek
|
||||
sessions always normalized to cache_read_tokens=0 — cache hits were
|
||||
invisible in accounting and billed at the full input rate (#61871)."""
|
||||
usage = SimpleNamespace(
|
||||
prompt_tokens=2000,
|
||||
completion_tokens=400,
|
||||
prompt_cache_hit_tokens=1500,
|
||||
prompt_cache_miss_tokens=500,
|
||||
)
|
||||
|
||||
normalized = normalize_usage(usage, provider="deepseek", api_mode="chat_completions")
|
||||
|
||||
assert normalized.cache_read_tokens == 1500
|
||||
# prompt_tokens includes cached; input = 2000 - 1500 = the miss bucket
|
||||
assert normalized.input_tokens == 500
|
||||
assert normalized.output_tokens == 400
|
||||
|
||||
|
||||
def test_normalize_usage_nested_details_win_over_deepseek_top_level():
|
||||
"""When a proxy forwards both shapes, the OpenAI nested value wins and
|
||||
the DeepSeek top-level field is not double-read."""
|
||||
usage = SimpleNamespace(
|
||||
prompt_tokens=2000,
|
||||
completion_tokens=100,
|
||||
prompt_tokens_details=SimpleNamespace(cached_tokens=900),
|
||||
prompt_cache_hit_tokens=1500,
|
||||
)
|
||||
|
||||
normalized = normalize_usage(usage, provider="deepseek", api_mode="chat_completions")
|
||||
|
||||
assert normalized.cache_read_tokens == 900
|
||||
assert normalized.input_tokens == 1100
|
||||
|
||||
|
||||
def test_normalize_usage_openai_reads_top_level_anthropic_cache_fields():
|
||||
"""Some OpenAI-compatible proxies (OpenRouter, Cline) expose
|
||||
Anthropic-style cache token counts at the top level of the usage object when
|
||||
|
|
|
|||
|
|
@ -1115,6 +1115,33 @@ class TestChatCompletionsCacheStats:
|
|||
result = transport.extract_cache_stats(r)
|
||||
assert result == {"cached_tokens": 500, "creation_tokens": 100}
|
||||
|
||||
def test_deepseek_native_top_level_cache_hit_tokens(self, transport):
|
||||
"""DeepSeek's native API (api.deepseek.com) reports cache hits as
|
||||
top-level prompt_cache_hit_tokens, not the OpenAI nested shape —
|
||||
the extractor must read it or direct DeepSeek sessions show 0%
|
||||
cache hit rate (#61871)."""
|
||||
r = SimpleNamespace(
|
||||
usage=SimpleNamespace(
|
||||
prompt_tokens_details=None,
|
||||
prompt_cache_hit_tokens=1500,
|
||||
prompt_cache_miss_tokens=500,
|
||||
)
|
||||
)
|
||||
result = transport.extract_cache_stats(r)
|
||||
assert result == {"cached_tokens": 1500, "creation_tokens": 0}
|
||||
|
||||
def test_nested_details_win_over_deepseek_top_level(self, transport):
|
||||
"""When both shapes are present, the OpenAI nested value wins."""
|
||||
details = SimpleNamespace(cached_tokens=800, cache_write_tokens=0)
|
||||
r = SimpleNamespace(
|
||||
usage=SimpleNamespace(
|
||||
prompt_tokens_details=details,
|
||||
prompt_cache_hit_tokens=1500,
|
||||
)
|
||||
)
|
||||
result = transport.extract_cache_stats(r)
|
||||
assert result == {"cached_tokens": 800, "creation_tokens": 0}
|
||||
|
||||
|
||||
class TestChatCompletionsGeminiNativeExtraBodyStrip:
|
||||
"""Profile extra_body (e.g. Nous portal tags) must not reach a native
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue