mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-22 16:25:58 +00:00
eb6aa03609
* feat(analytics): record auxiliary model usage per task in session accounting Auxiliary LLM calls (vision, compression, title_generation, web_extract, session_search, ...) discarded their token usage, leaving dashboard analytics blind to aux model spend (issue #23270). - hermes_state.py: session_model_usage gains a task PK dimension (''=main loop) via v22 table-rebuild migration (SQLite can't alter a PK); record_auxiliary_usage() writes per-(model,provider,task) deltas WITHOUT touching sessions counters (gateway overwrites those with absolute main-loop totals — folding aux in would double-count or be clobbered). Aux rows never inherit the session's main-loop route. - agent/aux_accounting.py: ContextVar ambient accounting context (mirrors the portal_tags conversation context); record_aux_usage() normalizes usage via usage_pricing.normalize_usage, estimates cost, and is strictly best-effort. moa_reference/moa_aggregator excluded — conversation_loop already folds MoA usage+cost into the main delta. - agent/auxiliary_client.py: _validate_llm_response is the recording chokepoint — every successful non-streaming aux response passes through it exactly once, sync and async, including fallback paths (model read from the response itself stays accurate across fallbacks). - run_agent.py: run_conversation publishes/resets the accounting context; agent/title_generator.py republishes on its bare thread. - hermes_cli/web_server.py: /api/analytics/usage folds aux rows into by_model (aux-only models finally appear) and adds a by_task summary; /api/analytics/models surfaces aux rows on the Models page. Design per review of PR #62850 by @eeksock (thread-local + separate auxiliary_usage table): rebuilt on ContextVar (async-safe — thread-local cross-attributes concurrent coroutines on one event loop) and the existing session_model_usage table instead of a parallel accounting path, extended beyond vision to every aux task, and wired the analytics endpoints so the dashboard actually shows it. Credit to @eeksock for the approach and @tboatman for the detailed root-cause analysis. * test(moa): match _validate_llm_response mock to new accounting-hint signature * test(aux): accept accounting-hint kwargs in remaining _validate_llm_response mocks
246 lines
11 KiB
Python
246 lines
11 KiB
Python
"""Regression tests for the universal "unsupported temperature" retry in
|
|
``agent.auxiliary_client``.
|
|
|
|
Auxiliary callers (context compression, session search,
|
|
web extract summarisation, etc.) hardcode ``temperature=0.3`` for historical
|
|
reasons. Several provider/model combinations reject ``temperature`` with a
|
|
400:
|
|
|
|
* OpenAI Responses (gpt-5/o-series reasoning models)
|
|
* Copilot Responses (reasoning models)
|
|
* OpenRouter reasoning models (gpt-5.5, some anthropic via OAI-compat)
|
|
* Anthropic Opus 4.7+ via OpenAI-compat endpoints
|
|
* Kimi/Moonshot (server-managed)
|
|
|
|
``_fixed_temperature_for_model`` catches Kimi up front, and
|
|
``build_chat_completion_kwargs`` drops temperature for Anthropic Opus 4.7+,
|
|
but the same backend can accept ``temperature`` for some models and reject
|
|
it for others (for example gpt-5.4 accepts but gpt-5.5 rejects on the same
|
|
endpoint). An allow/deny-list is not maintainable across providers.
|
|
|
|
The universal fix is reactive: when a call returns an
|
|
``Unsupported parameter: temperature`` 400, retry once without temperature.
|
|
These tests lock in that behaviour for both sync and async paths.
|
|
"""
|
|
|
|
from unittest.mock import patch, MagicMock, AsyncMock
|
|
|
|
import pytest
|
|
|
|
from agent.auxiliary_client import (
|
|
call_llm,
|
|
async_call_llm,
|
|
_is_unsupported_temperature_error,
|
|
)
|
|
|
|
|
|
class TestIsUnsupportedTemperatureError:
|
|
"""The detector must match the phrasings providers actually return."""
|
|
|
|
@pytest.mark.parametrize("message", [
|
|
# OpenAI / Codex Responses
|
|
"HTTP 400: Unsupported parameter: temperature",
|
|
"Error code: 400 - {'error': {'message': \"Unsupported parameter: 'temperature'\"}}",
|
|
# Copilot / OpenAI error-code form
|
|
"Error code: 400 - {'error': {'code': 'unsupported_parameter', 'param': 'temperature'}}",
|
|
# OpenRouter-style
|
|
"Provider returned error: temperature is not supported for this model",
|
|
"this model does not support temperature",
|
|
# Anthropic-style via OAI-compat
|
|
"temperature: unknown parameter",
|
|
# Some gateways
|
|
"unrecognized request argument supplied: temperature",
|
|
])
|
|
def test_matches_real_provider_messages(self, message):
|
|
assert _is_unsupported_temperature_error(RuntimeError(message)) is True
|
|
|
|
@pytest.mark.parametrize("message", [
|
|
# Unrelated 400s must NOT trigger a silent-retry
|
|
"HTTP 400: Invalid value: 'tool'. Supported values are: 'assistant'...",
|
|
"max_tokens is too large for this model",
|
|
"Rate limit exceeded",
|
|
"Connection reset by peer",
|
|
# Temperature value error is a different class of problem
|
|
"temperature must be between 0 and 2",
|
|
])
|
|
def test_does_not_match_unrelated_errors(self, message):
|
|
assert _is_unsupported_temperature_error(RuntimeError(message)) is False
|
|
|
|
|
|
def _dummy_response():
|
|
# The real code calls _validate_llm_response which inspects
|
|
# response.choices[0].message. The tests here patch that out, so
|
|
# any sentinel object is fine.
|
|
return {"ok": True}
|
|
|
|
|
|
class TestCallLlmUnsupportedTemperatureRetry:
|
|
"""``call_llm`` retries once without temperature and returns on success."""
|
|
|
|
def _setup(self, first_exc):
|
|
client = MagicMock()
|
|
client.base_url = "https://api.openai.com/v1"
|
|
client.chat.completions.create.side_effect = [first_exc, _dummy_response()]
|
|
return client
|
|
|
|
@pytest.mark.parametrize("error_message", [
|
|
"HTTP 400: Unsupported parameter: temperature",
|
|
"Error code: 400 - {'error': {'code': 'unsupported_parameter', 'param': 'temperature'}}",
|
|
"Provider error: this model does not support temperature",
|
|
])
|
|
def test_retries_once_without_temperature(self, error_message):
|
|
client = self._setup(RuntimeError(error_message))
|
|
|
|
with (
|
|
patch("agent.auxiliary_client._resolve_task_provider_model",
|
|
return_value=("openai-codex", "gpt-5.5", None, None, None)),
|
|
patch("agent.auxiliary_client._get_cached_client",
|
|
return_value=(client, "gpt-5.5")),
|
|
patch("agent.auxiliary_client._validate_llm_response",
|
|
side_effect=lambda resp, _task, **_kw: resp),
|
|
):
|
|
result = call_llm(
|
|
task="compression",
|
|
messages=[{"role": "user", "content": "remember this"}],
|
|
temperature=0.3,
|
|
max_tokens=500,
|
|
)
|
|
|
|
assert result == {"ok": True}
|
|
assert client.chat.completions.create.call_count == 2
|
|
first_kwargs = client.chat.completions.create.call_args_list[0].kwargs
|
|
retry_kwargs = client.chat.completions.create.call_args_list[1].kwargs
|
|
assert first_kwargs["temperature"] == 0.3
|
|
assert "temperature" not in retry_kwargs
|
|
# max_tokens is intentionally omitted on OpenAI-compatible endpoints
|
|
# (#34530) — auxiliary calls let the model max out its own output — so
|
|
# it must be absent in BOTH the first and retry kwargs. Use a kwarg that
|
|
# actually survives (model) to prove the retry preserves the rest.
|
|
assert "max_tokens" not in first_kwargs
|
|
assert "max_tokens" not in retry_kwargs
|
|
assert retry_kwargs["model"] == first_kwargs["model"]
|
|
|
|
def test_non_temperature_400_does_not_retry_as_temperature(self):
|
|
"""Unrelated 400s (e.g. bad tool role) must not silently drop temp."""
|
|
client = MagicMock()
|
|
client.base_url = "https://api.openai.com/v1"
|
|
non_temp_err = RuntimeError(
|
|
"HTTP 400: Invalid value: 'tool'. Supported values are: 'assistant'..."
|
|
)
|
|
client.chat.completions.create.side_effect = non_temp_err
|
|
|
|
with (
|
|
patch("agent.auxiliary_client._resolve_task_provider_model",
|
|
return_value=("openai-codex", "gpt-5.5", None, None, None)),
|
|
patch("agent.auxiliary_client._get_cached_client",
|
|
return_value=(client, "gpt-5.5")),
|
|
patch("agent.auxiliary_client._validate_llm_response",
|
|
side_effect=lambda resp, _task, **_kw: resp),
|
|
patch("agent.auxiliary_client._try_payment_fallback",
|
|
return_value=None),
|
|
):
|
|
with pytest.raises(RuntimeError, match="Invalid value"):
|
|
call_llm(
|
|
task="compression",
|
|
messages=[{"role": "user", "content": "x"}],
|
|
temperature=0.3,
|
|
max_tokens=500,
|
|
)
|
|
# Should NOT have retried (non-temperature 400 doesn't match)
|
|
assert client.chat.completions.create.call_count == 1
|
|
|
|
def test_no_retry_when_temperature_not_in_kwargs(self):
|
|
"""If caller didn't send temperature, don't invent a temperature-retry."""
|
|
client = MagicMock()
|
|
client.base_url = "https://api.openai.com/v1"
|
|
# Provider complains about temperature even though we didn't send it.
|
|
# (Pathological but possible with misleading error text.) The guard
|
|
# ``"temperature" in kwargs`` must prevent an unnecessary retry.
|
|
err = RuntimeError("HTTP 400: Unsupported parameter: temperature")
|
|
client.chat.completions.create.side_effect = err
|
|
|
|
with (
|
|
patch("agent.auxiliary_client._resolve_task_provider_model",
|
|
return_value=("openai-codex", "gpt-5.5", None, None, None)),
|
|
patch("agent.auxiliary_client._get_cached_client",
|
|
return_value=(client, "gpt-5.5")),
|
|
patch("agent.auxiliary_client._validate_llm_response",
|
|
side_effect=lambda resp, _task, **_kw: resp),
|
|
patch("agent.auxiliary_client._try_payment_fallback",
|
|
return_value=None),
|
|
):
|
|
with pytest.raises(RuntimeError):
|
|
call_llm(
|
|
task="compression",
|
|
messages=[{"role": "user", "content": "x"}],
|
|
temperature=None, # explicit: no temperature sent
|
|
max_tokens=500,
|
|
)
|
|
assert client.chat.completions.create.call_count == 1
|
|
|
|
|
|
class TestAsyncCallLlmUnsupportedTemperatureRetry:
|
|
"""``async_call_llm`` mirror of the sync retry semantics."""
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_retries_once_without_temperature(self):
|
|
client = MagicMock()
|
|
client.base_url = "https://api.openai.com/v1"
|
|
client.chat.completions.create = AsyncMock(side_effect=[
|
|
RuntimeError("HTTP 400: Unsupported parameter: temperature"),
|
|
_dummy_response(),
|
|
])
|
|
|
|
with (
|
|
patch("agent.auxiliary_client._resolve_task_provider_model",
|
|
return_value=("openai-codex", "gpt-5.5", None, None, None)),
|
|
patch("agent.auxiliary_client._get_cached_client",
|
|
return_value=(client, "gpt-5.5")),
|
|
patch("agent.auxiliary_client._validate_llm_response",
|
|
side_effect=lambda resp, _task, **_kw: resp),
|
|
):
|
|
result = await async_call_llm(
|
|
task="session_search",
|
|
messages=[{"role": "user", "content": "query"}],
|
|
temperature=0.3,
|
|
max_tokens=500,
|
|
)
|
|
|
|
assert result == {"ok": True}
|
|
assert client.chat.completions.create.await_count == 2
|
|
first_kwargs = client.chat.completions.create.call_args_list[0].kwargs
|
|
retry_kwargs = client.chat.completions.create.call_args_list[1].kwargs
|
|
assert first_kwargs["temperature"] == 0.3
|
|
assert "temperature" not in retry_kwargs
|
|
# max_tokens is intentionally omitted on OpenAI-compatible endpoints
|
|
# (#34530); assert it's absent and that model survives the retry.
|
|
assert "max_tokens" not in first_kwargs
|
|
assert "max_tokens" not in retry_kwargs
|
|
assert retry_kwargs["model"] == first_kwargs["model"]
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_async_non_temperature_400_does_not_retry(self):
|
|
client = MagicMock()
|
|
client.base_url = "https://api.openai.com/v1"
|
|
client.chat.completions.create = AsyncMock(
|
|
side_effect=RuntimeError("HTTP 400: Invalid value: 'tool'"),
|
|
)
|
|
|
|
with (
|
|
patch("agent.auxiliary_client._resolve_task_provider_model",
|
|
return_value=("openai-codex", "gpt-5.5", None, None, None)),
|
|
patch("agent.auxiliary_client._get_cached_client",
|
|
return_value=(client, "gpt-5.5")),
|
|
patch("agent.auxiliary_client._validate_llm_response",
|
|
side_effect=lambda resp, _task, **_kw: resp),
|
|
patch("agent.auxiliary_client._try_payment_fallback",
|
|
return_value=None),
|
|
):
|
|
with pytest.raises(RuntimeError, match="Invalid value"):
|
|
await async_call_llm(
|
|
task="session_search",
|
|
messages=[{"role": "user", "content": "x"}],
|
|
temperature=0.3,
|
|
max_tokens=500,
|
|
)
|
|
assert client.chat.completions.create.await_count == 1
|