mirror of
https://github.com/NousResearch/hermes-agent.git
synced 2026-07-21 16:18:55 +00:00
fix(ollama-cloud): capability-gate reasoning_effort + correct disable semantics
Three follow-up fixes to the salvaged reasoning_effort support, all verified
live against ollama.com /v1/chat/completions + /api/show on deepseek-v4-pro,
gemma3, and qwen3-coder:
1. Capability-gate on /api/show 'thinking'. The original ignored the
supports_reasoning flag and emitted reasoning_effort for every model. Now
gated: only models whose native /api/show capabilities list contains
'thinking' (deepseek-v4 yes; gemma3 / qwen3-coder no) get reasoning_effort.
Mirrors the LM Studio pattern — capability resolved once per (model,
base_url) in run_agent._supports_reasoning_extra_body via a cached probe
(hermes_cli.models.ollama_model_supports_thinking), threaded into the
profile hook as supports_reasoning. No live HTTP in the per-request path.
2. Disable actually disables. Ollama Cloud defaults to thinking ON and IGNORES
the extra_body.thinking:{type:disabled} shape (verified: still returned
reasoning). The only working off switch is top-level reasoning_effort:'none'.
The salvaged code returned ({}, {}) for enabled:false / effort:none, leaving
thinking ON. Now emits {'reasoning_effort': 'none'}.
3. Omit unrecognized effort. The original forwarded any unknown string verbatim
including 'minimal' (a real Hermes effort level). Ollama Cloud rejects
unrecognized values with a hard HTTP 400 (accepted set: low/medium/high/
max/none), so forwarding 'minimal' would break the request. Now omitted.
Core touches (run_agent.py, hermes_cli/models.py) add the capability probe;
the plugin profile only consumes the resolved flag. 24/24 profile tests green;
194 provider/transport tests unaffected.
This commit is contained in:
parent
4759362188
commit
5d9a72b7c2
4 changed files with 241 additions and 21 deletions
|
|
@ -3306,6 +3306,54 @@ def lmstudio_model_reasoning_options(
|
|||
return []
|
||||
|
||||
|
||||
def ollama_model_supports_thinking(
|
||||
model: str,
|
||||
base_url: Optional[str],
|
||||
api_key: Optional[str] = None,
|
||||
timeout: float = 5.0,
|
||||
) -> Optional[bool]:
|
||||
"""Return True if an Ollama (Cloud or local) model advertises ``thinking``.
|
||||
|
||||
Probes the native ``/api/show`` endpoint and checks the ``capabilities``
|
||||
list, which Ollama populates from the model's metadata (e.g.
|
||||
``deepseek-v4-pro`` → ``["completion", "tools", "thinking"]`` while
|
||||
``gemma3:27b`` → ``["completion", "vision"]``). This is the authoritative
|
||||
capability source — the OpenAI-compat ``/v1/models`` endpoint omits it.
|
||||
|
||||
Returns:
|
||||
True — the model declares the ``thinking`` capability.
|
||||
False — ``/api/show`` succeeded but the model has no ``thinking`` cap.
|
||||
None — the probe failed (unreachable / non-Ollama / error); the caller
|
||||
decides the fallback (we treat None as "don't emit").
|
||||
"""
|
||||
import httpx
|
||||
|
||||
server_url = (base_url or "").strip().rstrip("/")
|
||||
if server_url.endswith("/v1"):
|
||||
server_url = server_url[:-3]
|
||||
if not server_url:
|
||||
return None
|
||||
|
||||
bare_model = _strip_ollama_cloud_suffix((model or "").strip())
|
||||
if not bare_model:
|
||||
return None
|
||||
|
||||
token = str(api_key or "").strip()
|
||||
headers = {"Authorization": f"Bearer {token}"} if token else {}
|
||||
|
||||
try:
|
||||
with httpx.Client(timeout=timeout, headers=headers) as client:
|
||||
resp = client.post(f"{server_url}/api/show", json={"name": bare_model})
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
caps = resp.json().get("capabilities")
|
||||
if isinstance(caps, list):
|
||||
return "thinking" in caps
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_github_models(api_key: Optional[str] = None, timeout: float = 5.0) -> Optional[list[str]]:
|
||||
catalog = fetch_github_model_catalog(api_key=api_key, timeout=timeout)
|
||||
if not catalog:
|
||||
|
|
|
|||
|
|
@ -30,34 +30,50 @@ class OllamaCloudProfile(ProviderProfile):
|
|||
self,
|
||||
*,
|
||||
reasoning_config: dict | None = None,
|
||||
supports_reasoning: bool = False,
|
||||
**ctx: Any,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
"""Emit top-level ``reasoning_effort`` for Ollama Cloud.
|
||||
"""Emit top-level ``reasoning_effort`` for Ollama Cloud thinking models.
|
||||
|
||||
The ``supports_reasoning`` flag passed by the transport is
|
||||
deliberately ignored — this profile always handles reasoning
|
||||
when ``reasoning_config`` is present.
|
||||
Gated on ``supports_reasoning``, which the transport resolves from the
|
||||
model's native ``/api/show`` ``capabilities`` (``thinking``). Models
|
||||
without the thinking capability (e.g. ``gemma3``, ``qwen3-coder``) get
|
||||
no ``reasoning_effort`` at all — emitting it there is a no-op the API
|
||||
ignores, and gating avoids sending a meaningless field.
|
||||
"""
|
||||
top_level: dict[str, Any] = {}
|
||||
|
||||
if not supports_reasoning:
|
||||
return {}, {}
|
||||
|
||||
if reasoning_config and isinstance(reasoning_config, dict):
|
||||
enabled = reasoning_config.get("enabled", True)
|
||||
if enabled is False:
|
||||
return {}, {} # omit → model runs without thinking
|
||||
# Ollama Cloud defaults to thinking ON, and ignores the
|
||||
# extra_body.thinking:{type:disabled} shape (verified live).
|
||||
# The ONLY way to actually suppress thinking on its
|
||||
# /v1/chat/completions endpoint is top-level
|
||||
# reasoning_effort:"none" — omitting the field leaves
|
||||
# thinking on.
|
||||
return {}, {"reasoning_effort": "none"}
|
||||
|
||||
effort = (reasoning_config.get("effort") or "").strip().lower()
|
||||
if not effort:
|
||||
# No explicit effort requested — let the model decide
|
||||
# (Ollama Cloud's server default is thinking ON).
|
||||
return {}, {}
|
||||
if effort == "none":
|
||||
return {}, {} # explicit none → suppress thinking
|
||||
return {}, {"reasoning_effort": "none"} # explicit off switch
|
||||
if effort in ("xhigh", "max", "ultra"):
|
||||
top_level["reasoning_effort"] = "max"
|
||||
elif effort in ("low", "medium", "high"):
|
||||
top_level["reasoning_effort"] = effort
|
||||
else:
|
||||
# Unknown value — forward as-is, let the API decide
|
||||
top_level["reasoning_effort"] = effort
|
||||
# Any other value (including "minimal", which Ollama Cloud's
|
||||
# /v1/chat/completions rejects with HTTP 400 — its accepted set is
|
||||
# {low, medium, high, max, none}) is omitted so the model applies
|
||||
# its own default rather than triggering a hard 400. Matches the
|
||||
# sibling deepseek / opencode-zen profiles, which target the same
|
||||
# backend and omit unrecognized efforts rather than send garbage.
|
||||
|
||||
return {}, top_level
|
||||
|
||||
|
|
|
|||
37
run_agent.py
37
run_agent.py
|
|
@ -5445,6 +5445,12 @@ class AIAgent:
|
|||
opts = self._lmstudio_reasoning_options_cached()
|
||||
# "off-only" (or absent) means no real reasoning capability.
|
||||
return any(opt and opt != "off" for opt in opts)
|
||||
# Ollama Cloud (and any Ollama-compatible server): the native
|
||||
# /api/show capabilities list is authoritative — emit reasoning_effort
|
||||
# only for models that declare the "thinking" capability. deepseek-v4
|
||||
# has it; gemma3 / qwen3-coder don't. Cached per (model, base_url).
|
||||
if base_url_host_matches(self._base_url_lower, "ollama.com"):
|
||||
return self._ollama_supports_thinking_cached()
|
||||
if "openrouter" not in self._base_url_lower:
|
||||
return False
|
||||
if "api.mistral.ai" in self._base_url_lower:
|
||||
|
|
@ -5498,6 +5504,37 @@ class AIAgent:
|
|||
cache[key] = (opts, _time.monotonic())
|
||||
return opts
|
||||
|
||||
def _ollama_supports_thinking_cached(self) -> bool:
|
||||
"""Probe Ollama's ``/api/show`` capabilities once per (model, base_url).
|
||||
|
||||
Returns True only when the model declares the ``thinking`` capability.
|
||||
Caching mirrors the LM Studio probe: a True/False result is permanent
|
||||
(capabilities don't change), while a probe failure (None) is cached
|
||||
with a 60-second TTL so a transient outage doesn't suppress reasoning
|
||||
for the rest of the session but also doesn't round-trip every turn.
|
||||
"""
|
||||
import time as _time
|
||||
|
||||
cache = getattr(self, "_ollama_thinking_cache", None)
|
||||
if cache is None:
|
||||
cache = self._ollama_thinking_cache = {}
|
||||
key = (self.model, self.base_url)
|
||||
cached = cache.get(key)
|
||||
if cached is not None:
|
||||
supported, ts = cached
|
||||
# Definitive True/False → permanent. Unknown (None) → 60s TTL.
|
||||
if supported is not None or (_time.monotonic() - ts) < 60:
|
||||
return bool(supported)
|
||||
try:
|
||||
from hermes_cli.models import ollama_model_supports_thinking
|
||||
supported = ollama_model_supports_thinking(
|
||||
self.model, self.base_url, getattr(self, "api_key", "")
|
||||
)
|
||||
except Exception:
|
||||
supported = None
|
||||
cache[key] = (supported, _time.monotonic())
|
||||
return bool(supported)
|
||||
|
||||
def _resolve_lmstudio_summary_reasoning_effort(self) -> Optional[str]:
|
||||
"""Resolve a safe top-level ``reasoning_effort`` for LM Studio.
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@ class TestOllamaCloudReasoningEffort:
|
|||
@pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "])
|
||||
def test_xhigh_and_max_normalize_to_max(self, ollama_cloud_profile, effort):
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
)
|
||||
assert extra_body == {}
|
||||
|
|
@ -51,39 +52,47 @@ class TestOllamaCloudReasoningEffort:
|
|||
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
|
||||
def test_standard_efforts_pass_through(self, ollama_cloud_profile, effort):
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": effort},
|
||||
)
|
||||
assert top_level == {"reasoning_effort": effort}
|
||||
|
||||
# ── disabled → no reasoning_effort emitted ─────────────────────
|
||||
# ── disabled → reasoning_effort:"none" (the only working off switch) ──
|
||||
|
||||
def test_explicitly_disabled_emits_nothing(self, ollama_cloud_profile):
|
||||
def test_explicitly_disabled_sends_none(self, ollama_cloud_profile):
|
||||
"""Ollama Cloud defaults to thinking ON and ignores extra_body.thinking,
|
||||
so disabling requires top-level reasoning_effort:"none" (verified live);
|
||||
omitting the field would leave thinking on."""
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": False},
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
assert top_level == {"reasoning_effort": "none"}
|
||||
|
||||
def test_disabled_ignores_effort_field(self, ollama_cloud_profile):
|
||||
"""Effort silently dropped when thinking is off."""
|
||||
"""Effort is overridden by the disable off switch when thinking is off."""
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": False, "effort": "high"},
|
||||
)
|
||||
assert top_level == {}
|
||||
assert top_level == {"reasoning_effort": "none"}
|
||||
|
||||
# ── none effort → no reasoning_effort ──────────────────────────
|
||||
# ── none effort → reasoning_effort:"none" ──────────────────────
|
||||
|
||||
def test_none_effort_emits_nothing(self, ollama_cloud_profile):
|
||||
def test_none_effort_sends_none(self, ollama_cloud_profile):
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": "none"},
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
assert top_level == {"reasoning_effort": "none"}
|
||||
|
||||
# ── missing / empty effort → let model default ─────────────────
|
||||
|
||||
def test_no_reasoning_config_emits_nothing(self, ollama_cloud_profile):
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config=None,
|
||||
)
|
||||
assert extra_body == {}
|
||||
|
|
@ -91,6 +100,7 @@ class TestOllamaCloudReasoningEffort:
|
|||
|
||||
def test_empty_effort_emits_nothing(self, ollama_cloud_profile):
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": ""},
|
||||
)
|
||||
assert top_level == {}
|
||||
|
|
@ -98,17 +108,32 @@ class TestOllamaCloudReasoningEffort:
|
|||
def test_no_effort_key_emits_nothing(self, ollama_cloud_profile):
|
||||
"""When effort key is absent, let the model use its default."""
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True},
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
# ── unknown effort → forwarded as-is ───────────────────────────
|
||||
# ── unknown / minimal effort → omitted (server default) ────────
|
||||
|
||||
def test_unknown_effort_forwarded(self, ollama_cloud_profile):
|
||||
def test_unknown_effort_omitted(self, ollama_cloud_profile):
|
||||
"""Unrecognized effort is omitted, not forwarded verbatim, so the
|
||||
model applies its own default. Matches the sibling deepseek profile,
|
||||
which targets the same backend."""
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": "future-tier"},
|
||||
)
|
||||
assert top_level == {"reasoning_effort": "future-tier"}
|
||||
assert top_level == {}
|
||||
|
||||
def test_minimal_effort_omitted(self, ollama_cloud_profile):
|
||||
"""``minimal`` is a real Hermes effort level but is not documented for
|
||||
Ollama Cloud's /v1/chat/completions, so it is omitted rather than sent
|
||||
verbatim (which could trigger a 400)."""
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
supports_reasoning=True,
|
||||
reasoning_config={"enabled": True, "effort": "minimal"},
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestOllamaCloudFullKwargsIntegration:
|
||||
|
|
@ -125,6 +150,7 @@ class TestOllamaCloudFullKwargsIntegration:
|
|||
reasoning_config={"enabled": True, "effort": "xhigh"},
|
||||
base_url="https://ollama.com/v1",
|
||||
provider_name="ollama-cloud",
|
||||
supports_reasoning=True,
|
||||
)
|
||||
assert kwargs["model"] == "deepseek-v4-pro:cloud"
|
||||
assert kwargs["reasoning_effort"] == "max"
|
||||
|
|
@ -142,8 +168,101 @@ class TestOllamaCloudFullKwargsIntegration:
|
|||
reasoning_config={"enabled": False},
|
||||
base_url="https://ollama.com/v1",
|
||||
provider_name="ollama-cloud",
|
||||
supports_reasoning=True,
|
||||
)
|
||||
# Disabling requires the explicit off switch — Ollama Cloud defaults to
|
||||
# thinking ON, so omitting reasoning_effort would NOT disable it.
|
||||
assert kwargs["reasoning_effort"] == "none"
|
||||
|
||||
|
||||
class TestOllamaCloudCapabilityGating:
|
||||
"""reasoning_effort is gated on the model's thinking capability."""
|
||||
|
||||
def test_non_thinking_model_emits_nothing(self, ollama_cloud_profile):
|
||||
"""A model that doesn't support thinking (supports_reasoning=False)
|
||||
gets no reasoning_effort, even when an effort is requested — Ollama
|
||||
resolves thinking capability from /api/show, and we don't send a
|
||||
meaningless field to e.g. gemma3 / qwen3-coder."""
|
||||
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": True, "effort": "xhigh"},
|
||||
supports_reasoning=False,
|
||||
)
|
||||
assert extra_body == {}
|
||||
assert top_level == {}
|
||||
|
||||
def test_non_thinking_model_ignores_disable(self, ollama_cloud_profile):
|
||||
"""Even a disable request is a no-op for a non-thinking model."""
|
||||
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
|
||||
reasoning_config={"enabled": False},
|
||||
supports_reasoning=False,
|
||||
)
|
||||
assert top_level == {}
|
||||
|
||||
|
||||
class TestOllamaModelSupportsThinking:
|
||||
"""The /api/show capability probe used to resolve supports_reasoning."""
|
||||
|
||||
def _patch_show(self, monkeypatch, *, status=200, capabilities=None, raise_exc=None):
|
||||
import httpx
|
||||
|
||||
class _Resp:
|
||||
status_code = status
|
||||
|
||||
def json(self):
|
||||
return {"capabilities": capabilities} if capabilities is not None else {}
|
||||
|
||||
class _Client:
|
||||
def __init__(self, *a, **k):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
def post(self, *a, **k):
|
||||
if raise_exc:
|
||||
raise raise_exc
|
||||
return _Resp()
|
||||
|
||||
monkeypatch.setattr(httpx, "Client", _Client)
|
||||
|
||||
def test_thinking_capability_true(self, monkeypatch):
|
||||
from hermes_cli.models import ollama_model_supports_thinking
|
||||
|
||||
self._patch_show(monkeypatch, capabilities=["completion", "tools", "thinking"])
|
||||
assert (
|
||||
ollama_model_supports_thinking(
|
||||
"deepseek-v4-pro", "https://ollama.com/v1", "key"
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
def test_no_thinking_capability_false(self, monkeypatch):
|
||||
from hermes_cli.models import ollama_model_supports_thinking
|
||||
|
||||
self._patch_show(monkeypatch, capabilities=["completion", "vision"])
|
||||
assert (
|
||||
ollama_model_supports_thinking("gemma3:27b", "https://ollama.com/v1", "key")
|
||||
is False
|
||||
)
|
||||
|
||||
def test_probe_failure_returns_none(self, monkeypatch):
|
||||
from hermes_cli.models import ollama_model_supports_thinking
|
||||
|
||||
self._patch_show(monkeypatch, status=404)
|
||||
assert (
|
||||
ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None
|
||||
)
|
||||
|
||||
def test_exception_returns_none(self, monkeypatch):
|
||||
from hermes_cli.models import ollama_model_supports_thinking
|
||||
|
||||
self._patch_show(monkeypatch, raise_exc=RuntimeError("boom"))
|
||||
assert (
|
||||
ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None
|
||||
)
|
||||
assert "reasoning_effort" not in kwargs
|
||||
|
||||
|
||||
class TestOllamaCloudAuxModel:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue