fix(ollama-cloud): capability-gate reasoning_effort + correct disable semantics

Three follow-up fixes to the salvaged reasoning_effort support, all verified
live against ollama.com /v1/chat/completions + /api/show on deepseek-v4-pro,
gemma3, and qwen3-coder:

1. Capability-gate on /api/show 'thinking'. The original ignored the
   supports_reasoning flag and emitted reasoning_effort for every model. Now
   gated: only models whose native /api/show capabilities list contains
   'thinking' (deepseek-v4 yes; gemma3 / qwen3-coder no) get reasoning_effort.
   Mirrors the LM Studio pattern — capability resolved once per (model,
   base_url) in run_agent._supports_reasoning_extra_body via a cached probe
   (hermes_cli.models.ollama_model_supports_thinking), threaded into the
   profile hook as supports_reasoning. No live HTTP in the per-request path.

2. Disable actually disables. Ollama Cloud defaults to thinking ON and IGNORES
   the extra_body.thinking:{type:disabled} shape (verified: still returned
   reasoning). The only working off switch is top-level reasoning_effort:'none'.
   The salvaged code returned ({}, {}) for enabled:false / effort:none, leaving
   thinking ON. Now emits {'reasoning_effort': 'none'}.

3. Omit unrecognized effort. The original forwarded any unknown string verbatim
   including 'minimal' (a real Hermes effort level). Ollama Cloud rejects
   unrecognized values with a hard HTTP 400 (accepted set: low/medium/high/
   max/none), so forwarding 'minimal' would break the request. Now omitted.

Core touches (run_agent.py, hermes_cli/models.py) add the capability probe;
the plugin profile only consumes the resolved flag. 24/24 profile tests green;
194 provider/transport tests unaffected.
This commit is contained in:
kshitijk4poor 2026-06-23 23:40:41 +05:30 committed by Teknium
parent 4759362188
commit 5d9a72b7c2
4 changed files with 241 additions and 21 deletions

View file

@ -3306,6 +3306,54 @@ def lmstudio_model_reasoning_options(
return []
def ollama_model_supports_thinking(
model: str,
base_url: Optional[str],
api_key: Optional[str] = None,
timeout: float = 5.0,
) -> Optional[bool]:
"""Return True if an Ollama (Cloud or local) model advertises ``thinking``.
Probes the native ``/api/show`` endpoint and checks the ``capabilities``
list, which Ollama populates from the model's metadata (e.g.
``deepseek-v4-pro`` ``["completion", "tools", "thinking"]`` while
``gemma3:27b`` ``["completion", "vision"]``). This is the authoritative
capability source the OpenAI-compat ``/v1/models`` endpoint omits it.
Returns:
True the model declares the ``thinking`` capability.
False ``/api/show`` succeeded but the model has no ``thinking`` cap.
None the probe failed (unreachable / non-Ollama / error); the caller
decides the fallback (we treat None as "don't emit").
"""
import httpx
server_url = (base_url or "").strip().rstrip("/")
if server_url.endswith("/v1"):
server_url = server_url[:-3]
if not server_url:
return None
bare_model = _strip_ollama_cloud_suffix((model or "").strip())
if not bare_model:
return None
token = str(api_key or "").strip()
headers = {"Authorization": f"Bearer {token}"} if token else {}
try:
with httpx.Client(timeout=timeout, headers=headers) as client:
resp = client.post(f"{server_url}/api/show", json={"name": bare_model})
if resp.status_code != 200:
return None
caps = resp.json().get("capabilities")
if isinstance(caps, list):
return "thinking" in caps
except Exception:
return None
return None
def _fetch_github_models(api_key: Optional[str] = None, timeout: float = 5.0) -> Optional[list[str]]:
catalog = fetch_github_model_catalog(api_key=api_key, timeout=timeout)
if not catalog:

View file

@ -30,34 +30,50 @@ class OllamaCloudProfile(ProviderProfile):
self,
*,
reasoning_config: dict | None = None,
supports_reasoning: bool = False,
**ctx: Any,
) -> tuple[dict[str, Any], dict[str, Any]]:
"""Emit top-level ``reasoning_effort`` for Ollama Cloud.
"""Emit top-level ``reasoning_effort`` for Ollama Cloud thinking models.
The ``supports_reasoning`` flag passed by the transport is
deliberately ignored this profile always handles reasoning
when ``reasoning_config`` is present.
Gated on ``supports_reasoning``, which the transport resolves from the
model's native ``/api/show`` ``capabilities`` (``thinking``). Models
without the thinking capability (e.g. ``gemma3``, ``qwen3-coder``) get
no ``reasoning_effort`` at all emitting it there is a no-op the API
ignores, and gating avoids sending a meaningless field.
"""
top_level: dict[str, Any] = {}
if not supports_reasoning:
return {}, {}
if reasoning_config and isinstance(reasoning_config, dict):
enabled = reasoning_config.get("enabled", True)
if enabled is False:
return {}, {} # omit → model runs without thinking
# Ollama Cloud defaults to thinking ON, and ignores the
# extra_body.thinking:{type:disabled} shape (verified live).
# The ONLY way to actually suppress thinking on its
# /v1/chat/completions endpoint is top-level
# reasoning_effort:"none" — omitting the field leaves
# thinking on.
return {}, {"reasoning_effort": "none"}
effort = (reasoning_config.get("effort") or "").strip().lower()
if not effort:
# No explicit effort requested — let the model decide
# (Ollama Cloud's server default is thinking ON).
return {}, {}
if effort == "none":
return {}, {} # explicit none → suppress thinking
return {}, {"reasoning_effort": "none"} # explicit off switch
if effort in ("xhigh", "max", "ultra"):
top_level["reasoning_effort"] = "max"
elif effort in ("low", "medium", "high"):
top_level["reasoning_effort"] = effort
else:
# Unknown value — forward as-is, let the API decide
top_level["reasoning_effort"] = effort
# Any other value (including "minimal", which Ollama Cloud's
# /v1/chat/completions rejects with HTTP 400 — its accepted set is
# {low, medium, high, max, none}) is omitted so the model applies
# its own default rather than triggering a hard 400. Matches the
# sibling deepseek / opencode-zen profiles, which target the same
# backend and omit unrecognized efforts rather than send garbage.
return {}, top_level

View file

@ -5445,6 +5445,12 @@ class AIAgent:
opts = self._lmstudio_reasoning_options_cached()
# "off-only" (or absent) means no real reasoning capability.
return any(opt and opt != "off" for opt in opts)
# Ollama Cloud (and any Ollama-compatible server): the native
# /api/show capabilities list is authoritative — emit reasoning_effort
# only for models that declare the "thinking" capability. deepseek-v4
# has it; gemma3 / qwen3-coder don't. Cached per (model, base_url).
if base_url_host_matches(self._base_url_lower, "ollama.com"):
return self._ollama_supports_thinking_cached()
if "openrouter" not in self._base_url_lower:
return False
if "api.mistral.ai" in self._base_url_lower:
@ -5498,6 +5504,37 @@ class AIAgent:
cache[key] = (opts, _time.monotonic())
return opts
def _ollama_supports_thinking_cached(self) -> bool:
"""Probe Ollama's ``/api/show`` capabilities once per (model, base_url).
Returns True only when the model declares the ``thinking`` capability.
Caching mirrors the LM Studio probe: a True/False result is permanent
(capabilities don't change), while a probe failure (None) is cached
with a 60-second TTL so a transient outage doesn't suppress reasoning
for the rest of the session but also doesn't round-trip every turn.
"""
import time as _time
cache = getattr(self, "_ollama_thinking_cache", None)
if cache is None:
cache = self._ollama_thinking_cache = {}
key = (self.model, self.base_url)
cached = cache.get(key)
if cached is not None:
supported, ts = cached
# Definitive True/False → permanent. Unknown (None) → 60s TTL.
if supported is not None or (_time.monotonic() - ts) < 60:
return bool(supported)
try:
from hermes_cli.models import ollama_model_supports_thinking
supported = ollama_model_supports_thinking(
self.model, self.base_url, getattr(self, "api_key", "")
)
except Exception:
supported = None
cache[key] = (supported, _time.monotonic())
return bool(supported)
def _resolve_lmstudio_summary_reasoning_effort(self) -> Optional[str]:
"""Resolve a safe top-level ``reasoning_effort`` for LM Studio.

View file

@ -41,6 +41,7 @@ class TestOllamaCloudReasoningEffort:
@pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "])
def test_xhigh_and_max_normalize_to_max(self, ollama_cloud_profile, effort):
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": effort},
)
assert extra_body == {}
@ -51,39 +52,47 @@ class TestOllamaCloudReasoningEffort:
@pytest.mark.parametrize("effort", ["low", "medium", "high"])
def test_standard_efforts_pass_through(self, ollama_cloud_profile, effort):
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": effort},
)
assert top_level == {"reasoning_effort": effort}
# ── disabled → no reasoning_effort emitted ─────────────────────
# ── disabled → reasoning_effort:"none" (the only working off switch) ──
def test_explicitly_disabled_emits_nothing(self, ollama_cloud_profile):
def test_explicitly_disabled_sends_none(self, ollama_cloud_profile):
"""Ollama Cloud defaults to thinking ON and ignores extra_body.thinking,
so disabling requires top-level reasoning_effort:"none" (verified live);
omitting the field would leave thinking on."""
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": False},
)
assert extra_body == {}
assert top_level == {}
assert top_level == {"reasoning_effort": "none"}
def test_disabled_ignores_effort_field(self, ollama_cloud_profile):
"""Effort silently dropped when thinking is off."""
"""Effort is overridden by the disable off switch when thinking is off."""
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": False, "effort": "high"},
)
assert top_level == {}
assert top_level == {"reasoning_effort": "none"}
# ── none effort → no reasoning_effort ──────────────────────────
# ── none effort → reasoning_effort:"none" ──────────────────────
def test_none_effort_emits_nothing(self, ollama_cloud_profile):
def test_none_effort_sends_none(self, ollama_cloud_profile):
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": "none"},
)
assert extra_body == {}
assert top_level == {}
assert top_level == {"reasoning_effort": "none"}
# ── missing / empty effort → let model default ─────────────────
def test_no_reasoning_config_emits_nothing(self, ollama_cloud_profile):
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config=None,
)
assert extra_body == {}
@ -91,6 +100,7 @@ class TestOllamaCloudReasoningEffort:
def test_empty_effort_emits_nothing(self, ollama_cloud_profile):
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": ""},
)
assert top_level == {}
@ -98,17 +108,32 @@ class TestOllamaCloudReasoningEffort:
def test_no_effort_key_emits_nothing(self, ollama_cloud_profile):
"""When effort key is absent, let the model use its default."""
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True},
)
assert top_level == {}
# ── unknown effort → forwarded as-is ───────────────────────────
# ── unknown / minimal effort → omitted (server default) ────────
def test_unknown_effort_forwarded(self, ollama_cloud_profile):
def test_unknown_effort_omitted(self, ollama_cloud_profile):
"""Unrecognized effort is omitted, not forwarded verbatim, so the
model applies its own default. Matches the sibling deepseek profile,
which targets the same backend."""
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": "future-tier"},
)
assert top_level == {"reasoning_effort": "future-tier"}
assert top_level == {}
def test_minimal_effort_omitted(self, ollama_cloud_profile):
"""``minimal`` is a real Hermes effort level but is not documented for
Ollama Cloud's /v1/chat/completions, so it is omitted rather than sent
verbatim (which could trigger a 400)."""
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
supports_reasoning=True,
reasoning_config={"enabled": True, "effort": "minimal"},
)
assert top_level == {}
class TestOllamaCloudFullKwargsIntegration:
@ -125,6 +150,7 @@ class TestOllamaCloudFullKwargsIntegration:
reasoning_config={"enabled": True, "effort": "xhigh"},
base_url="https://ollama.com/v1",
provider_name="ollama-cloud",
supports_reasoning=True,
)
assert kwargs["model"] == "deepseek-v4-pro:cloud"
assert kwargs["reasoning_effort"] == "max"
@ -142,8 +168,101 @@ class TestOllamaCloudFullKwargsIntegration:
reasoning_config={"enabled": False},
base_url="https://ollama.com/v1",
provider_name="ollama-cloud",
supports_reasoning=True,
)
# Disabling requires the explicit off switch — Ollama Cloud defaults to
# thinking ON, so omitting reasoning_effort would NOT disable it.
assert kwargs["reasoning_effort"] == "none"
class TestOllamaCloudCapabilityGating:
"""reasoning_effort is gated on the model's thinking capability."""
def test_non_thinking_model_emits_nothing(self, ollama_cloud_profile):
"""A model that doesn't support thinking (supports_reasoning=False)
gets no reasoning_effort, even when an effort is requested Ollama
resolves thinking capability from /api/show, and we don't send a
meaningless field to e.g. gemma3 / qwen3-coder."""
extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras(
reasoning_config={"enabled": True, "effort": "xhigh"},
supports_reasoning=False,
)
assert extra_body == {}
assert top_level == {}
def test_non_thinking_model_ignores_disable(self, ollama_cloud_profile):
"""Even a disable request is a no-op for a non-thinking model."""
_, top_level = ollama_cloud_profile.build_api_kwargs_extras(
reasoning_config={"enabled": False},
supports_reasoning=False,
)
assert top_level == {}
class TestOllamaModelSupportsThinking:
"""The /api/show capability probe used to resolve supports_reasoning."""
def _patch_show(self, monkeypatch, *, status=200, capabilities=None, raise_exc=None):
import httpx
class _Resp:
status_code = status
def json(self):
return {"capabilities": capabilities} if capabilities is not None else {}
class _Client:
def __init__(self, *a, **k):
pass
def __enter__(self):
return self
def __exit__(self, *a):
return False
def post(self, *a, **k):
if raise_exc:
raise raise_exc
return _Resp()
monkeypatch.setattr(httpx, "Client", _Client)
def test_thinking_capability_true(self, monkeypatch):
from hermes_cli.models import ollama_model_supports_thinking
self._patch_show(monkeypatch, capabilities=["completion", "tools", "thinking"])
assert (
ollama_model_supports_thinking(
"deepseek-v4-pro", "https://ollama.com/v1", "key"
)
is True
)
def test_no_thinking_capability_false(self, monkeypatch):
from hermes_cli.models import ollama_model_supports_thinking
self._patch_show(monkeypatch, capabilities=["completion", "vision"])
assert (
ollama_model_supports_thinking("gemma3:27b", "https://ollama.com/v1", "key")
is False
)
def test_probe_failure_returns_none(self, monkeypatch):
from hermes_cli.models import ollama_model_supports_thinking
self._patch_show(monkeypatch, status=404)
assert (
ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None
)
def test_exception_returns_none(self, monkeypatch):
from hermes_cli.models import ollama_model_supports_thinking
self._patch_show(monkeypatch, raise_exc=RuntimeError("boom"))
assert (
ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None
)
assert "reasoning_effort" not in kwargs
class TestOllamaCloudAuxModel: