From 5d9a72b7c219b5e96dfb308f51920296e0d369cb Mon Sep 17 00:00:00 2001 From: kshitijk4poor <82637225+kshitijk4poor@users.noreply.github.com> Date: Tue, 23 Jun 2026 23:40:41 +0530 Subject: [PATCH] fix(ollama-cloud): capability-gate reasoning_effort + correct disable semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three follow-up fixes to the salvaged reasoning_effort support, all verified live against ollama.com /v1/chat/completions + /api/show on deepseek-v4-pro, gemma3, and qwen3-coder: 1. Capability-gate on /api/show 'thinking'. The original ignored the supports_reasoning flag and emitted reasoning_effort for every model. Now gated: only models whose native /api/show capabilities list contains 'thinking' (deepseek-v4 yes; gemma3 / qwen3-coder no) get reasoning_effort. Mirrors the LM Studio pattern — capability resolved once per (model, base_url) in run_agent._supports_reasoning_extra_body via a cached probe (hermes_cli.models.ollama_model_supports_thinking), threaded into the profile hook as supports_reasoning. No live HTTP in the per-request path. 2. Disable actually disables. Ollama Cloud defaults to thinking ON and IGNORES the extra_body.thinking:{type:disabled} shape (verified: still returned reasoning). The only working off switch is top-level reasoning_effort:'none'. The salvaged code returned ({}, {}) for enabled:false / effort:none, leaving thinking ON. Now emits {'reasoning_effort': 'none'}. 3. Omit unrecognized effort. The original forwarded any unknown string verbatim including 'minimal' (a real Hermes effort level). Ollama Cloud rejects unrecognized values with a hard HTTP 400 (accepted set: low/medium/high/ max/none), so forwarding 'minimal' would break the request. Now omitted. Core touches (run_agent.py, hermes_cli/models.py) add the capability probe; the plugin profile only consumes the resolved flag. 24/24 profile tests green; 194 provider/transport tests unaffected. --- hermes_cli/models.py | 48 ++++++ .../model-providers/ollama-cloud/__init__.py | 34 +++-- run_agent.py | 37 +++++ .../test_ollama_cloud_profile.py | 143 ++++++++++++++++-- 4 files changed, 241 insertions(+), 21 deletions(-) diff --git a/hermes_cli/models.py b/hermes_cli/models.py index 47552624cfe1..48fd28887aac 100644 --- a/hermes_cli/models.py +++ b/hermes_cli/models.py @@ -3306,6 +3306,54 @@ def lmstudio_model_reasoning_options( return [] +def ollama_model_supports_thinking( + model: str, + base_url: Optional[str], + api_key: Optional[str] = None, + timeout: float = 5.0, +) -> Optional[bool]: + """Return True if an Ollama (Cloud or local) model advertises ``thinking``. + + Probes the native ``/api/show`` endpoint and checks the ``capabilities`` + list, which Ollama populates from the model's metadata (e.g. + ``deepseek-v4-pro`` → ``["completion", "tools", "thinking"]`` while + ``gemma3:27b`` → ``["completion", "vision"]``). This is the authoritative + capability source — the OpenAI-compat ``/v1/models`` endpoint omits it. + + Returns: + True — the model declares the ``thinking`` capability. + False — ``/api/show`` succeeded but the model has no ``thinking`` cap. + None — the probe failed (unreachable / non-Ollama / error); the caller + decides the fallback (we treat None as "don't emit"). + """ + import httpx + + server_url = (base_url or "").strip().rstrip("/") + if server_url.endswith("/v1"): + server_url = server_url[:-3] + if not server_url: + return None + + bare_model = _strip_ollama_cloud_suffix((model or "").strip()) + if not bare_model: + return None + + token = str(api_key or "").strip() + headers = {"Authorization": f"Bearer {token}"} if token else {} + + try: + with httpx.Client(timeout=timeout, headers=headers) as client: + resp = client.post(f"{server_url}/api/show", json={"name": bare_model}) + if resp.status_code != 200: + return None + caps = resp.json().get("capabilities") + if isinstance(caps, list): + return "thinking" in caps + except Exception: + return None + return None + + def _fetch_github_models(api_key: Optional[str] = None, timeout: float = 5.0) -> Optional[list[str]]: catalog = fetch_github_model_catalog(api_key=api_key, timeout=timeout) if not catalog: diff --git a/plugins/model-providers/ollama-cloud/__init__.py b/plugins/model-providers/ollama-cloud/__init__.py index a7856124b98c..e849c1da27f9 100644 --- a/plugins/model-providers/ollama-cloud/__init__.py +++ b/plugins/model-providers/ollama-cloud/__init__.py @@ -30,34 +30,50 @@ class OllamaCloudProfile(ProviderProfile): self, *, reasoning_config: dict | None = None, + supports_reasoning: bool = False, **ctx: Any, ) -> tuple[dict[str, Any], dict[str, Any]]: - """Emit top-level ``reasoning_effort`` for Ollama Cloud. + """Emit top-level ``reasoning_effort`` for Ollama Cloud thinking models. - The ``supports_reasoning`` flag passed by the transport is - deliberately ignored — this profile always handles reasoning - when ``reasoning_config`` is present. + Gated on ``supports_reasoning``, which the transport resolves from the + model's native ``/api/show`` ``capabilities`` (``thinking``). Models + without the thinking capability (e.g. ``gemma3``, ``qwen3-coder``) get + no ``reasoning_effort`` at all — emitting it there is a no-op the API + ignores, and gating avoids sending a meaningless field. """ top_level: dict[str, Any] = {} + if not supports_reasoning: + return {}, {} + if reasoning_config and isinstance(reasoning_config, dict): enabled = reasoning_config.get("enabled", True) if enabled is False: - return {}, {} # omit → model runs without thinking + # Ollama Cloud defaults to thinking ON, and ignores the + # extra_body.thinking:{type:disabled} shape (verified live). + # The ONLY way to actually suppress thinking on its + # /v1/chat/completions endpoint is top-level + # reasoning_effort:"none" — omitting the field leaves + # thinking on. + return {}, {"reasoning_effort": "none"} effort = (reasoning_config.get("effort") or "").strip().lower() if not effort: # No explicit effort requested — let the model decide + # (Ollama Cloud's server default is thinking ON). return {}, {} if effort == "none": - return {}, {} # explicit none → suppress thinking + return {}, {"reasoning_effort": "none"} # explicit off switch if effort in ("xhigh", "max", "ultra"): top_level["reasoning_effort"] = "max" elif effort in ("low", "medium", "high"): top_level["reasoning_effort"] = effort - else: - # Unknown value — forward as-is, let the API decide - top_level["reasoning_effort"] = effort + # Any other value (including "minimal", which Ollama Cloud's + # /v1/chat/completions rejects with HTTP 400 — its accepted set is + # {low, medium, high, max, none}) is omitted so the model applies + # its own default rather than triggering a hard 400. Matches the + # sibling deepseek / opencode-zen profiles, which target the same + # backend and omit unrecognized efforts rather than send garbage. return {}, top_level diff --git a/run_agent.py b/run_agent.py index 0b8ad99a1714..22718eeac406 100644 --- a/run_agent.py +++ b/run_agent.py @@ -5445,6 +5445,12 @@ class AIAgent: opts = self._lmstudio_reasoning_options_cached() # "off-only" (or absent) means no real reasoning capability. return any(opt and opt != "off" for opt in opts) + # Ollama Cloud (and any Ollama-compatible server): the native + # /api/show capabilities list is authoritative — emit reasoning_effort + # only for models that declare the "thinking" capability. deepseek-v4 + # has it; gemma3 / qwen3-coder don't. Cached per (model, base_url). + if base_url_host_matches(self._base_url_lower, "ollama.com"): + return self._ollama_supports_thinking_cached() if "openrouter" not in self._base_url_lower: return False if "api.mistral.ai" in self._base_url_lower: @@ -5498,6 +5504,37 @@ class AIAgent: cache[key] = (opts, _time.monotonic()) return opts + def _ollama_supports_thinking_cached(self) -> bool: + """Probe Ollama's ``/api/show`` capabilities once per (model, base_url). + + Returns True only when the model declares the ``thinking`` capability. + Caching mirrors the LM Studio probe: a True/False result is permanent + (capabilities don't change), while a probe failure (None) is cached + with a 60-second TTL so a transient outage doesn't suppress reasoning + for the rest of the session but also doesn't round-trip every turn. + """ + import time as _time + + cache = getattr(self, "_ollama_thinking_cache", None) + if cache is None: + cache = self._ollama_thinking_cache = {} + key = (self.model, self.base_url) + cached = cache.get(key) + if cached is not None: + supported, ts = cached + # Definitive True/False → permanent. Unknown (None) → 60s TTL. + if supported is not None or (_time.monotonic() - ts) < 60: + return bool(supported) + try: + from hermes_cli.models import ollama_model_supports_thinking + supported = ollama_model_supports_thinking( + self.model, self.base_url, getattr(self, "api_key", "") + ) + except Exception: + supported = None + cache[key] = (supported, _time.monotonic()) + return bool(supported) + def _resolve_lmstudio_summary_reasoning_effort(self) -> Optional[str]: """Resolve a safe top-level ``reasoning_effort`` for LM Studio. diff --git a/tests/plugins/model_providers/test_ollama_cloud_profile.py b/tests/plugins/model_providers/test_ollama_cloud_profile.py index 15e798a2cd67..2922808c3cd2 100644 --- a/tests/plugins/model_providers/test_ollama_cloud_profile.py +++ b/tests/plugins/model_providers/test_ollama_cloud_profile.py @@ -41,6 +41,7 @@ class TestOllamaCloudReasoningEffort: @pytest.mark.parametrize("effort", ["xhigh", "max", "MAX", " Max "]) def test_xhigh_and_max_normalize_to_max(self, ollama_cloud_profile, effort): extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, reasoning_config={"enabled": True, "effort": effort}, ) assert extra_body == {} @@ -51,39 +52,47 @@ class TestOllamaCloudReasoningEffort: @pytest.mark.parametrize("effort", ["low", "medium", "high"]) def test_standard_efforts_pass_through(self, ollama_cloud_profile, effort): _, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, reasoning_config={"enabled": True, "effort": effort}, ) assert top_level == {"reasoning_effort": effort} - # ── disabled → no reasoning_effort emitted ───────────────────── + # ── disabled → reasoning_effort:"none" (the only working off switch) ── - def test_explicitly_disabled_emits_nothing(self, ollama_cloud_profile): + def test_explicitly_disabled_sends_none(self, ollama_cloud_profile): + """Ollama Cloud defaults to thinking ON and ignores extra_body.thinking, + so disabling requires top-level reasoning_effort:"none" (verified live); + omitting the field would leave thinking on.""" extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, reasoning_config={"enabled": False}, ) assert extra_body == {} - assert top_level == {} + assert top_level == {"reasoning_effort": "none"} def test_disabled_ignores_effort_field(self, ollama_cloud_profile): - """Effort silently dropped when thinking is off.""" + """Effort is overridden by the disable off switch when thinking is off.""" _, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, reasoning_config={"enabled": False, "effort": "high"}, ) - assert top_level == {} + assert top_level == {"reasoning_effort": "none"} - # ── none effort → no reasoning_effort ────────────────────────── + # ── none effort → reasoning_effort:"none" ────────────────────── - def test_none_effort_emits_nothing(self, ollama_cloud_profile): + def test_none_effort_sends_none(self, ollama_cloud_profile): extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, reasoning_config={"enabled": True, "effort": "none"}, ) assert extra_body == {} - assert top_level == {} + assert top_level == {"reasoning_effort": "none"} # ── missing / empty effort → let model default ───────────────── def test_no_reasoning_config_emits_nothing(self, ollama_cloud_profile): extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, reasoning_config=None, ) assert extra_body == {} @@ -91,6 +100,7 @@ class TestOllamaCloudReasoningEffort: def test_empty_effort_emits_nothing(self, ollama_cloud_profile): _, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, reasoning_config={"enabled": True, "effort": ""}, ) assert top_level == {} @@ -98,17 +108,32 @@ class TestOllamaCloudReasoningEffort: def test_no_effort_key_emits_nothing(self, ollama_cloud_profile): """When effort key is absent, let the model use its default.""" _, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, reasoning_config={"enabled": True}, ) assert top_level == {} - # ── unknown effort → forwarded as-is ─────────────────────────── + # ── unknown / minimal effort → omitted (server default) ──────── - def test_unknown_effort_forwarded(self, ollama_cloud_profile): + def test_unknown_effort_omitted(self, ollama_cloud_profile): + """Unrecognized effort is omitted, not forwarded verbatim, so the + model applies its own default. Matches the sibling deepseek profile, + which targets the same backend.""" _, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, reasoning_config={"enabled": True, "effort": "future-tier"}, ) - assert top_level == {"reasoning_effort": "future-tier"} + assert top_level == {} + + def test_minimal_effort_omitted(self, ollama_cloud_profile): + """``minimal`` is a real Hermes effort level but is not documented for + Ollama Cloud's /v1/chat/completions, so it is omitted rather than sent + verbatim (which could trigger a 400).""" + _, top_level = ollama_cloud_profile.build_api_kwargs_extras( + supports_reasoning=True, + reasoning_config={"enabled": True, "effort": "minimal"}, + ) + assert top_level == {} class TestOllamaCloudFullKwargsIntegration: @@ -125,6 +150,7 @@ class TestOllamaCloudFullKwargsIntegration: reasoning_config={"enabled": True, "effort": "xhigh"}, base_url="https://ollama.com/v1", provider_name="ollama-cloud", + supports_reasoning=True, ) assert kwargs["model"] == "deepseek-v4-pro:cloud" assert kwargs["reasoning_effort"] == "max" @@ -142,8 +168,101 @@ class TestOllamaCloudFullKwargsIntegration: reasoning_config={"enabled": False}, base_url="https://ollama.com/v1", provider_name="ollama-cloud", + supports_reasoning=True, + ) + # Disabling requires the explicit off switch — Ollama Cloud defaults to + # thinking ON, so omitting reasoning_effort would NOT disable it. + assert kwargs["reasoning_effort"] == "none" + + +class TestOllamaCloudCapabilityGating: + """reasoning_effort is gated on the model's thinking capability.""" + + def test_non_thinking_model_emits_nothing(self, ollama_cloud_profile): + """A model that doesn't support thinking (supports_reasoning=False) + gets no reasoning_effort, even when an effort is requested — Ollama + resolves thinking capability from /api/show, and we don't send a + meaningless field to e.g. gemma3 / qwen3-coder.""" + extra_body, top_level = ollama_cloud_profile.build_api_kwargs_extras( + reasoning_config={"enabled": True, "effort": "xhigh"}, + supports_reasoning=False, + ) + assert extra_body == {} + assert top_level == {} + + def test_non_thinking_model_ignores_disable(self, ollama_cloud_profile): + """Even a disable request is a no-op for a non-thinking model.""" + _, top_level = ollama_cloud_profile.build_api_kwargs_extras( + reasoning_config={"enabled": False}, + supports_reasoning=False, + ) + assert top_level == {} + + +class TestOllamaModelSupportsThinking: + """The /api/show capability probe used to resolve supports_reasoning.""" + + def _patch_show(self, monkeypatch, *, status=200, capabilities=None, raise_exc=None): + import httpx + + class _Resp: + status_code = status + + def json(self): + return {"capabilities": capabilities} if capabilities is not None else {} + + class _Client: + def __init__(self, *a, **k): + pass + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def post(self, *a, **k): + if raise_exc: + raise raise_exc + return _Resp() + + monkeypatch.setattr(httpx, "Client", _Client) + + def test_thinking_capability_true(self, monkeypatch): + from hermes_cli.models import ollama_model_supports_thinking + + self._patch_show(monkeypatch, capabilities=["completion", "tools", "thinking"]) + assert ( + ollama_model_supports_thinking( + "deepseek-v4-pro", "https://ollama.com/v1", "key" + ) + is True + ) + + def test_no_thinking_capability_false(self, monkeypatch): + from hermes_cli.models import ollama_model_supports_thinking + + self._patch_show(monkeypatch, capabilities=["completion", "vision"]) + assert ( + ollama_model_supports_thinking("gemma3:27b", "https://ollama.com/v1", "key") + is False + ) + + def test_probe_failure_returns_none(self, monkeypatch): + from hermes_cli.models import ollama_model_supports_thinking + + self._patch_show(monkeypatch, status=404) + assert ( + ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None + ) + + def test_exception_returns_none(self, monkeypatch): + from hermes_cli.models import ollama_model_supports_thinking + + self._patch_show(monkeypatch, raise_exc=RuntimeError("boom")) + assert ( + ollama_model_supports_thinking("x", "https://ollama.com/v1", "key") is None ) - assert "reasoning_effort" not in kwargs class TestOllamaCloudAuxModel: