fix(custom-provider): emit reasoning_effort at the live profile path

PR #57601's original branch added a top-level reasoning_effort emit to the
LEGACY build_kwargs path (agent/transports/chat_completions.py), but
provider=custom resolves to CustomProfile (plugins/model-providers/custom/),
so chat_completion_helpers takes the profile path and returns early — the
added branch was unreachable dead code for every custom endpoint.

Move the fix to its real site, CustomProfile.build_api_kwargs_extras(), and
follow the DeepSeek/Zai profile precedent:
  - disabled            -> extra_body.think = False (unchanged)
  - enabled + effort    -> TOP-LEVEL reasoning_effort (the OpenAI-compatible
                           format GLM-5.2/ARK expect), passed through verbatim
                           incl. max/xhigh
  - enabled + no effort -> omit, so the endpoint's server default applies
                           (avoids silently forcing 'medium' as the original
                           branch did)

Deliberately does NOT force think=True on enable — that flag is Ollama-only
and risks a 400 on GLM/vLLM endpoints that don't recognize it; thinking is
already server-default-on for these backends.

Verified end-to-end through the real profile dispatch (temp HERMES_HOME):
custom+high -> reasoning_effort=high; custom+max -> reasoning_effort=max;
custom+none -> think=False; custom+unset -> nothing; num_ctx composes.

Adds tests/plugins/model_providers/test_custom_profile.py (13 cases).
Addresses the custom-provider half of #55276.

Co-authored-by: huanshan5195 <huanshan5195@users.noreply.github.com>

agent/transports/chat_completions.py

@@ -377,22 +377,6 @@ class ChatCompletionsTransport(ProviderTransport):
             if _lm_effort is not None:
                 api_kwargs["reasoning_effort"] = _lm_effort
 
-        # Custom provider (e.g. GLM-5.2 on ARK): send top-level reasoning_effort
-        # string so the upstream API receives its native parameter format.
-        if params.get("is_custom_provider", False):
-            _custom_thinking_off = bool(
-                reasoning_config
-                and isinstance(reasoning_config, dict)
-                and reasoning_config.get("enabled") is False
-            )
-            if not _custom_thinking_off:
-                _custom_effort = "medium"
-                if reasoning_config and isinstance(reasoning_config, dict):
-                    _custom_effort = (
-                        reasoning_config.get("effort", "medium") or "medium"
-                    )
-                api_kwargs["reasoning_effort"] = _custom_effort
-
         # extra_body assembly
         extra_body: dict[str, Any] = {}
 

plugins/model-providers/custom/__init__.py

@@ -1,9 +1,13 @@
 """Custom / Ollama (local) provider profile.
 
 Covers any endpoint registered as provider="custom", including local
-Ollama instances. Key quirks:
+Ollama instances and OpenAI-compatible reasoning endpoints (GLM-5.2 on
+Volcengine ARK, vLLM, llama.cpp). Key quirks:
   - ollama_num_ctx → extra_body.options.num_ctx (local context window)
   - reasoning_config disabled → extra_body.think = False
+  - reasoning_config enabled + effort → top-level reasoning_effort
+    (the native OpenAI-compatible format GLM/ARK expect; unset omits it
+    so the endpoint's server default applies)
 """
 
 from typing import Any
@@ -23,6 +27,7 @@ class CustomProfile(ProviderProfile):
         **ctx: Any,
     ) -> tuple[dict[str, Any], dict[str, Any]]:
         extra_body: dict[str, Any] = {}
+        top_level: dict[str, Any] = {}
 
         # Ollama context window
         if ollama_num_ctx:
@@ -30,14 +35,29 @@ class CustomProfile(ProviderProfile):
             options["num_ctx"] = ollama_num_ctx
             extra_body["options"] = options
 
-        # Disable thinking when reasoning is turned off
+        # Reasoning / thinking control for custom OpenAI-compatible endpoints
+        # (GLM-5.2 on Volcengine ARK, vLLM, Ollama, llama.cpp, …).
+        #
+        #   - disabled  → extra_body.think = False (Ollama's thinking-off flag)
+        #   - enabled + effort set → TOP-LEVEL reasoning_effort string, the
+        #     format GLM-5.2/ARK and other OpenAI-compatible reasoning APIs
+        #     expect (GLM documents "high" and "max"; "max" is its default).
+        #   - enabled + no effort  → omit both, so the endpoint applies its own
+        #     server-side default (do NOT force a level the user didn't pick).
+        #
+        # We deliberately do NOT emit ``think=True`` on enable: it is an
+        # Ollama-only flag and thinking is already server-default-on for these
+        # backends, so forcing it risks a 400 on GLM/vLLM endpoints that don't
+        # recognize it. Mirrors the DeepSeek/Zai profile precedent.
         if reasoning_config and isinstance(reasoning_config, dict):
             _effort = (reasoning_config.get("effort") or "").strip().lower()
             _enabled = reasoning_config.get("enabled", True)
             if _effort == "none" or _enabled is False:
                 extra_body["think"] = False
+            elif _effort:
+                top_level["reasoning_effort"] = _effort
 
-        return extra_body, {}
+        return extra_body, top_level
 
     def fetch_models(
         self,

tests/plugins/model_providers/test_custom_profile.py

@@ -0,0 +1,118 @@
+"""Unit tests for the custom provider profile's reasoning wiring.
+
+``provider=custom`` covers any OpenAI-compatible endpoint the user points
+Hermes at — local Ollama, vLLM, llama.cpp, and hosted reasoning APIs like
+GLM-5.2 on Volcengine ARK. Before #57601's salvage, ``CustomProfile`` emitted
+nothing when reasoning was *enabled*, so a configured ``reasoning_effort``
+was silently dropped for every custom endpoint.
+
+These tests pin the wire-shape contract:
+  - disabled            → extra_body.think = False
+  - enabled + effort    → top-level reasoning_effort (native OpenAI-compat
+                          format GLM/ARK expect), passed through verbatim
+                          including ``max``/``xhigh``
+  - enabled + no effort → nothing emitted (endpoint's server default applies)
+  - ollama_num_ctx      → extra_body.options.num_ctx, orthogonal to reasoning
+"""
+
+from __future__ import annotations
+
+import pytest
+
+
+@pytest.fixture
+def custom_profile():
+    """Resolve the registered custom profile via the global registry.
+
+    Importing ``model_tools`` triggers plugin discovery, which registers the
+    ``custom`` profile. Going through ``get_provider_profile`` keeps the test
+    honest — if the registered class is ever downgraded to a plain
+    ``ProviderProfile``, the assertions below collapse.
+    """
+    import model_tools  # noqa: F401
+    import providers
+
+    profile = providers.get_provider_profile("custom")
+    assert profile is not None, "custom provider profile must be registered"
+    return profile
+
+
+class TestCustomReasoningWireShape:
+    """``build_api_kwargs_extras`` produces the correct wire format."""
+
+    def test_no_reasoning_config_emits_nothing(self, custom_profile):
+        """Unset reasoning → omit everything so the endpoint's default applies."""
+        eb, tl = custom_profile.build_api_kwargs_extras(
+            reasoning_config=None, model="glm-5.2"
+        )
+        assert eb == {}
+        assert tl == {}
+
+    def test_disabled_sends_think_false(self, custom_profile):
+        """enabled=False → extra_body.think = False (Ollama thinking-off flag)."""
+        eb, tl = custom_profile.build_api_kwargs_extras(
+            reasoning_config={"enabled": False}, model="glm-5.2"
+        )
+        assert eb == {"think": False}
+        assert tl == {}
+
+    def test_effort_none_sends_think_false(self, custom_profile):
+        """effort='none' is the disable alias → think=False, no effort."""
+        eb, tl = custom_profile.build_api_kwargs_extras(
+            reasoning_config={"enabled": True, "effort": "none"}, model="glm-5.2"
+        )
+        assert eb == {"think": False}
+        assert tl == {}
+
+    @pytest.mark.parametrize(
+        "effort", ["minimal", "low", "medium", "high", "xhigh", "max"]
+    )
+    def test_enabled_effort_goes_top_level(self, custom_profile, effort):
+        """enabled + effort → TOP-LEVEL reasoning_effort, passed through verbatim.
+
+        GLM-5.2/ARK and OpenAI-compatible reasoning APIs read reasoning_effort
+        as a top-level string, not nested in extra_body. ``max`` is GLM's
+        native deep-reasoning level and must survive.
+        """
+        eb, tl = custom_profile.build_api_kwargs_extras(
+            reasoning_config={"enabled": True, "effort": effort}, model="glm-5.2"
+        )
+        assert tl == {"reasoning_effort": effort}
+        assert "reasoning_effort" not in eb
+        assert "think" not in eb
+
+    def test_enabled_without_effort_emits_nothing(self, custom_profile):
+        """enabled but no effort → omit; do NOT force a level the user didn't pick."""
+        eb, tl = custom_profile.build_api_kwargs_extras(
+            reasoning_config={"enabled": True}, model="glm-5.2"
+        )
+        assert eb == {}
+        assert tl == {}
+
+    def test_does_not_force_think_true_on_enable(self, custom_profile):
+        """We must never send think=True on enable — it's Ollama-only and
+        would 400 on GLM/vLLM endpoints that don't recognize it."""
+        eb, _ = custom_profile.build_api_kwargs_extras(
+            reasoning_config={"enabled": True, "effort": "high"}, model="glm-5.2"
+        )
+        assert eb.get("think") is not True
+
+
+class TestCustomReasoningWithNumCtx:
+    """Ollama num_ctx and reasoning are independent and compose."""
+
+    def test_num_ctx_alone(self, custom_profile):
+        eb, tl = custom_profile.build_api_kwargs_extras(
+            reasoning_config=None, ollama_num_ctx=8192, model="qwen3"
+        )
+        assert eb == {"options": {"num_ctx": 8192}}
+        assert tl == {}
+
+    def test_num_ctx_with_effort(self, custom_profile):
+        eb, tl = custom_profile.build_api_kwargs_extras(
+            reasoning_config={"enabled": True, "effort": "high"},
+            ollama_num_ctx=8192,
+            model="qwen3",
+        )
+        assert eb == {"options": {"num_ctx": 8192}}
+        assert tl == {"reasoning_effort": "high"}