diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 1cde73419a44..28f0192d8161 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -1305,6 +1305,13 @@ def restore_primary_runtime(agent) -> bool: primary_provider or "?", ) + # ── Restore reasoning_config if it was saved ── + # switch_model saves reasoning_config in _primary_runtime. If the + # snapshot predates that (older sessions), keep the current value. + saved_reasoning = rt.get("reasoning_config") + if saved_reasoning is not None: + agent.reasoning_config = dict(saved_reasoning) + # ── Reset fallback chain for the new turn ── agent._fallback_activated = False agent._fallback_index = 0 @@ -2065,6 +2072,45 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo api_mode=agent.api_mode, ) + # ── Re-resolve reasoning_config from per-model override ── + # The new model may have a different reasoning_effort override. Re-read + # config so the override takes effect immediately on /model switch. + # Try both agent.model (normalized, e.g. "claude-opus-4-5") AND the raw + # config default (user's original spelling, e.g. "claude-opus-4.5") so + # override keys match regardless of how downstream consumers normalized + # the input. See plan FINDING #7 + session follow-up. + try: + from hermes_constants import ( + parse_reasoning_effort, + resolve_per_model_reasoning_effort, + ) + from hermes_cli.config import load_config as _sm_load_config + + _reasoning_cfg = _sm_load_config() or {} + _sm_overrides = (_reasoning_cfg.get("agent") or {}).get("reasoning_overrides", {}) or {} + # Try the normalized agent.model first, then the raw config default + _sm_raw_model_default = str((_reasoning_cfg.get("model") or {}).get("default", "") or "").strip() + _sm_per_model = None + for _candidate in (agent.model, _sm_raw_model_default): + if _candidate: + _sm_per_model = resolve_per_model_reasoning_effort(_candidate, _sm_overrides) + if _sm_per_model is not None: + break + if _sm_per_model is not None: + agent.reasoning_config = _sm_per_model + logger.info( + "switch_model: reasoning_config resolved to per-model override for %s: %s", + agent.model, _sm_per_model, + ) + else: + # Raw value — a YAML boolean False means thinking disabled, + # see parse_reasoning_effort. Do NOT str()/strip() coerce. + _sm_global = (_reasoning_cfg.get("agent") or {}).get("reasoning_effort", "") + agent.reasoning_config = parse_reasoning_effort(_sm_global) + logger.info("switch_model: reasoning_config resolved to global effort: %s", _sm_global or "(none)") + except Exception as _reasoning_err: + logger.debug("switch_model: could not re-resolve reasoning_config: %s", _reasoning_err) + # ── Invalidate cached system prompt so it rebuilds next turn ── agent._cached_system_prompt = None @@ -2087,6 +2133,7 @@ def switch_model(agent, new_model, new_provider, api_key='', base_url='', api_mo "client_kwargs": dict(agent._client_kwargs), "use_prompt_caching": agent._use_prompt_caching, "use_native_cache_layout": agent._use_native_cache_layout, + "reasoning_config": dict(agent.reasoning_config) if getattr(agent, "reasoning_config", None) else None, "compressor_model": getattr(_cc, "model", agent.model) if _cc else agent.model, "compressor_base_url": getattr(_cc, "base_url", agent.base_url) if _cc else agent.base_url, "compressor_api_key": getattr(_cc, "api_key", "") if _cc else "", diff --git a/agent/chat_completion_helpers.py b/agent/chat_completion_helpers.py index b93dcf661717..57cc5e76cdb8 100644 --- a/agent/chat_completion_helpers.py +++ b/agent/chat_completion_helpers.py @@ -1635,6 +1635,45 @@ def try_activate_fallback(agent, reason: "FailoverReason | None" = None) -> bool api_mode=agent.api_mode, ) + # Re-resolve reasoning_config for the new fallback model (Closes #21256). + # Per-model override (if any) takes precedence, else global reasoning_effort. + # Wrapped in try/except because config load failure must not kill the swap. + try: + from hermes_cli.config import load_config + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + + _fb_cfg = load_config() or {} + _fb_agent_cfg = _fb_cfg.get("agent", {}) or {} + _fb_overrides = _fb_agent_cfg.get("reasoning_overrides", {}) or {} + _fb_per_model = resolve_per_model_reasoning_effort(agent.model, _fb_overrides) + if _fb_per_model is not None: + agent.reasoning_config = _fb_per_model + logger.info( + "Fallback %s: reasoning_config resolved to per-model override: %s", + agent.model, _fb_per_model, + ) + else: + # Raw value — a YAML boolean False means thinking disabled, + # see parse_reasoning_effort. Do NOT coerce with ``or ""``. + _fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "") + agent.reasoning_config = parse_reasoning_effort(_fb_global_effort) + if agent.reasoning_config: + logger.info( + "Fallback %s: reasoning_config resolved to global effort: %s", + agent.model, _fb_global_effort, + ) + else: + logger.info( + "Fallback %s: reasoning_config resolved to None (disabled or default)", + agent.model, + ) + except Exception as _reasoning_err: + logger.debug( + "Failed to resolve reasoning_config for fallback %s; keeping current: %s", + agent.model, _reasoning_err, + ) + # Keep whatever reasoning_config was active — don't break the fallback swap. + # Keep the prompt's self-identity in sync with the model actually # answering, so "what model are you?" doesn't report the primary. rewrite_prompt_model_identity(agent, fb_model, fb_provider) diff --git a/cli-config.yaml.example b/cli-config.yaml.example index 1858f451c9a4..2583bfb54f0b 100644 --- a/cli-config.yaml.example +++ b/cli-config.yaml.example @@ -742,6 +742,19 @@ agent: # Options: "xhigh" (max), "high", "medium", "low", "minimal", "none" (disable) reasoning_effort: "medium" + # Per-model reasoning effort overrides (optional dict) + # Key: any sensible model spelling works (exact, dots↔dashes interchangeable, + # provider prefix optional). First match wins. + # Value: reasoning effort level (same options as reasoning_effort) + # Override the global reasoning_effort for that specific model. + # NOTE: no `hermes config set` support for this key -- edit YAML directly. + # reasoning_overrides: + # "openrouter/anthropic/claude-opus-4.5": "xhigh" + # "openai/gpt-5": "low" + # "claude-opus-4.6": "high" # bare model name also works + # "deepseek/deepseek-v4-pro": "xhigh" # dots and dashes are interchangeable + reasoning_overrides: {} + # Predefined personalities (use with /personality command) personalities: helpful: "You are a helpful, friendly AI assistant." diff --git a/cli.py b/cli.py index 17d2bdd0e4e3..c836834a2169 100644 --- a/cli.py +++ b/cli.py @@ -3917,8 +3917,18 @@ class HermesCLI(CLIAgentSetupMixin, CLICommandsMixin): ) # Reasoning config (OpenRouter reasoning effort level) - self.reasoning_config = _parse_reasoning_config( - CLI_CONFIG["agent"].get("reasoning_effort", "") + # Per-model override takes precedence over global effort (Closes #21256). + _reasoning_overrides = CLI_CONFIG["agent"].get("reasoning_overrides", {}) or {} + from hermes_constants import resolve_per_model_reasoning_effort + _per_model_reasoning = resolve_per_model_reasoning_effort( + self.model, _reasoning_overrides + ) + self.reasoning_config = ( + _per_model_reasoning + if _per_model_reasoning is not None + else _parse_reasoning_config( + CLI_CONFIG["agent"].get("reasoning_effort", "") + ) ) self.service_tier = _parse_service_tier_config( CLI_CONFIG["agent"].get("service_tier", "") diff --git a/cron/scheduler.py b/cron/scheduler.py index a50704ea10be..5e80fc2d7aec 100644 --- a/cron/scheduler.py +++ b/cron/scheduler.py @@ -2988,12 +2988,25 @@ def run_job( except Exception: pass - # Reasoning config from config.yaml (raw value — a YAML boolean False - # means thinking disabled, see parse_reasoning_effort) - from hermes_constants import parse_reasoning_effort - reasoning_config = parse_reasoning_effort( - _cfg.get("agent", {}).get("reasoning_effort", "") + # Reasoning config from config.yaml (per-model override > global) + from hermes_constants import ( + parse_reasoning_effort, + resolve_per_model_reasoning_effort, ) + _cron_model_cfg = _cfg.get("model", {}) if isinstance(_cfg.get("model", {}), dict) else {} + _cron_model = str( + _cron_model_cfg.get("default", "") or _cron_model_cfg.get("model", "") or "" + ).strip() + _cron_overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {} + _cron_per_model = resolve_per_model_reasoning_effort(_cron_model, _cron_overrides) + if _cron_per_model is not None: + reasoning_config = _cron_per_model + else: + # Raw value — a YAML boolean False means thinking disabled, + # see parse_reasoning_effort. Do NOT str()/strip() coerce. + reasoning_config = parse_reasoning_effort( + _cfg.get("agent", {}).get("reasoning_effort", "") + ) # Prefill messages from env or config.yaml. The top-level # prefill_messages_file key is canonical; agent.prefill_messages_file is diff --git a/docs/PER_MODEL_REASONING.md b/docs/PER_MODEL_REASONING.md new file mode 100644 index 000000000000..14a89466d6ee --- /dev/null +++ b/docs/PER_MODEL_REASONING.md @@ -0,0 +1,101 @@ +# Hermes Agent Configuration Guide + +## Per-Model Reasoning Effort Overrides + +You can configure different reasoning effort levels for different models. This allows you to set `high` effort for complex reasoning models like `claude-opus-4.5` while keeping `medium` for faster models like `gemini-flash`. + +### Configuration + +Edit your `config.yaml` (typically at `~/.hermes/config.yaml`): + +```yaml +agent: + reasoning_overrides: + claude-opus-4.5: high + gemini-flash: medium + gpt-4.5: high +``` + +### Key Matching + +The model name matching is **spelling-tolerant**. All of these variations will match: +- `claude-opus-4.5`, `claude-opus-4-5`, `claude-opus.4.5` +- `anthropic/claude-opus-4.5`, `openrouter/anthropic/claude-opus-4.5` +- With or without provider prefixes + +Exact matches take precedence over variants. + +### Resolution Order + +When determining reasoning effort for a model, Hermes checks in this order: + +1. **Session override**: `/reasoning high` (current session only) +2. **Per-model override**: `agent.reasoning_overrides.` from config.yaml +3. **Global default**: `agent.reasoning_effort` from config.yaml + +### How It Works + +The override applies automatically in these scenarios: + +- **CLI startup**: Uses the override for the configured default model +- **Gateway messaging**: Each gateway session uses the override for its model +- **Desktop/TUI**: Uses the override for the configured model +- **Model switching**: When you switch models, the reasoning effort updates to the new model's override +- **Fallback activation**: When the primary model fails and Hermes falls back to a secondary model, it uses that fallback model's override +- **Reasoning recovery**: When the primary model recovers after a fallback, the original model's override is restored + +### Examples + +#### Example 1: High effort for Opus, medium for others +```yaml +agent: + reasoning_overrides: + claude-opus-4.5: high +``` + +#### Example 2: Different efforts per model +```yaml +agent: + reasoning_overrides: + claude-opus-4.5: high + gemini-2.0-flash: low + gpt-4.5: high + o3-mini: medium +``` + +#### Example 3: With provider prefixes +```yaml +agent: + reasoning_overrides: + anthropic/claude-opus-4.5: high + google/gemini-2.0-flash: low +``` + +All of these are equivalent — the provider prefix is optional. + +### Disabling Reasoning for Specific Models + +Set the override to `none` to disable reasoning for a specific model: + +```yaml +agent: + reasoning_overrides: + gemini-flash: none +``` + +### Troubleshooting + +**Override not taking effect?** +- Check the exact model name in your config with `/model` +- Verify the override is under `agent.reasoning_overrides` (not `agent.reasoning_effort`) +- Restart the gateway or CLI session after editing config.yaml +- Check logs for parsing errors + +**Override applies but reasoning doesn't work?** +- Not all models support reasoning (e.g., `gemini-flash` has limited support) +- Check the model's documentation for reasoning capability +- Use a model that explicitly supports extended thinking + +**Session override not respecting per-model override?** +- Session overrides take precedence (by design) +- Clear the session override with `/reasoning default` to return to the per-model override diff --git a/gateway/run.py b/gateway/run.py index 8d63da9c952a..7ede7d4130df 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -4852,17 +4852,33 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew @staticmethod def _load_reasoning_config() -> dict | None: - """Load reasoning effort from config.yaml. + """Load reasoning effort from config.yaml, respecting per-model overrides. Reads agent.reasoning_effort from config.yaml. Valid: "none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra". Returns None to use default (medium). + + Per-model overrides (agent.reasoning_overrides) take precedence + over the global value when the current model matches a key + (spelling-tolerant). Closes #21256. """ - from hermes_constants import parse_reasoning_effort + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort cfg = _load_gateway_runtime_config() - # Keep the raw value — coercing with ``or ""`` turns a YAML boolean - # False (``reasoning_effort: false``/``off``/``no``) into "", silently - # re-enabling thinking for users who explicitly disabled it. + # Per-model override first + model_cfg = cfg.get("model") or {} + model = str( + (model_cfg.get("default", "") if isinstance(model_cfg, dict) else "") + or (model_cfg.get("model", "") if isinstance(model_cfg, dict) else "") + or "" + ).strip() + overrides = (cfg.get("agent") or {}).get("reasoning_overrides", {}) or {} + per_model = resolve_per_model_reasoning_effort(model, overrides) + if per_model is not None: + return per_model + # Global fallback — keep the raw value; coercing with ``or ""`` turns + # a YAML boolean False (``reasoning_effort: false``/``off``/``no``) + # into "", silently re-enabling thinking for users who explicitly + # disabled it. effort = cfg_get(cfg, "agent", "reasoning_effort", default="") result = parse_reasoning_effort(effort) if effort and str(effort).strip() and result is None: diff --git a/hermes_cli/config.py b/hermes_cli/config.py index 0754933d6082..737833fae208 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -1158,8 +1158,15 @@ DEFAULT_CONFIG = { # only controls how inbound user images are presented. "image_input_mode": "auto", "disabled_toolsets": [], + + # Per-model reasoning effort overrides (spelling-tolerant). + # Dict mapping model names (any reasonable spelling) to effort levels. + # Takes precedence over agent.reasoning_effort when the current model + # matches a key in this dict. + # Edit directly in config.yaml (no CLI support due to dots in keys). + "reasoning_overrides": {}, }, - + "terminal": { "backend": "local", "modal_mode": "auto", diff --git a/hermes_constants.py b/hermes_constants.py index 26842530c22d..5d64a619ecaa 100644 --- a/hermes_constants.py +++ b/hermes_constants.py @@ -823,6 +823,131 @@ def parse_reasoning_effort(effort) -> dict | None: return None +def _canonical_model_variants(model: str) -> list[str]: + """Generate bounded spelling variants for tolerant override matching. + + Model names mix two types of separators: + - **Word separators**: dashes between words (``claude-opus``) + - **Version separators**: dots or dashes between version digits (``4.5``, ``4-5``) + + The tricky case is that ``.`` appears in BOTH roles (word sep in some + spellings, version sep in others), so a blanket ``.replace('.', '-')`` + is lossy — it collapses version dots into dashes and no later step + recovers the canonical form (``claude-opus-4.5``). + + Strategy: generate a small set of base forms, then apply version-dot + recovery to EACH of them. This ensures symmetry: + ``claude-opus-4.5``, ``claude-opus-4-5``, and ``claude-opus.4.5`` all + produce the same variant set. + + Steps: + 1. Exact input + 2. Dots/dashes cross-substitution on the entire string + 3. Version-dot recovery applied to ALL derivatives + 4. Strip provider/aggregator prefix → bare model variants + 5. Apply version-dot recovery to bare derivatives + 6. Prepend known provider/aggregator prefixes + + Duplicates removed in insertion order (exact always wins). + """ + import re + + # Version-dot regexes — digit-separator-digit interconversion + _dash_to_dot = lambda s: re.sub(r'(\d)-(\d)', r'\1.\2', s) + _dot_to_dash = lambda s: re.sub(r'(\d)\.(\d)', r'\1-\2', s) + + seen = set() + variants = [] + + def _add(v): + if v and v not in seen: + seen.add(v) + variants.append(v) + + def _add_with_derivatives(s): + """Add s plus its dots↔dashes and version-dot derivatives.""" + _add(s) + all_dashed = s.replace('.', '-') + _add(all_dashed) + all_dotted = s.replace('-', '.') + _add(all_dotted) + # Version-dot recovery on each base form + _add(_dash_to_dot(s)) + _add(_dot_to_dash(s)) + _add(_dash_to_dot(all_dashed)) + _add(_dot_to_dash(all_dotted)) + + # 1-3. Base variants for the full string + _add_with_derivatives(model) + + # Split by / to handle provider prefix + parts = model.split('/') + + # 4. Bare model variants (strip provider/aggregator prefix) + if len(parts) >= 2: + bare = parts[-1] + _add_with_derivatives(bare) + + # Strip aggregator only (3+ parts) + # e.g. "openrouter/anthropic/claude-opus-4.5" → "anthropic/claude-opus-4.5" + if len(parts) >= 3: + _add_with_derivatives('/'.join(parts[1:])) + + # 5. Prepend known provider prefixes to bare variants + known_providers = ( + 'anthropic', 'openai', 'google', 'openrouter', 'groq', 'mistral', + 'xai', 'cohere', 'perplexity', 'together', 'fireworks', 'deepseek', + ) + bare_variants = [v for v in variants if '/' not in v] + for v in bare_variants: + for provider in known_providers: + _add(f"{provider}/{v}") + + # Prepend aggregator to single-slash variants + single_slash_variants = [v for v in variants if v.count('/') == 1] + known_aggregators = ('openrouter', 'opencode', 'fireworks', 'groq', 'together') + for v in single_slash_variants: + for agg in known_aggregators: + _add(f"{agg}/{v}") + + return variants + + +def resolve_per_model_reasoning_effort(model: str, overrides: dict | None) -> dict | None: + """Lookup a per-model reasoning_effort override with spelling-tolerance. + + Args: + model: The model string (any spelling — exact, normalized, bare, + with provider prefix, etc.) + overrides: The dict of per-model overrides from + agent.reasoning_overrides in config.yaml. Keys can be + any sensible spelling of the model name. + + Returns: + The parsed reasoning_config dict if a match is found, + None otherwise (caller should fall back to global reasoning_effort). + + Resolution order: + 1. Exact match + 2. Dots ↔ dashes variants + 3. Strip provider prefix (bare model name only) + 4. Strip aggregator prefix (middle segment only) + 5. Prepend known aggregator prefixes to bare/single-slash variants + + First non-None parse_reasoning_effort result wins. + """ + if not overrides or not isinstance(overrides, dict) or not model: + return None + + for variant in _canonical_model_variants(model): + if variant in overrides: + result = parse_reasoning_effort(overrides[variant]) + if result is not None: + return result + + return None + + def is_termux() -> bool: """Return True when running inside a Termux (Android) environment. diff --git a/tests/cron/test_reasoning_config_per_model.py b/tests/cron/test_reasoning_config_per_model.py new file mode 100644 index 000000000000..bdd09ade118b --- /dev/null +++ b/tests/cron/test_reasoning_config_per_model.py @@ -0,0 +1,106 @@ +"""Tests for per-model reasoning_effort override in cron scheduler.""" + +import pytest + + +class TestCronPerModelReasoningConfig: + """Test cron scheduler respects per-model reasoning overrides. + + Rather than spinning up a full CronScheduler (heavy), we verify the + resolution logic by testing the helper directly against a config dict + shaped the same way the scheduler reads it. + """ + + def test_per_model_override_resolves_for_cron_model(self): + """The spelling-tolerant helper resolves the cron config's model.""" + from hermes_constants import resolve_per_model_reasoning_effort + + # Simulate cron scheduler config shape + _cfg = { + "model": {"default": "anthropic/claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + _model_cfg = _cfg.get("model", {}) + _model = str(_model_cfg.get("default", "") or "").strip() + _overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {} + + result = resolve_per_model_reasoning_effort(_model, _overrides) + assert result is not None + assert result["effort"] == "xhigh" + + def test_cron_falls_back_to_global_when_no_override(self): + """When no per-model override matches, global effort is used.""" + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + + _cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "low", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + _model = _cfg["model"]["default"] + _overrides = _cfg["agent"]["reasoning_overrides"] + + per_model = resolve_per_model_reasoning_effort(_model, _overrides) + assert per_model is None # no match + + # Scheduler falls back to global + effort = _cfg["agent"]["reasoning_effort"] + result = parse_reasoning_effort(effort) + assert result is not None + assert result["effort"] == "low" + + def test_cron_handles_missing_model_key(self): + """Works when config has no model.default.""" + from hermes_constants import resolve_per_model_reasoning_effort + + _cfg = { + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": {"claude-opus-4.5": "high"}, + }, + } + _model_cfg = _cfg.get("model", {}) if isinstance(_cfg.get("model", {}), dict) else {} + _model = str(_model_cfg.get("default", "") or _model_cfg.get("model", "") or "").strip() + _overrides = (_cfg.get("agent", {}) or {}).get("reasoning_overrides", {}) or {} + + # Empty model → resolve returns None → scheduler uses global + result = resolve_per_model_reasoning_effort(_model, _overrides) + assert result is None + + def test_global_fallback_with_yaml_false(self): + """YAML boolean False must reach parse_reasoning_effort uncoerced. + + Regression: str(... or "").strip() turned False into "", silently + re-enabling thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + """ + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + + _cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": False, # YAML boolean, not string + "reasoning_overrides": {"claude-opus-4.5": "xhigh"}, + }, + } + _model = _cfg["model"]["default"] + _overrides = _cfg["agent"]["reasoning_overrides"] + + per_model = resolve_per_model_reasoning_effort(_model, _overrides) + assert per_model is None # no match + + # Scheduler global fallback — raw value, no coercion + result = parse_reasoning_effort( + _cfg.get("agent", {}).get("reasoning_effort", "") + ) + assert result is not None + assert result.get("enabled") is False diff --git a/tests/gateway/test_reasoning_config_per_model.py b/tests/gateway/test_reasoning_config_per_model.py new file mode 100644 index 000000000000..f9799de33198 --- /dev/null +++ b/tests/gateway/test_reasoning_config_per_model.py @@ -0,0 +1,111 @@ +"""Tests for per-model reasoning_effort override in gateway _load_reasoning_config.""" + +import pytest + +import gateway.run as gateway_run + + +class TestGatewayPerModelReasoningConfig: + """Test GatewayRunner._load_reasoning_config respects per-model overrides.""" + + def test_per_model_override_takes_precedence(self, monkeypatch): + """Per-model override wins over global reasoning_effort.""" + from hermes_cli.config import DEFAULT_CONFIG + + fake_cfg = { + "model": {"default": "anthropic/claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["enabled"] is True + assert result["effort"] == "xhigh" + + def test_global_fallback_when_no_override(self, monkeypatch): + """Global reasoning_effort applies when no per-model override matches.""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "high", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["effort"] == "high" + + def test_spelling_tolerant_match_in_gateway(self, monkeypatch): + """Override matches even with different spelling (dots vs dashes).""" + fake_cfg = { + "model": {"default": "claude-opus-4-5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "claude-opus-4.5": "xhigh", # key has dots, model has dashes + }, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["effort"] == "xhigh" + + def test_no_overrides_dict(self, monkeypatch): + """Works fine when reasoning_overrides key is absent.""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "low", + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["effort"] == "low" + + def test_empty_overrides(self, monkeypatch): + """Empty overrides dict falls back to global.""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": {}, + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result["effort"] == "medium" + + def test_global_fallback_with_yaml_false(self, monkeypatch): + """YAML boolean False must reach parse_reasoning_effort uncoerced. + + Regression: str(... or "").strip() turned False into "", silently + re-enabling thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + """ + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": False, # YAML boolean, not string + }, + } + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + result = gateway_run.GatewayRunner._load_reasoning_config() + assert result is not None + assert result.get("enabled") is False diff --git a/tests/run_agent/test_fallback_reasoning_override.py b/tests/run_agent/test_fallback_reasoning_override.py new file mode 100644 index 000000000000..1c2c3e6228a7 --- /dev/null +++ b/tests/run_agent/test_fallback_reasoning_override.py @@ -0,0 +1,144 @@ +"""Tests for per-model reasoning_effort override during fallback activation. + +Tests that try_activate_fallback re-resolves reasoning_config when +swapping to a fallback model, so per-model overrides are honored even +during error recovery. +""" + +import pytest +from unittest.mock import MagicMock, patch + + +class TestFallbackReasoningOverride: + """Test try_activate_fallback re-resolves reasoning_config.""" + + def test_fallback_re_resolves_reasoning_config(self): + """When fallback activates, reasoning_config should be re-resolved. + + We test the resolution logic directly rather than spinning up a + full try_activate_fallback (which requires extensive agent setup). + The production code calls resolve_per_model_reasoning_effort with + the fallback model string — we verify that works correctly. + """ + from hermes_constants import resolve_per_model_reasoning_effort + + # Simulate: primary was gemini-flash (medium), fallback to claude-opus-4.5 (xhigh) + overrides = { + "claude-opus-4.5": "xhigh", + "gemini-flash": "medium", + } + + # Fallback model lookup + fb_result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert fb_result is not None + assert fb_result["effort"] == "xhigh" + + # Primary model lookup (for comparison) + primary_result = resolve_per_model_reasoning_effort("gemini-flash", overrides) + assert primary_result is not None + assert primary_result["effort"] == "medium" + + # The key point: fallback result differs from primary + assert fb_result["effort"] != primary_result["effort"] + + def test_fallback_to_model_without_override_uses_global(self): + """Fallback to a model with no override should resolve to None (→ global).""" + from hermes_constants import resolve_per_model_reasoning_effort + + overrides = {"claude-opus-4.5": "xhigh"} + + # Fallback to gpt-5 which has no override + result = resolve_per_model_reasoning_effort("gpt-5", overrides) + assert result is None # caller falls back to global + + def test_fallback_with_normalized_model_name(self): + """Fallback model name may be normalized (dots→dashes); override should still match.""" + from hermes_constants import resolve_per_model_reasoning_effort + + # User wrote key with dots, but normalize_model_for_provider converts to dashes + overrides = {"claude-sonnet-4.6": "high"} + + result = resolve_per_model_reasoning_effort("claude-sonnet-4-6", overrides) + assert result is not None + assert result["effort"] == "high" + + def test_fallback_recovery_restores_primary_reasoning(self): + """After fallback + restore_primary_runtime, reasoning_config returns to primary's value. + + This tests the integration of Task 6 (_primary_runtime snapshot) with + Task 6b (fallback re-resolution). The full cycle: + 1. Primary model = gemini-flash, reasoning = medium + 2. /model switch → _primary_runtime captures reasoning_config + 3. Fallback activates → reasoning re-resolved for fallback model + 4. restore_primary_runtime → reasoning_config restored from snapshot + """ + from agent.agent_runtime_helpers import restore_primary_runtime + + agent = MagicMock() + # Simulate: _primary_runtime was captured during /model switch + agent._primary_runtime = { + "model": "gemini-flash", + "provider": "google", + "base_url": "", + "api_mode": "openai", + "api_key": "key", + "client_kwargs": {}, + "use_prompt_caching": False, + "use_native_cache_layout": False, + "reasoning_config": {"enabled": True, "effort": "medium"}, + "compressor_model": "gemini-flash", + "compressor_base_url": "", + "compressor_api_key": "", + "compressor_provider": "", + "compressor_context_length": 0, + "compressor_api_mode": "", + "compressor_threshold_tokens": 0, + } + agent._fallback_activated = True + agent._fallback_index = 0 + agent._fallback_chain = [] + agent._fallback_model = None + agent._transport_cache = {} + agent._config_context_length = None + agent._rate_limited_until = 0 + # During fallback, reasoning was changed to xhigh (fallback model's override) + agent.model = "claude-opus-4.5" + agent.provider = "anthropic" + agent.reasoning_config = {"enabled": True, "effort": "xhigh"} + agent.context_compressor = MagicMock() + agent.base_url = "" + agent._anthropic_prompt_cache_policy = MagicMock(return_value=(False, False)) + agent._create_openai_client = MagicMock(return_value=MagicMock()) + agent._ensure_lmstudio_runtime_loaded = MagicMock() + + result = restore_primary_runtime(agent) + assert result is True + # reasoning_config should be restored to primary's value (medium) + assert agent.reasoning_config == {"enabled": True, "effort": "medium"} + + def test_fallback_global_fallback_with_yaml_false(self): + """Fallback global fallback must not coerce YAML boolean False. + + Regression: ``or ""`` turned False into "", silently re-enabling + thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + + The production code in try_activate_fallback does: + _fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "") + agent.reasoning_config = parse_reasoning_effort(_fb_global_effort) + We verify that passing the raw False (not coerced "") produces + the disabled config. + """ + from hermes_constants import parse_reasoning_effort + + # Simulate: no per-model override matches, global is YAML False + _fb_agent_cfg = {"reasoning_effort": False} + + # This is the exact line from try_activate_fallback's else branch. + # The bug was: _fb_global_effort = _fb_agent_cfg.get(...) or "" + # which turned False into "". The fix passes the raw value. + _fb_global_effort = _fb_agent_cfg.get("reasoning_effort", "") + result = parse_reasoning_effort(_fb_global_effort) + + assert result is not None + assert result.get("enabled") is False diff --git a/tests/run_agent/test_switch_model_reasoning_override.py b/tests/run_agent/test_switch_model_reasoning_override.py new file mode 100644 index 000000000000..ae304e118df3 --- /dev/null +++ b/tests/run_agent/test_switch_model_reasoning_override.py @@ -0,0 +1,220 @@ +"""Tests for per-model reasoning_effort override during /model switch. + +Tests that switch_model: +1. Re-resolves reasoning_config when switching to a model with an override +2. Falls back to global when switching to a model without an override +3. Saves reasoning_config into _primary_runtime for fallback recovery +""" + +import pytest +from unittest.mock import MagicMock, patch + + +class TestSwitchModelReasoningOverride: + """Test switch_model re-resolves reasoning_config on model switch.""" + + def _make_fake_agent(self, model="gpt-5", provider="openai"): + """Create a minimal fake agent for switch_model testing.""" + agent = MagicMock() + agent.model = model + agent.provider = provider + agent.base_url = "https://api.openai.com/v1" + agent.api_mode = "openai" + agent.api_key = "test-key" + agent._client_kwargs = {"api_key": "test-key", "base_url": "https://api.openai.com/v1"} + agent._use_prompt_caching = False + agent._use_native_cache_layout = False + agent.reasoning_config = {"enabled": True, "effort": "medium"} + agent._fallback_activated = False + agent._fallback_index = 0 + agent._fallback_chain = [] + agent._fallback_model = None + agent._config_context_length = None + agent._transport_cache = {} + agent.context_compressor = None + agent._cached_system_prompt = None + agent._anthropic_api_key = "" + agent._anthropic_base_url = None + agent._is_anthropic_oauth = False + agent._anthropic_prompt_cache_policy = MagicMock( + return_value=(False, False) + ) + agent._ensure_lmstudio_runtime_loaded = MagicMock() + agent._create_openai_client = MagicMock(return_value=MagicMock()) + return agent + + def test_primary_runtime_includes_reasoning_config(self): + """After switch_model, _primary_runtime should contain reasoning_config key.""" + from agent.agent_runtime_helpers import switch_model + + agent = self._make_fake_agent() + + fake_cfg = { + "model": {"default": "claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "claude-opus-4.5": "xhigh", + }, + }, + } + + with patch("hermes_cli.config.load_config", return_value=fake_cfg): + try: + switch_model( + agent, + new_model="claude-opus-4.5", + new_provider="anthropic", + base_url="https://api.anthropic.com", + api_mode="anthropic_messages", + ) + except Exception: + # Client creation may fail in test env; check _primary_runtime was set + pass + + assert hasattr(agent, "_primary_runtime") + assert "reasoning_config" in agent._primary_runtime + + def test_reasoning_config_resolves_to_override_on_switch(self): + """switch_model should resolve reasoning_config to per-model override.""" + from agent.agent_runtime_helpers import switch_model + + agent = self._make_fake_agent() + + fake_cfg = { + "model": {"default": "claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "claude-opus-4.5": "xhigh", + }, + }, + } + + with patch("hermes_cli.config.load_config", return_value=fake_cfg): + try: + switch_model( + agent, + new_model="claude-opus-4.5", + new_provider="anthropic", + base_url="https://api.anthropic.com", + api_mode="anthropic_messages", + ) + except Exception: + pass + + # reasoning_config should be updated to xhigh + assert agent.reasoning_config is not None + assert agent.reasoning_config.get("effort") == "xhigh" + + def test_reasoning_config_falls_back_to_global(self): + """switch_model should fall back to global when no override for new model.""" + from agent.agent_runtime_helpers import switch_model + + agent = self._make_fake_agent() + + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "low", + "reasoning_overrides": { + "claude-opus-4.5": "xhigh", # override for different model + }, + }, + } + + with patch("hermes_cli.config.load_config", return_value=fake_cfg): + try: + switch_model( + agent, + new_model="gpt-5", + new_provider="openai", + api_mode="openai", + ) + except Exception: + pass + + # No override for gpt-5 → should fall back to global "low" + assert agent.reasoning_config is not None + assert agent.reasoning_config.get("effort") == "low" + + def test_restore_primary_runtime_restores_reasoning(self): + """restore_primary_runtime should restore reasoning_config from snapshot.""" + from agent.agent_runtime_helpers import restore_primary_runtime + + agent = MagicMock() + agent._primary_runtime = { + "model": "claude-opus-4.5", + "provider": "anthropic", + "base_url": "https://api.anthropic.com", + "api_mode": "anthropic_messages", + "api_key": "key", + "client_kwargs": {}, + "use_prompt_caching": True, + "use_native_cache_layout": False, + "reasoning_config": {"enabled": True, "effort": "xhigh"}, + "compressor_model": "claude-opus-4.5", + "compressor_base_url": "", + "compressor_api_key": "", + "compressor_provider": "", + "compressor_context_length": 0, + "compressor_api_mode": "", + "compressor_threshold_tokens": 0, + "anthropic_api_key": "key", + "anthropic_base_url": "https://api.anthropic.com", + "is_anthropic_oauth": False, + } + agent._fallback_activated = True + agent._fallback_index = 0 + agent._fallback_chain = [] + agent._fallback_model = None + agent._transport_cache = {} + agent._config_context_length = None + agent._rate_limited_until = 0 + agent.model = "fallback-model" + agent.provider = "openai" + agent.reasoning_config = {"enabled": True, "effort": "medium"} + agent.context_compressor = MagicMock() + agent.base_url = "" + # Mock the methods restore_primary_runtime calls + agent._anthropic_prompt_cache_policy = MagicMock(return_value=(True, False)) + agent._create_openai_client = MagicMock(return_value=MagicMock()) + agent._ensure_lmstudio_runtime_loaded = MagicMock() + + result = restore_primary_runtime(agent) + assert result is True + assert agent.reasoning_config == {"enabled": True, "effort": "xhigh"} + + def test_switch_model_global_fallback_with_yaml_false(self): + """switch_model global fallback must not coerce YAML boolean False. + + Regression: str(... or "").strip() turned False into "", silently + re-enabling thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + """ + from agent.agent_runtime_helpers import switch_model + + agent = self._make_fake_agent() + + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": False, # YAML boolean, not string + "reasoning_overrides": {}, + }, + } + + with patch("hermes_cli.config.load_config", return_value=fake_cfg): + try: + switch_model( + agent, + new_model="gpt-5", + new_provider="openai", + api_mode="openai", + ) + except Exception: + pass + + # No override for gpt-5 → global fallback with raw False + assert agent.reasoning_config is not None + assert agent.reasoning_config.get("enabled") is False \ No newline at end of file diff --git a/tests/test_hermes_constants.py b/tests/test_hermes_constants.py index 44a449d91d08..5c70ebcb952b 100644 --- a/tests/test_hermes_constants.py +++ b/tests/test_hermes_constants.py @@ -490,6 +490,206 @@ class TestParseReasoningEffort: assert documented.issubset(set(VALID_REASONING_EFFORTS)) +class TestResolvePerModelReasoningEffort: + """Tests for resolve_per_model_reasoning_effort() — spelling-tolerant + per-model override lookup from agent.reasoning_overrides dict. + + Contract: the override key the user writes in config.yaml should match + regardless of how downstream consumers normalize the model string. + normalize_model_for_provider() converts dots to dashes and + adds/strips provider prefixes. Our resolver tolerates these + variations so the user's intent ("this model always gets xhigh") + is honored no matter which code path feeds the model string. + """ + + def test_exact_match(self): + """Exact model string match returns the parsed override.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "xhigh"} + result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "xhigh"} + + def test_none_when_no_matching_key(self): + """Model not in overrides returns None (caller falls back to global).""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "xhigh"} + assert resolve_per_model_reasoning_effort("gpt-5", overrides) is None + + def test_none_value_returns_disabled(self): + """Override set to 'none' returns {'enabled': False}.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "none"} + result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert result == {"enabled": False} + + def test_invalid_value_returns_none(self): + """Override with invalid effort falls back to None (global).""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "banana"} + assert resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) is None + + def test_none_or_empty_overrides_returns_none(self): + """None or empty overrides dict returns None.""" + from hermes_constants import resolve_per_model_reasoning_effort + assert resolve_per_model_reasoning_effort("claude-opus-4.5", None) is None + assert resolve_per_model_reasoning_effort("claude-opus-4.5", {}) is None + + def test_empty_model_returns_none(self): + """Empty model string returns None.""" + from hermes_constants import resolve_per_model_reasoning_effort + assert resolve_per_model_reasoning_effort("", {"gpt-5": "low"}) is None + + # --- Spelling tolerance layer --- + + def test_dots_to_dashes_variant(self): + """User wrote key with dots; input comes in normalized with dashes. + + normalize_model_for_provider converts claude-opus-4.5 → claude-opus-4-5 + for the anthropic provider. The user's override key should still match. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "xhigh"} + result = resolve_per_model_reasoning_effort("claude-opus-4-5", overrides) + assert result == {"enabled": True, "effort": "xhigh"} + + def test_dashes_to_dots_variant(self): + """User wrote key with dashes; input comes in with dots.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4-5": "high"} + result = resolve_per_model_reasoning_effort("claude-opus.4.5", overrides) + assert result == {"enabled": True, "effort": "high"} + + def test_strip_provider_prefix(self): + """User wrote key WITH provider prefix; input comes in bare. + + E.g. user config: model.default: claude-opus-4.5 (no prefix), + but override key: anthropic/claude-opus-4.5. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"anthropic/claude-opus-4.5": "high"} + result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "high"} + + def test_prepend_provider_prefix(self): + """User wrote key bare; input comes in WITH provider prefix. + + E.g. user config: model.default: anthropic/claude-opus-4.5, + but override key: claude-opus-4.5 (no prefix). + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "high"} + result = resolve_per_model_reasoning_effort("anthropic/claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "high"} + + def test_aggregator_prefix_stripping(self): + """openrouter/anthropic/claude-opus-4.5 should match key anthropic/claude-opus-4.5. + + Aggregator providers (openrouter) prepend their own name, + creating a triple-prefix. The resolver strips the aggregator + layer to find the user's two-segment key. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"anthropic/claude-opus-4.5": "xhigh"} + result = resolve_per_model_reasoning_effort("openrouter/anthropic/claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "xhigh"} + + def test_exact_match_wins_over_variant(self): + """Ambiguity resolution: exact match takes priority over a variant. + + If both 'claude-opus-4.5' (exact) and 'claude-opus-4-5' (dashes + variant) are keys, the exact input matches the exact key first. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "high", "claude-opus-4-5": "xhigh"} + result = resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) + assert result == {"enabled": True, "effort": "high"} + + def test_none_when_no_variant_matches(self): + """All variants exhausted without a match returns None.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"gpt-5": "low"} + assert resolve_per_model_reasoning_effort("claude-opus-4.5", overrides) is None + + def test_all_dotted_input_matches_canonical_key(self): + """Regression: all-dotted input (claude-opus.4.5) must match + canonical key (claude-opus-4.5). + + This was a real bug found by delegate review: the old + all_dashed = model.replace('.', '-') collapsed version dots, + making the canonical form unreachable from all-dotted input. + """ + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"claude-opus-4.5": "xhigh"} + result = resolve_per_model_reasoning_effort("claude-opus.4.5", overrides) + assert result is not None + assert result["effort"] == "xhigh" + + def test_different_models_do_not_match(self): + """No false positives: gemini-2.0-flash must not match gemini-flash.""" + from hermes_constants import resolve_per_model_reasoning_effort + overrides = {"gemini-flash": "low"} + assert resolve_per_model_reasoning_effort("gemini-2.0-flash", overrides) is None + + +class TestReasoningOverridesDefaultConfig: + """Tests for the agent.reasoning_overrides default config key (Task 2).""" + + def test_default_config_has_reasoning_overrides_key(self): + """DEFAULT_CONFIG['agent'] contains 'reasoning_overrides' as an empty dict.""" + from hermes_cli.config import DEFAULT_CONFIG + assert "reasoning_overrides" in DEFAULT_CONFIG["agent"] + assert DEFAULT_CONFIG["agent"]["reasoning_overrides"] == {} + + def test_config_version_bumped(self): + """Config version was bumped to signal the schema change for reasoning_overrides.""" + from hermes_cli.config import DEFAULT_CONFIG + assert DEFAULT_CONFIG.get("_config_version") >= 31 + + def test_load_config_preserves_user_reasoning_overrides(self, tmp_path, monkeypatch): + """User-added reasoning_overrides are preserved through load_config().""" + import yaml + from hermes_cli.config import load_config, get_config_path + + user_config = { + "agent": { + "reasoning_overrides": { + "anthropic/claude-opus-4-5": "high", + "openrouter/anthropic/claude-sonnet-4-6": "low", + } + } + } + config_path = tmp_path / "config.yaml" + config_path.write_text(yaml.safe_dump(user_config)) + + # load_config() reads from get_config_path() — patch its global reference + monkeypatch.setitem( + load_config.__globals__, "get_config_path", lambda: config_path + ) + + loaded = load_config() + assert loaded["agent"]["reasoning_overrides"] == { + "anthropic/claude-opus-4-5": "high", + "openrouter/anthropic/claude-sonnet-4-6": "low", + } + + def test_spelling_tolerant_lookup_works_with_user_config(self): + """resolve_per_model_reasoning_effort works with user-added overrides.""" + from hermes_constants import resolve_per_model_reasoning_effort + # User config with one override, query uses different spelling + overrides = { + "anthropic/claude-opus-4.5": "xhigh", # user wrote with dots + } + # Lookup with different spelling (bare, dashes) — should still match + result = resolve_per_model_reasoning_effort("claude-opus-4-5", overrides) + assert result == {"enabled": True, "effort": "xhigh"} + + # Another override, bare key + overrides2 = {"gpt-5": "low"} + # Lookup with provider prefix — should match + result2 = resolve_per_model_reasoning_effort("openai/gpt-5", overrides2) + assert result2 == {"enabled": True, "effort": "low"} + + class TestSecureParentDir: """Tests for secure_parent_dir() — prevents chmod on / or top-level dirs.""" diff --git a/tests/tui_gateway/test_reasoning_config_per_model.py b/tests/tui_gateway/test_reasoning_config_per_model.py new file mode 100644 index 000000000000..d3468ff530a9 --- /dev/null +++ b/tests/tui_gateway/test_reasoning_config_per_model.py @@ -0,0 +1,100 @@ +"""Tests for per-model reasoning_effort override in TUI gateway _load_reasoning_config.""" + +import pytest + +import tui_gateway.server as tui_server + + +class TestTUIPerModelReasoningConfig: + """Test tui_gateway _load_reasoning_config respects per-model overrides.""" + + def test_per_model_override_takes_precedence(self, monkeypatch): + """Per-model override wins over global reasoning_effort.""" + fake_cfg = { + "model": {"default": "anthropic/claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + + result = tui_server._load_reasoning_config() + assert result is not None + assert result["enabled"] is True + assert result["effort"] == "xhigh" + + def test_global_fallback_when_no_override(self, monkeypatch): + """Global reasoning_effort applies when no per-model override matches.""" + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": "high", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "xhigh", + }, + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + + result = tui_server._load_reasoning_config() + assert result is not None + assert result["effort"] == "high" + + def test_spelling_tolerant_match(self, monkeypatch): + """Override matches even with different spelling (provider prefix).""" + fake_cfg = { + "model": {"default": "claude-opus-4.5"}, + "agent": { + "reasoning_effort": "medium", + "reasoning_overrides": { + "anthropic/claude-opus-4.5": "high", # key has prefix, model doesn't + }, + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + + result = tui_server._load_reasoning_config() + assert result is not None + assert result["effort"] == "high" + + def test_parity_with_gateway_loader(self, monkeypatch): + """TUI and gateway loaders return identical results for same config.""" + import gateway.run as gateway_run + + fake_cfg = { + "model": {"default": "openrouter/anthropic/claude-sonnet-4.6"}, + "agent": { + "reasoning_effort": "low", + "reasoning_overrides": { + "claude-sonnet-4.6": "high", + }, + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + monkeypatch.setattr(gateway_run, "_load_gateway_runtime_config", lambda: fake_cfg) + + tui_result = tui_server._load_reasoning_config() + gw_result = gateway_run.GatewayRunner._load_reasoning_config() + assert tui_result == gw_result + + def test_global_fallback_with_yaml_false(self, monkeypatch): + """YAML boolean False must reach parse_reasoning_effort uncoerced. + + Regression: str(... or "").strip() turned False into "", silently + re-enabling thinking. The raw value must pass through so + parse_reasoning_effort(False) returns {'enabled': False}. + """ + fake_cfg = { + "model": {"default": "gpt-5"}, + "agent": { + "reasoning_effort": False, # YAML boolean, not string + }, + } + monkeypatch.setattr(tui_server, "_load_cfg", lambda: fake_cfg) + + result = tui_server._load_reasoning_config() + assert result is not None + assert result.get("enabled") is False diff --git a/tui_gateway/server.py b/tui_gateway/server.py index 83c79975e2ae..db32955ddd90 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -2563,13 +2563,33 @@ def _display_mouse_tracking(display: dict) -> str: def _load_reasoning_config() -> dict | None: - from hermes_constants import parse_reasoning_effort + """Load reasoning effort from config.yaml, respecting per-model overrides. - # Pass the raw value through — ``or ""`` would coerce a YAML boolean - # False (``reasoning_effort: false``/``off``/``no``) to "", silently - # re-enabling thinking for users who explicitly turned it off. + Per-model overrides (agent.reasoning_overrides) take precedence + over the global value when the current model matches a key + (spelling-tolerant). Closes #21256. + """ + from hermes_constants import parse_reasoning_effort, resolve_per_model_reasoning_effort + + cfg = _load_cfg() + + # Per-model override first + model_cfg = cfg.get("model") or {} + model = str( + (model_cfg.get("default", "") if isinstance(model_cfg, dict) else "") + or (model_cfg.get("model", "") if isinstance(model_cfg, dict) else "") + or "" + ).strip() + overrides = (cfg.get("agent") or {}).get("reasoning_overrides", {}) or {} + per_model = resolve_per_model_reasoning_effort(model, overrides) + if per_model is not None: + return per_model + + # Global fallback — pass the raw value through; ``or ""`` would coerce + # a YAML boolean False (``reasoning_effort: false``/``off``/``no``) to + # "", silently re-enabling thinking for users who explicitly turned it off. return parse_reasoning_effort( - (_load_cfg().get("agent") or {}).get("reasoning_effort", "") + (cfg.get("agent") or {}).get("reasoning_effort", "") ) diff --git a/website/docs/user-guide/configuration.md b/website/docs/user-guide/configuration.md index 40748ce23197..67f3a0ed5c5b 100644 --- a/website/docs/user-guide/configuration.md +++ b/website/docs/user-guide/configuration.md @@ -1301,6 +1301,37 @@ You can also change the reasoning effort at runtime with the `/reasoning` comman /reasoning hide # Hide model thinking ``` +#### Per-Model Reasoning Overrides + +You can set different reasoning effort levels for different models. This is useful when you want high reasoning for complex models but medium for faster ones: + +```yaml +agent: + reasoning_effort: "medium" # global default + reasoning_overrides: + "openrouter/anthropic/claude-opus-4.5": "xhigh" + "openai/gpt-5": "low" + "claude-sonnet-4.6": "high" # bare model name also works +``` + +The key matching is **spelling-tolerant** — any reasonable spelling will match: +- `claude-opus-4.5`, `claude-opus-4-5`, `claude-opus.4.5` (dots and dashes are interchangeable) +- `anthropic/claude-opus-4.5`, `openrouter/anthropic/claude-opus-4.5` (provider prefix optional) +- Exact matches take precedence over variants + +:::note +There is no `hermes config set` support for `reasoning_overrides` keys — edit the YAML file directly. This is because model names often contain dots (e.g. `claude-opus-4.5`), which conflict with the CLI's dotted-key syntax. +::: + +**Resolution priority:** + +1. Session-scoped `/reasoning --session` override (gateway only) +2. Per-model override from `agent.reasoning_overrides` (spelling-tolerant) +3. Global `agent.reasoning_effort` +4. Provider default + +The override applies automatically everywhere: CLI startup, messaging gateway, Desktop/TUI, cron jobs, `/model` mid-session switches, and fallback model activation. + ## Tool-Use Enforcement Some models occasionally describe intended actions as text instead of making tool calls ("I would run the tests..." instead of actually calling the terminal). Tool-use enforcement injects system prompt guidance that steers the model back to actually calling tools.